[vm, simarm64] Handle FFI callbacks.

TEST=ci
Bug: https://github.com/dart-lang/sdk/issues/60204
Change-Id: Ib2b5c9e9a8079258143be69a409a141e92a039a5
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/413560
Reviewed-by: Daco Harkes <dacoharkes@google.com>
Commit-Queue: Ryan Macnak <rmacnak@google.com>

runtime/vm/bss_relocs.cc

@@ -29,10 +29,10 @@ void BSS::InitializeBSSEntry(BSS::Relocation relocation,
 
 void BSS::Initialize(Thread* current, uword* bss_start, bool vm) {
   // TODO(https://dartbug.com/52579): Remove.
-  InitializeBSSEntry(Relocation::DRT_GetFfiCallbackMetadata,
+  InitializeBSSEntry(Relocation::DLRT_GetFfiCallbackMetadata,
                      reinterpret_cast<uword>(DLRT_GetFfiCallbackMetadata),
                      bss_start);
-  InitializeBSSEntry(Relocation::DRT_ExitTemporaryIsolate,
+  InitializeBSSEntry(Relocation::DLRT_ExitTemporaryIsolate,
                      reinterpret_cast<uword>(DLRT_ExitTemporaryIsolate),
                      bss_start);
 }

runtime/vm/bss_relocs.h

@@ -16,8 +16,8 @@ class BSS : public AllStatic {
   // portion of the BSS segment, so just the indices are shared, not the values
   // stored at the index.
   enum class Relocation : intptr_t {
-    DRT_GetFfiCallbackMetadata,  // TODO(https://dartbug.com/52579): Remove.
-    DRT_ExitTemporaryIsolate,    // TODO(https://dartbug.com/52579): Remove.
+    DLRT_GetFfiCallbackMetadata,  // TODO(https://dartbug.com/52579): Remove.
+    DLRT_ExitTemporaryIsolate,    // TODO(https://dartbug.com/52579): Remove.
     EndOfVmEntries,
 
     // We don't have any isolate group specific entries at the moment.

runtime/vm/compiler/stub_code_compiler_arm64.cc

@@ -488,6 +488,7 @@ void StubCodeCompiler::GenerateLoadFfiCallbackMetadataRuntimeFunction(
 void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
 #if defined(USING_SIMULATOR) && !defined(DART_PRECOMPILER)
   // TODO(37299): FFI is not supported in SIMARM64.
+  // See Simulator::DoDirectedFfiCallback.
   __ Breakpoint();
 #else
   Label body;
@@ -552,7 +553,8 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
 #if defined(DART_TARGET_OS_FUCHSIA)
     // TODO(https://dartbug.com/52579): Remove.
     if (FLAG_precompiled_mode) {
-      GenerateLoadBSSEntry(BSS::Relocation::DRT_GetFfiCallbackMetadata, R4, R9);
+      GenerateLoadBSSEntry(BSS::Relocation::DLRT_GetFfiCallbackMetadata, R4,
+                           R9);
     } else {
       Label call;
       __ ldr(R4, compiler::Address::PC(2 * Instr::kInstrSize));
@@ -630,7 +632,7 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
 #if defined(DART_TARGET_OS_FUCHSIA)
     // TODO(https://dartbug.com/52579): Remove.
     if (FLAG_precompiled_mode) {
-      GenerateLoadBSSEntry(BSS::Relocation::DRT_ExitTemporaryIsolate, R4, R9);
+      GenerateLoadBSSEntry(BSS::Relocation::DLRT_ExitTemporaryIsolate, R4, R9);
     } else {
       Label call;
       __ ldr(R4, compiler::Address::PC(2 * Instr::kInstrSize));

runtime/vm/compiler/stub_code_compiler_riscv.cc

@@ -400,7 +400,8 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
 #if defined(DART_TARGET_OS_FUCHSIA)
     // TODO(https://dartbug.com/52579): Remove.
     if (FLAG_precompiled_mode) {
-      GenerateLoadBSSEntry(BSS::Relocation::DRT_GetFfiCallbackMetadata, T1, T2);
+      GenerateLoadBSSEntry(BSS::Relocation::DLRT_GetFfiCallbackMetadata, T1,
+                           T2);
     } else {
       const intptr_t kPCRelativeLoadOffset = 12;
       intptr_t start = __ CodeSize();
@@ -486,7 +487,7 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
 #if defined(DART_TARGET_OS_FUCHSIA)
     // TODO(https://dartbug.com/52579): Remove.
     if (FLAG_precompiled_mode) {
-      GenerateLoadBSSEntry(BSS::Relocation::DRT_ExitTemporaryIsolate, T1, T2);
+      GenerateLoadBSSEntry(BSS::Relocation::DLRT_ExitTemporaryIsolate, T1, T2);
     } else {
       const intptr_t kPCRelativeLoadOffset = 12;
       intptr_t start = __ CodeSize();

runtime/vm/compiler/stub_code_compiler_x64.cc

@@ -507,7 +507,7 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
 #if defined(DART_TARGET_OS_FUCHSIA)
     // TODO(https://dartbug.com/52579): Remove.
     if (FLAG_precompiled_mode) {
-      GenerateLoadBSSEntry(BSS::Relocation::DRT_GetFfiCallbackMetadata, RAX,
+      GenerateLoadBSSEntry(BSS::Relocation::DLRT_GetFfiCallbackMetadata, RAX,
                            TMP);
     } else {
       __ movq(RAX, Immediate(
@@ -586,7 +586,8 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
 #if defined(DART_TARGET_OS_FUCHSIA)
     // TODO(https://dartbug.com/52579): Remove.
     if (FLAG_precompiled_mode) {
-      GenerateLoadBSSEntry(BSS::Relocation::DRT_ExitTemporaryIsolate, RAX, TMP);
+      GenerateLoadBSSEntry(BSS::Relocation::DLRT_ExitTemporaryIsolate, RAX,
+                           TMP);
     } else {
       __ movq(RAX,
               Immediate(reinterpret_cast<int64_t>(DLRT_ExitTemporaryIsolate)));

runtime/vm/ffi_callback_metadata.cc

@@ -13,6 +13,11 @@
 
 namespace dart {
 
+#if defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64)
+extern "C" void SimulatorFfiCallbackTrampoline();
+extern "C" void SimulatorFfiCallbackTrampolineEnd();
+#endif
+
 FfiCallbackMetadata::FfiCallbackMetadata() {}
 
 void FfiCallbackMetadata::EnsureStubPageLocked() {
@@ -23,13 +28,20 @@ void FfiCallbackMetadata::EnsureStubPageLocked() {
 
   ASSERT_LESS_OR_EQUAL(VirtualMemory::PageSize(), kPageSize);
 
+#if defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64)
+  const uword code_start =
+      reinterpret_cast<uword>(SimulatorFfiCallbackTrampoline);
+  const uword code_end =
+      reinterpret_cast<uword>(SimulatorFfiCallbackTrampolineEnd);
+  const uword page_start = code_start & ~(VirtualMemory::PageSize() - 1);
+#else
   const Code& trampoline_code = StubCode::FfiCallbackTrampoline();
   const uword code_start = trampoline_code.EntryPoint();
   const uword code_end = code_start + trampoline_code.Size();
   const uword page_start = code_start & ~(VirtualMemory::PageSize() - 1);
-
   ASSERT_LESS_OR_EQUAL((code_start - page_start) + trampoline_code.Size(),
                        RXMappingSize());
+#endif
 
   // Stub page uses a tight (unaligned) bound for the end of the code area.
   // Otherwise we can read past the end of the code area when doing DuplicateRX.
@@ -38,16 +50,17 @@ void FfiCallbackMetadata::EnsureStubPageLocked() {
 
   offset_of_first_trampoline_in_page_ = code_start - page_start;
 
-#if defined(DART_TARGET_OS_FUCHSIA)
+#if defined(DART_TARGET_OS_FUCHSIA) ||                                         \
+    (defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64))
   // On Fuchsia we can't currently duplicate pages, so use the first page of
   // trampolines. Store the stub page's metadata in a separately allocated RW
   // page.
   // TODO(https://dartbug.com/52579): Remove.
-  fuchsia_metadata_page_ = VirtualMemory::AllocateAligned(
+  original_metadata_page_ = VirtualMemory::AllocateAligned(
       MappingSize(), MappingAlignment(), /*is_executable=*/false,
       /*is_compressed=*/false, "FfiCallbackMetadata::TrampolinePage");
   Metadata* metadata = reinterpret_cast<Metadata*>(
-      fuchsia_metadata_page_->start() + MetadataOffset());
+      original_metadata_page_->start() + MetadataOffset());
   for (intptr_t i = 0; i < NumCallbackTrampolinesPerPage(); ++i) {
     AddToFreeListLocked(&metadata[i]);
   }
@@ -61,9 +74,10 @@ FfiCallbackMetadata::~FfiCallbackMetadata() {
     delete trampoline_pages_[i];
   }
 
-#if defined(DART_TARGET_OS_FUCHSIA)
+#if defined(DART_TARGET_OS_FUCHSIA) ||                                         \
+    (defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64))
   // TODO(https://dartbug.com/52579): Remove.
-  delete fuchsia_metadata_page_;
+  delete original_metadata_page_;
 #endif  // defined(DART_TARGET_OS_FUCHSIA)
 }
 
@@ -92,7 +106,8 @@ void FfiCallbackMetadata::FillRuntimeFunction(VirtualMemory* page,
 }
 
 VirtualMemory* FfiCallbackMetadata::AllocateTrampolinePage() {
-#if defined(DART_TARGET_OS_FUCHSIA)
+#if defined(DART_TARGET_OS_FUCHSIA) ||                                         \
+    (defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64))
   // TODO(https://dartbug.com/52579): Remove.
   UNREACHABLE();
   return nullptr;
@@ -330,7 +345,10 @@ FfiCallbackMetadata::Trampoline FfiCallbackMetadata::TrampolineOfMetadata(
   const uword start = MappingStart(reinterpret_cast<uword>(metadata));
   Metadata* metadatas = reinterpret_cast<Metadata*>(start + MetadataOffset());
   const uword index = metadata - metadatas;
-#if defined(DART_TARGET_OS_FUCHSIA)
+#if defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64)
+  return reinterpret_cast<uword>(SimulatorFfiCallbackTrampoline) +
+         index * kNativeCallbackTrampolineSize;
+#elif defined(DART_TARGET_OS_FUCHSIA)
   return StubCode::FfiCallbackTrampoline().EntryPoint() +
          index * kNativeCallbackTrampolineSize;
 #else
@@ -341,7 +359,8 @@ FfiCallbackMetadata::Trampoline FfiCallbackMetadata::TrampolineOfMetadata(
 
 FfiCallbackMetadata::Metadata* FfiCallbackMetadata::MetadataOfTrampoline(
     Trampoline trampoline) const {
-#if defined(DART_TARGET_OS_FUCHSIA)
+#if defined(DART_TARGET_OS_FUCHSIA) ||                                         \
+    (defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64))
   // On Fuchsia the metadata page is separate to the trampoline page.
   // TODO(https://dartbug.com/52579): Remove.
   const uword page_start =
@@ -352,7 +371,7 @@ FfiCallbackMetadata::Metadata* FfiCallbackMetadata::MetadataOfTrampoline(
       kNativeCallbackTrampolineSize;
   ASSERT(index < NumCallbackTrampolinesPerPage());
   Metadata* metadata_table = reinterpret_cast<Metadata*>(
-      fuchsia_metadata_page_->start() + MetadataOffset());
+      original_metadata_page_->start() + MetadataOffset());
   return metadata_table + index;
 #else
   const uword start = MappingStart(trampoline);

runtime/vm/ffi_callback_metadata.h

@@ -311,9 +311,17 @@ class FfiCallbackMetadata {
   Metadata* free_list_head_ = nullptr;
   Metadata* free_list_tail_ = nullptr;
 
-#if defined(DART_TARGET_OS_FUCHSIA)
+#if defined(DART_TARGET_OS_FUCHSIA) ||                                         \
+    (defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64))
   // TODO(https://dartbug.com/52579): Remove.
-  VirtualMemory* fuchsia_metadata_page_ = nullptr;
+  // On Fuchsia, we cannot duplicate the page containing the trampoline stub
+  // unless we plumb through from the embedder the VMO handle that was used to
+  // load the VM isolate snapshot.
+  // On simulator FFI, SimulatorFfiCallbackTrampoline cannot be duplicated
+  // because it contains a PC-relative call. It would need to be replaced with
+  // something like normal stub's PC-relative loading to a corresponding data
+  // page, or if we can assume the initial-exec code model a TLS load.
+  VirtualMemory* original_metadata_page_ = nullptr;
 #endif  // defined(DART_TARGET_OS_FUCHSIA)
 
   DISALLOW_COPY_AND_ASSIGN(FfiCallbackMetadata);

runtime/vm/native_arguments.h

@@ -22,10 +22,10 @@ class Thread;
 #if defined(TESTING) || defined(DEBUG)
 
 #if defined(USING_SIMULATOR)
+// The simulator/isolate might not exist yet for an async FFI callback, so do
+// not check. An equivalent check exists in the simulator itself.
 #define CHECK_STACK_ALIGNMENT                                                  \
   {                                                                            \
-    uword current_sp = Simulator::Current()->get_register(SPREG);              \
-    ASSERT(Utils::IsAligned(current_sp, OS::ActivationFrameAlignment()));      \
   }
 #elif defined(DART_HOST_OS_WINDOWS)
 // The compiler may dynamically align the stack on Windows, so do not check.

runtime/vm/simulator_arm.cc

@@ -1414,6 +1414,9 @@ void Simulator::SupervisorCall(Instr* instr) {
   int svc = instr->SvcField();
   switch (svc) {
     case Instr::kSimulatorRedirectCode: {
+      ASSERT(Utils::IsAligned(get_register(SPREG),
+                              OS::ActivationFrameAlignment()));
+
       SimulatorSetjmpBuffer buffer(this);
 
       if (!DART_SETJMP(buffer.buffer_)) {

runtime/vm/simulator_arm64.cc

@@ -18,6 +18,7 @@
 #include "vm/image_snapshot.h"
 #include "vm/native_arguments.h"
 #include "vm/os_thread.h"
+#include "vm/runtime_entry.h"
 #include "vm/stack_frame.h"
 
 namespace dart {
@@ -1677,6 +1678,8 @@ void Simulator::DoRedirectedCall(Instr* instr) {
   // We can't instrument the runtime.
   memory_.FlushAll();
 
+  ASSERT(Utils::IsAligned(get_register(SPREG), OS::ActivationFrameAlignment()));
+
   SimulatorSetjmpBuffer buffer(this);
   if (!DART_SETJMP(buffer.buffer_)) {
     int64_t saved_lr = get_register(LR);
@@ -1758,7 +1761,7 @@ struct CalloutContext {
   uword target;
 };
 
-extern "C" void FfiCalloutTrampoline(CalloutContext*);
+extern "C" void SimulatorFfiCalloutTrampoline(CalloutContext*);
 
 void Simulator::DoRedirectedFfiCall(Instr* instr) {
 #if !defined(HOST_ARCH_ARM64)
@@ -1778,15 +1781,26 @@ void Simulator::DoRedirectedFfiCall(Instr* instr) {
     CalloutContext ctxt;
     ctxt.simulator_stack_pointer = get_register(R31);
     ctxt.simulator_frame_pointer = get_register(FP);
-    for (intptr_t i = 0; i < 8; i++) {
-      ctxt.integer_arguments[i] = get_register(static_cast<Register>(R0 + i));
-      ctxt.double_arguments[i] =
-          get_vregisterd(static_cast<VRegister>(V0 + i), 0);
-    }
+    ctxt.integer_arguments[0] = get_register(R0);
+    ctxt.integer_arguments[1] = get_register(R1);
+    ctxt.integer_arguments[2] = get_register(R2);
+    ctxt.integer_arguments[3] = get_register(R3);
+    ctxt.integer_arguments[4] = get_register(R4);
+    ctxt.integer_arguments[5] = get_register(R5);
+    ctxt.integer_arguments[6] = get_register(R6);
+    ctxt.integer_arguments[7] = get_register(R7);
+    ctxt.double_arguments[0] = get_vregisterd(V0, 0);
+    ctxt.double_arguments[1] = get_vregisterd(V1, 0);
+    ctxt.double_arguments[2] = get_vregisterd(V2, 0);
+    ctxt.double_arguments[3] = get_vregisterd(V3, 0);
+    ctxt.double_arguments[4] = get_vregisterd(V4, 0);
+    ctxt.double_arguments[5] = get_vregisterd(V5, 0);
+    ctxt.double_arguments[6] = get_vregisterd(V6, 0);
+    ctxt.double_arguments[7] = get_vregisterd(V7, 0);
     ctxt.r8 = get_register(R8);
     ctxt.target = external;
 
-    FfiCalloutTrampoline(&ctxt);
+    SimulatorFfiCalloutTrampoline(&ctxt);
 
     if (IsTracingExecution()) {
       THR_Print("Return from FFI function at 0x%" Px "\n", external);
@@ -1797,6 +1811,8 @@ void Simulator::DoRedirectedFfiCall(Instr* instr) {
     set_register(instr, R1, ctxt.integer_arguments[1]);
     set_vregisterd(V0, 0, ctxt.double_arguments[0]);
     set_vregisterd(V1, 0, ctxt.double_arguments[1]);
+    set_vregisterd(V2, 0, ctxt.double_arguments[2]);
+    set_vregisterd(V3, 0, ctxt.double_arguments[3]);
 
     // Skip over hlt and blr.
     set_pc(saved_pc + 2 * Instr::kInstrSize);
@@ -1806,6 +1822,104 @@ void Simulator::DoRedirectedFfiCall(Instr* instr) {
 #endif
 }
 
+struct CallbackContext {
+  uword integer_arguments[8];
+  uword double_arguments[8];
+  uword r8;
+  uword sp;
+};
+
+extern "C" void DoRedirectedFfiCallback(CallbackContext* ctxt,
+                                        uword trampoline) {
+  uword entry_point;
+  uword trampoline_type;
+  Thread* thread =
+      DLRT_GetFfiCallbackMetadata(trampoline, &entry_point, &trampoline_type);
+  if (thread == nullptr) {
+    // If GetFfiCallbackMetadata returned a null thread, it means that the async
+    // callback was invoked after it was deleted. In this case, do nothing.
+    return;
+  }
+
+  Simulator* sim = Simulator::Current();
+  ASSERT(sim != nullptr);
+  sim->DoRedirectedFfiCallback(thread, ctxt, entry_point, trampoline_type);
+}
+
+// Compare FfiCallbackTrampolineStub.
+void Simulator::DoRedirectedFfiCallback(Thread* thread,
+                                        CallbackContext* ctxt,
+                                        uword entry_point,
+                                        uword trampoline_type) {
+  // The C caller might not be using frame pointers, so we just hard-code a
+  // maximum frame size instead of using FP-SP like we do for callouts.
+  constexpr intptr_t kStackSlotsCopied = 128;
+
+  {
+    // <copy stack arguments>
+    // stp lr, thr, [sp, -16]!
+    uword* sp = reinterpret_cast<uword*>(get_register(R31, R31IsSP));
+    uword* sp_in = reinterpret_cast<uword*>(ctxt->sp);
+    for (intptr_t i = kStackSlotsCopied - 1; i >= 0; i--) {
+      *--sp = sp_in[i];
+    }
+    *--sp = get_register(LR);
+    *--sp = get_register(THR);
+    set_register(nullptr, R31, reinterpret_cast<uword>(sp));
+    COMPILE_ASSERT(FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta ==
+                   2);
+  }
+
+  set_register(nullptr, R0, ctxt->integer_arguments[0]);
+  set_register(nullptr, R1, ctxt->integer_arguments[1]);
+  set_register(nullptr, R2, ctxt->integer_arguments[2]);
+  set_register(nullptr, R3, ctxt->integer_arguments[3]);
+  set_register(nullptr, R4, ctxt->integer_arguments[4]);
+  set_register(nullptr, R5, ctxt->integer_arguments[5]);
+  set_register(nullptr, R6, ctxt->integer_arguments[6]);
+  set_register(nullptr, R7, ctxt->integer_arguments[7]);
+  set_vregisterd(V0, 0, ctxt->double_arguments[0]);
+  set_vregisterd(V1, 0, ctxt->double_arguments[1]);
+  set_vregisterd(V2, 0, ctxt->double_arguments[2]);
+  set_vregisterd(V3, 0, ctxt->double_arguments[3]);
+  set_vregisterd(V4, 0, ctxt->double_arguments[4]);
+  set_vregisterd(V5, 0, ctxt->double_arguments[5]);
+  set_vregisterd(V6, 0, ctxt->double_arguments[6]);
+  set_vregisterd(V7, 0, ctxt->double_arguments[7]);
+  set_register(nullptr, R8, ctxt->r8);
+  set_register(nullptr, THR, reinterpret_cast<uword>(thread));
+
+  set_register(nullptr, LR, kEndSimulatingPC);
+  set_pc(entry_point);
+  Execute();
+
+  ctxt->integer_arguments[0] = get_register(R0);
+  ctxt->integer_arguments[1] = get_register(R1);
+  ctxt->double_arguments[0] = get_vregisterd(V0, 0);
+  ctxt->double_arguments[1] = get_vregisterd(V1, 0);
+  ctxt->double_arguments[2] = get_vregisterd(V2, 0);
+  ctxt->double_arguments[3] = get_vregisterd(V3, 0);
+
+  {
+    // ldp lr, thr, [sp], 16!
+    // <drop arguments>
+    uword* sp = reinterpret_cast<uword*>(get_register(R31, R31IsSP));
+    set_register(nullptr, THR, *sp++);
+    set_register(nullptr, LR, *sp++);
+    sp += kStackSlotsCopied;
+    set_register(nullptr, R31, reinterpret_cast<uword>(sp));
+    COMPILE_ASSERT(FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta ==
+                   2);
+  }
+
+  if (trampoline_type ==
+      static_cast<uword>(FfiCallbackMetadata::TrampolineType::kAsync)) {
+    DLRT_ExitTemporaryIsolate();
+  } else {
+    thread->EnterSafepointToNative();
+  }
+}
+
 void Simulator::ClobberVolatileRegisters() {
   // Clear atomic reservation.
   exclusive_access_addr_ = exclusive_access_value_ = 0;
@@ -3820,6 +3934,9 @@ void Simulator::Execute() {
   } else {
     ExecuteTrace();
   }
+
+  // We can't instrument the runtime.
+  memory_.FlushAll();
 }
 
 void Simulator::ExecuteNoTrace() {
@@ -3952,9 +4069,6 @@ int64_t Simulator::Call(int64_t entry,
     return_value = get_register(R0);
   }
 
-  // We can't instrument the runtime.
-  memory_.FlushAll();
-
   return return_value;
 }
 

runtime/vm/simulator_arm64.h

@@ -26,6 +26,7 @@ class Isolate;
 class Mutex;
 class SimulatorSetjmpBuffer;
 class Thread;
+struct CallbackContext;
 
 typedef struct {
   union {
@@ -99,6 +100,10 @@ class Simulator {
                int64_t parameter3,
                bool fp_return = false,
                bool fp_args = false);
+  void DoRedirectedFfiCallback(Thread* thread,
+                               CallbackContext* ctxt,
+                               uword entry_point,
+                               uword trampoline_type);
 
   // Runtime and native call support.
   enum CallKind {

runtime/vm/simulator_arm64_trampolines.S

@@ -6,17 +6,17 @@
 // we need it be executable in contexts where we cannot JIT. (Alternatively, we
 // could start requiring the VM snapshot to be provided in every mode.)
 
-#if defined(__aarch64__)
+#if defined(__aarch64__) && defined(SIMULATOR_FFI)
 
 .text
 
 #if defined(__APPLE__)
-.globl _FfiCalloutTrampoline
-_FfiCalloutTrampoline:
+.globl _SimulatorFfiCalloutTrampoline
+_SimulatorFfiCalloutTrampoline:
 #else
-.globl FfiCalloutTrampoline
-.type FfiCalloutTrampoline, %function
-FfiCalloutTrampoline:
+.globl SimulatorFfiCalloutTrampoline
+.type SimulatorFfiCalloutTrampoline, %function
+SimulatorFfiCalloutTrampoline:
 #endif
   stp fp, lr, [sp, #-16]!
   mov fp, sp
@@ -39,22 +39,14 @@ FfiCalloutTrampoline:
   // Load the ABI argument registers. Note that Dart FFI does not support
   // full 128-bit SIMD arguments, so we don't need to set the full V
   // registers.
-  ldr x0, [x19, #32]  // CalloutContext.integer_arguments[0]
-  ldr x1, [x19, #40]
-  ldr x2, [x19, #48]
-  ldr x3, [x19, #56]
-  ldr x4, [x19, #64]
-  ldr x5, [x19, #72]
-  ldr x6, [x19, #80]
-  ldr x7, [x19, #88]
-  ldr d0, [x19, #96]  // CalloutContext.double_arguments[0]
-  ldr d1, [x19, #104]
-  ldr d2, [x19, #112]
-  ldr d3, [x19, #120]
-  ldr d4, [x19, #128]
-  ldr d5, [x19, #136]
-  ldr d6, [x19, #144]
-  ldr d7, [x19, #152]
+  ldp x0, x1, [x19, #32]  // CalloutContext.integer_arguments[0]
+  ldp x2, x3, [x19, #48]
+  ldp x4, x5, [x19, #64]
+  ldp x6, x7, [x19, #80]
+  ldp d0, d1, [x19, #96]  // CalloutContext.double_arguments[0]
+  ldp d2, d3, [x19, #112]
+  ldp d4, d5, [x19, #128]
+  ldp d6, d7, [x19, #144]
   ldr x8, [x19, #160]  // CalloutContext.r8
 
   // Call target.
@@ -62,17 +54,79 @@ FfiCalloutTrampoline:
   blr lr
 
   // Save the ABI result registers.
-  str x0, [x19, #32]  // CalloutContext.integer_arguments[0]
-  str x1, [x19, #40]
-  str d0, [x19, #96]  // CalloutContext.double_arguments[0]
-  str d1, [x19, #104]
+  stp x0, x1, [x19, #32]  // CalloutContext.integer_arguments[0]
+  stp d0, d1, [x19, #96]  // CalloutContext.double_arguments[0]
+  stp d2, d3, [x19, #112]
 
   add sp, fp, -16
   ldp x19, x20, [sp], #16
   ldp fp, lr, [sp], #16
   ret
 #if !defined(__APPLE__)
-.size FfiCalloutTrampoline,.-FfiCalloutTrampoline
+.size SimulatorFfiCalloutTrampoline,.-SimulatorFfiCalloutTrampoline
 #endif
 
-#endif  // defined(__aarch64__)
+#if defined(__APPLE__)
+.globl _SimulatorFfiCallbackTrampoline
+_SimulatorFfiCallbackTrampoline:
+#else
+.globl SimulatorFfiCallbackTrampoline
+.type SimulatorFfiCallbackTrampoline, %function
+SimulatorFfiCallbackTrampoline:
+#endif
+  // FfiCallbackMetadata::NumCallbackTrampolinesPerPage() == 8150
+  .rept 8150
+  adr x9, 0
+  b .Lbody
+  .endr
+.Lbody:
+  mov x10, sp
+  stp fp, lr, [sp, #-16]!
+  mov fp, sp
+
+  add sp, sp, -144  // Allocate CallbackContext
+
+  // Save ABI argument registers.
+  stp x0, x1, [sp, #0]  // CallbackContext.integer_arguments[0]
+  stp x2, x3, [sp, #16]
+  stp x4, x5, [sp, #32]
+  stp x6, x7, [sp, #48]
+  stp d0, d1, [sp, #64]  // CallbackContext.double_arguments[0]
+  stp d2, d3, [sp, #80]
+  stp d4, d5, [sp, #96]
+  stp d6, d7, [sp, #112]
+  str x8, [sp, #128]  // CallbackContext.r8
+  str x10, [sp, #136]  // CallbackContext.sp
+
+  // Pass arguments registers and thunk address to the runtime.
+  mov x0, sp
+  mov x1, x9  // I.e., which callback.
+#if defined(__APPLE__)
+  bl _DoRedirectedFfiCallback
+#else
+  bl DoRedirectedFfiCallback
+#endif
+
+  // Load ABI result registers.
+  ldp x0, x1, [sp, #0]  // CallbackContext.integer_arguments[0]
+  ldp d0, d1, [sp, #64]  // CallbackContext.double_arguments[0]
+  ldp d2, d3, [sp, #80]
+
+  mov sp, fp
+  ldp fp, lr, [sp], #16
+  ret
+#if !defined(__APPLE__)
+.size SimulatorFfiCallbackTrampoline,.-SimulatorFfiCallbackTrampoline
+#endif
+
+#if defined(__APPLE__)
+.globl _SimulatorFfiCallbackTrampolineEnd
+_SimulatorFfiCallbackTrampolineEnd:
+#else
+.globl SimulatorFfiCallbackTrampolineEnd
+.type SimulatorFfiCallbackTrampolineEnd, %function
+SimulatorFfiCallbackTrampolineEnd:
+#endif
+  brk 0
+
+#endif  // defined(__aarch64__) && defined(SIMULATOR_FFI)