diff --git a/runtime/vm/bss_relocs.cc b/runtime/vm/bss_relocs.cc index b76eb86bd5e..56d8456051d 100644 --- a/runtime/vm/bss_relocs.cc +++ b/runtime/vm/bss_relocs.cc @@ -29,10 +29,10 @@ void BSS::InitializeBSSEntry(BSS::Relocation relocation, void BSS::Initialize(Thread* current, uword* bss_start, bool vm) { // TODO(https://dartbug.com/52579): Remove. - InitializeBSSEntry(Relocation::DRT_GetFfiCallbackMetadata, + InitializeBSSEntry(Relocation::DLRT_GetFfiCallbackMetadata, reinterpret_cast(DLRT_GetFfiCallbackMetadata), bss_start); - InitializeBSSEntry(Relocation::DRT_ExitTemporaryIsolate, + InitializeBSSEntry(Relocation::DLRT_ExitTemporaryIsolate, reinterpret_cast(DLRT_ExitTemporaryIsolate), bss_start); } diff --git a/runtime/vm/bss_relocs.h b/runtime/vm/bss_relocs.h index 37e21734fb4..97647e214c6 100644 --- a/runtime/vm/bss_relocs.h +++ b/runtime/vm/bss_relocs.h @@ -16,8 +16,8 @@ class BSS : public AllStatic { // portion of the BSS segment, so just the indices are shared, not the values // stored at the index. enum class Relocation : intptr_t { - DRT_GetFfiCallbackMetadata, // TODO(https://dartbug.com/52579): Remove. - DRT_ExitTemporaryIsolate, // TODO(https://dartbug.com/52579): Remove. + DLRT_GetFfiCallbackMetadata, // TODO(https://dartbug.com/52579): Remove. + DLRT_ExitTemporaryIsolate, // TODO(https://dartbug.com/52579): Remove. EndOfVmEntries, // We don't have any isolate group specific entries at the moment. diff --git a/runtime/vm/compiler/stub_code_compiler_arm64.cc b/runtime/vm/compiler/stub_code_compiler_arm64.cc index daffde03d50..e6b8cc44da6 100644 --- a/runtime/vm/compiler/stub_code_compiler_arm64.cc +++ b/runtime/vm/compiler/stub_code_compiler_arm64.cc @@ -488,6 +488,7 @@ void StubCodeCompiler::GenerateLoadFfiCallbackMetadataRuntimeFunction( void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { #if defined(USING_SIMULATOR) && !defined(DART_PRECOMPILER) // TODO(37299): FFI is not supported in SIMARM64. + // See Simulator::DoDirectedFfiCallback. __ Breakpoint(); #else Label body; @@ -552,7 +553,8 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { #if defined(DART_TARGET_OS_FUCHSIA) // TODO(https://dartbug.com/52579): Remove. if (FLAG_precompiled_mode) { - GenerateLoadBSSEntry(BSS::Relocation::DRT_GetFfiCallbackMetadata, R4, R9); + GenerateLoadBSSEntry(BSS::Relocation::DLRT_GetFfiCallbackMetadata, R4, + R9); } else { Label call; __ ldr(R4, compiler::Address::PC(2 * Instr::kInstrSize)); @@ -630,7 +632,7 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { #if defined(DART_TARGET_OS_FUCHSIA) // TODO(https://dartbug.com/52579): Remove. if (FLAG_precompiled_mode) { - GenerateLoadBSSEntry(BSS::Relocation::DRT_ExitTemporaryIsolate, R4, R9); + GenerateLoadBSSEntry(BSS::Relocation::DLRT_ExitTemporaryIsolate, R4, R9); } else { Label call; __ ldr(R4, compiler::Address::PC(2 * Instr::kInstrSize)); diff --git a/runtime/vm/compiler/stub_code_compiler_riscv.cc b/runtime/vm/compiler/stub_code_compiler_riscv.cc index dbd557df457..407492a5abd 100644 --- a/runtime/vm/compiler/stub_code_compiler_riscv.cc +++ b/runtime/vm/compiler/stub_code_compiler_riscv.cc @@ -400,7 +400,8 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { #if defined(DART_TARGET_OS_FUCHSIA) // TODO(https://dartbug.com/52579): Remove. if (FLAG_precompiled_mode) { - GenerateLoadBSSEntry(BSS::Relocation::DRT_GetFfiCallbackMetadata, T1, T2); + GenerateLoadBSSEntry(BSS::Relocation::DLRT_GetFfiCallbackMetadata, T1, + T2); } else { const intptr_t kPCRelativeLoadOffset = 12; intptr_t start = __ CodeSize(); @@ -486,7 +487,7 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { #if defined(DART_TARGET_OS_FUCHSIA) // TODO(https://dartbug.com/52579): Remove. if (FLAG_precompiled_mode) { - GenerateLoadBSSEntry(BSS::Relocation::DRT_ExitTemporaryIsolate, T1, T2); + GenerateLoadBSSEntry(BSS::Relocation::DLRT_ExitTemporaryIsolate, T1, T2); } else { const intptr_t kPCRelativeLoadOffset = 12; intptr_t start = __ CodeSize(); diff --git a/runtime/vm/compiler/stub_code_compiler_x64.cc b/runtime/vm/compiler/stub_code_compiler_x64.cc index b397e955197..1cae8552cd5 100644 --- a/runtime/vm/compiler/stub_code_compiler_x64.cc +++ b/runtime/vm/compiler/stub_code_compiler_x64.cc @@ -507,7 +507,7 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { #if defined(DART_TARGET_OS_FUCHSIA) // TODO(https://dartbug.com/52579): Remove. if (FLAG_precompiled_mode) { - GenerateLoadBSSEntry(BSS::Relocation::DRT_GetFfiCallbackMetadata, RAX, + GenerateLoadBSSEntry(BSS::Relocation::DLRT_GetFfiCallbackMetadata, RAX, TMP); } else { __ movq(RAX, Immediate( @@ -586,7 +586,8 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { #if defined(DART_TARGET_OS_FUCHSIA) // TODO(https://dartbug.com/52579): Remove. if (FLAG_precompiled_mode) { - GenerateLoadBSSEntry(BSS::Relocation::DRT_ExitTemporaryIsolate, RAX, TMP); + GenerateLoadBSSEntry(BSS::Relocation::DLRT_ExitTemporaryIsolate, RAX, + TMP); } else { __ movq(RAX, Immediate(reinterpret_cast(DLRT_ExitTemporaryIsolate))); diff --git a/runtime/vm/ffi_callback_metadata.cc b/runtime/vm/ffi_callback_metadata.cc index 93febd1c534..8d25f1542ec 100644 --- a/runtime/vm/ffi_callback_metadata.cc +++ b/runtime/vm/ffi_callback_metadata.cc @@ -13,6 +13,11 @@ namespace dart { +#if defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64) +extern "C" void SimulatorFfiCallbackTrampoline(); +extern "C" void SimulatorFfiCallbackTrampolineEnd(); +#endif + FfiCallbackMetadata::FfiCallbackMetadata() {} void FfiCallbackMetadata::EnsureStubPageLocked() { @@ -23,13 +28,20 @@ void FfiCallbackMetadata::EnsureStubPageLocked() { ASSERT_LESS_OR_EQUAL(VirtualMemory::PageSize(), kPageSize); +#if defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64) + const uword code_start = + reinterpret_cast(SimulatorFfiCallbackTrampoline); + const uword code_end = + reinterpret_cast(SimulatorFfiCallbackTrampolineEnd); + const uword page_start = code_start & ~(VirtualMemory::PageSize() - 1); +#else const Code& trampoline_code = StubCode::FfiCallbackTrampoline(); const uword code_start = trampoline_code.EntryPoint(); const uword code_end = code_start + trampoline_code.Size(); const uword page_start = code_start & ~(VirtualMemory::PageSize() - 1); - ASSERT_LESS_OR_EQUAL((code_start - page_start) + trampoline_code.Size(), RXMappingSize()); +#endif // Stub page uses a tight (unaligned) bound for the end of the code area. // Otherwise we can read past the end of the code area when doing DuplicateRX. @@ -38,16 +50,17 @@ void FfiCallbackMetadata::EnsureStubPageLocked() { offset_of_first_trampoline_in_page_ = code_start - page_start; -#if defined(DART_TARGET_OS_FUCHSIA) +#if defined(DART_TARGET_OS_FUCHSIA) || \ + (defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64)) // On Fuchsia we can't currently duplicate pages, so use the first page of // trampolines. Store the stub page's metadata in a separately allocated RW // page. // TODO(https://dartbug.com/52579): Remove. - fuchsia_metadata_page_ = VirtualMemory::AllocateAligned( + original_metadata_page_ = VirtualMemory::AllocateAligned( MappingSize(), MappingAlignment(), /*is_executable=*/false, /*is_compressed=*/false, "FfiCallbackMetadata::TrampolinePage"); Metadata* metadata = reinterpret_cast( - fuchsia_metadata_page_->start() + MetadataOffset()); + original_metadata_page_->start() + MetadataOffset()); for (intptr_t i = 0; i < NumCallbackTrampolinesPerPage(); ++i) { AddToFreeListLocked(&metadata[i]); } @@ -61,9 +74,10 @@ FfiCallbackMetadata::~FfiCallbackMetadata() { delete trampoline_pages_[i]; } -#if defined(DART_TARGET_OS_FUCHSIA) +#if defined(DART_TARGET_OS_FUCHSIA) || \ + (defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64)) // TODO(https://dartbug.com/52579): Remove. - delete fuchsia_metadata_page_; + delete original_metadata_page_; #endif // defined(DART_TARGET_OS_FUCHSIA) } @@ -92,7 +106,8 @@ void FfiCallbackMetadata::FillRuntimeFunction(VirtualMemory* page, } VirtualMemory* FfiCallbackMetadata::AllocateTrampolinePage() { -#if defined(DART_TARGET_OS_FUCHSIA) +#if defined(DART_TARGET_OS_FUCHSIA) || \ + (defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64)) // TODO(https://dartbug.com/52579): Remove. UNREACHABLE(); return nullptr; @@ -330,7 +345,10 @@ FfiCallbackMetadata::Trampoline FfiCallbackMetadata::TrampolineOfMetadata( const uword start = MappingStart(reinterpret_cast(metadata)); Metadata* metadatas = reinterpret_cast(start + MetadataOffset()); const uword index = metadata - metadatas; -#if defined(DART_TARGET_OS_FUCHSIA) +#if defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64) + return reinterpret_cast(SimulatorFfiCallbackTrampoline) + + index * kNativeCallbackTrampolineSize; +#elif defined(DART_TARGET_OS_FUCHSIA) return StubCode::FfiCallbackTrampoline().EntryPoint() + index * kNativeCallbackTrampolineSize; #else @@ -341,7 +359,8 @@ FfiCallbackMetadata::Trampoline FfiCallbackMetadata::TrampolineOfMetadata( FfiCallbackMetadata::Metadata* FfiCallbackMetadata::MetadataOfTrampoline( Trampoline trampoline) const { -#if defined(DART_TARGET_OS_FUCHSIA) +#if defined(DART_TARGET_OS_FUCHSIA) || \ + (defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64)) // On Fuchsia the metadata page is separate to the trampoline page. // TODO(https://dartbug.com/52579): Remove. const uword page_start = @@ -352,7 +371,7 @@ FfiCallbackMetadata::Metadata* FfiCallbackMetadata::MetadataOfTrampoline( kNativeCallbackTrampolineSize; ASSERT(index < NumCallbackTrampolinesPerPage()); Metadata* metadata_table = reinterpret_cast( - fuchsia_metadata_page_->start() + MetadataOffset()); + original_metadata_page_->start() + MetadataOffset()); return metadata_table + index; #else const uword start = MappingStart(trampoline); diff --git a/runtime/vm/ffi_callback_metadata.h b/runtime/vm/ffi_callback_metadata.h index dcc3640fcc2..2f596239b71 100644 --- a/runtime/vm/ffi_callback_metadata.h +++ b/runtime/vm/ffi_callback_metadata.h @@ -311,9 +311,17 @@ class FfiCallbackMetadata { Metadata* free_list_head_ = nullptr; Metadata* free_list_tail_ = nullptr; -#if defined(DART_TARGET_OS_FUCHSIA) +#if defined(DART_TARGET_OS_FUCHSIA) || \ + (defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64)) // TODO(https://dartbug.com/52579): Remove. - VirtualMemory* fuchsia_metadata_page_ = nullptr; + // On Fuchsia, we cannot duplicate the page containing the trampoline stub + // unless we plumb through from the embedder the VMO handle that was used to + // load the VM isolate snapshot. + // On simulator FFI, SimulatorFfiCallbackTrampoline cannot be duplicated + // because it contains a PC-relative call. It would need to be replaced with + // something like normal stub's PC-relative loading to a corresponding data + // page, or if we can assume the initial-exec code model a TLS load. + VirtualMemory* original_metadata_page_ = nullptr; #endif // defined(DART_TARGET_OS_FUCHSIA) DISALLOW_COPY_AND_ASSIGN(FfiCallbackMetadata); diff --git a/runtime/vm/native_arguments.h b/runtime/vm/native_arguments.h index fa1e2328cbf..1bbc371bfb9 100644 --- a/runtime/vm/native_arguments.h +++ b/runtime/vm/native_arguments.h @@ -22,10 +22,10 @@ class Thread; #if defined(TESTING) || defined(DEBUG) #if defined(USING_SIMULATOR) +// The simulator/isolate might not exist yet for an async FFI callback, so do +// not check. An equivalent check exists in the simulator itself. #define CHECK_STACK_ALIGNMENT \ { \ - uword current_sp = Simulator::Current()->get_register(SPREG); \ - ASSERT(Utils::IsAligned(current_sp, OS::ActivationFrameAlignment())); \ } #elif defined(DART_HOST_OS_WINDOWS) // The compiler may dynamically align the stack on Windows, so do not check. diff --git a/runtime/vm/simulator_arm.cc b/runtime/vm/simulator_arm.cc index f34890dda21..9aa8584a549 100644 --- a/runtime/vm/simulator_arm.cc +++ b/runtime/vm/simulator_arm.cc @@ -1414,6 +1414,9 @@ void Simulator::SupervisorCall(Instr* instr) { int svc = instr->SvcField(); switch (svc) { case Instr::kSimulatorRedirectCode: { + ASSERT(Utils::IsAligned(get_register(SPREG), + OS::ActivationFrameAlignment())); + SimulatorSetjmpBuffer buffer(this); if (!DART_SETJMP(buffer.buffer_)) { diff --git a/runtime/vm/simulator_arm64.cc b/runtime/vm/simulator_arm64.cc index 29b08965363..4c12e243ec7 100644 --- a/runtime/vm/simulator_arm64.cc +++ b/runtime/vm/simulator_arm64.cc @@ -18,6 +18,7 @@ #include "vm/image_snapshot.h" #include "vm/native_arguments.h" #include "vm/os_thread.h" +#include "vm/runtime_entry.h" #include "vm/stack_frame.h" namespace dart { @@ -1677,6 +1678,8 @@ void Simulator::DoRedirectedCall(Instr* instr) { // We can't instrument the runtime. memory_.FlushAll(); + ASSERT(Utils::IsAligned(get_register(SPREG), OS::ActivationFrameAlignment())); + SimulatorSetjmpBuffer buffer(this); if (!DART_SETJMP(buffer.buffer_)) { int64_t saved_lr = get_register(LR); @@ -1758,7 +1761,7 @@ struct CalloutContext { uword target; }; -extern "C" void FfiCalloutTrampoline(CalloutContext*); +extern "C" void SimulatorFfiCalloutTrampoline(CalloutContext*); void Simulator::DoRedirectedFfiCall(Instr* instr) { #if !defined(HOST_ARCH_ARM64) @@ -1778,15 +1781,26 @@ void Simulator::DoRedirectedFfiCall(Instr* instr) { CalloutContext ctxt; ctxt.simulator_stack_pointer = get_register(R31); ctxt.simulator_frame_pointer = get_register(FP); - for (intptr_t i = 0; i < 8; i++) { - ctxt.integer_arguments[i] = get_register(static_cast(R0 + i)); - ctxt.double_arguments[i] = - get_vregisterd(static_cast(V0 + i), 0); - } + ctxt.integer_arguments[0] = get_register(R0); + ctxt.integer_arguments[1] = get_register(R1); + ctxt.integer_arguments[2] = get_register(R2); + ctxt.integer_arguments[3] = get_register(R3); + ctxt.integer_arguments[4] = get_register(R4); + ctxt.integer_arguments[5] = get_register(R5); + ctxt.integer_arguments[6] = get_register(R6); + ctxt.integer_arguments[7] = get_register(R7); + ctxt.double_arguments[0] = get_vregisterd(V0, 0); + ctxt.double_arguments[1] = get_vregisterd(V1, 0); + ctxt.double_arguments[2] = get_vregisterd(V2, 0); + ctxt.double_arguments[3] = get_vregisterd(V3, 0); + ctxt.double_arguments[4] = get_vregisterd(V4, 0); + ctxt.double_arguments[5] = get_vregisterd(V5, 0); + ctxt.double_arguments[6] = get_vregisterd(V6, 0); + ctxt.double_arguments[7] = get_vregisterd(V7, 0); ctxt.r8 = get_register(R8); ctxt.target = external; - FfiCalloutTrampoline(&ctxt); + SimulatorFfiCalloutTrampoline(&ctxt); if (IsTracingExecution()) { THR_Print("Return from FFI function at 0x%" Px "\n", external); @@ -1797,6 +1811,8 @@ void Simulator::DoRedirectedFfiCall(Instr* instr) { set_register(instr, R1, ctxt.integer_arguments[1]); set_vregisterd(V0, 0, ctxt.double_arguments[0]); set_vregisterd(V1, 0, ctxt.double_arguments[1]); + set_vregisterd(V2, 0, ctxt.double_arguments[2]); + set_vregisterd(V3, 0, ctxt.double_arguments[3]); // Skip over hlt and blr. set_pc(saved_pc + 2 * Instr::kInstrSize); @@ -1806,6 +1822,104 @@ void Simulator::DoRedirectedFfiCall(Instr* instr) { #endif } +struct CallbackContext { + uword integer_arguments[8]; + uword double_arguments[8]; + uword r8; + uword sp; +}; + +extern "C" void DoRedirectedFfiCallback(CallbackContext* ctxt, + uword trampoline) { + uword entry_point; + uword trampoline_type; + Thread* thread = + DLRT_GetFfiCallbackMetadata(trampoline, &entry_point, &trampoline_type); + if (thread == nullptr) { + // If GetFfiCallbackMetadata returned a null thread, it means that the async + // callback was invoked after it was deleted. In this case, do nothing. + return; + } + + Simulator* sim = Simulator::Current(); + ASSERT(sim != nullptr); + sim->DoRedirectedFfiCallback(thread, ctxt, entry_point, trampoline_type); +} + +// Compare FfiCallbackTrampolineStub. +void Simulator::DoRedirectedFfiCallback(Thread* thread, + CallbackContext* ctxt, + uword entry_point, + uword trampoline_type) { + // The C caller might not be using frame pointers, so we just hard-code a + // maximum frame size instead of using FP-SP like we do for callouts. + constexpr intptr_t kStackSlotsCopied = 128; + + { + // + // stp lr, thr, [sp, -16]! + uword* sp = reinterpret_cast(get_register(R31, R31IsSP)); + uword* sp_in = reinterpret_cast(ctxt->sp); + for (intptr_t i = kStackSlotsCopied - 1; i >= 0; i--) { + *--sp = sp_in[i]; + } + *--sp = get_register(LR); + *--sp = get_register(THR); + set_register(nullptr, R31, reinterpret_cast(sp)); + COMPILE_ASSERT(FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta == + 2); + } + + set_register(nullptr, R0, ctxt->integer_arguments[0]); + set_register(nullptr, R1, ctxt->integer_arguments[1]); + set_register(nullptr, R2, ctxt->integer_arguments[2]); + set_register(nullptr, R3, ctxt->integer_arguments[3]); + set_register(nullptr, R4, ctxt->integer_arguments[4]); + set_register(nullptr, R5, ctxt->integer_arguments[5]); + set_register(nullptr, R6, ctxt->integer_arguments[6]); + set_register(nullptr, R7, ctxt->integer_arguments[7]); + set_vregisterd(V0, 0, ctxt->double_arguments[0]); + set_vregisterd(V1, 0, ctxt->double_arguments[1]); + set_vregisterd(V2, 0, ctxt->double_arguments[2]); + set_vregisterd(V3, 0, ctxt->double_arguments[3]); + set_vregisterd(V4, 0, ctxt->double_arguments[4]); + set_vregisterd(V5, 0, ctxt->double_arguments[5]); + set_vregisterd(V6, 0, ctxt->double_arguments[6]); + set_vregisterd(V7, 0, ctxt->double_arguments[7]); + set_register(nullptr, R8, ctxt->r8); + set_register(nullptr, THR, reinterpret_cast(thread)); + + set_register(nullptr, LR, kEndSimulatingPC); + set_pc(entry_point); + Execute(); + + ctxt->integer_arguments[0] = get_register(R0); + ctxt->integer_arguments[1] = get_register(R1); + ctxt->double_arguments[0] = get_vregisterd(V0, 0); + ctxt->double_arguments[1] = get_vregisterd(V1, 0); + ctxt->double_arguments[2] = get_vregisterd(V2, 0); + ctxt->double_arguments[3] = get_vregisterd(V3, 0); + + { + // ldp lr, thr, [sp], 16! + // + uword* sp = reinterpret_cast(get_register(R31, R31IsSP)); + set_register(nullptr, THR, *sp++); + set_register(nullptr, LR, *sp++); + sp += kStackSlotsCopied; + set_register(nullptr, R31, reinterpret_cast(sp)); + COMPILE_ASSERT(FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta == + 2); + } + + if (trampoline_type == + static_cast(FfiCallbackMetadata::TrampolineType::kAsync)) { + DLRT_ExitTemporaryIsolate(); + } else { + thread->EnterSafepointToNative(); + } +} + void Simulator::ClobberVolatileRegisters() { // Clear atomic reservation. exclusive_access_addr_ = exclusive_access_value_ = 0; @@ -3820,6 +3934,9 @@ void Simulator::Execute() { } else { ExecuteTrace(); } + + // We can't instrument the runtime. + memory_.FlushAll(); } void Simulator::ExecuteNoTrace() { @@ -3952,9 +4069,6 @@ int64_t Simulator::Call(int64_t entry, return_value = get_register(R0); } - // We can't instrument the runtime. - memory_.FlushAll(); - return return_value; } diff --git a/runtime/vm/simulator_arm64.h b/runtime/vm/simulator_arm64.h index bc812cf6fc3..9ef60d657dd 100644 --- a/runtime/vm/simulator_arm64.h +++ b/runtime/vm/simulator_arm64.h @@ -26,6 +26,7 @@ class Isolate; class Mutex; class SimulatorSetjmpBuffer; class Thread; +struct CallbackContext; typedef struct { union { @@ -99,6 +100,10 @@ class Simulator { int64_t parameter3, bool fp_return = false, bool fp_args = false); + void DoRedirectedFfiCallback(Thread* thread, + CallbackContext* ctxt, + uword entry_point, + uword trampoline_type); // Runtime and native call support. enum CallKind { diff --git a/runtime/vm/simulator_arm64_trampolines.S b/runtime/vm/simulator_arm64_trampolines.S index ccd19c79811..a927ec3ab54 100644 --- a/runtime/vm/simulator_arm64_trampolines.S +++ b/runtime/vm/simulator_arm64_trampolines.S @@ -6,17 +6,17 @@ // we need it be executable in contexts where we cannot JIT. (Alternatively, we // could start requiring the VM snapshot to be provided in every mode.) -#if defined(__aarch64__) +#if defined(__aarch64__) && defined(SIMULATOR_FFI) .text #if defined(__APPLE__) -.globl _FfiCalloutTrampoline -_FfiCalloutTrampoline: +.globl _SimulatorFfiCalloutTrampoline +_SimulatorFfiCalloutTrampoline: #else -.globl FfiCalloutTrampoline -.type FfiCalloutTrampoline, %function -FfiCalloutTrampoline: +.globl SimulatorFfiCalloutTrampoline +.type SimulatorFfiCalloutTrampoline, %function +SimulatorFfiCalloutTrampoline: #endif stp fp, lr, [sp, #-16]! mov fp, sp @@ -39,22 +39,14 @@ FfiCalloutTrampoline: // Load the ABI argument registers. Note that Dart FFI does not support // full 128-bit SIMD arguments, so we don't need to set the full V // registers. - ldr x0, [x19, #32] // CalloutContext.integer_arguments[0] - ldr x1, [x19, #40] - ldr x2, [x19, #48] - ldr x3, [x19, #56] - ldr x4, [x19, #64] - ldr x5, [x19, #72] - ldr x6, [x19, #80] - ldr x7, [x19, #88] - ldr d0, [x19, #96] // CalloutContext.double_arguments[0] - ldr d1, [x19, #104] - ldr d2, [x19, #112] - ldr d3, [x19, #120] - ldr d4, [x19, #128] - ldr d5, [x19, #136] - ldr d6, [x19, #144] - ldr d7, [x19, #152] + ldp x0, x1, [x19, #32] // CalloutContext.integer_arguments[0] + ldp x2, x3, [x19, #48] + ldp x4, x5, [x19, #64] + ldp x6, x7, [x19, #80] + ldp d0, d1, [x19, #96] // CalloutContext.double_arguments[0] + ldp d2, d3, [x19, #112] + ldp d4, d5, [x19, #128] + ldp d6, d7, [x19, #144] ldr x8, [x19, #160] // CalloutContext.r8 // Call target. @@ -62,17 +54,79 @@ FfiCalloutTrampoline: blr lr // Save the ABI result registers. - str x0, [x19, #32] // CalloutContext.integer_arguments[0] - str x1, [x19, #40] - str d0, [x19, #96] // CalloutContext.double_arguments[0] - str d1, [x19, #104] + stp x0, x1, [x19, #32] // CalloutContext.integer_arguments[0] + stp d0, d1, [x19, #96] // CalloutContext.double_arguments[0] + stp d2, d3, [x19, #112] add sp, fp, -16 ldp x19, x20, [sp], #16 ldp fp, lr, [sp], #16 ret #if !defined(__APPLE__) -.size FfiCalloutTrampoline,.-FfiCalloutTrampoline +.size SimulatorFfiCalloutTrampoline,.-SimulatorFfiCalloutTrampoline #endif -#endif // defined(__aarch64__) +#if defined(__APPLE__) +.globl _SimulatorFfiCallbackTrampoline +_SimulatorFfiCallbackTrampoline: +#else +.globl SimulatorFfiCallbackTrampoline +.type SimulatorFfiCallbackTrampoline, %function +SimulatorFfiCallbackTrampoline: +#endif + // FfiCallbackMetadata::NumCallbackTrampolinesPerPage() == 8150 + .rept 8150 + adr x9, 0 + b .Lbody + .endr +.Lbody: + mov x10, sp + stp fp, lr, [sp, #-16]! + mov fp, sp + + add sp, sp, -144 // Allocate CallbackContext + + // Save ABI argument registers. + stp x0, x1, [sp, #0] // CallbackContext.integer_arguments[0] + stp x2, x3, [sp, #16] + stp x4, x5, [sp, #32] + stp x6, x7, [sp, #48] + stp d0, d1, [sp, #64] // CallbackContext.double_arguments[0] + stp d2, d3, [sp, #80] + stp d4, d5, [sp, #96] + stp d6, d7, [sp, #112] + str x8, [sp, #128] // CallbackContext.r8 + str x10, [sp, #136] // CallbackContext.sp + + // Pass arguments registers and thunk address to the runtime. + mov x0, sp + mov x1, x9 // I.e., which callback. +#if defined(__APPLE__) + bl _DoRedirectedFfiCallback +#else + bl DoRedirectedFfiCallback +#endif + + // Load ABI result registers. + ldp x0, x1, [sp, #0] // CallbackContext.integer_arguments[0] + ldp d0, d1, [sp, #64] // CallbackContext.double_arguments[0] + ldp d2, d3, [sp, #80] + + mov sp, fp + ldp fp, lr, [sp], #16 + ret +#if !defined(__APPLE__) +.size SimulatorFfiCallbackTrampoline,.-SimulatorFfiCallbackTrampoline +#endif + +#if defined(__APPLE__) +.globl _SimulatorFfiCallbackTrampolineEnd +_SimulatorFfiCallbackTrampolineEnd: +#else +.globl SimulatorFfiCallbackTrampolineEnd +.type SimulatorFfiCallbackTrampolineEnd, %function +SimulatorFfiCallbackTrampolineEnd: +#endif + brk 0 + +#endif // defined(__aarch64__) && defined(SIMULATOR_FFI)