diff --git a/runtime/vm/ffi_callback_metadata.cc b/runtime/vm/ffi_callback_metadata.cc index ab7fc9a7d41..90b74d34b7d 100644 --- a/runtime/vm/ffi_callback_metadata.cc +++ b/runtime/vm/ffi_callback_metadata.cc @@ -29,28 +29,26 @@ void FfiCallbackMetadata::EnsureStubPageLocked() { ASSERT_LESS_OR_EQUAL(VirtualMemory::PageSize(), kPageSize); + uword code_start, code_end, code_size; #if defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64) - uword code_start, code_end, page_start; if (FLAG_use_simulator) { code_start = reinterpret_cast(SimulatorFfiCallbackTrampoline); code_end = reinterpret_cast(SimulatorFfiCallbackTrampolineEnd); - page_start = code_start & ~(VirtualMemory::PageSize() - 1); + code_size = code_end - code_start; } else { const Code& trampoline_code = StubCode::FfiCallbackTrampoline(); code_start = trampoline_code.EntryPoint(); code_end = code_start + trampoline_code.Size(); - page_start = code_start & ~(VirtualMemory::PageSize() - 1); - ASSERT_LESS_OR_EQUAL((code_start - page_start) + trampoline_code.Size(), - RXMappingSize()); + code_size = trampoline_code.Size(); } #else const Code& trampoline_code = StubCode::FfiCallbackTrampoline(); - const uword code_start = trampoline_code.EntryPoint(); - const uword code_end = code_start + trampoline_code.Size(); - const uword page_start = code_start & ~(VirtualMemory::PageSize() - 1); - ASSERT_LESS_OR_EQUAL((code_start - page_start) + trampoline_code.Size(), - RXMappingSize()); + code_start = trampoline_code.EntryPoint(); + code_end = code_start + trampoline_code.Size(); + code_size = trampoline_code.Size(); #endif + const uword page_start = code_start & ~(VirtualMemory::PageSize() - 1); + ASSERT_LESS_OR_EQUAL((code_start - page_start) + code_size, RXMappingSize()); // Stub page uses a tight (unaligned) bound for the end of the code area. // Otherwise we can read past the end of the code area when doing DuplicateRX. @@ -58,19 +56,6 @@ void FfiCallbackMetadata::EnsureStubPageLocked() { code_end - page_start); offset_of_first_trampoline_in_page_ = code_start - page_start; - -#if defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64) - if (FLAG_use_simulator) { - original_metadata_page_ = VirtualMemory::AllocateAligned( - MappingSize(), MappingAlignment(), /*is_executable=*/false, - /*is_compressed=*/false, "FfiCallbackMetadata::TrampolinePage"); - MetadataEntry* metadata_entry = reinterpret_cast( - original_metadata_page_->start() + MetadataOffset()); - for (intptr_t i = 0; i < NumCallbackTrampolinesPerPage(); ++i) { - AddToFreeListLocked(&metadata_entry[i]); - } - } -#endif // defined(DART_TARGET_OS_FUCHSIA) } FfiCallbackMetadata::~FfiCallbackMetadata() { @@ -79,11 +64,6 @@ FfiCallbackMetadata::~FfiCallbackMetadata() { for (intptr_t i = 0; i < trampoline_pages_.length(); ++i) { delete trampoline_pages_[i]; } - -#if defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64) - // TODO(https://dartbug.com/52579): Remove. - delete original_metadata_page_; -#endif // defined(DART_TARGET_OS_FUCHSIA) } void FfiCallbackMetadata::Init() { @@ -117,13 +97,6 @@ void FfiCallbackMetadata::FillRuntimeFunction(VirtualMemory* page, } VirtualMemory* FfiCallbackMetadata::AllocateTrampolinePage() { -#if defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64) - if (FLAG_use_simulator) { - UNREACHABLE(); - return nullptr; - } -#endif - #if defined(DART_HOST_OS_MACOS) && defined(DART_PRECOMPILED_RUNTIME) const bool should_remap_stub_page = true; #else @@ -196,6 +169,12 @@ VirtualMemory* FfiCallbackMetadata::AllocateTrampolinePage() { return new_page; } +#if defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64) +struct CallbackContext; +extern "C" void DoRedirectedFfiCallback(CallbackContext* ctxt, + uword trampoline); +#endif + void FfiCallbackMetadata::EnsureFreeListNotEmptyLocked() { ASSERT(lock_.IsOwnedByCurrentThread()); EnsureStubPageLocked(); @@ -213,6 +192,10 @@ void FfiCallbackMetadata::EnsureFreeListNotEmptyLocked() { // Fill in the runtime functions. FillRuntimeFunction(new_page, kGetFfiCallbackMetadata, reinterpret_cast(DLRT_GetFfiCallbackMetadata)); +#if defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64) + FillRuntimeFunction(new_page, kDoRedirectedFfiCallback, + reinterpret_cast(DoRedirectedFfiCallback)); +#endif // Add all the trampolines to the free list. const intptr_t trampolines_per_page = NumCallbackTrampolinesPerPage(); @@ -416,44 +399,12 @@ FfiCallbackMetadata::Trampoline FfiCallbackMetadata::TrampolineOfMetadataEntry( MetadataEntry* metadata_entries = reinterpret_cast(start + MetadataOffset()); const uword index = metadata_entry - metadata_entries; -#if defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64) - if (FLAG_use_simulator) { - return reinterpret_cast(SimulatorFfiCallbackTrampoline) + - index * kNativeCallbackTrampolineSize; - } else { - return start + offset_of_first_trampoline_in_page_ + - index * kNativeCallbackTrampolineSize; - } -#else return start + offset_of_first_trampoline_in_page_ + index * kNativeCallbackTrampolineSize; -#endif } FfiCallbackMetadata::MetadataEntry* FfiCallbackMetadata::MetadataEntryOfTrampoline(Trampoline trampoline) const { -#if defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64) - if (FLAG_use_simulator) { - const uword page_start = - Utils::RoundDown(trampoline - offset_of_first_trampoline_in_page_, - VirtualMemory::PageSize()); - const uword index = - (trampoline - offset_of_first_trampoline_in_page_ - page_start) / - kNativeCallbackTrampolineSize; - ASSERT(index < NumCallbackTrampolinesPerPage()); - MetadataEntry* metadata_etnry_table = reinterpret_cast( - original_metadata_page_->start() + MetadataOffset()); - return metadata_etnry_table + index; - } else { - const uword start = MappingStart(trampoline); - MetadataEntry* metadata_entries = - reinterpret_cast(start + MetadataOffset()); - const uword index = - (trampoline - start - offset_of_first_trampoline_in_page_) / - kNativeCallbackTrampolineSize; - return &metadata_entries[index]; - } -#else const uword start = MappingStart(trampoline); MetadataEntry* metadata_entries = reinterpret_cast(start + MetadataOffset()); @@ -461,7 +412,6 @@ FfiCallbackMetadata::MetadataEntryOfTrampoline(Trampoline trampoline) const { (trampoline - start - offset_of_first_trampoline_in_page_) / kNativeCallbackTrampolineSize; return &metadata_entries[index]; -#endif } FfiCallbackMetadata::Metadata diff --git a/runtime/vm/ffi_callback_metadata.h b/runtime/vm/ffi_callback_metadata.h index a323290bfd0..0b5e80609d6 100644 --- a/runtime/vm/ffi_callback_metadata.h +++ b/runtime/vm/ffi_callback_metadata.h @@ -62,10 +62,7 @@ class FfiCallbackMetadata { enum RuntimeFunctions { kGetFfiCallbackMetadata, - kExitTemporaryIsolate, - kExitIsolateGroupBoundIsolate, - kExitSyncCallbackTargetIsolate, - kExitSyncCallback, + kDoRedirectedFfiCallback, kNumRuntimeFunctions, }; @@ -397,15 +394,6 @@ class FfiCallbackMetadata { MetadataEntry* free_list_head_ = nullptr; MetadataEntry* free_list_tail_ = nullptr; -#if (defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64)) - // TODO(https://dartbug.com/52579): Remove. - // On simulator FFI, SimulatorFfiCallbackTrampoline cannot be duplicated - // because it contains a PC-relative call. It would need to be replaced with - // something like normal stub's PC-relative loading to a corresponding data - // page, or if we can assume the initial-exec code model a TLS load. - VirtualMemory* original_metadata_page_ = nullptr; -#endif // defined(DART_TARGET_OS_FUCHSIA) - DISALLOW_COPY_AND_ASSIGN(FfiCallbackMetadata); }; diff --git a/runtime/vm/ffi_trampolines_arm64.S b/runtime/vm/ffi_trampolines_arm64.S index 2871c454f9b..67aa7192860 100644 --- a/runtime/vm/ffi_trampolines_arm64.S +++ b/runtime/vm/ffi_trampolines_arm64.S @@ -6,6 +6,12 @@ // This is not written as VM stub because we need it be executable in contexts where we cannot JIT. // (Alternatively, we could start requiring the VM snapshot to be provided in every mode.) +#if defined(__APPLE__) +#define LOCAL_SYMBOL(x) L##x +#else +#define LOCAL_SYMBOL(x) .L##x +#endif + #if defined(__aarch64__) && (defined(SIMULATOR_FFI) || (defined(DART_DYNAMIC_MODULES) && !defined(DART_PRECOMPILED_RUNTIME))) .text @@ -31,11 +37,11 @@ FfiCallTrampoline: // Copy top frame from Dart stack to C stack ldr x0, [x19, #0] // FfiCallArguments.stack_area ldr x1, [x19, #8] // FfiCallArguments.stack_area_end -.Lcopy1: +LOCAL_SYMBOL(copy1): ldp x2, x3, [x1, #-16]! // From stack_area stp x2, x3, [sp, #-16]! // To C SP cmp x1, x0 - b.gt .Lcopy1 + b.gt LOCAL_SYMBOL(copy1) // Load the ABI argument registers. Note that Dart FFI does not support // full 128-bit SIMD arguments, so we don't need to set the full V @@ -92,11 +98,11 @@ SimulatorFfiCalloutTrampoline: ldr x1, [x19, #24] // CalloutContext.simulator_frame_pointer add x1, x1, 15 // Round up the frame pointer, since the Dart frame pointer and x1, x1, ~15 // is not double-word aligned. -.Lcopy2: +LOCAL_SYMBOL(copy2): ldp x2, x3, [x1, #-16]! // From Dart FP stp x2, x3, [sp, #-16]! // To C SP cmp x1, x0 - b.gt .Lcopy2 + b.gt LOCAL_SYMBOL(copy2) // Load the ABI argument registers. Note that Dart FFI does not support // full 128-bit SIMD arguments, so we don't need to set the full V @@ -137,12 +143,16 @@ _SimulatorFfiCallbackTrampoline: .type SimulatorFfiCallbackTrampoline, %function SimulatorFfiCallbackTrampoline: #endif - // FfiCallbackMetadata::NumCallbackTrampolinesPerPage() == 8150 - .rept 8150 +LOCAL_SYMBOL(start): +#if defined(__APPLE__) + .rept 2019 // FfiCallbackMetadata::NumCallbackTrampolinesPerPage() +#else + .rept 8163 // FfiCallbackMetadata::NumCallbackTrampolinesPerPage() +#endif adr x9, 0 - b .Lbody + b LOCAL_SYMBOL(body) .endr -.Lbody: +LOCAL_SYMBOL(body): mov x10, sp stp fp, lr, [sp, #-16]! mov fp, sp @@ -164,11 +174,17 @@ SimulatorFfiCallbackTrampoline: // Pass arguments registers and thunk address to the runtime. mov x0, sp mov x1, x9 // I.e., which callback. + + adr x2, LOCAL_SYMBOL(start) #if defined(__APPLE__) - bl _DoRedirectedFfiCallback + and x2, x2, ~(0x4000-1) // FfiCallbackMetadata::kPageSize = 16k + add x2, x2, 0x8000 #else - bl DoRedirectedFfiCallback + and x2, x2, ~(0x10000-1) // FfiCallbackMetadata::kPageSize = 64k + add x2, x2, 0x20000 #endif + ldr x2, [x2, #8] // FfiCallbackMetadata::kDoRedirectedFfiCallback + blr x2 // DoRedirectedFfiCallback // Load ABI result registers. ldp x0, x1, [sp, #0] // CallbackContext.integer_arguments[0] diff --git a/runtime/vm/simulator_arm64.cc b/runtime/vm/simulator_arm64.cc index b8134100d32..fa24636e917 100644 --- a/runtime/vm/simulator_arm64.cc +++ b/runtime/vm/simulator_arm64.cc @@ -1831,6 +1831,17 @@ struct CallbackContext { extern "C" void DoRedirectedFfiCallback(CallbackContext* ctxt, uword trampoline) { + // Assumptions in ffi_trampolines_arm64.S + COMPILE_ASSERT(sizeof(CallbackContext) == 144); + COMPILE_ASSERT(FfiCallbackMetadata::kDoRedirectedFfiCallback == 1); +#if defined(DART_TARGET_OS_MACOS) + COMPILE_ASSERT(FfiCallbackMetadata::kPageSize == 16 * KB); + COMPILE_ASSERT(FfiCallbackMetadata::NumCallbackTrampolinesPerPage() == 2019); +#else + COMPILE_ASSERT(FfiCallbackMetadata::kPageSize == 64 * KB); + COMPILE_ASSERT(FfiCallbackMetadata::NumCallbackTrampolinesPerPage() == 8163); +#endif + CallbackMetadata out; Thread* thread = DLRT_GetFfiCallbackMetadata(trampoline, &out); if (thread == nullptr) {