diff --git a/runtime/bin/dartvm_test_component.cml b/runtime/bin/dartvm_test_component.cml index a08f64fa0aa..463d84e66cf 100644 --- a/runtime/bin/dartvm_test_component.cml +++ b/runtime/bin/dartvm_test_component.cml @@ -4,7 +4,7 @@ { include: [ - "//runtime/vm.shard.cml", + "//runtime/vm-jit.shard.cml", ], program: { binary: "exe.stripped/dartvm", diff --git a/runtime/bin/run_vm_tests_test_component.cml b/runtime/bin/run_vm_tests_test_component.cml index fdb5e8f762b..7adedf8a778 100644 --- a/runtime/bin/run_vm_tests_test_component.cml +++ b/runtime/bin/run_vm_tests_test_component.cml @@ -4,7 +4,7 @@ { include: [ - "//runtime/vm.shard.cml", + "//runtime/vm-jit.shard.cml", ], program: { binary: "exe.stripped/run_vm_tests", diff --git a/runtime/vm-jit.shard.cml b/runtime/vm-jit.shard.cml new file mode 100644 index 00000000000..740cf21fd25 --- /dev/null +++ b/runtime/vm-jit.shard.cml @@ -0,0 +1,16 @@ +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +{ + include: [ + "//runtime/vm.shard.cml", + ], + use: [ + { + protocol: [ + "fuchsia.kernel.VmexResource", + ], + }, + ] +} diff --git a/runtime/vm.shard.cml b/runtime/vm.shard.cml index 3aad2368486..8fbd1c6c0af 100644 --- a/runtime/vm.shard.cml +++ b/runtime/vm.shard.cml @@ -31,7 +31,6 @@ "fuchsia.deprecatedtimezone.Timezone", "fuchsia.inspect.InspectSink", "fuchsia.intl.PropertyProvider", - "fuchsia.kernel.VmexResource", "fuchsia.logger.LogSink", "fuchsia.net.name.Lookup", "fuchsia.posix.socket.Provider", diff --git a/runtime/vm/compiler/backend/flow_graph_compiler.cc b/runtime/vm/compiler/backend/flow_graph_compiler.cc index 8d8b39d99a5..fea1f23e281 100644 --- a/runtime/vm/compiler/backend/flow_graph_compiler.cc +++ b/runtime/vm/compiler/backend/flow_graph_compiler.cc @@ -264,6 +264,13 @@ bool FlowGraphCompiler::CanOSRFunction() const { !is_optimizing(); } +void FlowGraphCompiler::InsertBSSRelocation(BSS::Relocation reloc) { + const intptr_t offset = assembler()->InsertAlignedRelocation(reloc); + AddDescriptor(UntaggedPcDescriptors::kBSSRelocation, /*pc_offset=*/offset, + /*deopt_id=*/DeoptId::kNone, InstructionSource(), + /*try_index=*/-1); +} + bool FlowGraphCompiler::ForceSlowPathForStackOverflow() const { #if !defined(PRODUCT) if (FLAG_stacktrace_every > 0 || FLAG_deoptimize_every > 0 || diff --git a/runtime/vm/compiler/backend/flow_graph_compiler.h b/runtime/vm/compiler/backend/flow_graph_compiler.h index bfaac2706b5..2f1d48583b4 100644 --- a/runtime/vm/compiler/backend/flow_graph_compiler.h +++ b/runtime/vm/compiler/backend/flow_graph_compiler.h @@ -478,6 +478,9 @@ class FlowGraphCompiler : public ValueObject { bool CanOSRFunction() const; bool is_optimizing() const { return is_optimizing_; } + void InsertBSSRelocation(BSS::Relocation reloc); + void LoadBSSEntry(BSS::Relocation relocation, Register dst, Register tmp); + // The function was fully intrinsified, so the body is unreachable. // // We still need to compile the body in unoptimized mode because the diff --git a/runtime/vm/compiler/backend/flow_graph_compiler_arm.cc b/runtime/vm/compiler/backend/flow_graph_compiler_arm.cc index 3d9f84acb80..29fb234eb70 100644 --- a/runtime/vm/compiler/backend/flow_graph_compiler_arm.cc +++ b/runtime/vm/compiler/backend/flow_graph_compiler_arm.cc @@ -1042,6 +1042,31 @@ void FlowGraphCompiler::EmitNativeLoad(Register dst, } } +void FlowGraphCompiler::LoadBSSEntry(BSS::Relocation relocation, + Register dst, + Register tmp) { + compiler::Label skip_reloc; + __ b(&skip_reloc); + InsertBSSRelocation(relocation); + __ Bind(&skip_reloc); + + // For historical reasons, the PC on ARM points 8 bytes (two instructions) + // past the current instruction. + __ sub(tmp, PC, + compiler::Operand(Instr::kPCReadOffset + compiler::target::kWordSize)); + + // tmp holds the address of the relocation. + __ ldr(dst, compiler::Address(tmp)); + + // dst holds the relocation itself: tmp - bss_start. + // tmp = tmp + (bss_start - tmp) = bss_start + __ add(tmp, tmp, compiler::Operand(dst)); + + // tmp holds the start of the BSS section. + // Load the "get-thread" routine: *bss_start. + __ ldr(dst, compiler::Address(tmp)); +} + #undef __ #define __ compiler_->assembler()-> diff --git a/runtime/vm/compiler/backend/flow_graph_compiler_arm64.cc b/runtime/vm/compiler/backend/flow_graph_compiler_arm64.cc index cbeb9398c0c..0f90cbc2b08 100644 --- a/runtime/vm/compiler/backend/flow_graph_compiler_arm64.cc +++ b/runtime/vm/compiler/backend/flow_graph_compiler_arm64.cc @@ -1068,6 +1068,28 @@ void FlowGraphCompiler::EmitNativeLoad(Register dst, } } +void FlowGraphCompiler::LoadBSSEntry(BSS::Relocation relocation, + Register dst, + Register tmp) { + compiler::Label skip_reloc; + __ b(&skip_reloc); + InsertBSSRelocation(relocation); + __ Bind(&skip_reloc); + + __ adr(tmp, compiler::Immediate(-compiler::target::kWordSize)); + + // tmp holds the address of the relocation. + __ ldr(dst, compiler::Address(tmp)); + + // dst holds the relocation itself: tmp - bss_start. + // tmp = tmp + (bss_start - tmp) = bss_start + __ add(tmp, tmp, compiler::Operand(dst)); + + // tmp holds the start of the BSS section. + // Load the "get-thread" routine: *bss_start. + __ ldr(dst, compiler::Address(tmp)); +} + #undef __ #define __ compiler_->assembler()-> diff --git a/runtime/vm/compiler/backend/flow_graph_compiler_riscv.cc b/runtime/vm/compiler/backend/flow_graph_compiler_riscv.cc index 40c9f18233d..18f4e2e8669 100644 --- a/runtime/vm/compiler/backend/flow_graph_compiler_riscv.cc +++ b/runtime/vm/compiler/backend/flow_graph_compiler_riscv.cc @@ -1141,6 +1141,29 @@ void FlowGraphCompiler::EmitNativeLoad(Register dst, __ PopRegister(tmp); } +void FlowGraphCompiler::LoadBSSEntry(BSS::Relocation relocation, + Register dst, + Register tmp) { + compiler::Label skip_reloc; + __ j(&skip_reloc, compiler::Assembler::kNearJump); + InsertBSSRelocation(relocation); + __ Bind(&skip_reloc); + + __ auipc(tmp, 0); + __ addi(tmp, tmp, -compiler::target::kWordSize); + + // tmp holds the address of the relocation. + __ lx(dst, compiler::Address(tmp)); + + // dst holds the relocation itself: tmp - bss_start. + // tmp = tmp + (bss_start - tmp) = bss_start + __ add(tmp, tmp, dst); + + // tmp holds the start of the BSS section. + // Load the "get-thread" routine: *bss_start. + __ lx(dst, compiler::Address(tmp)); +} + #undef __ #define __ compiler_->assembler()-> diff --git a/runtime/vm/compiler/backend/flow_graph_compiler_x64.cc b/runtime/vm/compiler/backend/flow_graph_compiler_x64.cc index 6eaef11a37c..b5badd6b725 100644 --- a/runtime/vm/compiler/backend/flow_graph_compiler_x64.cc +++ b/runtime/vm/compiler/backend/flow_graph_compiler_x64.cc @@ -1072,6 +1072,32 @@ void FlowGraphCompiler::EmitNativeLoad(Register dst, } } +void FlowGraphCompiler::LoadBSSEntry(BSS::Relocation relocation, + Register dst, + Register tmp) { + compiler::Label skip_reloc; + __ jmp(&skip_reloc); + InsertBSSRelocation(relocation); + const intptr_t reloc_end = __ CodeSize(); + __ Bind(&skip_reloc); + + const intptr_t kLeaqLength = 7; + __ leaq(dst, compiler::Address::AddressRIPRelative( + -kLeaqLength - compiler::target::kWordSize)); + ASSERT((__ CodeSize() - reloc_end) == kLeaqLength); + + // dst holds the address of the relocation. + __ movq(tmp, compiler::Address(dst, 0)); + + // tmp holds the relocation itself: dst - bss_start. + // dst = dst + (bss_start - dst) = bss_start + __ addq(dst, tmp); + + // dst holds the start of the BSS section. + // Load the routine. + __ movq(dst, compiler::Address(dst, 0)); +} + #undef __ #define __ compiler_->assembler()-> diff --git a/runtime/vm/compiler/stub_code_compiler.cc b/runtime/vm/compiler/stub_code_compiler.cc index f561a72b383..cf405a985df 100644 --- a/runtime/vm/compiler/stub_code_compiler.cc +++ b/runtime/vm/compiler/stub_code_compiler.cc @@ -2781,6 +2781,17 @@ void StubCodeCompiler::GenerateFfiAsyncCallbackSendStub() { __ Ret(); } +void StubCodeCompiler::InsertBSSRelocation(BSS::Relocation reloc) { + ASSERT(pc_descriptors_list_ != nullptr); + const intptr_t pc_offset = assembler->InsertAlignedRelocation(reloc); + pc_descriptors_list_->AddDescriptor( + UntaggedPcDescriptors::kBSSRelocation, pc_offset, + /*deopt_id=*/DeoptId::kNone, + /*token_pos=*/TokenPosition::kNoSource, + /*try_index=*/-1, + /*yield_index=*/UntaggedPcDescriptors::kInvalidYieldIndex); +} + #if !defined(TARGET_ARCH_IA32) static void GenerateSubtypeTestCacheLoopBody(Assembler* assembler, int n, diff --git a/runtime/vm/compiler/stub_code_compiler.h b/runtime/vm/compiler/stub_code_compiler.h index 48e441e372a..c73c03ef384 100644 --- a/runtime/vm/compiler/stub_code_compiler.h +++ b/runtime/vm/compiler/stub_code_compiler.h @@ -238,6 +238,11 @@ class StubCodeCompiler { intptr_t return_function_offset_in_object_store, intptr_t return_stub_offset_in_thread); + void GenerateLoadBSSEntry(BSS::Relocation relocation, + Register dst, + Register tmp); + void InsertBSSRelocation(BSS::Relocation reloc); + void GenerateLoadFfiCallbackMetadataRuntimeFunction(uword function_index, Register dst); diff --git a/runtime/vm/compiler/stub_code_compiler_arm.cc b/runtime/vm/compiler/stub_code_compiler_arm.cc index fdf0711101f..ed369f1efce 100644 --- a/runtime/vm/compiler/stub_code_compiler_arm.cc +++ b/runtime/vm/compiler/stub_code_compiler_arm.cc @@ -271,6 +271,31 @@ void StubCodeCompiler::GenerateFfiCallTrampolineStub() { __ Breakpoint(); // Not implemented. } +void StubCodeCompiler::GenerateLoadBSSEntry(BSS::Relocation relocation, + Register dst, + Register tmp) { + compiler::Label skip_reloc; + __ b(&skip_reloc); + InsertBSSRelocation(relocation); + __ Bind(&skip_reloc); + + // For historical reasons, the PC on ARM points 8 bytes (two instructions) + // past the current instruction. + __ sub(tmp, PC, + compiler::Operand(Instr::kPCReadOffset + compiler::target::kWordSize)); + + // tmp holds the address of the relocation. + __ ldr(dst, compiler::Address(tmp)); + + // dst holds the relocation itself: tmp - bss_start. + // tmp = tmp + (bss_start - tmp) = bss_start + __ add(tmp, tmp, compiler::Operand(dst)); + + // tmp holds the start of the BSS section. + // Load the "get-thread" routine: *bss_start. + __ ldr(dst, compiler::Address(tmp)); +} + void StubCodeCompiler::GenerateLoadFfiCallbackMetadataRuntimeFunction( uword function_index, Register dst) { diff --git a/runtime/vm/compiler/stub_code_compiler_arm64.cc b/runtime/vm/compiler/stub_code_compiler_arm64.cc index fea54932ce6..6f723752bb3 100644 --- a/runtime/vm/compiler/stub_code_compiler_arm64.cc +++ b/runtime/vm/compiler/stub_code_compiler_arm64.cc @@ -434,6 +434,28 @@ void StubCodeCompiler::GenerateFfiCallTrampolineStub() { __ Breakpoint(); // See ffi_trampolines_arm64.S } +void StubCodeCompiler::GenerateLoadBSSEntry(BSS::Relocation relocation, + Register dst, + Register tmp) { + compiler::Label skip_reloc; + __ b(&skip_reloc); + InsertBSSRelocation(relocation); + __ Bind(&skip_reloc); + + __ adr(tmp, compiler::Immediate(-compiler::target::kWordSize)); + + // tmp holds the address of the relocation. + __ ldr(dst, compiler::Address(tmp)); + + // dst holds the relocation itself: tmp - bss_start. + // tmp = tmp + (bss_start - tmp) = bss_start + __ add(tmp, tmp, compiler::Operand(dst)); + + // tmp holds the start of the BSS section. + // Load the "get-thread" routine: *bss_start. + __ ldr(dst, compiler::Address(tmp)); +} + void StubCodeCompiler::GenerateLoadFfiCallbackMetadataRuntimeFunction( uword function_index, Register dst) { @@ -499,8 +521,23 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { __ mov(R0, R9); __ mov(R1, SP); +#if defined(DART_TARGET_OS_FUCHSIA) + // TODO(https://dartbug.com/52579): Remove. + if (FLAG_precompiled_mode) { + GenerateLoadBSSEntry(BSS::Relocation::DLRT_GetFfiCallbackMetadata, R4, + R9); + } else { + Label call; + __ ldr(R4, Address::PC(2 * Instr::kInstrSize)); + __ b(&call); + __ Emit64(reinterpret_cast(&DLRT_GetFfiCallbackMetadata)); + __ Bind(&call); + } +#else GenerateLoadFfiCallbackMetadataRuntimeFunction( FfiCallbackMetadata::kGetFfiCallbackMetadata, R4); +#endif // defined(DART_TARGET_OS_FUCHSIA) + __ mov(CSP, SP); __ blr(R4); // DLRT_GetFfiCallbackMetadata __ mov(SP, CSP); diff --git a/runtime/vm/compiler/stub_code_compiler_ia32.cc b/runtime/vm/compiler/stub_code_compiler_ia32.cc index 884e4eba663..eb75dfe9f59 100644 --- a/runtime/vm/compiler/stub_code_compiler_ia32.cc +++ b/runtime/vm/compiler/stub_code_compiler_ia32.cc @@ -173,6 +173,13 @@ void StubCodeCompiler::GenerateExitSafepointStub() { __ ret(); } +void StubCodeCompiler::GenerateLoadBSSEntry(BSS::Relocation relocation, + Register dst, + Register tmp) { + // Only used in AOT. + __ Breakpoint(); +} + // Calls a native function inside a safepoint. // // On entry: diff --git a/runtime/vm/compiler/stub_code_compiler_riscv.cc b/runtime/vm/compiler/stub_code_compiler_riscv.cc index 7291d7e2630..654789f61f5 100644 --- a/runtime/vm/compiler/stub_code_compiler_riscv.cc +++ b/runtime/vm/compiler/stub_code_compiler_riscv.cc @@ -283,6 +283,29 @@ void StubCodeCompiler::GenerateFfiCallTrampolineStub() { __ Breakpoint(); // Not implemented. } +void StubCodeCompiler::GenerateLoadBSSEntry(BSS::Relocation relocation, + Register dst, + Register tmp) { + compiler::Label skip_reloc; + __ j(&skip_reloc, compiler::Assembler::kNearJump); + InsertBSSRelocation(relocation); + __ Bind(&skip_reloc); + + __ auipc(tmp, 0); + __ addi(tmp, tmp, -compiler::target::kWordSize); + + // tmp holds the address of the relocation. + __ lx(dst, compiler::Address(tmp)); + + // dst holds the relocation itself: tmp - bss_start. + // tmp = tmp + (bss_start - tmp) = bss_start + __ add(tmp, tmp, dst); + + // tmp holds the start of the BSS section. + // Load the "get-thread" routine: *bss_start. + __ lx(dst, compiler::Address(tmp)); +} + void StubCodeCompiler::GenerateLoadFfiCallbackMetadataRuntimeFunction( uword function_index, Register dst) { @@ -354,10 +377,36 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { __ mv(A0, T1); __ mv(A1, SP); + // Since DLRT_GetFfiCallbackMetadata can theoretically be loaded anywhere, + // we use the same trick as before to ensure a predictable instruction + // sequence. +#if defined(DART_TARGET_OS_FUCHSIA) + // TODO(https://dartbug.com/52579): Remove. + if (FLAG_precompiled_mode) { + GenerateLoadBSSEntry(BSS::Relocation::DLRT_GetFfiCallbackMetadata, T1, + T2); + } else { + const intptr_t kPCRelativeLoadOffset = 12; + intptr_t start = __ CodeSize(); + __ auipc(T1, 0); + __ lx(T1, Address(T1, kPCRelativeLoadOffset)); + Label call; + __ j(&call); + + ASSERT_EQUAL(__ CodeSize() - start, kPCRelativeLoadOffset); +#if XLEN == 32 + __ Emit32(reinterpret_cast(&DLRT_GetFfiCallbackMetadata)); +#else + __ Emit64(reinterpret_cast(&DLRT_GetFfiCallbackMetadata)); +#endif + __ Bind(&call); + } +#else GenerateLoadFfiCallbackMetadataRuntimeFunction( FfiCallbackMetadata::kGetFfiCallbackMetadata, T1); - __ jalr(T1); +#endif // defined(DART_TARGET_OS_FUCHSIA) + __ jalr(T1); __ mv(THR, A0); __ lx(T2, Address(SP, 0 * target::kWordSize)); // entry_point __ lx(T3, Address(SP, 1 * target::kWordSize)); // is_tail diff --git a/runtime/vm/compiler/stub_code_compiler_x64.cc b/runtime/vm/compiler/stub_code_compiler_x64.cc index 9a29d3e367f..2cc44fea877 100644 --- a/runtime/vm/compiler/stub_code_compiler_x64.cc +++ b/runtime/vm/compiler/stub_code_compiler_x64.cc @@ -459,6 +459,32 @@ void StubCodeCompiler::GenerateFfiCallTrampolineStub() { __ Ret(); } +void StubCodeCompiler::GenerateLoadBSSEntry(BSS::Relocation relocation, + Register dst, + Register tmp) { + compiler::Label skip_reloc; + __ jmp(&skip_reloc); + InsertBSSRelocation(relocation); + const intptr_t reloc_end = __ CodeSize(); + __ Bind(&skip_reloc); + + const intptr_t kLeaqLength = 7; + __ leaq(dst, compiler::Address::AddressRIPRelative( + -kLeaqLength - compiler::target::kWordSize)); + ASSERT((__ CodeSize() - reloc_end) == kLeaqLength); + + // dst holds the address of the relocation. + __ movq(tmp, compiler::Address(dst, 0)); + + // tmp holds the relocation itself: dst - bss_start. + // dst = dst + (bss_start - dst) = bss_start + __ addq(dst, tmp); + + // dst holds the start of the BSS section. + // Load the routine. + __ movq(dst, compiler::Address(dst, 0)); +} + void StubCodeCompiler::GenerateLoadFfiCallbackMetadataRuntimeFunction( uword function_index, Register dst) { @@ -527,8 +553,20 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { __ movq(CallingConventions::kArg1Reg, RAX); __ movq(CallingConventions::kArg2Reg, RSP); +#if defined(DART_TARGET_OS_FUCHSIA) + // TODO(https://dartbug.com/52579): Remove. + if (FLAG_precompiled_mode) { + GenerateLoadBSSEntry(BSS::Relocation::DLRT_GetFfiCallbackMetadata, RAX, + TMP); + } else { + __ movq(RAX, Immediate( + reinterpret_cast(DLRT_GetFfiCallbackMetadata))); + } +#else GenerateLoadFfiCallbackMetadataRuntimeFunction( FfiCallbackMetadata::kGetFfiCallbackMetadata, RAX); +#endif // defined(DART_TARGET_OS_FUCHSIA) + __ CallCFunction(RAX, /*restore_rsp=*/true); __ movq(THR, RAX); diff --git a/runtime/vm/ffi_callback_metadata.cc b/runtime/vm/ffi_callback_metadata.cc index 90b74d34b7d..a7eee874eea 100644 --- a/runtime/vm/ffi_callback_metadata.cc +++ b/runtime/vm/ffi_callback_metadata.cc @@ -29,26 +29,28 @@ void FfiCallbackMetadata::EnsureStubPageLocked() { ASSERT_LESS_OR_EQUAL(VirtualMemory::PageSize(), kPageSize); - uword code_start, code_end, code_size; #if defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64) + uword code_start, code_end, page_start; if (FLAG_use_simulator) { code_start = reinterpret_cast(SimulatorFfiCallbackTrampoline); code_end = reinterpret_cast(SimulatorFfiCallbackTrampolineEnd); - code_size = code_end - code_start; + page_start = code_start & ~(VirtualMemory::PageSize() - 1); } else { const Code& trampoline_code = StubCode::FfiCallbackTrampoline(); code_start = trampoline_code.EntryPoint(); code_end = code_start + trampoline_code.Size(); - code_size = trampoline_code.Size(); + page_start = code_start & ~(VirtualMemory::PageSize() - 1); + ASSERT_LESS_OR_EQUAL((code_start - page_start) + trampoline_code.Size(), + RXMappingSize()); } #else const Code& trampoline_code = StubCode::FfiCallbackTrampoline(); - code_start = trampoline_code.EntryPoint(); - code_end = code_start + trampoline_code.Size(); - code_size = trampoline_code.Size(); -#endif + const uword code_start = trampoline_code.EntryPoint(); + const uword code_end = code_start + trampoline_code.Size(); const uword page_start = code_start & ~(VirtualMemory::PageSize() - 1); - ASSERT_LESS_OR_EQUAL((code_start - page_start) + code_size, RXMappingSize()); + ASSERT_LESS_OR_EQUAL((code_start - page_start) + trampoline_code.Size(), + RXMappingSize()); +#endif // Stub page uses a tight (unaligned) bound for the end of the code area. // Otherwise we can read past the end of the code area when doing DuplicateRX. @@ -56,6 +58,32 @@ void FfiCallbackMetadata::EnsureStubPageLocked() { code_end - page_start); offset_of_first_trampoline_in_page_ = code_start - page_start; + +#if defined(DART_TARGET_OS_FUCHSIA) + // On Fuchsia we can't currently duplicate pages, so use the first page of + // trampolines. Store the stub page's metadata in a separately allocated RW + // page. + // TODO(https://dartbug.com/52579): Remove. + original_metadata_page_ = VirtualMemory::AllocateAligned( + MappingSize(), MappingAlignment(), /*is_executable=*/false, + /*is_compressed=*/false, "FfiCallbackMetadata::TrampolinePage"); + MetadataEntry* metadata_entry = reinterpret_cast( + original_metadata_page_->start() + MetadataOffset()); + for (intptr_t i = 0; i < NumCallbackTrampolinesPerPage(); ++i) { + AddToFreeListLocked(&metadata_entry[i]); + } +#elif defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64) + if (FLAG_use_simulator) { + original_metadata_page_ = VirtualMemory::AllocateAligned( + MappingSize(), MappingAlignment(), /*is_executable=*/false, + /*is_compressed=*/false, "FfiCallbackMetadata::TrampolinePage"); + MetadataEntry* metadata_entry = reinterpret_cast( + original_metadata_page_->start() + MetadataOffset()); + for (intptr_t i = 0; i < NumCallbackTrampolinesPerPage(); ++i) { + AddToFreeListLocked(&metadata_entry[i]); + } + } +#endif // defined(DART_TARGET_OS_FUCHSIA) } FfiCallbackMetadata::~FfiCallbackMetadata() { @@ -64,6 +92,12 @@ FfiCallbackMetadata::~FfiCallbackMetadata() { for (intptr_t i = 0; i < trampoline_pages_.length(); ++i) { delete trampoline_pages_[i]; } + +#if defined(DART_TARGET_OS_FUCHSIA) || \ + (defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64)) + // TODO(https://dartbug.com/52579): Remove. + delete original_metadata_page_; +#endif // defined(DART_TARGET_OS_FUCHSIA) } void FfiCallbackMetadata::Init() { @@ -97,6 +131,18 @@ void FfiCallbackMetadata::FillRuntimeFunction(VirtualMemory* page, } VirtualMemory* FfiCallbackMetadata::AllocateTrampolinePage() { +#if defined(DART_TARGET_OS_FUCHSIA) + // TODO(https://dartbug.com/52579): Remove. + UNREACHABLE(); + return nullptr; +#else +#if defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64) + if (FLAG_use_simulator) { + UNREACHABLE(); + return nullptr; + } +#endif + #if defined(DART_HOST_OS_MACOS) && defined(DART_PRECOMPILED_RUNTIME) const bool should_remap_stub_page = true; #else @@ -110,9 +156,6 @@ VirtualMemory* FfiCallbackMetadata::AllocateTrampolinePage() { // codesigning violation if hardened runtime is enabled or we will simply // not be able to execute trampoline code. const bool is_executable = !should_remap_stub_page; -#elif defined(DART_HOST_OS_FUCHSIA) - // The initial allocation needs to be marked executable. - const bool is_executable = true; #else // On other operating systems we can simply flip RW->RX as necessary. const bool is_executable = false; @@ -167,14 +210,9 @@ VirtualMemory* FfiCallbackMetadata::AllocateTrampolinePage() { #endif return new_page; +#endif // defined(DART_TARGET_OS_FUCHSIA) } -#if defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64) -struct CallbackContext; -extern "C" void DoRedirectedFfiCallback(CallbackContext* ctxt, - uword trampoline); -#endif - void FfiCallbackMetadata::EnsureFreeListNotEmptyLocked() { ASSERT(lock_.IsOwnedByCurrentThread()); EnsureStubPageLocked(); @@ -192,10 +230,6 @@ void FfiCallbackMetadata::EnsureFreeListNotEmptyLocked() { // Fill in the runtime functions. FillRuntimeFunction(new_page, kGetFfiCallbackMetadata, reinterpret_cast(DLRT_GetFfiCallbackMetadata)); -#if defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64) - FillRuntimeFunction(new_page, kDoRedirectedFfiCallback, - reinterpret_cast(DoRedirectedFfiCallback)); -#endif // Add all the trampolines to the free list. const intptr_t trampolines_per_page = NumCallbackTrampolinesPerPage(); @@ -399,12 +433,60 @@ FfiCallbackMetadata::Trampoline FfiCallbackMetadata::TrampolineOfMetadataEntry( MetadataEntry* metadata_entries = reinterpret_cast(start + MetadataOffset()); const uword index = metadata_entry - metadata_entries; +#if defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64) + if (FLAG_use_simulator) { + return reinterpret_cast(SimulatorFfiCallbackTrampoline) + + index * kNativeCallbackTrampolineSize; + } else { + return start + offset_of_first_trampoline_in_page_ + + index * kNativeCallbackTrampolineSize; + } +#elif defined(DART_TARGET_OS_FUCHSIA) + return StubCode::FfiCallbackTrampoline().EntryPoint() + + index * kNativeCallbackTrampolineSize; +#else return start + offset_of_first_trampoline_in_page_ + index * kNativeCallbackTrampolineSize; +#endif } FfiCallbackMetadata::MetadataEntry* FfiCallbackMetadata::MetadataEntryOfTrampoline(Trampoline trampoline) const { +#if defined(DART_TARGET_OS_FUCHSIA) + // On Fuchsia the metadata page is separate to the trampoline page. + // TODO(https://dartbug.com/52579): Remove. + const uword page_start = + Utils::RoundDown(trampoline - offset_of_first_trampoline_in_page_, + VirtualMemory::PageSize()); + const uword index = + (trampoline - offset_of_first_trampoline_in_page_ - page_start) / + kNativeCallbackTrampolineSize; + ASSERT(index < NumCallbackTrampolinesPerPage()); + MetadataEntry* metadata_etnry_table = reinterpret_cast( + original_metadata_page_->start() + MetadataOffset()); + return metadata_etnry_table + index; +#elif defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64) + if (FLAG_use_simulator) { + const uword page_start = + Utils::RoundDown(trampoline - offset_of_first_trampoline_in_page_, + VirtualMemory::PageSize()); + const uword index = + (trampoline - offset_of_first_trampoline_in_page_ - page_start) / + kNativeCallbackTrampolineSize; + ASSERT(index < NumCallbackTrampolinesPerPage()); + MetadataEntry* metadata_etnry_table = reinterpret_cast( + original_metadata_page_->start() + MetadataOffset()); + return metadata_etnry_table + index; + } else { + const uword start = MappingStart(trampoline); + MetadataEntry* metadata_entries = + reinterpret_cast(start + MetadataOffset()); + const uword index = + (trampoline - start - offset_of_first_trampoline_in_page_) / + kNativeCallbackTrampolineSize; + return &metadata_entries[index]; + } +#else const uword start = MappingStart(trampoline); MetadataEntry* metadata_entries = reinterpret_cast(start + MetadataOffset()); @@ -412,6 +494,7 @@ FfiCallbackMetadata::MetadataEntryOfTrampoline(Trampoline trampoline) const { (trampoline - start - offset_of_first_trampoline_in_page_) / kNativeCallbackTrampolineSize; return &metadata_entries[index]; +#endif } FfiCallbackMetadata::Metadata diff --git a/runtime/vm/ffi_callback_metadata.h b/runtime/vm/ffi_callback_metadata.h index 0b5e80609d6..0b762e78cef 100644 --- a/runtime/vm/ffi_callback_metadata.h +++ b/runtime/vm/ffi_callback_metadata.h @@ -62,7 +62,10 @@ class FfiCallbackMetadata { enum RuntimeFunctions { kGetFfiCallbackMetadata, - kDoRedirectedFfiCallback, + kExitTemporaryIsolate, + kExitIsolateGroupBoundIsolate, + kExitSyncCallbackTargetIsolate, + kExitSyncCallback, kNumRuntimeFunctions, }; @@ -284,6 +287,10 @@ class FfiCallbackMetadata { static constexpr intptr_t kPageSize = 64 * KB; #elif defined(DART_TARGET_OS_MACOS) && defined(TARGET_ARCH_ARM64) static constexpr intptr_t kPageSize = 16 * KB; +#elif defined(DART_TARGET_OS_FUCHSIA) + // Fuchsia only gets one page, so make it big. + // TODO(https://dartbug.com/52579): Remove. + static constexpr intptr_t kPageSize = 64 * KB; #else static constexpr intptr_t kPageSize = 4 * KB; #endif @@ -394,6 +401,19 @@ class FfiCallbackMetadata { MetadataEntry* free_list_head_ = nullptr; MetadataEntry* free_list_tail_ = nullptr; +#if defined(DART_TARGET_OS_FUCHSIA) || \ + (defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64)) + // TODO(https://dartbug.com/52579): Remove. + // On Fuchsia, we cannot duplicate the page containing the trampoline stub + // unless we plumb through from the embedder the VMO handle that was used to + // load the VM isolate snapshot. + // On simulator FFI, SimulatorFfiCallbackTrampoline cannot be duplicated + // because it contains a PC-relative call. It would need to be replaced with + // something like normal stub's PC-relative loading to a corresponding data + // page, or if we can assume the initial-exec code model a TLS load. + VirtualMemory* original_metadata_page_ = nullptr; +#endif // defined(DART_TARGET_OS_FUCHSIA) + DISALLOW_COPY_AND_ASSIGN(FfiCallbackMetadata); }; diff --git a/runtime/vm/ffi_trampolines_arm64.S b/runtime/vm/ffi_trampolines_arm64.S index db4d4a791b1..2871c454f9b 100644 --- a/runtime/vm/ffi_trampolines_arm64.S +++ b/runtime/vm/ffi_trampolines_arm64.S @@ -6,12 +6,6 @@ // This is not written as VM stub because we need it be executable in contexts where we cannot JIT. // (Alternatively, we could start requiring the VM snapshot to be provided in every mode.) -#if defined(__APPLE__) -#define LOCAL_SYMBOL(x) L##x -#else -#define LOCAL_SYMBOL(x) .L##x -#endif - #if defined(__aarch64__) && (defined(SIMULATOR_FFI) || (defined(DART_DYNAMIC_MODULES) && !defined(DART_PRECOMPILED_RUNTIME))) .text @@ -37,11 +31,11 @@ FfiCallTrampoline: // Copy top frame from Dart stack to C stack ldr x0, [x19, #0] // FfiCallArguments.stack_area ldr x1, [x19, #8] // FfiCallArguments.stack_area_end -LOCAL_SYMBOL(copy1): +.Lcopy1: ldp x2, x3, [x1, #-16]! // From stack_area stp x2, x3, [sp, #-16]! // To C SP cmp x1, x0 - b.gt LOCAL_SYMBOL(copy1) + b.gt .Lcopy1 // Load the ABI argument registers. Note that Dart FFI does not support // full 128-bit SIMD arguments, so we don't need to set the full V @@ -98,11 +92,11 @@ SimulatorFfiCalloutTrampoline: ldr x1, [x19, #24] // CalloutContext.simulator_frame_pointer add x1, x1, 15 // Round up the frame pointer, since the Dart frame pointer and x1, x1, ~15 // is not double-word aligned. -LOCAL_SYMBOL(copy2): +.Lcopy2: ldp x2, x3, [x1, #-16]! // From Dart FP stp x2, x3, [sp, #-16]! // To C SP cmp x1, x0 - b.gt LOCAL_SYMBOL(copy2) + b.gt .Lcopy2 // Load the ABI argument registers. Note that Dart FFI does not support // full 128-bit SIMD arguments, so we don't need to set the full V @@ -143,18 +137,12 @@ _SimulatorFfiCallbackTrampoline: .type SimulatorFfiCallbackTrampoline, %function SimulatorFfiCallbackTrampoline: #endif -LOCAL_SYMBOL(start): -#if defined(__Fuchsia__) - .rept 483 // FfiCallbackMetadata::NumCallbackTrampolinesPerPage() -#elif defined(__APPLE__) - .rept 2019 // FfiCallbackMetadata::NumCallbackTrampolinesPerPage() -#else - .rept 8163 // FfiCallbackMetadata::NumCallbackTrampolinesPerPage() -#endif + // FfiCallbackMetadata::NumCallbackTrampolinesPerPage() == 8150 + .rept 8150 adr x9, 0 - b LOCAL_SYMBOL(body) + b .Lbody .endr -LOCAL_SYMBOL(body): +.Lbody: mov x10, sp stp fp, lr, [sp, #-16]! mov fp, sp @@ -176,20 +164,11 @@ LOCAL_SYMBOL(body): // Pass arguments registers and thunk address to the runtime. mov x0, sp mov x1, x9 // I.e., which callback. - - adr x2, LOCAL_SYMBOL(start) -#if defined(__Fuchsia__) - and x2, x2, ~(0x1000-1) // FfiCallbackMetadata::kPageSize = 4k - add x2, x2, 0x2000 -#elif defined(__APPLE__) - and x2, x2, ~(0x4000-1) // FfiCallbackMetadata::kPageSize = 16k - add x2, x2, 0x8000 +#if defined(__APPLE__) + bl _DoRedirectedFfiCallback #else - and x2, x2, ~(0x10000-1) // FfiCallbackMetadata::kPageSize = 64k - add x2, x2, 0x20000 + bl DoRedirectedFfiCallback #endif - ldr x2, [x2, #8] // FfiCallbackMetadata::kDoRedirectedFfiCallback - blr x2 // DoRedirectedFfiCallback // Load ABI result registers. ldp x0, x1, [sp, #0] // CallbackContext.integer_arguments[0] diff --git a/runtime/vm/simulator_arm64.cc b/runtime/vm/simulator_arm64.cc index 71022e78b85..b8134100d32 100644 --- a/runtime/vm/simulator_arm64.cc +++ b/runtime/vm/simulator_arm64.cc @@ -1829,24 +1829,8 @@ struct CallbackContext { uword sp; }; -#if defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64) - extern "C" void DoRedirectedFfiCallback(CallbackContext* ctxt, uword trampoline) { - // Assumptions in ffi_trampolines_arm64.S - COMPILE_ASSERT(sizeof(CallbackContext) == 144); - COMPILE_ASSERT(FfiCallbackMetadata::kDoRedirectedFfiCallback == 1); -#if defined(DART_TARGET_OS_FUCHSIA) - COMPILE_ASSERT(FfiCallbackMetadata::kPageSize == 4 * KB); - COMPILE_ASSERT(FfiCallbackMetadata::NumCallbackTrampolinesPerPage() == 483); -#elif defined(DART_TARGET_OS_MACOS) - COMPILE_ASSERT(FfiCallbackMetadata::kPageSize == 16 * KB); - COMPILE_ASSERT(FfiCallbackMetadata::NumCallbackTrampolinesPerPage() == 2019); -#else - COMPILE_ASSERT(FfiCallbackMetadata::kPageSize == 64 * KB); - COMPILE_ASSERT(FfiCallbackMetadata::NumCallbackTrampolinesPerPage() == 8163); -#endif - CallbackMetadata out; Thread* thread = DLRT_GetFfiCallbackMetadata(trampoline, &out); if (thread == nullptr) { @@ -1860,8 +1844,6 @@ extern "C" void DoRedirectedFfiCallback(CallbackContext* ctxt, sim->DoRedirectedFfiCallback(thread, ctxt, &out); } -#endif // defined(SIMULATOR_FFI) && defined(HOST_ARCH_ARM64) - // Compare FfiCallbackTrampolineStub. void Simulator::DoRedirectedFfiCallback(Thread* thread, CallbackContext* ctxt,