dart-lang/sdk
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Revert "[vm] Migrate FFI callbacks to the new metadata system."

Browse Source 
Roll back https://dart-review.googlesource.com/c/sdk/+/302903
and https://dart-review.googlesource.com/c/sdk/+/306124

Since the bug is in VirtualMemory::DuplicateRX, also roll back
https://dart-review.googlesource.com/c/sdk/+/303960
https://dart-review.googlesource.com/c/sdk/+/304400
https://dart-review.googlesource.com/c/sdk/+/304620

Bug: https://buganizer.corp.google.com/issues/284959841
Change-Id: Id212de209fb57b2c3395bb61064dc8f569832884
TEST=CI
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/306674
Commit-Queue: Liam Appelbe <liama@google.com>
Reviewed-by: Ryan Macnak <rmacnak@google.com>
This commit is contained in:
Liam Appelbe
2023-06-01 01:00:58 +00:00
committed by Commit Queue
parent f5aa52af8d
commit 8ea3e9fd0d
43 changed files with 974 additions and 1253 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/lib/ffi.cc
+25 -3
View File
@@ -13,7 +13,6 @@
#include "vm/class_id.h"
#include "vm/compiler/ffi/native_type.h"
#include "vm/exceptions.h"
#include "vm/ffi_callback_metadata.h"
#include "vm/flags.h"
#include "vm/heap/gc_shared.h"
#include "vm/log.h"
@@ -40,8 +39,31 @@ DEFINE_NATIVE_ENTRY(Ffi_asFunctionInternal, 2, 2) {
DEFINE_NATIVE_ENTRY(Ffi_pointerFromFunction, 1, 1) {
const auto& function = Function::CheckedHandle(zone, arguments->NativeArg0());
void* pointer = isolate->CreateSyncFfiCallback(zone, function);
return Pointer::New(reinterpret_cast<uword>(pointer));
const auto& code =
Code::Handle(zone, FLAG_precompiled_mode ? function.CurrentCode()
: function.EnsureHasCode());
ASSERT(!code.IsNull());
uword entry_point = code.EntryPoint();
// In JIT we use one more indirection:
// * AOT: Native -> Ffi Trampoline -> Dart function
// * JIT: Native -> Jit trampoline -> Ffi Trampoline -> Dart function
//
// We do that since ffi trampoline code lives in Dart heap. During GC we can
// flip page protections from RX to RW to GC JITed code. During that time
// machine code on such pages cannot be executed. Native code therefore has to
// perform the safepoint transition before executing code in Dart heap (which
// is why we use the jit trampoline).
#if !defined(DART_PRECOMPILED_RUNTIME)
if (NativeCallbackTrampolines::Enabled()) {
entry_point =
isolate->group()->native_callback_trampolines()->TrampolineForId(
function.FfiCallbackId());
}
#endif
return Pointer::New(entry_point);
}
DEFINE_NATIVE_ENTRY(DartNativeApiFunctionPointer, 0, 1) {
runtime/platform/assert.h
-15
View File
@@ -271,17 +271,6 @@ void Expect::Null(const T p) {
} \
} while (false)
#define ASSERT_LESS_OR_EQUAL(actual, expected) \
do { \
if ((actual) > (expected)) { \
const std::string actual_str = std::to_string(actual); \
const std::string expected_str = std::to_string(expected); \
dart::Assert(__FILE__, __LINE__) \
.Fail("expected \"%s\" = %s >= actual \"%s\" = %s", #expected, \
expected_str.c_str(), #actual, actual_str.c_str()); \
} \
} while (false)
#define ASSERT_IMPLIES(antecedent, consequent) \
do { \
if (antecedent) { \
@@ -310,10 +299,6 @@ void Expect::Null(const T p) {
do { \
} while (false && ((expected) != (actual)))
#define ASSERT_LESS_OR_EQUAL(expected, actual) \
do { \
} while (false && ((actual) > (expected)))
#define ASSERT_IMPLIES(antecedent, consequent) \
do { \
} while (false && (!(antecedent) || (consequent)))
runtime/vm/allocation.cc
+1
View File
@@ -44,6 +44,7 @@ void StackResource::Init(ThreadState* thread) {
// We can only have longjumps and exceptions when there is a current
// thread and isolate. If there is no current thread, we don't need to
// protect this case.
// TODO(23807): Eliminate this special case.
if (thread != nullptr) {
ASSERT(Thread::Current() == thread);
thread_ = thread;
runtime/vm/app_snapshot.cc
+15
View File
@@ -6470,6 +6470,21 @@ class ProgramDeserializationRoots : public DeserializationRoots {
unit.set_base_objects(refs);
}
#if !defined(DART_PRECOMPILED_RUNTIME)
// The JIT trampolines are allocated at runtime and not part of the
// snapshot. So if the snapshot contains ffi-callback-trampolines we'll
// have to allocate corresponding JIT trampolines.
auto* const tramps = isolate_group->native_callback_trampolines();
const auto& ffi_callback_code =
GrowableObjectArray::Handle(object_store->ffi_callback_code());
if (!ffi_callback_code.IsNull()) {
const intptr_t len = ffi_callback_code.Length();
while (tramps->next_callback_id() < len) {
tramps->AllocateTrampoline();
}
}
#endif // !defined(DART_PRECOMPILED_RUNTIME)
// Setup native resolver for bootstrap impl.
Bootstrap::SetupNativeResolver();
}
runtime/vm/bss_relocs.cc
+7
View File
@@ -37,6 +37,13 @@ void BSS::Initialize(Thread* current, uword* bss_start, bool vm) {
InitializeBSSEntry(Relocation::InstructionsRelocatedAddress,
instructions - dso_base, bss_start);
}
if (!vm) {
// Fill values at isolate-only indices.
InitializeBSSEntry(Relocation::DRT_GetThreadForNativeCallback,
reinterpret_cast<uword>(DLRT_GetThreadForNativeCallback),
bss_start);
}
}
} // namespace dart
runtime/vm/bss_relocs.h
+6 -7
View File
@@ -17,17 +17,16 @@ class BSS : public AllStatic {
// stored at the index.
enum class Relocation : intptr_t {
InstructionsRelocatedAddress,
EndOfVmEntries,
// We don't have any isolate group specific entries at the moment.
EndOfIsolateGroupEntries = EndOfVmEntries,
// End of shared entries.
DRT_GetThreadForNativeCallback,
// End of isolate-only entries.
};
static constexpr intptr_t kVmEntryCount =
static_cast<intptr_t>(Relocation::EndOfVmEntries);
static_cast<intptr_t>(Relocation::InstructionsRelocatedAddress) + 1;
static constexpr intptr_t kIsolateGroupEntryCount =
static_cast<intptr_t>(Relocation::EndOfIsolateGroupEntries);
static constexpr intptr_t kIsolateEntryCount =
static_cast<intptr_t>(Relocation::DRT_GetThreadForNativeCallback) + 1;
static constexpr intptr_t RelocationIndex(Relocation reloc) {
return static_cast<intptr_t>(reloc);
runtime/vm/compiler/backend/il_arm.cc
+25 -7
View File
@@ -1646,10 +1646,11 @@ void NativeReturnInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
__ Pop(vm_tag_reg);
// The trampoline that called us will enter the safepoint on our behalf.
__ TransitionGeneratedToNative(vm_tag_reg, old_exit_frame_reg,
old_exit_through_ffi_reg, tmp,
/*enter_safepoint=*/false);
// If we were called by a trampoline, it will enter the safepoint on our
// behalf.
__ TransitionGeneratedToNative(
vm_tag_reg, old_exit_frame_reg, old_exit_through_ffi_reg, tmp,
/*enter_safepoint=*/!NativeCallbackTrampolines::Enabled());
__ PopNativeCalleeSavedRegisters();
@@ -1696,6 +1697,24 @@ void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
__ PushNativeCalleeSavedRegisters();
// Load the thread object. If we were called by a JIT trampoline, the thread
// is already loaded.
const intptr_t callback_id = marshaller_.dart_signature().FfiCallbackId();
if (!NativeCallbackTrampolines::Enabled()) {
compiler->LoadBSSEntry(BSS::Relocation::DRT_GetThreadForNativeCallback, R1,
R0);
// Create another frame to align the frame before continuing in "native"
// code.
__ EnterFrame(1 << FP, 0);
__ ReserveAlignedFrameSpace(0);
__ LoadImmediate(R0, callback_id);
__ blx(R1);
__ mov(THR, compiler::Operand(R0));
__ LeaveFrame(1 << FP);
}
// Save the current VMTag on the stack.
__ LoadFromOffset(R0, THR, compiler::target::Thread::vm_tag_offset());
__ Push(R0);
@@ -1720,7 +1739,8 @@ void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
__ EmitEntryFrameVerification(R0);
// The callback trampoline (caller) has already left the safepoint for us.
// Either DLRT_GetThreadForNativeCallback or the callback trampoline (caller)
// will leave the safepoint for us.
__ TransitionNativeToGenerated(/*scratch0=*/R0, /*scratch1=*/R1,
/*exit_safepoint=*/false);
@@ -1728,8 +1748,6 @@ void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
// handles.
// Load the code object.
const Function& target_function = marshaller_.dart_signature();
const intptr_t callback_id = target_function.FfiCallbackId();
__ LoadFromOffset(R0, THR, compiler::target::Thread::isolate_group_offset());
__ LoadFromOffset(R0, R0,
compiler::target::IsolateGroup::object_store_offset());
runtime/vm/compiler/backend/il_arm64.cc
+26 -7
View File
@@ -1519,11 +1519,13 @@ void NativeReturnInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
__ StoreToOffset(tmp, THR, compiler::target::Thread::top_resource_offset());
// Reset the exit frame info to old_exit_frame_reg *before* entering the
// safepoint. The trampoline that called us will enter the safepoint on our
// safepoint.
//
// If we were called by a trampoline, it will enter the safepoint on our
// behalf.
__ TransitionGeneratedToNative(vm_tag_reg, old_exit_frame_reg,
old_exit_through_ffi_reg,
/*enter_safepoint=*/false);
__ TransitionGeneratedToNative(
vm_tag_reg, old_exit_frame_reg, old_exit_through_ffi_reg,
/*enter_safepoint=*/!NativeCallbackTrampolines::Enabled());
__ PopNativeCalleeSavedRegisters();
@@ -1571,6 +1573,24 @@ void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
__ PushNativeCalleeSavedRegisters();
// Load the thread object. If we were called by a JIT trampoline, the thread
// is already loaded.
const intptr_t callback_id = marshaller_.dart_signature().FfiCallbackId();
if (!NativeCallbackTrampolines::Enabled()) {
compiler->LoadBSSEntry(BSS::Relocation::DRT_GetThreadForNativeCallback, R1,
R0);
// Create another frame to align the frame before continuing in "native"
// code.
__ EnterFrame(0);
__ ReserveAlignedFrameSpace(0);
__ LoadImmediate(R0, callback_id);
__ blr(R1);
__ mov(THR, R0);
__ LeaveFrame();
}
// Now that we have THR, we can set CSP.
__ SetupCSPFromThread(THR);
@@ -1607,15 +1627,14 @@ void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
// correct offset from FP.
__ EmitEntryFrameVerification();
// The callback trampoline (caller) has already left the safepoint for us.
// Either DLRT_GetThreadForNativeCallback or the callback trampoline (caller)
// will leave the safepoint for us.
__ TransitionNativeToGenerated(R0, /*exit_safepoint=*/false);
// Now that the safepoint has ended, we can touch Dart objects without
// handles.
// Load the code object.
const Function& target_function = marshaller_.dart_signature();
const intptr_t callback_id = target_function.FfiCallbackId();
__ LoadFromOffset(R0, THR, compiler::target::Thread::isolate_group_offset());
__ LoadFromOffset(R0, R0,
compiler::target::IsolateGroup::object_store_offset());
runtime/vm/compiler/backend/il_ia32.cc
+14 -8
View File
@@ -366,12 +366,14 @@ void NativeReturnInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
__ movl(old_exit_through_ffi_reg, vm_tag_reg);
__ popl(vm_tag_reg);
// Reset the exit frame info to old_exit_frame_reg *before* entering the
// safepoint. The trampoline that called us will enter the safepoint on our
// This will reset the exit frame info to old_exit_frame_reg *before* entering
// the safepoint.
//
// If we were called by a trampoline, it will enter the safepoint on our
// behalf.
__ TransitionGeneratedToNative(vm_tag_reg, old_exit_frame_reg,
old_exit_through_ffi_reg,
/*enter_safepoint=*/false);
__ TransitionGeneratedToNative(
vm_tag_reg, old_exit_frame_reg, old_exit_through_ffi_reg,
/*enter_safepoint=*/!NativeCallbackTrampolines::Enabled());
// Move XMM0 into ST0 if needed.
if (return_in_st0) {
@@ -1251,6 +1253,11 @@ void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
__ pushl(ESI);
__ pushl(EDI);
const intptr_t callback_id = marshaller_.dart_signature().FfiCallbackId();
// The thread object was already loaded by a JIT trampoline.
ASSERT(NativeCallbackTrampolines::Enabled());
// Save the current VMTag on the stack.
__ movl(ECX, compiler::Assembler::VMTagAddress());
__ pushl(ECX);
@@ -1273,14 +1280,13 @@ void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
// correct offset from FP.
__ EmitEntryFrameVerification();
// The callback trampoline (caller) has already left the safepoint for us.
// Either DLRT_GetThreadForNativeCallback or the callback trampoline (caller)
// will leave the safepoint for us.
__ TransitionNativeToGenerated(EAX, /*exit_safepoint=*/false);
// Now that the safepoint has ended, we can hold Dart objects with bare hands.
// Load the code object.
const Function& target_function = marshaller_.dart_signature();
const intptr_t callback_id = target_function.FfiCallbackId();
__ movl(EAX, compiler::Address(
THR, compiler::target::Thread::isolate_group_offset()));
__ movl(EAX, compiler::Address(
runtime/vm/compiler/backend/il_riscv.cc
+36 -7
View File
@@ -1693,11 +1693,13 @@ void NativeReturnInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
__ StoreToOffset(tmp, THR, compiler::target::Thread::top_resource_offset());
// Reset the exit frame info to old_exit_frame_reg *before* entering the
// safepoint. The trampoline that called us will enter the safepoint on our
// safepoint.
//
// If we were called by a trampoline, it will enter the safepoint on our
// behalf.
__ TransitionGeneratedToNative(vm_tag_reg, old_exit_frame_reg,
old_exit_through_ffi_reg,
/*enter_safepoint=*/false);
__ TransitionGeneratedToNative(
vm_tag_reg, old_exit_frame_reg, old_exit_through_ffi_reg,
/*enter_safepoint=*/!NativeCallbackTrampolines::Enabled());
__ PopNativeCalleeSavedRegisters();
@@ -1736,6 +1738,34 @@ void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
__ PushNativeCalleeSavedRegisters();
// Load the thread object. If we were called by a trampoline, the thread is
// already loaded.
if (FLAG_precompiled_mode) {
compiler->LoadBSSEntry(BSS::Relocation::DRT_GetThreadForNativeCallback, A1,
A0);
} else if (!NativeCallbackTrampolines::Enabled()) {
// In JIT mode, we can just paste the address of the runtime entry into the
// generated code directly. This is not a problem since we don't save
// callbacks into JIT snapshots.
__ LoadImmediate(
A1, reinterpret_cast<int64_t>(DLRT_GetThreadForNativeCallback));
}
const intptr_t callback_id = marshaller_.dart_signature().FfiCallbackId();
if (!NativeCallbackTrampolines::Enabled()) {
// Create another frame to align the frame before continuing in "native"
// code.
__ EnterFrame(0);
__ ReserveAlignedFrameSpace(0);
__ LoadImmediate(A0, callback_id);
__ jalr(A1);
__ mv(THR, A0);
__ LeaveFrame();
}
#if defined(USING_SHADOW_CALL_STACK)
#error Unimplemented
#endif
@@ -1765,15 +1795,14 @@ void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
// correct offset from FP.
__ EmitEntryFrameVerification();
// The callback trampoline (caller) has already left the safepoint for us.
// Either DLRT_GetThreadForNativeCallback or the callback trampoline (caller)
// will leave the safepoint for us.
__ TransitionNativeToGenerated(A0, /*exit_safepoint=*/false);
// Now that the safepoint has ended, we can touch Dart objects without
// handles.
// Load the code object.
const Function& target_function = marshaller_.dart_signature();
const intptr_t callback_id = target_function.FfiCallbackId();
__ LoadFromOffset(A0, THR, compiler::target::Thread::isolate_group_offset());
__ LoadFromOffset(A0, A0,
compiler::target::IsolateGroup::object_store_offset());
runtime/vm/compiler/backend/il_x64.cc
+24 -7
View File
@@ -482,10 +482,11 @@ void NativeReturnInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
__ popq(vm_tag_reg);
// The trampoline that called us will enter the safepoint on our behalf.
__ TransitionGeneratedToNative(vm_tag_reg, old_exit_frame_reg,
old_exit_through_ffi_reg,
/*enter_safepoint=*/false);
// If we were called by a trampoline, it will enter the safepoint on our
// behalf.
__ TransitionGeneratedToNative(
vm_tag_reg, old_exit_frame_reg, old_exit_through_ffi_reg,
/*enter_safepoint=*/!NativeCallbackTrampolines::Enabled());
// Restore C++ ABI callee-saved registers.
__ PopRegisters(kCalleeSaveRegistersSet);
@@ -1491,6 +1492,23 @@ void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
// Save ABI callee-saved registers.
__ PushRegisters(kCalleeSaveRegistersSet);
// Load the thread object. If we were called by a JIT trampoline, the thread
// is already loaded.
const intptr_t callback_id = marshaller_.dart_signature().FfiCallbackId();
if (!NativeCallbackTrampolines::Enabled()) {
compiler->LoadBSSEntry(BSS::Relocation::DRT_GetThreadForNativeCallback, RAX,
RCX);
__ EnterFrame(0);
__ ReserveAlignedFrameSpace(0);
COMPILE_ASSERT(RAX != CallingConventions::kArg1Reg);
__ movq(CallingConventions::kArg1Reg, compiler::Immediate(callback_id));
__ CallCFunction(RAX);
__ movq(THR, RAX);
__ LeaveFrame();
}
// Save the current VMTag on the stack.
__ movq(RAX, compiler::Assembler::VMTagAddress());
__ pushq(RAX);
@@ -1513,12 +1531,11 @@ void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
// correct offset from FP.
__ EmitEntryFrameVerification();
// The callback trampoline (caller) has already left the safepoint for us.
// Either DLRT_GetThreadForNativeCallback or the callback trampoline (caller)
// will leave the safepoint for us.
__ TransitionNativeToGenerated(/*exit_safepoint=*/false);
// Load the code object.
const Function& target_function = marshaller_.dart_signature();
const intptr_t callback_id = target_function.FfiCallbackId();
__ movq(RAX, compiler::Address(
THR, compiler::target::Thread::isolate_group_offset()));
__ movq(RAX, compiler::Address(
runtime/vm/compiler/ffi/callback.cc
+52
View File
@@ -116,13 +116,52 @@ static void EnsureFfiCallbackMetadata(Thread* thread, intptr_t callback_id) {
GrowableObjectArray::New(kInitialCallbackIdsReserved, Heap::kOld);
object_store->set_ffi_callback_code(code_array);
}
#if defined(TARGET_ARCH_IA32)
auto& stack_array =
TypedData::Handle(zone, object_store->ffi_callback_stack_return());
if (stack_array.IsNull()) {
stack_array = TypedData::New(kTypedDataInt8ArrayCid,
kInitialCallbackIdsReserved, Heap::kOld);
object_store->set_ffi_callback_stack_return(stack_array);
}
ASSERT(code_array.Length() <= stack_array.Length());
#endif
#if !defined(DART_PRECOMPILED_RUNTIME) && !defined(DART_PRECOMPILER)
auto* const tramps = thread->isolate_group()->native_callback_trampolines();
ASSERT(code_array.Length() == tramps->next_callback_id());
#endif
if (code_array.Length() <= callback_id) {
// Ensure we've enough space in the arrays.
while (!(callback_id < code_array.Length())) {
code_array.Add(Code::null_object());
#if !defined(DART_PRECOMPILED_RUNTIME) && !defined(DART_PRECOMPILER)
tramps->AllocateTrampoline();
#endif
}
#if defined(TARGET_ARCH_IA32)
if (callback_id >= stack_array.Length()) {
const int32_t capacity = stack_array.Length();
if (callback_id >= capacity) {
// Ensure both that we grow enough and an exponential growth strategy.
const int32_t new_capacity =
Utils::Maximum(callback_id + 1, capacity * 2);
stack_array = TypedData::Grow(stack_array, new_capacity);
object_store->set_ffi_callback_stack_return(stack_array);
}
}
#endif // defined(TARGET_ARCH_IA32)
}
#if !defined(DART_PRECOMPILED_RUNTIME) && !defined(DART_PRECOMPILER)
// Verify invariants of the 3 arrays (still) hold.
ASSERT(code_array.Length() == tramps->next_callback_id());
#if defined(TARGET_ARCH_IA32)
ASSERT(code_array.Length() <= stack_array.Length());
#endif
#endif
ASSERT(callback_id < code_array.Length());
}
@@ -138,6 +177,19 @@ void SetFfiCallbackCode(Thread* thread,
const auto& code_array =
GrowableObjectArray::Handle(zone, object_store->ffi_callback_code());
code_array.SetAt(callback_id, code);
#if defined(TARGET_ARCH_IA32)
// On ia32, store the stack delta that we need to use when returning.
const intptr_t stack_return_delta =
ffi_trampoline.FfiCSignatureReturnsStruct() &&
CallingConventions::kUsesRet4
? compiler::target::kWordSize
: 0;
const auto& stack_delta_array =
TypedData::Handle(zone, object_store->ffi_callback_stack_return());
stack_delta_array.SetUint8(callback_id, stack_return_delta);
#endif // defined(TARGET_ARCH_IA32)
}
} // namespace ffi
runtime/vm/compiler/ffi/marshaller.cc
+7 -6
View File
@@ -12,7 +12,6 @@
#include "vm/compiler/ffi/native_location.h"
#include "vm/compiler/ffi/native_type.h"
#include "vm/exceptions.h"
#include "vm/ffi_callback_metadata.h"
#include "vm/log.h"
#include "vm/object_store.h"
#include "vm/raw_object.h"
@@ -638,11 +637,13 @@ class CallbackArgumentTranslator : public ValueObject {
// shadow space if present (factored into
// kCallbackSlotsBeforeSavedArguments).
//
// Finally, for NativeCallbackTrampolines, factor in the extra stack space
// corresponding to those trampolines' frames (above the entry frame).
const intptr_t stack_delta =
kCallbackSlotsBeforeSavedArguments +
FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta;
// Finally, if we are using NativeCallbackTrampolines, factor in the extra
// stack space corresponding to those trampolines' frames (above the entry
// frame).
intptr_t stack_delta = kCallbackSlotsBeforeSavedArguments;
if (NativeCallbackTrampolines::Enabled()) {
stack_delta += StubCodeCompiler::kNativeCallbackTrampolineStackDelta;
}
FrameRebase rebase(
zone,
/*old_base=*/SPREG, /*new_base=*/SPREG,
runtime/vm/compiler/runtime_api.h
+1
View File
@@ -1317,6 +1317,7 @@ class ObjectStore : public AllStatic {
static word type_type_offset();
static word ffi_callback_code_offset();
static word ffi_callback_stack_return_offset();
static word suspend_state_await_offset();
static word suspend_state_await_with_type_check_offset();
runtime/vm/compiler/runtime_offsets_extracted.h
+60
View File
@@ -260,6 +260,8 @@ static constexpr dart::compiler::target::word ObjectStore_type_type_offset =
0x88;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_code_offset = 0x280;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_stack_return_offset = 0x284;
static constexpr dart::compiler::target::word
ObjectStore_suspend_state_await_offset = 0x224;
static constexpr dart::compiler::target::word
@@ -966,6 +968,8 @@ static constexpr dart::compiler::target::word ObjectStore_type_type_offset =
0x110;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_code_offset = 0x500;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_stack_return_offset = 0x508;
static constexpr dart::compiler::target::word
ObjectStore_suspend_state_await_offset = 0x448;
static constexpr dart::compiler::target::word
@@ -1672,6 +1676,8 @@ static constexpr dart::compiler::target::word ObjectStore_type_type_offset =
0x88;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_code_offset = 0x280;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_stack_return_offset = 0x284;
static constexpr dart::compiler::target::word
ObjectStore_suspend_state_await_offset = 0x224;
static constexpr dart::compiler::target::word
@@ -2377,6 +2383,8 @@ static constexpr dart::compiler::target::word ObjectStore_type_type_offset =
0x110;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_code_offset = 0x500;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_stack_return_offset = 0x508;
static constexpr dart::compiler::target::word
ObjectStore_suspend_state_await_offset = 0x448;
static constexpr dart::compiler::target::word
@@ -3087,6 +3095,8 @@ static constexpr dart::compiler::target::word ObjectStore_type_type_offset =
0x110;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_code_offset = 0x500;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_stack_return_offset = 0x508;
static constexpr dart::compiler::target::word
ObjectStore_suspend_state_await_offset = 0x448;
static constexpr dart::compiler::target::word
@@ -3794,6 +3804,8 @@ static constexpr dart::compiler::target::word ObjectStore_type_type_offset =
0x110;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_code_offset = 0x500;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_stack_return_offset = 0x508;
static constexpr dart::compiler::target::word
ObjectStore_suspend_state_await_offset = 0x448;
static constexpr dart::compiler::target::word
@@ -4501,6 +4513,8 @@ static constexpr dart::compiler::target::word ObjectStore_type_type_offset =
0x88;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_code_offset = 0x280;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_stack_return_offset = 0x284;
static constexpr dart::compiler::target::word
ObjectStore_suspend_state_await_offset = 0x224;
static constexpr dart::compiler::target::word
@@ -5208,6 +5222,8 @@ static constexpr dart::compiler::target::word ObjectStore_type_type_offset =
0x110;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_code_offset = 0x500;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_stack_return_offset = 0x508;
static constexpr dart::compiler::target::word
ObjectStore_suspend_state_await_offset = 0x448;
static constexpr dart::compiler::target::word
@@ -5909,6 +5925,8 @@ static constexpr dart::compiler::target::word ObjectStore_type_type_offset =
0x88;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_code_offset = 0x280;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_stack_return_offset = 0x284;
static constexpr dart::compiler::target::word
ObjectStore_suspend_state_await_offset = 0x224;
static constexpr dart::compiler::target::word
@@ -6607,6 +6625,8 @@ static constexpr dart::compiler::target::word ObjectStore_type_type_offset =
0x110;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_code_offset = 0x500;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_stack_return_offset = 0x508;
static constexpr dart::compiler::target::word
ObjectStore_suspend_state_await_offset = 0x448;
static constexpr dart::compiler::target::word
@@ -7305,6 +7325,8 @@ static constexpr dart::compiler::target::word ObjectStore_type_type_offset =
0x88;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_code_offset = 0x280;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_stack_return_offset = 0x284;
static constexpr dart::compiler::target::word
ObjectStore_suspend_state_await_offset = 0x224;
static constexpr dart::compiler::target::word
@@ -8002,6 +8024,8 @@ static constexpr dart::compiler::target::word ObjectStore_type_type_offset =
0x110;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_code_offset = 0x500;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_stack_return_offset = 0x508;
static constexpr dart::compiler::target::word
ObjectStore_suspend_state_await_offset = 0x448;
static constexpr dart::compiler::target::word
@@ -8704,6 +8728,8 @@ static constexpr dart::compiler::target::word ObjectStore_type_type_offset =
0x110;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_code_offset = 0x500;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_stack_return_offset = 0x508;
static constexpr dart::compiler::target::word
ObjectStore_suspend_state_await_offset = 0x448;
static constexpr dart::compiler::target::word
@@ -9403,6 +9429,8 @@ static constexpr dart::compiler::target::word ObjectStore_type_type_offset =
0x110;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_code_offset = 0x500;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_stack_return_offset = 0x508;
static constexpr dart::compiler::target::word
ObjectStore_suspend_state_await_offset = 0x448;
static constexpr dart::compiler::target::word
@@ -10102,6 +10130,8 @@ static constexpr dart::compiler::target::word ObjectStore_type_type_offset =
0x88;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_code_offset = 0x280;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_stack_return_offset = 0x284;
static constexpr dart::compiler::target::word
ObjectStore_suspend_state_await_offset = 0x224;
static constexpr dart::compiler::target::word
@@ -10801,6 +10831,8 @@ static constexpr dart::compiler::target::word ObjectStore_type_type_offset =
0x110;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_code_offset = 0x500;
static constexpr dart::compiler::target::word
ObjectStore_ffi_callback_stack_return_offset = 0x508;
static constexpr dart::compiler::target::word
ObjectStore_suspend_state_await_offset = 0x448;
static constexpr dart::compiler::target::word
@@ -11536,6 +11568,8 @@ static constexpr dart::compiler::target::word AOT_ObjectStore_type_type_offset =
0x88;
static constexpr dart::compiler::target::word
AOT_ObjectStore_ffi_callback_code_offset = 0x280;
static constexpr dart::compiler::target::word
AOT_ObjectStore_ffi_callback_stack_return_offset = 0x284;
static constexpr dart::compiler::target::word
AOT_ObjectStore_suspend_state_await_offset = 0x224;
static constexpr dart::compiler::target::word
@@ -12317,6 +12351,8 @@ static constexpr dart::compiler::target::word AOT_ObjectStore_type_type_offset =
0x110;
static constexpr dart::compiler::target::word
AOT_ObjectStore_ffi_callback_code_offset = 0x500;
static constexpr dart::compiler::target::word
AOT_ObjectStore_ffi_callback_stack_return_offset = 0x508;
static constexpr dart::compiler::target::word
AOT_ObjectStore_suspend_state_await_offset = 0x448;
static constexpr dart::compiler::target::word
@@ -13101,6 +13137,8 @@ static constexpr dart::compiler::target::word AOT_ObjectStore_type_type_offset =
0x110;
static constexpr dart::compiler::target::word
AOT_ObjectStore_ffi_callback_code_offset = 0x500;
static constexpr dart::compiler::target::word
AOT_ObjectStore_ffi_callback_stack_return_offset = 0x508;
static constexpr dart::compiler::target::word
AOT_ObjectStore_suspend_state_await_offset = 0x448;
static constexpr dart::compiler::target::word
@@ -13884,6 +13922,8 @@ static constexpr dart::compiler::target::word AOT_ObjectStore_type_type_offset =
0x110;
static constexpr dart::compiler::target::word
AOT_ObjectStore_ffi_callback_code_offset = 0x500;
static constexpr dart::compiler::target::word
AOT_ObjectStore_ffi_callback_stack_return_offset = 0x508;
static constexpr dart::compiler::target::word
AOT_ObjectStore_suspend_state_await_offset = 0x448;
static constexpr dart::compiler::target::word
@@ -14667,6 +14707,8 @@ static constexpr dart::compiler::target::word AOT_ObjectStore_type_type_offset =
0x110;
static constexpr dart::compiler::target::word
AOT_ObjectStore_ffi_callback_code_offset = 0x500;
static constexpr dart::compiler::target::word
AOT_ObjectStore_ffi_callback_stack_return_offset = 0x508;
static constexpr dart::compiler::target::word
AOT_ObjectStore_suspend_state_await_offset = 0x448;
static constexpr dart::compiler::target::word
@@ -15452,6 +15494,8 @@ static constexpr dart::compiler::target::word AOT_ObjectStore_type_type_offset =
0x88;
static constexpr dart::compiler::target::word
AOT_ObjectStore_ffi_callback_code_offset = 0x280;
static constexpr dart::compiler::target::word
AOT_ObjectStore_ffi_callback_stack_return_offset = 0x284;
static constexpr dart::compiler::target::word
AOT_ObjectStore_suspend_state_await_offset = 0x224;
static constexpr dart::compiler::target::word
@@ -16234,6 +16278,8 @@ static constexpr dart::compiler::target::word AOT_ObjectStore_type_type_offset =
0x110;
static constexpr dart::compiler::target::word
AOT_ObjectStore_ffi_callback_code_offset = 0x500;
static constexpr dart::compiler::target::word
AOT_ObjectStore_ffi_callback_stack_return_offset = 0x508;
static constexpr dart::compiler::target::word
AOT_ObjectStore_suspend_state_await_offset = 0x448;
static constexpr dart::compiler::target::word
@@ -17009,6 +17055,8 @@ static constexpr dart::compiler::target::word AOT_ObjectStore_type_type_offset =
0x88;
static constexpr dart::compiler::target::word
AOT_ObjectStore_ffi_callback_code_offset = 0x280;
static constexpr dart::compiler::target::word
AOT_ObjectStore_ffi_callback_stack_return_offset = 0x284;
static constexpr dart::compiler::target::word
AOT_ObjectStore_suspend_state_await_offset = 0x224;
static constexpr dart::compiler::target::word
@@ -17781,6 +17829,8 @@ static constexpr dart::compiler::target::word AOT_ObjectStore_type_type_offset =
0x110;
static constexpr dart::compiler::target::word
AOT_ObjectStore_ffi_callback_code_offset = 0x500;
static constexpr dart::compiler::target::word
AOT_ObjectStore_ffi_callback_stack_return_offset = 0x508;
static constexpr dart::compiler::target::word
AOT_ObjectStore_suspend_state_await_offset = 0x448;
static constexpr dart::compiler::target::word
@@ -18556,6 +18606,8 @@ static constexpr dart::compiler::target::word AOT_ObjectStore_type_type_offset =
0x110;
static constexpr dart::compiler::target::word
AOT_ObjectStore_ffi_callback_code_offset = 0x500;
static constexpr dart::compiler::target::word
AOT_ObjectStore_ffi_callback_stack_return_offset = 0x508;
static constexpr dart::compiler::target::word
AOT_ObjectStore_suspend_state_await_offset = 0x448;
static constexpr dart::compiler::target::word
@@ -19330,6 +19382,8 @@ static constexpr dart::compiler::target::word AOT_ObjectStore_type_type_offset =
0x110;
static constexpr dart::compiler::target::word
AOT_ObjectStore_ffi_callback_code_offset = 0x500;
static constexpr dart::compiler::target::word
AOT_ObjectStore_ffi_callback_stack_return_offset = 0x508;
static constexpr dart::compiler::target::word
AOT_ObjectStore_suspend_state_await_offset = 0x448;
static constexpr dart::compiler::target::word
@@ -20104,6 +20158,8 @@ static constexpr dart::compiler::target::word AOT_ObjectStore_type_type_offset =
0x110;
static constexpr dart::compiler::target::word
AOT_ObjectStore_ffi_callback_code_offset = 0x500;
static constexpr dart::compiler::target::word
AOT_ObjectStore_ffi_callback_stack_return_offset = 0x508;
static constexpr dart::compiler::target::word
AOT_ObjectStore_suspend_state_await_offset = 0x448;
static constexpr dart::compiler::target::word
@@ -20880,6 +20936,8 @@ static constexpr dart::compiler::target::word AOT_ObjectStore_type_type_offset =
0x88;
static constexpr dart::compiler::target::word
AOT_ObjectStore_ffi_callback_code_offset = 0x280;
static constexpr dart::compiler::target::word
AOT_ObjectStore_ffi_callback_stack_return_offset = 0x284;
static constexpr dart::compiler::target::word
AOT_ObjectStore_suspend_state_await_offset = 0x224;
static constexpr dart::compiler::target::word
@@ -21653,6 +21711,8 @@ static constexpr dart::compiler::target::word AOT_ObjectStore_type_type_offset =
0x110;
static constexpr dart::compiler::target::word
AOT_ObjectStore_ffi_callback_code_offset = 0x500;
static constexpr dart::compiler::target::word
AOT_ObjectStore_ffi_callback_stack_return_offset = 0x508;
static constexpr dart::compiler::target::word
AOT_ObjectStore_suspend_state_await_offset = 0x448;
static constexpr dart::compiler::target::word
runtime/vm/compiler/runtime_offsets_list.h
+1
View File
@@ -197,6 +197,7 @@
FIELD(ObjectStore, string_type_offset) \
FIELD(ObjectStore, type_type_offset) \
FIELD(ObjectStore, ffi_callback_code_offset) \
FIELD(ObjectStore, ffi_callback_stack_return_offset) \
FIELD(ObjectStore, suspend_state_await_offset) \
FIELD(ObjectStore, suspend_state_await_with_type_check_offset) \
FIELD(ObjectStore, suspend_state_handle_exception_offset) \
runtime/vm/compiler/stub_code_compiler.h
+38 -3
View File
@@ -104,6 +104,44 @@ class StubCodeCompiler {
void GenerateUsageCounterIncrement(Register temp_reg);
void GenerateOptimizedUsageCounterIncrement();
#if defined(TARGET_ARCH_X64)
static constexpr intptr_t kNativeCallbackTrampolineSize = 10;
#if defined(DART_COMPRESSED_POINTERS)
static constexpr intptr_t kNativeCallbackSharedStubSize = 236;
#else
static constexpr intptr_t kNativeCallbackSharedStubSize = 228;
#endif
static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 2;
#elif defined(TARGET_ARCH_IA32)
static constexpr intptr_t kNativeCallbackTrampolineSize = 10;
static constexpr intptr_t kNativeCallbackSharedStubSize = 152;
static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 4;
#elif defined(TARGET_ARCH_ARM)
static constexpr intptr_t kNativeCallbackTrampolineSize = 12;
static constexpr intptr_t kNativeCallbackSharedStubSize = 148;
static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 4;
#elif defined(TARGET_ARCH_ARM64)
static constexpr intptr_t kNativeCallbackTrampolineSize = 12;
#if defined(DART_COMPRESSED_POINTERS)
static constexpr intptr_t kNativeCallbackSharedStubSize = 268;
#else
static constexpr intptr_t kNativeCallbackSharedStubSize = 244;
#endif
static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 2;
#elif defined(TARGET_ARCH_RISCV32)
static constexpr intptr_t kNativeCallbackTrampolineSize = 8;
static constexpr intptr_t kNativeCallbackSharedStubSize = 238;
static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 2;
#elif defined(TARGET_ARCH_RISCV64)
static constexpr intptr_t kNativeCallbackTrampolineSize = 8;
static constexpr intptr_t kNativeCallbackSharedStubSize = 210;
static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 2;
#else
#error What architecture?
#endif
void GenerateJITCallbackTrampolines(intptr_t next_callback_id);
// Calculates the offset (in words) from FP to the provided [cpu_register].
//
// Assumes
@@ -162,9 +200,6 @@ class StubCodeCompiler {
intptr_t return_function_offset_in_object_store,
intptr_t return_stub_offset_in_thread);
void GenerateLoadFfiCallbackMetadataRuntimeFunction(uword function_index,
Register dst);
DISALLOW_COPY_AND_ASSIGN(StubCodeCompiler);
};
runtime/vm/compiler/stub_code_compiler_arm.cc
+61 -66
View File
@@ -21,7 +21,6 @@
#include "vm/compiler/assembler/assembler.h"
#include "vm/compiler/backend/locations.h"
#include "vm/constants.h"
#include "vm/ffi_callback_metadata.h"
#include "vm/instructions.h"
#include "vm/static_type_exactness_state.h"
#include "vm/tags.h"
@@ -373,59 +372,45 @@ void StubCodeCompiler::GenerateCallNativeThroughSafepointStub() {
__ bx(R4);
}
void StubCodeCompiler::GenerateLoadFfiCallbackMetadataRuntimeFunction(
uword function_index,
Register dst) {
// Keep in sync with FfiCallbackMetadata::EnsureFirstTrampolinePageLocked.
// Note: If the stub was aligned, this could be a single PC relative load.
// Load a pointer to the beginning of the stub into dst.
const intptr_t code_size = __ CodeSize();
__ SubImmediate(dst, PC, Instr::kPCReadOffset + code_size);
// Round dst down to the page size.
__ AndImmediate(dst, dst, FfiCallbackMetadata::kPageMask);
// Load the function from the function table.
__ LoadFromOffset(
dst,
Address(dst, FfiCallbackMetadata::RuntimeFunctionOffset(function_index)));
}
void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
#if defined(USING_SIMULATOR) && !defined(DART_PRECOMPILER)
// TODO(37299): FFI is not supported in SIMARM.
#if !defined(DART_PRECOMPILER)
void StubCodeCompiler::GenerateJITCallbackTrampolines(
intptr_t next_callback_id) {
#if defined(USING_SIMULATOR)
// TODO(37299): FFI is not support in SIMARM.
__ Breakpoint();
#else
Label body;
Label done;
// TMP is volatile and not used for passing any arguments.
COMPILE_ASSERT(!IsCalleeSavedRegister(TMP) && !IsArgumentRegister(TMP));
for (intptr_t i = 0; i < FfiCallbackMetadata::NumCallbackTrampolinesPerPage();
++i) {
// The FfiCallbackMetadata table is keyed by the trampoline entry point. So
// look up the current PC, then jump to the shared section. The PC is offset
// by Instr::kPCReadOffset, which is subtracted below.
__ mov(TMP, Operand(PC));
__ b(&body);
for (intptr_t i = 0;
i < NativeCallbackTrampolines::NumCallbackTrampolinesPerPage(); ++i) {
// We don't use LoadImmediate because we need the trampoline size to be
// fixed independently of the callback ID.
//
// PC points two instructions ahead of the current one -- directly where we
// store the callback ID.
__ ldr(TMP, Address(PC, 0));
__ b(&done);
__ Emit(next_callback_id + i);
}
ASSERT(__ CodeSize() ==
FfiCallbackMetadata::kNativeCallbackTrampolineSize *
FfiCallbackMetadata::NumCallbackTrampolinesPerPage());
kNativeCallbackTrampolineSize *
NativeCallbackTrampolines::NumCallbackTrampolinesPerPage());
__ Bind(&body);
__ Bind(&done);
const intptr_t shared_stub_start = __ CodeSize();
// Save THR (callee-saved), R4 & R5 (temporaries, callee-saved), and LR.
COMPILE_ASSERT(FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta == 4);
COMPILE_ASSERT(StubCodeCompiler::kNativeCallbackTrampolineStackDelta == 4);
SPILLS_LR_TO_FRAME(
__ PushList((1 << LR) | (1 << THR) | (1 << R4) | (1 << R5)));
// The PC is in TMP, but is offset by kPCReadOffset. To get the actual
// trampoline entry point we need to subtract that.
__ sub(R4, TMP, Operand(Instr::kPCReadOffset));
// Don't rely on TMP being preserved by assembler macros anymore.
__ mov(R4, Operand(TMP));
COMPILE_ASSERT(IsCalleeSavedRegister(R4));
COMPILE_ASSERT(!IsArgumentRegister(THR));
@@ -436,42 +421,52 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
// Load the thread, verify the callback ID and exit the safepoint.
//
// We exit the safepoint inside DLRT_GetFfiCallbackMetadata in order to save
// code size on this shared stub.
// We exit the safepoint inside DLRT_GetThreadForNativeCallbackTrampoline
// in order to safe code size on this shared stub.
{
__ mov(R0, Operand(R4));
// We also need to look up the entry point for the trampoline. This is
// returned using a pointer passed to the second arg of the C function
// below. We aim that pointer at a reserved stack slot.
__ sub(SP, SP, Operand(compiler::target::kWordSize));
__ mov(R1, Operand(SP));
// We also need to know if this is a sync or async callback. This is also
// returned by pointer.
__ sub(SP, SP, Operand(compiler::target::kWordSize));
__ mov(R2, Operand(SP));
__ EnterFrame(1 << FP, 0);
__ ReserveAlignedFrameSpace(0);
GenerateLoadFfiCallbackMetadataRuntimeFunction(
FfiCallbackMetadata::kGetFfiCallbackMetadata, R4);
__ mov(R0, Operand(R4));
__ blx(R4);
// Since DLRT_GetThreadForNativeCallbackTrampoline can theoretically be
// loaded anywhere, we use the same trick as before to ensure a predictable
// instruction sequence.
Label call;
__ ldr(R1, Address(PC, 0));
__ b(&call);
__ Emit(
reinterpret_cast<intptr_t>(&DLRT_GetThreadForNativeCallbackTrampoline));
__ Bind(&call);
__ blx(R1);
__ mov(THR, Operand(R0));
__ LeaveFrame(1 << FP);
// The trampoline type is at the top of the stack. Pop it into R4.
__ Pop(R4);
// Entry point is now at the top of the stack. Pop it into R5.
__ Pop(R5);
}
__ PopRegisters(argument_registers);
COMPILE_ASSERT(!IsArgumentRegister(R8));
// Load the code object.
__ LoadFromOffset(R5, THR, compiler::target::Thread::isolate_group_offset());
__ LoadFromOffset(R5, R5,
compiler::target::IsolateGroup::object_store_offset());
__ LoadFromOffset(R5, R5,
compiler::target::ObjectStore::ffi_callback_code_offset());
__ LoadFieldFromOffset(R5, R5,
compiler::target::GrowableObjectArray::data_offset());
__ ldr(R5, __ ElementAddressForRegIndex(
/*is_load=*/true,
/*external=*/false,
/*array_cid=*/kArrayCid,
/*index_scale, smi-tagged=*/compiler::target::kWordSize * 2,
/*index_unboxed=*/false,
/*array=*/R5,
/*index=*/R4));
__ LoadFieldFromOffset(R5, R5, compiler::target::Code::entry_point_offset());
// On entry to the function, there will be four extra slots on the stack:
// saved THR, R4, R5 and the return address. The target will know to skip
// them.
@@ -483,17 +478,17 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
// Returns.
__ PopList((1 << PC) | (1 << THR) | (1 << R4) | (1 << R5));
ASSERT_LESS_OR_EQUAL(__ CodeSize() - shared_stub_start,
FfiCallbackMetadata::kNativeCallbackSharedStubSize);
ASSERT_LESS_OR_EQUAL(__ CodeSize(), FfiCallbackMetadata::kPageSize);
ASSERT((__ CodeSize() - shared_stub_start) == kNativeCallbackSharedStubSize);
ASSERT(__ CodeSize() <= VirtualMemory::PageSize());
#if defined(DEBUG)
while (__ CodeSize() < FfiCallbackMetadata::kPageSize) {
while (__ CodeSize() < VirtualMemory::PageSize()) {
__ Breakpoint();
}
#endif
#endif
}
#endif // !defined(DART_PRECOMPILER)
void StubCodeCompiler::GenerateDispatchTableNullErrorStub() {
__ EnterStubFrame();
runtime/vm/compiler/stub_code_compiler_arm64.cc
+79 -64
View File
@@ -20,7 +20,6 @@
#include "vm/compiler/assembler/assembler.h"
#include "vm/compiler/backend/locations.h"
#include "vm/constants.h"
#include "vm/ffi_callback_metadata.h"
#include "vm/instructions.h"
#include "vm/static_type_exactness_state.h"
#include "vm/tags.h"
@@ -426,53 +425,44 @@ void StubCodeCompiler::GenerateCallNativeThroughSafepointStub() {
__ ret(R19);
}
void StubCodeCompiler::GenerateLoadFfiCallbackMetadataRuntimeFunction(
uword function_index,
Register dst) {
// Keep in sync with FfiCallbackMetadata::EnsureFirstTrampolinePageLocked.
// Note: If the stub was aligned, this could be a single PC relative load.
// Load a pointer to the beginning of the stub into dst.
const intptr_t code_size = __ CodeSize();
__ adr(dst, Immediate(-code_size));
// Round dst down to the page size.
__ andi(dst, dst, Immediate(FfiCallbackMetadata::kPageMask));
// Load the function from the function table.
__ LoadFromOffset(
dst,
Address(dst, FfiCallbackMetadata::RuntimeFunctionOffset(function_index)));
}
void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
#if defined(USING_SIMULATOR) && !defined(DART_PRECOMPILER)
// TODO(37299): FFI is not supported in SIMARM64.
#if !defined(DART_PRECOMPILER)
void StubCodeCompiler::GenerateJITCallbackTrampolines(
intptr_t next_callback_id) {
#if !defined(HOST_ARCH_ARM64)
// TODO(37299): FFI is not support in SIMARM64.
__ Breakpoint();
#else
Label body;
Label done;
// R9 is volatile and not used for passing any arguments.
COMPILE_ASSERT(!IsCalleeSavedRegister(R9) && !IsArgumentRegister(R9));
for (intptr_t i = 0; i < FfiCallbackMetadata::NumCallbackTrampolinesPerPage();
++i) {
// The FfiCallbackMetadata table is keyed by the trampoline entry point. So
// look up the current PC, then jump to the shared section.
__ adr(R9, Immediate(0));
__ b(&body);
for (intptr_t i = 0;
i < NativeCallbackTrampolines::NumCallbackTrampolinesPerPage(); ++i) {
// We don't use LoadImmediate because we need the trampoline size to be
// fixed independently of the callback ID.
//
// Instead we paste the callback ID directly in the code load it
// PC-relative.
__ ldr(R9, compiler::Address::PC(2 * Instr::kInstrSize));
__ b(&done);
__ Emit(next_callback_id + i);
}
ASSERT_EQUAL(__ CodeSize(),
FfiCallbackMetadata::kNativeCallbackTrampolineSize *
FfiCallbackMetadata::NumCallbackTrampolinesPerPage());
ASSERT(__ CodeSize() ==
kNativeCallbackTrampolineSize *
NativeCallbackTrampolines::NumCallbackTrampolinesPerPage());
__ Bind(&body);
__ Bind(&done);
const intptr_t shared_stub_start = __ CodeSize();
// The load of the callback ID might have incorrect higher-order bits, since
// we only emit a 32-bit callback ID.
__ uxtw(R9, R9);
// Save THR (callee-saved) and LR on the real C stack (CSP). Keeps it
// aligned.
COMPILE_ASSERT(FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta == 2);
COMPILE_ASSERT(StubCodeCompiler::kNativeCallbackTrampolineStackDelta == 2);
SPILLS_LR_TO_FRAME(__ stp(
THR, LR, Address(CSP, -2 * target::kWordSize, Address::PairPreIndex)));
@@ -483,55 +473,80 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
all_registers.Add(Location::RegisterLocation(
CallingConventions::kPointerToReturnStructRegisterCall));
// The call below might clobber R9 (volatile, holding callback_id).
all_registers.Add(Location::RegisterLocation(R9));
// Load the thread, verify the callback ID and exit the safepoint.
//
// We exit the safepoint inside DLRT_GetFfiCallbackMetadata in order to save
// code size on this shared stub.
// We exit the safepoint inside DLRT_GetThreadForNativeCallbackTrampoline
// in order to safe code size on this shared stub.
{
__ mov(SP, CSP);
__ EnterFrame(0);
__ PushRegisters(all_registers);
__ mov(R0, R9);
// We also need to look up the entry point for the trampoline. This is
// returned using a pointer passed to the second arg of the C function
// below. We aim that pointer at a reserved stack slot.
__ AddImmediate(SP, SP, -compiler::target::kWordSize);
__ mov(R1, SP);
// We also need to know if this is a sync or async callback. This is also
// returned by pointer.
__ AddImmediate(SP, SP, -compiler::target::kWordSize);
__ mov(R2, SP);
__ EnterFrame(0);
__ ReserveAlignedFrameSpace(0);
__ mov(CSP, SP);
GenerateLoadFfiCallbackMetadataRuntimeFunction(
FfiCallbackMetadata::kGetFfiCallbackMetadata, R4);
// Since DLRT_GetThreadForNativeCallbackTrampoline can theoretically be
// loaded anywhere, we use the same trick as before to ensure a predictable
// instruction sequence.
Label call;
__ mov(R0, R9);
__ ldr(R1, compiler::Address::PC(2 * Instr::kInstrSize));
__ b(&call);
__ blr(R4);
__ Emit64(
reinterpret_cast<int64_t>(&DLRT_GetThreadForNativeCallbackTrampoline));
__ Bind(&call);
__ blr(R1);
__ mov(THR, R0);
__ LeaveFrame();
// The trampoline type is at the top of the stack. Pop it into R9.
__ Pop(R9);
// Entry point is now at the top of the stack. Pop it into R10.
COMPILE_ASSERT(!IsCalleeSavedRegister(R10) && !IsArgumentRegister(R10));
__ Pop(R10);
__ PopRegisters(all_registers);
__ LeaveFrame();
__ mov(CSP, SP);
}
COMPILE_ASSERT(!IsCalleeSavedRegister(R10) && !IsArgumentRegister(R10));
// Load the code object.
__ LoadFromOffset(R10, THR, compiler::target::Thread::isolate_group_offset());
__ LoadFromOffset(R10, R10,
compiler::target::IsolateGroup::object_store_offset());
__ LoadFromOffset(R10, R10,
compiler::target::ObjectStore::ffi_callback_code_offset());
#if defined(DART_COMPRESSED_POINTERS)
// Partially setup HEAP_BITS for LoadCompressed[FieldFromOffset].
ASSERT(IsAbiPreservedRegister(HEAP_BITS)); // Need to save and restore.
__ Push(HEAP_BITS);
__ ldr(HEAP_BITS, compiler::Address(THR, target::Thread::heap_base_offset()));
__ LsrImmediate(HEAP_BITS, HEAP_BITS, 32);
#endif
__ LoadCompressedFieldFromOffset(
R10, R10, compiler::target::GrowableObjectArray::data_offset());
__ LoadCompressed(
R10,
__ ElementAddressForRegIndex(
/*external=*/false,
/*array_cid=*/kArrayCid,
/*index_scale, smi-tagged=*/compiler::target::kCompressedWordSize * 2,
/*index_unboxed=*/false,
/*array=*/R10,
/*index=*/R9,
/*temp=*/TMP));
#if defined(DART_COMPRESSED_POINTERS)
__ Pop(HEAP_BITS);
#endif
__ LoadFieldFromOffset(R10, R10,
compiler::target::Code::entry_point_offset());
// Clobbers all volatile registers, including the callback ID in R9.
// Resets CSP and SP, important for EnterSafepoint below.
__ blr(R10);
@@ -545,17 +560,17 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
__ ret();
ASSERT_LESS_OR_EQUAL(__ CodeSize() - shared_stub_start,
FfiCallbackMetadata::kNativeCallbackSharedStubSize);
ASSERT_LESS_OR_EQUAL(__ CodeSize(), FfiCallbackMetadata::kPageSize);
ASSERT((__ CodeSize() - shared_stub_start) == kNativeCallbackSharedStubSize);
ASSERT(__ CodeSize() <= VirtualMemory::PageSize());
#if defined(DEBUG)
while (__ CodeSize() < FfiCallbackMetadata::kPageSize) {
while (__ CodeSize() < VirtualMemory::PageSize()) {
__ Breakpoint();
}
#endif
#endif // !defined(HOST_ARCH_ARM64)
}
#endif // !defined(DART_PRECOMPILER)
// R1: The extracted method.
// R4: The type_arguments_field_offset (or 0)
runtime/vm/compiler/stub_code_compiler_ia32.cc
+88 -67
View File
@@ -19,7 +19,6 @@
#include "vm/compiler/assembler/assembler.h"
#include "vm/compiler/backend/locations.h"
#include "vm/constants.h"
#include "vm/ffi_callback_metadata.h"
#include "vm/instructions.h"
#include "vm/static_type_exactness_state.h"
#include "vm/tags.h"
@@ -211,108 +210,131 @@ void StubCodeCompiler::GenerateCallNativeThroughSafepointStub() {
__ jmp(EBX);
}
void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
Label ret_4;
void StubCodeCompiler::GenerateJITCallbackTrampolines(
intptr_t next_callback_id) {
Label done, ret_4;
// EAX is volatile and doesn't hold any arguments.
COMPILE_ASSERT(!IsArgumentRegister(EAX) && !IsCalleeSavedRegister(EAX));
Label body, load_tramp_addr;
const intptr_t kCallLength = 5;
for (intptr_t i = 0; i < FfiCallbackMetadata::NumCallbackTrampolinesPerPage();
++i) {
// The FfiCallbackMetadata table is keyed by the trampoline entry point. So
// look up the current PC, then jump to the shared section. There's no easy
// way to get the PC in ia32 so we have to do a call, grab the return adress
// from the stack, then return here (mismatched call/ret causes problems),
// then jump to the shared section.
const intptr_t size_before = __ CodeSize();
__ call(&load_tramp_addr);
const intptr_t size_after = __ CodeSize();
ASSERT_EQUAL(size_after - size_before, kCallLength);
__ jmp(&body);
for (intptr_t i = 0;
i < NativeCallbackTrampolines::NumCallbackTrampolinesPerPage(); ++i) {
__ movl(EAX, compiler::Immediate(next_callback_id + i));
__ jmp(&done);
}
ASSERT_EQUAL(__ CodeSize(),
FfiCallbackMetadata::kNativeCallbackTrampolineSize *
FfiCallbackMetadata::NumCallbackTrampolinesPerPage());
ASSERT(__ CodeSize() ==
kNativeCallbackTrampolineSize *
NativeCallbackTrampolines::NumCallbackTrampolinesPerPage());
__ Bind(&done);
const intptr_t shared_stub_start = __ CodeSize();
__ Bind(&load_tramp_addr);
// Load the return adress into EAX, and subtract the size of the call
// instruction. This is our original trampoline address.
__ movl(EAX, Address(SPREG, 0));
__ subl(EAX, Immediate(kCallLength));
__ ret();
__ Bind(&body);
// Save THR and EBX which are callee-saved.
__ pushl(THR);
__ pushl(EBX);
// We need the callback ID after the call for return stack.
__ pushl(EAX);
// THR & return address
COMPILE_ASSERT(FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta == 4);
COMPILE_ASSERT(StubCodeCompiler::kNativeCallbackTrampolineStackDelta == 4);
// Load the thread, verify the callback ID and exit the safepoint.
//
// We exit the safepoint inside DLRT_GetFfiCallbackMetadata in order to safe
// code size on this shared stub.
// We exit the safepoint inside DLRT_GetThreadForNativeCallbackTrampoline
// in order to save code size on this shared stub.
{
__ EnterFrame(0);
// entry_point, trampoline_type, &trampoline_type, &entry_point, trampoline
// ^------ GetFfiCallbackMetadata args ------^
__ ReserveAlignedFrameSpace(5 * target::kWordSize);
__ ReserveAlignedFrameSpace(compiler::target::kWordSize);
// Trampoline arg.
__ movl(Address(SPREG, 0 * target::kWordSize), EAX);
// Pointer to trampoline type stack slot.
__ movl(EAX, SPREG);
__ addl(EAX, Immediate(3 * target::kWordSize));
__ movl(Address(SPREG, 2 * target::kWordSize), EAX);
// Pointer to entry point stack slot.
__ addl(EAX, Immediate(target::kWordSize));
__ movl(Address(SPREG, 1 * target::kWordSize), EAX);
__ movl(EAX,
Immediate(reinterpret_cast<int64_t>(DLRT_GetFfiCallbackMetadata)));
__ movl(compiler::Address(SPREG, 0), EAX);
__ movl(EAX, compiler::Immediate(reinterpret_cast<int64_t>(
DLRT_GetThreadForNativeCallbackTrampoline)));
__ call(EAX);
__ movl(THR, EAX);
// Save the trampoline type in EBX, and the entry point in ECX.
__ movl(EBX, Address(SPREG, 3 * target::kWordSize));
__ movl(ECX, Address(SPREG, 4 * target::kWordSize));
__ movl(EAX, compiler::Address(SPREG, 0));
__ LeaveFrame();
// Save the trampoline type to the stack, because we'll need it after the
// call to decide whether to ret() or ret(4).
__ pushl(EBX);
}
COMPILE_ASSERT(!IsCalleeSavedRegister(ECX) && !IsArgumentRegister(ECX));
COMPILE_ASSERT(ECX != THR);
// Load the target from the thread.
__ movl(ECX, compiler::Address(
THR, compiler::target::Thread::isolate_group_offset()));
__ movl(ECX, compiler::Address(
ECX, compiler::target::IsolateGroup::object_store_offset()));
__ movl(ECX,
compiler::Address(
ECX, compiler::target::ObjectStore::ffi_callback_code_offset()));
__ movl(ECX, compiler::FieldAddress(
ECX, compiler::target::GrowableObjectArray::data_offset()));
__ movl(ECX, __ ElementAddressForRegIndex(
/*external=*/false,
/*array_cid=*/kArrayCid,
/*index, smi-tagged=*/compiler::target::kWordSize * 2,
/*index_unboxed=*/false,
/*array=*/ECX,
/*index=*/EAX));
__ movl(ECX, compiler::FieldAddress(
ECX, compiler::target::Code::entry_point_offset()));
// On entry to the function, there will be two extra slots on the stack:
// the saved THR and the return address. The target will know to skip them.
__ call(ECX);
// Register state:
// - callee saved registers (should be restored)
// - EBX available as scratch because we restore it later.
// - ESI(THR) contains thread
// - EDI
// - return registers (should not be touched)
// - EAX
// - EDX
// - available scratch registers
// - ECX free
// Load the return stack delta from the thread.
__ movl(ECX, compiler::Address(
THR, compiler::target::Thread::isolate_group_offset()));
__ movl(ECX, compiler::Address(
ECX, compiler::target::IsolateGroup::object_store_offset()));
__ movl(
ECX,
compiler::Address(
ECX,
compiler::target::ObjectStore::ffi_callback_stack_return_offset()));
__ popl(EBX); // Compiler callback id.
__ movzxb(EBX, __ ElementAddressForRegIndex(
/*external=*/false,
/*array_cid=*/kTypedDataUint8ArrayCid,
/*index=*/1,
/*index_unboxed=*/false,
/*array=*/ECX,
/*index=*/EBX));
#if defined(DEBUG)
// Stack delta should be either 0 or 4.
Label check_done;
__ BranchIfZero(EBX, &check_done);
__ CompareImmediate(EBX, compiler::target::kWordSize);
__ BranchIf(EQUAL, &check_done);
__ Breakpoint();
__ Bind(&check_done);
#endif
// Takes care to not clobber *any* registers (besides scratch).
__ EnterFullSafepoint(/*scratch=*/ECX);
// Pop the trampoline type into ECX.
__ popl(ECX);
// Restore callee-saved registers.
__ movl(ECX, EBX);
__ popl(EBX);
__ popl(THR);
__ cmpl(ECX, Immediate(static_cast<uword>(
FfiCallbackMetadata::TrampolineType::kSync)));
__ j(NOT_EQUAL, &ret_4, Assembler::kNearJump);
__ cmpl(ECX, compiler::Immediate(Smi::RawValue(0)));
__ j(NOT_EQUAL, &ret_4, compiler::Assembler::kNearJump);
__ ret();
__ Bind(&ret_4);
@@ -320,12 +342,11 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
// 'kNativeCallbackSharedStubSize' is an upper bound because the exact
// instruction size can vary slightly based on OS calling conventions.
ASSERT_LESS_OR_EQUAL(__ CodeSize() - shared_stub_start,
FfiCallbackMetadata::kNativeCallbackSharedStubSize);
ASSERT_LESS_OR_EQUAL(__ CodeSize(), FfiCallbackMetadata::kPageSize);
ASSERT((__ CodeSize() - shared_stub_start) <= kNativeCallbackSharedStubSize);
ASSERT(__ CodeSize() <= VirtualMemory::PageSize());
#if defined(DEBUG)
while (__ CodeSize() < FfiCallbackMetadata::kPageSize) {
while (__ CodeSize() < VirtualMemory::PageSize()) {
__ Breakpoint();
}
#endif
runtime/vm/compiler/stub_code_compiler_riscv.cc
+65 -55
View File
@@ -20,7 +20,6 @@
#include "vm/compiler/assembler/assembler.h"
#include "vm/compiler/backend/locations.h"
#include "vm/constants.h"
#include "vm/ffi_callback_metadata.h"
#include "vm/instructions.h"
#include "vm/static_type_exactness_state.h"
#include "vm/tags.h"
@@ -293,53 +292,37 @@ void StubCodeCompiler::GenerateCallNativeThroughSafepointStub() {
__ jr(S3);
}
void StubCodeCompiler::GenerateLoadFfiCallbackMetadataRuntimeFunction(
uword function_index,
Register dst) {
// Keep in sync with FfiCallbackMetadata::EnsureFirstTrampolinePageLocked.
// Note: If the stub was aligned, this could be a single PC relative load.
// Load a pointer to the beginning of the stub into dst.
const intptr_t code_size = __ CodeSize();
__ auipc(dst, 0);
__ AddImmediate(dst, -code_size);
// Round dst down to the page size.
__ AndImmediate(dst, FfiCallbackMetadata::kPageMask);
// Load the function from the function table.
__ LoadFromOffset(
dst,
Address(dst, FfiCallbackMetadata::RuntimeFunctionOffset(function_index)));
}
void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
#if defined(USING_SIMULATOR) && !defined(DART_PRECOMPILER)
// TODO(37299): FFI is not supported in SIMRISCV32/64.
#if !defined(DART_PRECOMPILER)
void StubCodeCompiler::GenerateJITCallbackTrampolines(
intptr_t next_callback_id) {
#if defined(USING_SIMULATOR)
// TODO(37299): FFI is not support in SIMRISCV32/64.
__ ebreak();
#else
Label body;
Label loaded_callback_id_hi;
// T1 is volatile and not used for passing any arguments.
COMPILE_ASSERT(!IsCalleeSavedRegister(T1) && !IsArgumentRegister(T1));
for (intptr_t i = 0; i < FfiCallbackMetadata::NumCallbackTrampolinesPerPage();
++i) {
// The FfiCallbackMetadata table is keyed by the trampoline entry point. So
// look up the current PC, then jump to the shared section.
__ auipc(T1, 0);
__ j(&body);
for (intptr_t i = 0;
i < NativeCallbackTrampolines::NumCallbackTrampolinesPerPage(); ++i) {
// We don't use LoadImmediate because we need the trampoline size to be
// fixed independently of the callback ID.
// lui has 20 bits of range.
__ lui_fixed(T1, (next_callback_id + i) << 12);
__ j(&loaded_callback_id_hi);
}
ASSERT_EQUAL(__ CodeSize(),
FfiCallbackMetadata::kNativeCallbackTrampolineSize *
FfiCallbackMetadata::NumCallbackTrampolinesPerPage());
ASSERT(__ CodeSize() ==
kNativeCallbackTrampolineSize *
NativeCallbackTrampolines::NumCallbackTrampolinesPerPage());
const intptr_t shared_stub_start = __ CodeSize();
__ Bind(&body);
__ Bind(&loaded_callback_id_hi);
__ srai(T1, T1, 12);
// Save THR (callee-saved) and RA. Keeps stack aligned.
COMPILE_ASSERT(FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta == 2);
COMPILE_ASSERT(StubCodeCompiler::kNativeCallbackTrampolineStackDelta == 2);
__ PushRegisterPair(RA, THR);
COMPILE_ASSERT(!IsArgumentRegister(THR));
@@ -351,31 +334,37 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
// Load the thread, verify the callback ID and exit the safepoint.
//
// We exit the safepoint inside DLRT_GetFfiCallbackMetadata in order to save
// code size on this shared stub.
// We exit the safepoint inside DLRT_GetThreadForNativeCallbackTrampoline
// in order to safe code size on this shared stub.
{
__ PushRegisters(all_registers);
__ EnterFrame(0);
// Reserve one slot for the entry point and one for the tramp abi.
__ ReserveAlignedFrameSpace(2 * target::kWordSize);
__ ReserveAlignedFrameSpace(0);
// Since DLRT_GetFfiCallbackMetadata can theoretically be loaded anywhere,
// we use the same trick as before to ensure a predictable instruction
// sequence.
// Since DLRT_GetThreadForNativeCallbackTrampoline can theoretically be
// loaded anywhere, we use the same trick as before to ensure a predictable
// instruction sequence.
Label call;
__ mv(A0, T1); // trampoline
__ mv(A1, SPREG); // out_entry_point
__ addi(A2, SPREG, target::kWordSize); // out_trampoline_type
__ mv(A0, T1);
GenerateLoadFfiCallbackMetadataRuntimeFunction(
FfiCallbackMetadata::kGetFfiCallbackMetadata, T1);
const intptr_t kPCRelativeLoadOffset = 12;
intptr_t start = __ CodeSize();
__ auipc(T1, 0);
__ lx(T1, Address(T1, kPCRelativeLoadOffset));
__ j(&call);
ASSERT_EQUAL(__ CodeSize() - start, kPCRelativeLoadOffset);
#if XLEN == 32
__ Emit32(
reinterpret_cast<int32_t>(&DLRT_GetThreadForNativeCallbackTrampoline));
#else
__ Emit64(
reinterpret_cast<int64_t>(&DLRT_GetThreadForNativeCallbackTrampoline));
#endif
__ Bind(&call);
__ jalr(T1);
__ mv(THR, A0);
__ lx(T2, Address(SPREG, 0)); // entry_point
__ lx(T3, Address(SPREG, target::kWordSize)); // trampoline_type
__ LeaveFrame();
@@ -385,6 +374,26 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
COMPILE_ASSERT(!IsCalleeSavedRegister(T2) && !IsArgumentRegister(T2));
COMPILE_ASSERT(!IsCalleeSavedRegister(T3) && !IsArgumentRegister(T3));
// Load the code object.
__ LoadFromOffset(T2, THR, compiler::target::Thread::isolate_group_offset());
__ LoadFromOffset(T2, T2,
compiler::target::IsolateGroup::object_store_offset());
__ LoadFromOffset(T2, T2,
compiler::target::ObjectStore::ffi_callback_code_offset());
__ LoadCompressedFieldFromOffset(
T2, T2, compiler::target::GrowableObjectArray::data_offset());
__ LoadCompressed(
T2,
__ ElementAddressForRegIndex(
/*external=*/false,
/*array_cid=*/kArrayCid,
/*index_scale, smi-tagged=*/compiler::target::kCompressedWordSize * 2,
/*index_unboxed=*/false,
/*array=*/T2,
/*index=*/T1,
/*temp=*/T3));
__ LoadFieldFromOffset(T2, T2, compiler::target::Code::entry_point_offset());
// Clobbers all volatile registers, including the callback ID in T1.
__ jalr(T2);
@@ -394,17 +403,18 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
__ PopRegisterPair(RA, THR);
__ ret();
ASSERT_LESS_OR_EQUAL(__ CodeSize() - shared_stub_start,
FfiCallbackMetadata::kNativeCallbackSharedStubSize);
ASSERT_LESS_OR_EQUAL(__ CodeSize(), FfiCallbackMetadata::kPageSize);
ASSERT_EQUAL((__ CodeSize() - shared_stub_start),
kNativeCallbackSharedStubSize);
ASSERT(__ CodeSize() <= VirtualMemory::PageSize());
#if defined(DEBUG)
while (__ CodeSize() < FfiCallbackMetadata::kPageSize) {
while (__ CodeSize() < VirtualMemory::PageSize()) {
__ ebreak();
}
#endif
#endif
}
#endif // !defined(DART_PRECOMPILER)
// T1: The extracted method.
// T4: The type_arguments_field_offset (or 0)
runtime/vm/compiler/stub_code_compiler_x64.cc
+55 -71
View File
@@ -23,7 +23,6 @@
#include "vm/compiler/api/type_check_mode.h"
#include "vm/compiler/assembler/assembler.h"
#include "vm/constants.h"
#include "vm/ffi_callback_metadata.h"
#include "vm/instructions.h"
#include "vm/static_type_exactness_state.h"
#include "vm/tags.h"
@@ -386,54 +385,29 @@ void StubCodeCompiler::GenerateCallNativeThroughSafepointStub() {
__ ret();
}
void StubCodeCompiler::GenerateLoadFfiCallbackMetadataRuntimeFunction(
uword function_index,
Register dst) {
// Keep in sync with FfiCallbackMetadata::EnsureFirstTrampolinePageLocked.
// Note: If the stub was aligned, this could be a single PC relative load.
// Load a pointer to the beginning of the stub into dst.
const intptr_t kLeaqLength = 7;
const intptr_t code_size = __ CodeSize();
__ leaq(dst, Address::AddressRIPRelative(-kLeaqLength - code_size));
// Round dst down to the page size.
__ andq(dst, Immediate(FfiCallbackMetadata::kPageMask));
// Load the function from the function table.
__ LoadFromOffset(
dst,
Address(dst, FfiCallbackMetadata::RuntimeFunctionOffset(function_index)));
}
#if !defined(DART_PRECOMPILER)
static const RegisterSet kArgumentRegisterSet(
CallingConventions::kArgumentRegisters,
CallingConventions::kFpuArgumentRegisters);
void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
void StubCodeCompiler::GenerateJITCallbackTrampolines(
intptr_t next_callback_id) {
Label done;
// RAX is volatile and not used for passing any arguments.
COMPILE_ASSERT(!IsCalleeSavedRegister(RAX) && !IsArgumentRegister(RAX));
Label body;
for (intptr_t i = 0; i < FfiCallbackMetadata::NumCallbackTrampolinesPerPage();
++i) {
// The FfiCallbackMetadata table is keyed by the trampoline entry point. So
// look up the current PC, then jump to the shared section. RIP gives us the
// address of the next instruction, so to get the true entry point, we have
// to subtract the size of the leaq instruction.
const intptr_t kLeaqLength = 7;
const intptr_t size_before = __ CodeSize();
__ leaq(RAX, Address::AddressRIPRelative(-kLeaqLength));
const intptr_t size_after = __ CodeSize();
ASSERT_EQUAL(size_after - size_before, kLeaqLength);
__ jmp(&body);
for (intptr_t i = 0;
i < NativeCallbackTrampolines::NumCallbackTrampolinesPerPage(); ++i) {
__ movq(RAX, compiler::Immediate(next_callback_id + i));
__ jmp(&done);
}
ASSERT_EQUAL(__ CodeSize(),
FfiCallbackMetadata::kNativeCallbackTrampolineSize *
FfiCallbackMetadata::NumCallbackTrampolinesPerPage());
kNativeCallbackTrampolineSize *
NativeCallbackTrampolines::NumCallbackTrampolinesPerPage());
__ Bind(&body);
__ Bind(&done);
const intptr_t shared_stub_start = __ CodeSize();
@@ -441,53 +415,38 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
__ pushq(THR);
// 2 = THR & return address
COMPILE_ASSERT(2 == FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta);
COMPILE_ASSERT(2 == StubCodeCompiler::kNativeCallbackTrampolineStackDelta);
// Save the callback ID.
__ pushq(RAX);
// Save all registers which might hold arguments.
__ PushRegisters(kArgumentRegisterSet);
// Load the thread, verify the callback ID and exit the safepoint.
//
// We exit the safepoint inside DLRT_GetFfiCallbackMetadata in order to safe
// code size on this shared stub.
// We exit the safepoint inside DLRT_GetThreadForNativeCallbackTrampoline
// in order to save code size on this shared stub.
{
COMPILE_ASSERT(RAX != CallingConventions::kArg1Reg);
__ movq(CallingConventions::kArg1Reg, RAX);
// We also need to look up the entry point for the trampoline. This is
// returned using a pointer passed to the second arg of the C function
// below. We aim that pointer at a reserved stack slot.
COMPILE_ASSERT(RAX != CallingConventions::kArg2Reg);
__ pushq(Immediate(0)); // Reserve a stack slot for the entry point.
__ movq(CallingConventions::kArg2Reg, RSP);
// We also need to know if this is a sync or async callback. This is also
// returned by pointer.
COMPILE_ASSERT(RAX != CallingConventions::kArg3Reg);
__ pushq(Immediate(0)); // Reserve a stack slot for the trampoline type.
__ movq(CallingConventions::kArg3Reg, RSP);
GenerateLoadFfiCallbackMetadataRuntimeFunction(
FfiCallbackMetadata::kGetFfiCallbackMetadata, RAX);
__ EnterFrame(0);
__ ReserveAlignedFrameSpace(0);
COMPILE_ASSERT(RAX != CallingConventions::kArg1Reg);
__ movq(CallingConventions::kArg1Reg, RAX);
__ movq(RAX, compiler::Immediate(reinterpret_cast<int64_t>(
DLRT_GetThreadForNativeCallbackTrampoline)));
__ CallCFunction(RAX);
__ movq(THR, RAX);
__ LeaveFrame();
// The trampoline type is at the top of the stack. Pop it into RAX.
__ popq(RAX);
// Entry point is now at the top of the stack. Pop it into TMP.
__ popq(TMP);
}
// Restore the arguments.
__ PopRegisters(kArgumentRegisterSet);
// Restore the callback ID.
__ popq(RAX);
// Current state:
//
// Stack:
@@ -495,9 +454,34 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
// <return address>
// <saved THR>
//
// Registers: Like entry, except TMP == target, RAX == abi, and THR == thread
// Registers: Like entry, except RAX == callback_id and THR == thread
// All argument registers are untouched.
COMPILE_ASSERT(!IsCalleeSavedRegister(TMP) && !IsArgumentRegister(TMP));
// Load the target from the thread.
__ movq(TMP, compiler::Address(
THR, compiler::target::Thread::isolate_group_offset()));
__ movq(TMP, compiler::Address(
TMP, compiler::target::IsolateGroup::object_store_offset()));
__ movq(TMP,
compiler::Address(
TMP, compiler::target::ObjectStore::ffi_callback_code_offset()));
__ LoadCompressed(
TMP, compiler::FieldAddress(
TMP, compiler::target::GrowableObjectArray::data_offset()));
__ LoadCompressed(
TMP,
__ ElementAddressForRegIndex(
/*external=*/false,
/*array_cid=*/kArrayCid,
/*index_scale, smi-tagged=*/compiler::target::kCompressedWordSize * 2,
/*index_unboxed=*/false,
/*array=*/TMP,
/*index=*/RAX));
__ movq(TMP, compiler::FieldAddress(
TMP, compiler::target::Code::entry_point_offset()));
// On entry to the function, there will be two extra slots on the stack:
// the saved THR and the return address. The target will know to skip them.
__ call(TMP);
@@ -512,16 +496,16 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
// 'kNativeCallbackSharedStubSize' is an upper bound because the exact
// instruction size can vary slightly based on OS calling conventions.
ASSERT_LESS_OR_EQUAL(__ CodeSize() - shared_stub_start,
FfiCallbackMetadata::kNativeCallbackSharedStubSize);
ASSERT_LESS_OR_EQUAL(__ CodeSize(), FfiCallbackMetadata::kPageSize);
ASSERT((__ CodeSize() - shared_stub_start) <= kNativeCallbackSharedStubSize);
ASSERT(__ CodeSize() <= VirtualMemory::PageSize());
#if defined(DEBUG)
while (__ CodeSize() < FfiCallbackMetadata::kPageSize) {
while (__ CodeSize() < VirtualMemory::PageSize()) {
__ Breakpoint();
}
#endif
}
#endif // !defined(DART_PRECOMPILER)
// RBX: The extracted method.
// RDX: The type_arguments_field_offset (or 0)
runtime/vm/dart.cc
-3
View File
@@ -20,7 +20,6 @@
#if defined(DART_PRECOMPILED_RUNTIME) && defined(DART_TARGET_OS_LINUX)
#include "vm/elf.h"
#endif
#include "vm/ffi_callback_metadata.h"
#include "vm/flags.h"
#include "vm/handles.h"
#include "vm/heap/become.h"
@@ -342,7 +341,6 @@ char* Dart::DartInit(const Dart_InitializeParams* params) {
StoreBuffer::Init();
MarkingStack::Init();
TargetCPUFeatures::Init();
FfiCallbackMetadata::Init();
#if defined(USING_SIMULATOR)
Simulator::Init();
@@ -766,7 +764,6 @@ char* Dart::Cleanup() {
SubtypeTestCache::Cleanup();
ArgumentsDescriptor::Cleanup();
OffsetsTable::Cleanup();
FfiCallbackMetadata::Cleanup();
TargetCPUFeatures::Cleanup();
MarkingStack::Cleanup();
StoreBuffer::Cleanup();
runtime/vm/elf.cc
+1 -1
View File
@@ -1192,7 +1192,7 @@ void Elf::CreateBSS() {
label = kVmBssLabel;
} else if (strcmp(portion.symbol_name,
kIsolateSnapshotInstructionsAsmSymbol) == 0) {
size = BSS::kIsolateGroupEntryCount * compiler::target::kWordSize;
size = BSS::kIsolateEntryCount * compiler::target::kWordSize;
symbol_name = kIsolateSnapshotBssAsmSymbol;
label = kIsolateBssLabel;
} else {
runtime/vm/ffi_callback_metadata.cc
-256
View File
@@ -1,256 +0,0 @@
// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#include "vm/ffi_callback_metadata.h"
#include "vm/flag_list.h"
#include "vm/object.h"
#include "vm/runtime_entry.h"
#include "vm/stub_code.h"
namespace dart {
FfiCallbackMetadata::FfiCallbackMetadata() {}
void FfiCallbackMetadata::EnsureStubPageLocked() {
// Assumes lock_ is already locked for writing.
if (stub_page_ != nullptr) {
return;
}
// Keep in sync with GenerateLoadFfiCallbackMetadataRuntimeFunction.
// The FfiCallbackTrampoline stub is designed to take up 1 page of memory. At
// the moment it's not aligned though, so we need to do some alignment math
// here. So when we duplicate it below, we're wasting some memory because the
// stub probably straddles 2 aligned pages. It would be better to align the
// stub inside the stub code compiler, but we don't have a way of doing that
// at the moment.
// TODO(52498): Align the stub.
// | page | page | pages |
// [ alignment ][ stub ][ alignment ][ functions ][ metadata ][ alignment ]
ASSERT_LESS_OR_EQUAL(VirtualMemory::PageSize(), kPageSize);
const Code& trampoline_code = StubCode::FfiCallbackTrampoline();
const uword code_start = trampoline_code.EntryPoint();
const uword page_start = Utils::RoundDown(code_start, kPageSize);
const uword code_end_aligned = page_start + 2 * kPageSize;
ASSERT_LESS_OR_EQUAL(code_start + trampoline_code.Size(), code_end_aligned);
const uword functions_start = code_end_aligned;
const uword functions_size =
kNumRuntimeFunctions * compiler::target::kWordSize;
const uword metadata_start = functions_start + functions_size;
const uword metadata_size =
NumCallbackTrampolinesPerPage() * sizeof(Metadata);
const uword metadata_end = metadata_start + metadata_size;
const uword page_end = Utils::RoundUp(metadata_end, kPageSize);
stub_page_ = VirtualMemory::ForImagePage(reinterpret_cast<void*>(page_start),
code_end_aligned - page_start);
offset_of_first_trampoline_in_page_ = code_start - page_start;
offset_of_first_runtime_function_in_page_ = functions_start - page_start;
offset_of_first_metadata_in_page_ = metadata_start - page_start;
size_of_trampoline_page_ = page_end - page_start;
}
FfiCallbackMetadata::~FfiCallbackMetadata() {
// Unmap all the trampoline pages. 'VirtualMemory's are new-allocated.
delete stub_page_;
for (intptr_t i = 0; i < trampoline_pages_.length(); ++i) {
delete trampoline_pages_[i];
}
}
void FfiCallbackMetadata::Init() {
ASSERT(singleton_ == nullptr);
singleton_ = new FfiCallbackMetadata();
}
void FfiCallbackMetadata::Cleanup() {
ASSERT(singleton_ != nullptr);
delete singleton_;
singleton_ = nullptr;
}
FfiCallbackMetadata* FfiCallbackMetadata::Instance() {
ASSERT(singleton_ != nullptr);
return singleton_;
}
void FfiCallbackMetadata::FillRuntimeFunction(VirtualMemory* page,
uword index,
void* function) {
uword offset = offset_of_first_runtime_function_in_page_ +
index * compiler::target::kWordSize;
void** slot = reinterpret_cast<void**>(page->start() + offset);
*slot = function;
}
VirtualMemory* FfiCallbackMetadata::AllocateTrampolinePage() {
VirtualMemory* new_page = VirtualMemory::AllocateAligned(
size_of_trampoline_page_, kPageSize, /*is_executable=*/false,
/*is_compressed=*/false, "FfiCallbackMetadata::TrampolinePage");
if (new_page == nullptr) {
return nullptr;
}
if (!stub_page_->DuplicateRX(new_page)) {
delete new_page;
return nullptr;
}
return new_page;
}
void FfiCallbackMetadata::EnsureFreeListNotEmptyLocked() {
// Assumes lock_ is already locked for writing.
if (free_list_head_ != nullptr) {
return;
}
EnsureStubPageLocked();
VirtualMemory* new_page = AllocateTrampolinePage();
if (new_page == nullptr) {
Exceptions::ThrowOOM();
}
trampoline_pages_.Add(new_page);
// Fill in the runtime functions.
FillRuntimeFunction(new_page, kGetFfiCallbackMetadata,
reinterpret_cast<void*>(DLRT_GetFfiCallbackMetadata));
// Add all the trampolines to the free list.
const intptr_t trampolines_per_page = NumCallbackTrampolinesPerPage();
for (intptr_t i = 0; i < trampolines_per_page; ++i) {
const Trampoline trampoline = reinterpret_cast<Trampoline>(
new_page->start() + offset_of_first_trampoline_in_page_ +
i * kNativeCallbackTrampolineSize);
AddToFreeListLocked(trampoline, LookupEntryLocked(trampoline));
}
}
FfiCallbackMetadata::Trampoline
FfiCallbackMetadata::AllocateTrampolineLocked() {
// Assumes lock_ is already locked for writing.
EnsureFreeListNotEmptyLocked();
ASSERT(free_list_head_ != nullptr);
const Trampoline trampoline = free_list_head_;
auto* entry = LookupEntryLocked(trampoline);
free_list_head_ = entry->free_list_next_;
if (free_list_head_ == nullptr) {
ASSERT(free_list_tail_ == trampoline);
free_list_tail_ = nullptr;
}
return trampoline;
}
void FfiCallbackMetadata::AddToFreeListLocked(Trampoline trampoline,
Metadata* entry) {
// Assumes lock_ is already locked for writing.
if (free_list_tail_ == nullptr) {
ASSERT(free_list_head_ == nullptr);
free_list_head_ = free_list_tail_ = trampoline;
} else {
ASSERT(free_list_head_ != nullptr);
auto* tail = LookupEntryLocked(free_list_tail_);
ASSERT(!tail->IsLive());
ASSERT(tail->free_list_next_ == nullptr);
tail->free_list_next_ = trampoline;
free_list_tail_ = trampoline;
}
entry->target_isolate_ = nullptr;
entry->free_list_next_ = nullptr;
}
void FfiCallbackMetadata::DeleteSyncTrampolines(Trampoline* sync_list_head) {
WriteRwLocker locker(Thread::Current(), &lock_);
for (Trampoline trampoline = *sync_list_head; trampoline != nullptr;) {
auto* entry = LookupEntryLocked(trampoline);
ASSERT(entry != nullptr);
const Trampoline next_trampoline = entry->sync_list_next();
AddToFreeListLocked(trampoline, entry);
trampoline = next_trampoline;
}
*sync_list_head = nullptr;
}
FfiCallbackMetadata::Trampoline FfiCallbackMetadata::CreateFfiCallback(
Isolate* isolate,
Zone* zone,
const Function& function,
Trampoline* sync_list_head) {
const auto& code =
Code::Handle(zone, FLAG_precompiled_mode ? function.CurrentCode()
: function.EnsureHasCode());
ASSERT(!code.IsNull());
const uword target_entry_point = code.EntryPoint();
const Trampoline sync_list_next = *sync_list_head;
TrampolineType trampoline_type = TrampolineType::kSync;
#if defined(TARGET_ARCH_IA32)
// On ia32, store the stack delta that we need to use when returning.
const intptr_t stack_return_delta =
function.FfiCSignatureReturnsStruct() && CallingConventions::kUsesRet4
? compiler::target::kWordSize
: 0;
if (stack_return_delta != 0) {
ASSERT(stack_return_delta == 4);
trampoline_type = TrampolineType::kSyncStackDelta4;
}
#endif
WriteRwLocker locker(Thread::Current(), &lock_);
const Trampoline trampoline = AllocateTrampolineLocked();
*sync_list_head = trampoline;
*LookupEntryLocked(trampoline) =
Metadata(isolate, target_entry_point, sync_list_next, trampoline_type);
return trampoline;
}
FfiCallbackMetadata::Trampoline FfiCallbackMetadata::CreateSyncFfiCallback(
Isolate* isolate,
Zone* zone,
const Function& function,
Trampoline* sync_list_head) {
return CreateFfiCallback(isolate, zone, function, sync_list_head);
}
FfiCallbackMetadata::Metadata* FfiCallbackMetadata::LookupEntryLocked(
Trampoline trampoline) const {
// Assumes lock_ is already locked for reading or writing.
const uword location = reinterpret_cast<uword>(trampoline);
// The location that the trampoline would be if the code page was aligned.
const uword aligned_location = location - offset_of_first_trampoline_in_page_;
// Since the code page isn't aligned, the trampoline may actually be in the
// following page. So round down the aligned_location, not the raw location.
const uword page_start = Utils::RoundDown(aligned_location, kPageSize);
const uword offset = aligned_location - page_start;
ASSERT_EQUAL(offset % kNativeCallbackTrampolineSize, 0);
const intptr_t index = offset / kNativeCallbackTrampolineSize;
ASSERT(index < NumCallbackTrampolinesPerPage());
const uword metadata_table = page_start + offset_of_first_metadata_in_page_;
return reinterpret_cast<Metadata*>(metadata_table) + index;
}
FfiCallbackMetadata::Metadata FfiCallbackMetadata::LookupMetadataForTrampoline(
Trampoline trampoline) const {
// Note: The locker's thread may be null because this method is explicitly
// designed to be usable outside of a VM thread.
ReadRwLocker locker(Thread::Current(), &lock_);
return *LookupEntryLocked(trampoline);
}
FfiCallbackMetadata* FfiCallbackMetadata::singleton_ = nullptr;
} // namespace dart
runtime/vm/ffi_callback_metadata.h
-229
View File
@@ -1,229 +0,0 @@
// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#ifndef RUNTIME_VM_FFI_CALLBACK_METADATA_H_
#define RUNTIME_VM_FFI_CALLBACK_METADATA_H_
#include "platform/growable_array.h"
#include "platform/utils.h"
#include "vm/hash_map.h"
#include "vm/lockers.h"
#include "vm/virtual_memory.h"
namespace dart {
// Stores metadata related to FFI callbacks (Dart functions that are assigned a
// function pointer that can be invoked by native code). This is essentially a
// map from trampoline pointer to Metadata, with some logic to assign and memory
// manage those trampolines.
//
// In the past, callbacks were primarily identified by an integer ID, but in
// this class we identify them by their trampoline pointer to solve a very
// specific issue. The trampolines are allocated in pages. On iOS in AOT mode,
// we can't create new executable memory, but we can duplicate existing memory.
// When we were using numeric IDs to identify the trampolines, each trampoline
// page was different, because the IDs were embedded in the machine code. So we
// couldn't use trampolines in AOT mode. But if we key the metadata table by the
// trampoline pointer, then the trampoline just has to look up the PC at the
// start of the trampoline function, so the machine code will always be the
// same. This means we can just duplicate the trampoline page, allowing us to
// unify the FFI callback implementation across JIT and AOT, even on iOS.
class FfiCallbackMetadata {
public:
using Trampoline = void*;
enum class TrampolineType : uint8_t {
kSync = 0,
kAsync = 1,
#if defined(TARGET_ARCH_IA32)
kSyncStackDelta4 = 2,
#endif
};
enum RuntimeFunctions {
kGetFfiCallbackMetadata,
kNumRuntimeFunctions,
};
static void Init();
static void Cleanup();
// Returns the FfiCallbackMetadata singleton.
static FfiCallbackMetadata* Instance();
// Creates a sync callback trampoline for the given function.
Trampoline CreateSyncFfiCallback(Isolate* isolate,
Zone* zone,
const Function& function,
Trampoline* sync_list_head);
// Deletes all the sync trampolines in the list.
void DeleteSyncTrampolines(Trampoline* sync_list_head);
// FFI callback metadata for any sync or async trampoline.
class Metadata {
Isolate* target_isolate_ = nullptr;
union {
// IsLive()
struct {
// Note: This is a pointer into an an Instructions object. This is only
// safe because Instructions objects are never moved by the GC.
uword target_entry_point_;
Trampoline sync_list_next_;
TrampolineType trampoline_type_;
};
// !IsLive()
Trampoline free_list_next_;
};
Metadata()
: target_entry_point_(0),
sync_list_next_(nullptr),
trampoline_type_(TrampolineType::kSync) {}
Metadata(Isolate* target_isolate,
uword target_entry_point,
Trampoline sync_list_next,
TrampolineType trampoline_type)
: target_isolate_(target_isolate),
target_entry_point_(target_entry_point),
sync_list_next_(sync_list_next),
trampoline_type_(trampoline_type) {}
public:
friend class FfiCallbackMetadata;
bool operator==(const Metadata& other) const {
return target_isolate_ == other.target_isolate_ &&
target_entry_point_ == other.target_entry_point_ &&
sync_list_next_ == other.sync_list_next_ &&
trampoline_type_ == other.trampoline_type_;
}
bool operator!=(const Metadata& other) const { return !(*this == other); }
// Whether the callback is still alive.
bool IsLive() const { return target_isolate_ != 0; }
// The target isolate. The isolate that owns the callback. Sync callbacks
// must be invoked on this isolate. Async callbacks will send a message to
// this isolate.
Isolate* target_isolate() const {
ASSERT(IsLive());
return target_isolate_;
}
// The Dart entrypoint for the callback, which the trampoline invokes.
uword target_entry_point() const {
ASSERT(IsLive());
return target_entry_point_;
}
// To efficiently delete all the sync callbacks for a isolate, they are
// stored in a singly-linked list. This is the next link in that list.
Trampoline sync_list_next() const {
ASSERT(IsLive());
return sync_list_next_;
}
// Tells FfiCallbackTrampolineStub how to call into the entry point. Mostly
// it's just a flag for whether this is a sync or async callback, but on
// IA32 it also encodes whether there's a stack delta of 4 to deal with.
TrampolineType trampoline_type() const {
ASSERT(IsLive());
return trampoline_type_;
}
};
// Returns the Metadata object for the given trampoline.
Metadata LookupMetadataForTrampoline(Trampoline trampoline) const;
// The number of trampolines that can be stored on a single page.
static intptr_t NumCallbackTrampolinesPerPage() {
return (kPageSize - kNativeCallbackSharedStubSize) /
kNativeCallbackTrampolineSize;
}
// Size of the trampoline page. Ideally we'd use VirtualMemory::PageSize(),
// but that varies across machines, and we need it to be consistent between
// host and target since it affects stub code generation. So kPageSize may be
// an overestimate of the target's VirtualMemory::PageSize(), but we try to
// get it as close as possible to avoid wasting memory.
#if defined(DART_TARGET_OS_LINUX) && defined(TARGET_ARCH_ARM64)
static constexpr intptr_t kPageSize = 64 * KB;
#elif defined(DART_TARGET_OS_MACOS) && defined(TARGET_ARCH_ARM64)
static constexpr intptr_t kPageSize = 16 * KB;
#else
static constexpr intptr_t kPageSize = 4 * KB;
#endif
static constexpr intptr_t kPageMask = ~(kPageSize - 1);
// Offset from the start of the trampoline code page to a specific slot in the
// runtime function table.
static uword RuntimeFunctionOffset(uword function_index) {
return 2 * kPageSize + function_index * compiler::target::kWordSize;
}
#if defined(TARGET_ARCH_X64)
static constexpr intptr_t kNativeCallbackTrampolineSize = 12;
static constexpr intptr_t kNativeCallbackSharedStubSize = 289;
static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 2;
#elif defined(TARGET_ARCH_IA32)
static constexpr intptr_t kNativeCallbackTrampolineSize = 10;
static constexpr intptr_t kNativeCallbackSharedStubSize = 142;
static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 4;
#elif defined(TARGET_ARCH_ARM)
static constexpr intptr_t kNativeCallbackTrampolineSize = 8;
static constexpr intptr_t kNativeCallbackSharedStubSize = 196;
static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 4;
#elif defined(TARGET_ARCH_ARM64)
static constexpr intptr_t kNativeCallbackTrampolineSize = 8;
static constexpr intptr_t kNativeCallbackSharedStubSize = 296;
static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 2;
#elif defined(TARGET_ARCH_RISCV32)
static constexpr intptr_t kNativeCallbackTrampolineSize = 8;
static constexpr intptr_t kNativeCallbackSharedStubSize = 230;
static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 2;
#elif defined(TARGET_ARCH_RISCV64)
static constexpr intptr_t kNativeCallbackTrampolineSize = 8;
static constexpr intptr_t kNativeCallbackSharedStubSize = 202;
static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 2;
#else
#error What architecture?
#endif
private:
FfiCallbackMetadata();
~FfiCallbackMetadata();
void EnsureStubPageLocked();
void AddToFreeListLocked(Trampoline trampoline, Metadata* entry);
void FillRuntimeFunction(VirtualMemory* page, uword index, void* function);
VirtualMemory* AllocateTrampolinePage();
void EnsureFreeListNotEmptyLocked();
Trampoline AllocateTrampolineLocked();
Trampoline TryAllocateFromFreeListLocked();
Trampoline CreateFfiCallback(Isolate* isolate,
Zone* zone,
const Function& function,
Trampoline* sync_list_head);
Metadata* LookupEntryLocked(Trampoline trampoline) const;
static FfiCallbackMetadata* singleton_;
mutable RwLock lock_;
VirtualMemory* stub_page_ = nullptr;
MallocGrowableArray<VirtualMemory*> trampoline_pages_;
uword offset_of_first_trampoline_in_page_ = 0;
uword offset_of_first_runtime_function_in_page_ = 0;
uword offset_of_first_metadata_in_page_ = 0;
uword size_of_trampoline_page_ = 0;
Trampoline free_list_head_ = nullptr;
Trampoline free_list_tail_ = nullptr;
DISALLOW_COPY_AND_ASSIGN(FfiCallbackMetadata);
};
} // namespace dart
#endif // RUNTIME_VM_FFI_CALLBACK_METADATA_H_
runtime/vm/ffi_callback_metadata_test.cc
-227
View File
@@ -1,227 +0,0 @@
// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#include "vm/ffi_callback_metadata.h"
#include <memory>
#include <thread> // NOLINT(build/c++11)
#include <unordered_set>
#include <vector>
#include "include/dart_api.h"
#include "platform/assert.h"
#include "vm/class_finalizer.h"
#include "vm/compiler/ffi/callback.h"
#include "vm/compiler/jit/compiler.h"
#include "vm/message_handler.h"
#include "vm/object.h"
#include "vm/port.h"
#include "vm/symbols.h"
#include "vm/unit_test.h"
namespace dart {
FunctionPtr CreateTestFunction() {
const auto& ffi_lib = Library::Handle(Library::FfiLibrary());
const auto& ffi_void = Class::Handle(ffi_lib.LookupClass(Symbols::FfiVoid()));
const auto& ffi_void_type =
Type::Handle(Type::NewNonParameterizedType(ffi_void));
auto* thread = Thread::Current();
const char* kScriptChars =
R"(
void testFunction() {
}
)";
Dart_Handle library;
{
TransitionVMToNative transition(thread);
library = TestCase::LoadTestScript(kScriptChars, nullptr);
EXPECT_VALID(library);
}
const auto& lib =
Library::Handle(Library::RawCast(Api::UnwrapHandle(library)));
EXPECT(ClassFinalizer::ProcessPendingClasses());
const auto& cls = Class::Handle(lib.toplevel_class());
EXPECT(!cls.IsNull());
const auto& error = cls.EnsureIsFinalized(thread);
EXPECT(error == Error::null());
auto& function_name = String::Handle(String::New("testFunction"));
const auto& func = Function::Handle(cls.LookupStaticFunction(function_name));
EXPECT(!func.IsNull());
FunctionType& signature = FunctionType::Handle(FunctionType::New());
signature.set_result_type(ffi_void_type);
signature.SetIsFinalized();
signature ^= signature.Canonicalize(thread);
const auto& callback = Function::Handle(compiler::ffi::NativeCallbackFunction(
signature, func, Instance::Handle(Instance::null())));
const auto& result = Object::Handle(
thread->zone(), Compiler::CompileFunction(thread, callback));
EXPECT(!result.IsError());
return callback.ptr();
}
class FakeMessageHandler : public MessageHandler {
public:
MessageStatus HandleMessage(std::unique_ptr<Message> message) override {
return MessageHandler::kOK;
}
};
VM_UNIT_TEST_CASE(FfiCallbackMetadata_CreateSyncFfiCallback) {
auto* fcm = FfiCallbackMetadata::Instance();
FfiCallbackMetadata::Trampoline tramp1 = nullptr;
FfiCallbackMetadata::Trampoline tramp2 = nullptr;
{
TestIsolateScope isolate_scope;
Thread* thread = Thread::Current();
Isolate* isolate = thread->isolate();
ASSERT(isolate == isolate_scope.isolate());
TransitionNativeToVM transition(thread);
StackZone stack_zone(thread);
HandleScope handle_scope(thread);
auto* zone = thread->zone();
const auto& func = Function::Handle(CreateTestFunction());
const auto& code = Code::Handle(func.EnsureHasCode());
EXPECT(!code.IsNull());
tramp1 = isolate->CreateSyncFfiCallback(zone, func);
EXPECT_NE(tramp1, nullptr);
FfiCallbackMetadata::Metadata m1 = fcm->LookupMetadataForTrampoline(tramp1);
EXPECT(m1.IsLive());
EXPECT_EQ(m1.target_isolate(), isolate);
EXPECT_EQ(m1.target_entry_point(), code.EntryPoint());
EXPECT_EQ(static_cast<uint8_t>(m1.trampoline_type()),
static_cast<uint8_t>(FfiCallbackMetadata::TrampolineType::kSync));
EXPECT_EQ(isolate->ffi_callback_sync_list_head(), tramp1);
EXPECT_EQ(m1.sync_list_next(), nullptr);
tramp2 = isolate->CreateSyncFfiCallback(zone, func);
EXPECT_NE(tramp2, nullptr);
EXPECT_NE(tramp2, tramp1);
FfiCallbackMetadata::Metadata m2 = fcm->LookupMetadataForTrampoline(tramp2);
EXPECT(m2.IsLive());
EXPECT_EQ(m2.target_isolate(), isolate);
EXPECT_EQ(m2.target_entry_point(), code.EntryPoint());
EXPECT_EQ(static_cast<uint8_t>(m2.trampoline_type()),
static_cast<uint8_t>(FfiCallbackMetadata::TrampolineType::kSync));
EXPECT_EQ(isolate->ffi_callback_sync_list_head(), tramp2);
EXPECT_EQ(m2.sync_list_next(), tramp1);
EXPECT_EQ(m1.sync_list_next(), nullptr);
}
{
// Isolate has shut down, so all sync callbacks should be deleted.
FfiCallbackMetadata::Metadata m1 = fcm->LookupMetadataForTrampoline(tramp1);
EXPECT(!m1.IsLive());
FfiCallbackMetadata::Metadata m2 = fcm->LookupMetadataForTrampoline(tramp2);
EXPECT(!m2.IsLive());
}
}
static void RunBigRandomMultithreadedTest(uint64_t seed) {
static constexpr int kIterations = 1000;
TestIsolateScope isolate_scope;
Thread* thread = Thread::Current();
Isolate* isolate = thread->isolate();
ASSERT(isolate == isolate_scope.isolate());
TransitionNativeToVM transition(thread);
StackZone stack_zone(thread);
HandleScope handle_scope(thread);
auto* fcm = FfiCallbackMetadata::Instance();
Random random(seed);
std::unordered_set<FfiCallbackMetadata::Trampoline> sync_tramps;
FfiCallbackMetadata::Trampoline sync_list_head = nullptr;
const auto& sync_func = Function::Handle(CreateTestFunction());
const auto& sync_code = Code::Handle(sync_func.EnsureHasCode());
EXPECT(!sync_code.IsNull());
for (int itr = 0; itr < kIterations; ++itr) {
// Do a random action:
// - Allocate a sync callback from one of the threads
// - Allocate an async callback from one of the threads
// - Delete an async callback
// - Delete all the sync callbacks for an isolate
// Sync callbacks. Randomly create and destroy them, but make destruction
// rare since all sync callbacks for the isolate are deleted at once.
if ((random.NextUInt32() % 100) == 0) {
// Delete.
fcm->DeleteSyncTrampolines(&sync_list_head);
for (FfiCallbackMetadata::Trampoline tramp : sync_tramps) {
EXPECT(!fcm->LookupMetadataForTrampoline(tramp).IsLive());
}
sync_tramps.clear();
EXPECT_EQ(sync_list_head, nullptr);
} else {
// Create.
sync_tramps.insert(fcm->CreateSyncFfiCallback(
isolate, thread->zone(), sync_func, &sync_list_head));
}
// Verify all the sync callbacks.
for (FfiCallbackMetadata::Trampoline tramp : sync_tramps) {
auto metadata = fcm->LookupMetadataForTrampoline(tramp);
EXPECT(metadata.IsLive());
EXPECT_EQ(metadata.target_isolate(), isolate);
EXPECT_EQ(metadata.target_entry_point(), sync_code.EntryPoint());
EXPECT_EQ(
static_cast<uint8_t>(metadata.trampoline_type()),
static_cast<uint8_t>(FfiCallbackMetadata::TrampolineType::kSync));
}
// Verify the isolate's list of sync callbacks.
uword sync_list_length = 0;
for (FfiCallbackMetadata::Trampoline tramp = sync_list_head; tramp != 0;) {
++sync_list_length;
auto metadata = fcm->LookupMetadataForTrampoline(tramp);
EXPECT(metadata.IsLive());
EXPECT_EQ(metadata.target_isolate(), isolate);
EXPECT_EQ(sync_tramps.count(tramp), 1u);
tramp = metadata.sync_list_next();
}
EXPECT_EQ(sync_list_length, sync_tramps.size());
}
// Delete all remaining callbacks.
fcm->DeleteSyncTrampolines(&sync_list_head);
for (FfiCallbackMetadata::Trampoline tramp : sync_tramps) {
EXPECT(!fcm->LookupMetadataForTrampoline(tramp).IsLive());
}
}
ISOLATE_UNIT_TEST_CASE(FfiCallbackMetadata_BigRandomMultithreadedTest) {
static constexpr int kThreads = 5;
std::vector<std::thread> threads;
Random random;
for (int i = 0; i < kThreads; ++i) {
threads.push_back(
std::thread(RunBigRandomMultithreadedTest, random.NextUInt64()));
}
for (auto& thread : threads) {
thread.join();
}
}
} // namespace dart
runtime/vm/ffi_callback_trampolines.cc
+107
View File
@@ -0,0 +1,107 @@
// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#include "vm/ffi_callback_trampolines.h"
#if !defined(DART_PRECOMPILED_RUNTIME)
#include "vm/code_comments.h"
#include "vm/code_observers.h"
#include "vm/compiler/assembler/assembler.h"
#include "vm/compiler/assembler/disassembler.h"
#include "vm/exceptions.h"
#endif // !defined(DART_PRECOMPILED_RUNTIME)
namespace dart {
#if !defined(DART_PRECOMPILED_RUNTIME)
uword NativeCallbackTrampolines::TrampolineForId(int32_t callback_id) {
#if defined(DART_PRECOMPILER)
ASSERT(!Enabled());
UNREACHABLE();
#else
const intptr_t trampolines_per_page = NumCallbackTrampolinesPerPage();
const intptr_t page_index = callback_id / trampolines_per_page;
const uword entry_point = trampoline_pages_[page_index]->start();
return entry_point +
(callback_id % trampolines_per_page) *
compiler::StubCodeCompiler::kNativeCallbackTrampolineSize;
#endif
}
void NativeCallbackTrampolines::AllocateTrampoline() {
#if defined(DART_PRECOMPILER)
ASSERT(!Enabled());
UNREACHABLE();
#else
// Callback IDs are limited to 32-bits for trampoline compactness.
if (kWordSize == 8 &&
!Utils::IsInt(32, next_callback_id_ + NumCallbackTrampolinesPerPage())) {
Exceptions::ThrowOOM();
}
if (trampolines_left_on_page_ == 0) {
// Fuchsia requires memory to be allocated with ZX_RIGHT_EXECUTE in order
// to be flipped to kReadExecute after being kReadWrite.
VirtualMemory* const memory = VirtualMemory::AllocateAligned(
/*size=*/VirtualMemory::PageSize(),
/*alignment=*/VirtualMemory::PageSize(),
/*is_executable=*/true,
/*is_compressed=*/false,
/*name=*/"Dart VM FFI callback trampolines");
memory->Protect(VirtualMemory::kReadWrite);
if (memory == nullptr) {
Exceptions::ThrowOOM();
}
trampoline_pages_.Add(memory);
compiler::Assembler assembler(/*object_pool_builder=*/nullptr);
compiler::StubCodeCompiler stubCodeCompiler(&assembler);
stubCodeCompiler.GenerateJITCallbackTrampolines(next_callback_id_);
MemoryRegion region(memory->address(), memory->size());
assembler.FinalizeInstructions(region);
memory->Protect(VirtualMemory::kReadExecute);
#if !defined(PRODUCT)
const char* name = "FfiJitCallbackTrampolines";
if (CodeObservers::AreActive()) {
const auto& comments = CreateCommentsFrom(&assembler);
CodeObservers::NotifyAll(name,
/*base=*/memory->start(),
/*prologue_offset=*/0,
/*size=*/assembler.CodeSize(),
/*optimized=*/false, // not really relevant
&comments);
}
#endif
#if !defined(PRODUCT) || defined(FORCE_INCLUDE_DISASSEMBLER)
if (FLAG_disassemble_stubs && FLAG_support_disassembler) {
DisassembleToStdout formatter;
THR_Print(
"Code for native callback trampolines "
"[%" Pd " -> %" Pd "]: {\n",
next_callback_id_,
next_callback_id_ + NumCallbackTrampolinesPerPage() - 1);
const auto& comments = CreateCommentsFrom(&assembler);
Disassembler::Disassemble(memory->start(),
memory->start() + assembler.CodeSize(),
&formatter, &comments);
}
#endif
trampolines_left_on_page_ = NumCallbackTrampolinesPerPage();
}
trampolines_left_on_page_--;
next_callback_id_++;
#endif // defined(DART_PRECOMPILER)
}
#endif // !defined(DART_PRECOMPILED_RUNTIME)
} // namespace dart
runtime/vm/ffi_callback_trampolines.h
+78
View File
@@ -0,0 +1,78 @@
// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#ifndef RUNTIME_VM_FFI_CALLBACK_TRAMPOLINES_H_
#define RUNTIME_VM_FFI_CALLBACK_TRAMPOLINES_H_
#include "platform/allocation.h"
#include "platform/growable_array.h"
#include "vm/flag_list.h"
#include "vm/virtual_memory.h"
#if !defined(DART_PRECOMPILED_RUNTIME)
#include "vm/compiler/stub_code_compiler.h"
#endif // !defined(DART_PRECOMPILED_RUNTIME)
namespace dart {
#if !defined(DART_PRECOMPILED_RUNTIME)
// In JIT mode, when write-protection is enabled without dual-mapping, we cannot
// rely on Instructions generated in the Isolate's heap to be executable while
// native code is running in a safepoint. This means that native code cannot
// directly invoke FFI callback trampolines.
//
// To solve this, we create trampolines tied to consecutive sequences of
// callback IDs which leave the safepoint before invoking the FFI callback,
// and re-enter the safepoint on return from the callback.
//
// Since we can never map these trampolines RX -> RW, we eagerly generate as
// many as will fit on a single page, since pages are the smallest granularity
// of memory protection.
//
// See also:
// - StubCodeCompiler::GenerateJITCallbackTrampolines
// - {NativeEntryInstr, NativeReturnInstr}::EmitNativeCode
DECLARE_FLAG(bool, write_protect_code);
class NativeCallbackTrampolines : public ValueObject {
public:
static bool Enabled() { return !FLAG_precompiled_mode; }
static intptr_t NumCallbackTrampolinesPerPage() {
return (VirtualMemory::PageSize() -
compiler::StubCodeCompiler::kNativeCallbackSharedStubSize) /
compiler::StubCodeCompiler::kNativeCallbackTrampolineSize;
}
NativeCallbackTrampolines() {}
~NativeCallbackTrampolines() {
// Unmap all the trampoline pages. 'VirtualMemory's are new-allocated.
for (intptr_t i = 0; i < trampoline_pages_.length(); ++i) {
delete trampoline_pages_[i];
}
}
// For each callback ID, we have an entry in Thread::ffi_callback_code_ and
// a trampoline here. These arrays must be kept in sync and this method is
// exposed to assert that.
intptr_t next_callback_id() const { return next_callback_id_; }
// Allocates a callback trampoline corresponding to the callback id
// 'next_callback_id()'. Returns an entrypoint to the trampoline.
void AllocateTrampoline();
// Get the entrypoint for a previously allocated callback ID.
uword TrampolineForId(int32_t callback_id);
private:
MallocGrowableArray<VirtualMemory*> trampoline_pages_;
intptr_t trampolines_left_on_page_ = 0;
intptr_t next_callback_id_ = 0;
DISALLOW_COPY_AND_ASSIGN(NativeCallbackTrampolines);
};
#endif // !defined(DART_PRECOMPILED_RUNTIME)
} // namespace dart
#endif // RUNTIME_VM_FFI_CALLBACK_TRAMPOLINES_H_
runtime/vm/image_snapshot.cc
+1 -2
View File
@@ -1361,8 +1361,7 @@ const char* ImageWriter::Deobfuscate(Zone* zone,
void AssemblyImageWriter::WriteBss(bool vm) {
EnterSection(ProgramSection::Bss, vm, ImageWriter::kBssAlignment);
auto const entry_count =
vm ? BSS::kVmEntryCount : BSS::kIsolateGroupEntryCount;
auto const entry_count = vm ? BSS::kVmEntryCount : BSS::kIsolateEntryCount;
for (intptr_t i = 0; i < entry_count; i++) {
// All bytes in the .bss section must be zero.
WriteTargetWord(0);
runtime/vm/isolate.cc
+4 -14
View File
@@ -20,7 +20,6 @@
#include "vm/debugger.h"
#include "vm/deopt_instructions.h"
#include "vm/dispatch_table.h"
#include "vm/ffi_callback_metadata.h"
#include "vm/flags.h"
#include "vm/heap/heap.h"
#include "vm/heap/pointer_block.h"
@@ -336,6 +335,9 @@ IsolateGroup::IsolateGroup(std::shared_ptr<IsolateGroupSource> source,
start_time_micros_(OS::GetCurrentMonotonicMicros()),
is_system_isolate_group_(source->flags.is_system_isolate),
random_(),
#if !defined(DART_PRECOMPILED_RUNTIME)
native_callback_trampolines_(),
#endif
#if !defined(PRODUCT) && !defined(DART_PRECOMPILED_RUNTIME)
last_reload_timestamp_(OS::GetCurrentTimeMillis()),
reload_every_n_stack_overflow_checks_(FLAG_reload_every),
@@ -2306,13 +2308,6 @@ void Isolate::LowLevelShutdown() {
delete message_handler();
set_message_handler(nullptr);
// Clean up any synchronous FFI callbacks registered with this isolate. Skip
// if this isolate never registered any.
if (ffi_callback_sync_list_head_ != nullptr) {
FfiCallbackMetadata::Instance()->DeleteSyncTrampolines(
&ffi_callback_sync_list_head_);
}
#if !defined(PRODUCT)
if (FLAG_dump_megamorphic_stats) {
MegamorphicCacheTable::PrintSizes(this);
@@ -2506,7 +2501,7 @@ void Isolate::LowLevelCleanup(Isolate* isolate) {
// memory might have become unreachable. We should evaluate how to best
// inform the GC about this situation.
}
}
} // namespace dart
Dart_InitializeIsolateCallback Isolate::initialize_callback_ = nullptr;
Dart_IsolateGroupCreateCallback Isolate::create_group_callback_ = nullptr;
@@ -3519,11 +3514,6 @@ void Isolate::WaitForOutstandingSpawns() {
}
}
void* Isolate::CreateSyncFfiCallback(Zone* zone, const Function& function) {
return FfiCallbackMetadata::Instance()->CreateSyncFfiCallback(
this, zone, function, &ffi_callback_sync_list_head_);
}
#if !defined(PRODUCT)
void IsolateGroup::CloneClassTableForReload() {
RELEASE_ASSERT(class_table_ == heap_walk_class_table_);
runtime/vm/isolate.h
+14 -6
View File
@@ -37,6 +37,10 @@
#include "vm/token_position.h"
#include "vm/virtual_memory.h"
#if !defined(DART_PRECOMPILED_RUNTIME)
#include "vm/ffi_callback_trampolines.h"
#endif // !defined(DART_PRECOMPILED_RUNTIME)
namespace dart {
// Forward declarations.
@@ -416,6 +420,12 @@ class IsolateGroup : public IntrusiveDListEntry<IsolateGroup> {
bool is_system_isolate_group() const { return is_system_isolate_group_; }
#if !defined(DART_PRECOMPILED_RUNTIME)
NativeCallbackTrampolines* native_callback_trampolines() {
return &native_callback_trampolines_;
}
#endif
// IsolateGroup-specific flag handling.
static void FlagsInitialize(Dart_IsolateFlags* api_flags);
void FlagsCopyTo(Dart_IsolateFlags* api_flags);
@@ -827,6 +837,10 @@ class IsolateGroup : public IntrusiveDListEntry<IsolateGroup> {
bool is_system_isolate_group_;
Random random_;
#if !defined(DART_PRECOMPILED_RUNTIME)
NativeCallbackTrampolines native_callback_trampolines_;
#endif
#if !defined(PRODUCT) && !defined(DART_PRECOMPILED_RUNTIME)
int64_t last_reload_timestamp_;
std::shared_ptr<IsolateGroupReloadContext> group_reload_context_;
@@ -1242,11 +1256,6 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry<Isolate> {
deopt_context_ = value;
}
void* CreateSyncFfiCallback(Zone* zone, const Function& function);
// Visible for testing.
void* ffi_callback_sync_list_head() { return ffi_callback_sync_list_head_; }
intptr_t BlockClassFinalization() {
ASSERT(defer_finalization_count_ >= 0);
return defer_finalization_count_++;
@@ -1637,7 +1646,6 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry<Isolate> {
MessageHandler* message_handler_ = nullptr;
intptr_t defer_finalization_count_ = 0;
DeoptContext* deopt_context_ = nullptr;
void* ffi_callback_sync_list_head_ = nullptr;
GrowableObjectArrayPtr tag_table_;
runtime/vm/object_store.h
+1
View File
@@ -206,6 +206,7 @@ class ObjectPointerVisitor;
RW(Array, unique_dynamic_targets) \
RW(GrowableObjectArray, megamorphic_cache_table) \
RW(GrowableObjectArray, ffi_callback_code) \
RW(TypedData, ffi_callback_stack_return) \
RW(Code, build_generic_method_extractor_code) \
RW(Code, build_nongeneric_method_extractor_code) \
RW(Code, dispatch_table_null_error_stub) \
runtime/vm/runtime_entry.cc
+68 -30
View File
@@ -17,7 +17,6 @@
#include "vm/debugger.h"
#include "vm/double_conversion.h"
#include "vm/exceptions.h"
#include "vm/ffi_callback_metadata.h"
#include "vm/flags.h"
#include "vm/heap/verifier.h"
#include "vm/instructions.h"
@@ -3845,31 +3844,44 @@ DEFINE_RAW_LEAF_RUNTIME_ENTRY(ExitSafepointIgnoreUnwindInProgress,
false,
&DFLRT_ExitSafepointIgnoreUnwindInProgress);
#if defined(DART_HOST_OS_WINDOWS)
#pragma intrinsic(_ReturnAddress)
#endif
// Ensure that 'callback_id' refers to a valid callback.
//
// If "entry != 0", additionally checks that entry is inside the instructions
// of this callback.
//
// Aborts if any of these conditions fails.
static void VerifyCallbackIdMetadata(Thread* thread,
int32_t callback_id,
uword entry) {
NoSafepointScope _;
// This is called by a native callback trampoline
// (see StubCodeCompiler::GenerateFfiCallbackTrampolineStub). Not registered as
// a runtime entry because we can't use Thread to look it up.
extern "C" Thread* DLRT_GetFfiCallbackMetadata(void* trampoline,
uword* out_entry_point,
uword* out_trampoline_type) {
CHECK_STACK_ALIGNMENT;
TRACE_RUNTIME_CALL("GetFfiCallbackMetadata %p", trampoline);
auto metadata =
FfiCallbackMetadata::Instance()->LookupMetadataForTrampoline(trampoline);
if (!metadata.IsLive()) {
FATAL("Callback invoked after it has been deleted.");
const GrowableObjectArrayPtr array =
thread->isolate_group()->object_store()->ffi_callback_code();
if (array == GrowableObjectArray::null()) {
FATAL("Cannot invoke callback on incorrect isolate.");
}
Isolate* target_isolate = metadata.target_isolate();
ASSERT(out_entry_point != nullptr);
*out_entry_point = metadata.target_entry_point();
ASSERT(out_trampoline_type != nullptr);
*out_trampoline_type = static_cast<uword>(metadata.trampoline_type());
const SmiPtr length_smi = GrowableObjectArray::NoSafepointLength(array);
const intptr_t length = Smi::Value(length_smi);
if (callback_id < 0 || callback_id >= length) {
FATAL("Cannot invoke callback on incorrect isolate.");
}
if (entry != 0) {
CompressedObjectPtr* const code_array =
Array::DataOf(GrowableObjectArray::NoSafepointData(array));
const CodePtr code =
Code::RawCast(code_array[callback_id].Decompress(array.heap_base()));
if (!Code::ContainsInstructionAt(code, entry)) {
FATAL("Cannot invoke callback on incorrect isolate.");
}
}
}
// Not registered as a runtime entry because we can't use Thread to look it up.
static Thread* GetThreadForNativeCallback(uword callback_id,
uword return_address) {
Thread* const thread = Thread::Current();
if (thread == nullptr) {
FATAL("Cannot invoke native callback outside an isolate.");
@@ -3883,9 +3895,6 @@ extern "C" Thread* DLRT_GetFfiCallbackMetadata(void* trampoline,
if (!thread->IsDartMutatorThread()) {
FATAL("Native callbacks must be invoked on the mutator thread.");
}
if (thread->isolate() != target_isolate) {
FATAL("Cannot invoke native callback from a different isolate.");
}
// Set the execution state to VM while waiting for the safepoint to end.
// This isn't strictly necessary but enables tests to check that we're not
@@ -3893,15 +3902,44 @@ extern "C" Thread* DLRT_GetFfiCallbackMetadata(void* trampoline,
thread->set_execution_state(Thread::kThreadInVM);
thread->ExitSafepoint();
VerifyCallbackIdMetadata(thread, callback_id, return_address);
TRACE_RUNTIME_CALL("GetFfiCallbackMetadata thread %p", thread);
TRACE_RUNTIME_CALL("GetFfiCallbackMetadata entry_point %p",
(void*)*out_entry_point);
TRACE_RUNTIME_CALL("GetFfiCallbackMetadata trampoline_type %p",
(void*)*out_trampoline_type);
return thread;
}
#if defined(DART_HOST_OS_WINDOWS)
#pragma intrinsic(_ReturnAddress)
#endif
// This is called directly by NativeEntryInstr. At the moment we enter this
// routine, the caller is generated code in the Isolate heap. Therefore we check
// that the return address (caller) corresponds to the declared callback ID's
// code within this Isolate.
extern "C" Thread* DLRT_GetThreadForNativeCallback(uword callback_id) {
CHECK_STACK_ALIGNMENT;
TRACE_RUNTIME_CALL("GetThreadForNativeCallback %" Pd, callback_id);
#if defined(DART_HOST_OS_WINDOWS)
void* return_address = _ReturnAddress();
#else
void* return_address = __builtin_return_address(0);
#endif
Thread* return_value = GetThreadForNativeCallback(
callback_id, reinterpret_cast<uword>(return_address));
TRACE_RUNTIME_CALL("GetThreadForNativeCallback returning %p", return_value);
return return_value;
}
// This is called by a native callback trampoline
// (see StubCodeCompiler::GenerateJITCallbackTrampolines). There is no need to
// check the return address because the trampoline will use the callback ID to
// look up the generated code. We still check that the callback ID is valid for
// this isolate.
extern "C" Thread* DLRT_GetThreadForNativeCallbackTrampoline(
uword callback_id) {
CHECK_STACK_ALIGNMENT;
return GetThreadForNativeCallback(callback_id, 0);
}
extern "C" ApiLocalScope* DLRT_EnterHandleScope(Thread* thread) {
CHECK_STACK_ALIGNMENT;
TRACE_RUNTIME_CALL("EnterHandleScope %p", thread);
runtime/vm/runtime_entry.h
+2 -3
View File
@@ -158,9 +158,8 @@ RUNTIME_ENTRY_LIST(DECLARE_RUNTIME_ENTRY)
LEAF_RUNTIME_ENTRY_LIST(DECLARE_LEAF_RUNTIME_ENTRY)
// Expected to be called inside a safepoint.
extern "C" Thread* DLRT_GetFfiCallbackMetadata(void* trampoline,
uword* out_entry_point,
uword* out_callback_kind);
extern "C" Thread* DLRT_GetThreadForNativeCallback(uword callback_id);
extern "C" Thread* DLRT_GetThreadForNativeCallbackTrampoline(uword callback_id);
// For creating scoped handles in FFI trampolines.
extern "C" ApiLocalScope* DLRT_EnterHandleScope(Thread* thread);
runtime/vm/stub_code_list.h
-1
View File
@@ -138,7 +138,6 @@ namespace dart {
V(ExitSafepoint) \
V(ExitSafepointIgnoreUnwindInProgress) \
V(CallNativeThroughSafepoint) \
V(FfiCallbackTrampoline) \
V(InitStaticField) \
V(InitLateStaticField) \
V(InitLateFinalStaticField) \
runtime/vm/thread.cc
+4
View File
@@ -27,6 +27,10 @@
#include "vm/timeline.h"
#include "vm/zone.h"
#if !defined(DART_PRECOMPILED_RUNTIME)
#include "vm/ffi_callback_trampolines.h"
#endif // !defined(DART_PRECOMPILED_RUNTIME)
namespace dart {
#if !defined(PRODUCT)
runtime/vm/virtual_memory.cc
-44
View File
@@ -7,10 +7,6 @@
#include "platform/assert.h"
#include "platform/utils.h"
#if defined(DART_HOST_OS_MACOS)
#include <mach/mach.h>
#endif
namespace dart {
bool VirtualMemory::InSamePage(uword address0, uword address1) {
@@ -47,44 +43,4 @@ VirtualMemory* VirtualMemory::ForImagePage(void* pointer, uword size) {
return memory;
}
bool VirtualMemory::DuplicateRX(VirtualMemory* target) {
ASSERT_LESS_OR_EQUAL(size(), target->size());
#if defined(DART_HOST_OS_MACOS)
// Mac is special cased because iOS doesn't allow allocating new executable
// memory, so the default approach would fail. We are allowed to make new
// mappings of existing executable memory using vm_remap though, which is
// effectively the same for non-writable memory.
const mach_port_t task = mach_task_self();
const vm_address_t source_address = reinterpret_cast<vm_address_t>(address());
const vm_size_t mem_size = size();
const vm_prot_t read_execute = VM_PROT_READ | VM_PROT_EXECUTE;
vm_prot_t current_protection = read_execute;
vm_prot_t max_protection = read_execute;
vm_address_t target_address =
reinterpret_cast<vm_address_t>(target->address());
kern_return_t status = vm_remap(
task, &target_address, mem_size,
/*mask=*/0,
/*flags=*/VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE, task, source_address,
/*copy=*/true, &current_protection, &max_protection,
/*inheritance=*/VM_INHERIT_NONE);
if (status != KERN_SUCCESS) {
return false;
}
ASSERT(reinterpret_cast<void*>(target_address) == target->address());
ASSERT_EQUAL(current_protection & read_execute, read_execute);
ASSERT_EQUAL(max_protection & read_execute, read_execute);
return true;
#else // defined(DART_HOST_OS_MACOS)
// TODO(52497): Use dual mapping on platforms where it's supported.
// Check that target doesn't overlap with this.
ASSERT(target->start() >= end() || target->end() <= start());
memcpy(target->address(), address(), size()); // NOLINT
Protect(target->address(), size(), kReadExecute);
return true;
#endif // defined(DART_HOST_OS_MACOS)
}
} // namespace dart
runtime/vm/virtual_memory.h
-9
View File
@@ -63,15 +63,6 @@ class VirtualMemory {
bool is_compressed,
const char* name);
// Duplicates `this` memory into the `target` memory. This is designed to work
// on all platforms, including iOS, which doesn't allow creating new
// executable memory.
//
// Assumes
// * `this` has RX protection.
// * `target` has RW protection, and is at least as large as `this`.
bool DuplicateRX(VirtualMemory* target);
// Returns the cached page size. Use only if Init() has been called.
static intptr_t PageSize() {
ASSERT(page_size_ != 0);
runtime/vm/virtual_memory_test.cc
-32
View File
@@ -89,36 +89,4 @@ VM_UNIT_TEST_CASE(FreeVirtualMemory) {
}
}
static int testFunction(int x) {
return x * 2;
}
NO_SANITIZE_UNDEFINED("function") // See #52440
VM_UNIT_TEST_CASE(DuplicateRXVirtualMemory) {
const uword page_size = VirtualMemory::PageSize();
const uword pointer = reinterpret_cast<uword>(&testFunction);
const uword page_start = Utils::RoundDown(pointer, page_size);
const uword offset = pointer - page_start;
// Grab 2 * page_size, in case testFunction happens to land near the end of
// the page.
VirtualMemory* vm = VirtualMemory::ForImagePage(
reinterpret_cast<void*>(page_start), 2 * page_size);
EXPECT_NE(nullptr, vm);
VirtualMemory* vm2 = VirtualMemory::AllocateAligned(
vm->size(), kPageSize, /*is_executable=*/false,
/*is_compressed=*/false, "FfiCallbackMetadata::TrampolinePage");
bool ok = vm->DuplicateRX(vm2);
EXPECT_EQ(true, ok);
auto testFunction2 = reinterpret_cast<int (*)(int)>(vm2->start() + offset);
EXPECT_NE(&testFunction, testFunction2);
EXPECT_EQ(246, testFunction2(123));
delete vm;
delete vm2;
}
} // namespace dart
runtime/vm/vm_sources.gni
+2 -3
View File
@@ -105,8 +105,8 @@ vm_sources = [
"exceptions.h",
"experimental_features.cc",
"experimental_features.h",
"ffi_callback_metadata.cc",
"ffi_callback_metadata.h",
"ffi_callback_trampolines.cc",
"ffi_callback_trampolines.h",
"field_table.cc",
"field_table.h",
"finalizable_data.h",
@@ -414,7 +414,6 @@ vm_sources_tests = [
"exceptions_test.cc",
"fixed_cache_test.cc",
"flags_test.cc",
"ffi_callback_metadata_test.cc",
"growable_array_test.cc",
"guard_field_test.cc",
"handles_test.cc",
tests/ffi/ffi_callback_unique_test.dart
+3
View File
@@ -16,6 +16,9 @@ void main() {
b.add(
Pointer.fromFunction<Int Function()>(nativeToDartCallback, 1).address);
}
// Another pointer from a different call site.
a.add(Pointer.fromFunction<Int Function()>(nativeToDartCallback, 0).address);
b.add(Pointer.fromFunction<Int Function()>(nativeToDartCallback, 1).address);
ensureEqualEntries(a);
ensureEqualEntries(b);
tests/ffi_2/ffi_callback_unique_test.dart
+3
View File
@@ -16,6 +16,9 @@ void main() {
b.add(
Pointer.fromFunction<Int Function()>(nativeToDartCallback, 1).address);
}
// Another pointer from a different call site.
a.add(Pointer.fromFunction<Int Function()>(nativeToDartCallback, 0).address);
b.add(Pointer.fromFunction<Int Function()>(nativeToDartCallback, 1).address);
ensureEqualEntries(a);
ensureEqualEntries(b);