dart-lang/sdk
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[vm] Shrink the FFI callback stub.

Browse Source 
Instead of switching over each of the different callback types and duplicating the call-dart-spill-call-runtime-restore in the stub, have the initial runtime call return a function pointer for the runtime call to end that kind of callback. A separate case for tail calls is still needed.

TEST=ci
Change-Id: I691487ca12337ddef1adaeec4c163c56e2feb09e
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/488261
Commit-Queue: Ryan Macnak <rmacnak@google.com>
Reviewed-by: Alexander Aprelev <aam@google.com>
This commit is contained in:
Ryan Macnak
2026-03-18 10:18:06 -07:00
committed by Commit Queue
parent 0476748b12
commit ec77e28f26
14 changed files with 313 additions and 1081 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/vm/bss_relocs.cc
-11
View File
@@ -31,17 +31,6 @@ void BSS::Initialize(Thread* current, uword* bss_start, bool vm) {
InitializeBSSEntry(Relocation::DLRT_GetFfiCallbackMetadata,
reinterpret_cast<uword>(DLRT_GetFfiCallbackMetadata),
bss_start);
InitializeBSSEntry(Relocation::DLRT_ExitTemporaryIsolate,
reinterpret_cast<uword>(DLRT_ExitTemporaryIsolate),
bss_start);
InitializeBSSEntry(Relocation::DLRT_ExitIsolateGroupBoundIsolate,
reinterpret_cast<uword>(DLRT_ExitIsolateGroupBoundIsolate),
bss_start);
InitializeBSSEntry(
Relocation::DLRT_ExitSyncCallbackTargetIsolate,
reinterpret_cast<uword>(DLRT_ExitSyncCallbackTargetIsolate), bss_start);
InitializeBSSEntry(Relocation::DLRT_ExitSyncCallback,
reinterpret_cast<uword>(DLRT_ExitSyncCallback), bss_start);
}
} // namespace dart
runtime/vm/bss_relocs.h
-4
View File
@@ -17,10 +17,6 @@ class BSS : public AllStatic {
// stored at the index.
enum class Relocation : intptr_t {
DLRT_GetFfiCallbackMetadata, // TODO(https://dartbug.com/52579): Remove.
DLRT_ExitTemporaryIsolate, // TODO(https://dartbug.com/52579): Remove.
DLRT_ExitIsolateGroupBoundIsolate, // TODO(https://dartbug.com/52579)
DLRT_ExitSyncCallbackTargetIsolate, // TODO(https://dartbug.com/52579)
DLRT_ExitSyncCallback, // TODO(https://dartbug.com/52579)
EndOfVmEntries,
// We don't have any isolate group specific entries at the moment.
runtime/vm/compiler/ffi/abi.h
+2
View File
@@ -42,6 +42,8 @@ enum class Abi {
kWindowsArm64,
kWindowsIA32,
kWindowsX64,
kMacOSRiscv64 = kLinuxRiscv64, // To build simriscv64 on Mac.
};
const int64_t num_abis = static_cast<int64_t>(Abi::kWindowsX64) + 1;
runtime/vm/compiler/stub_code_compiler_arm.cc
+32 -120
View File
@@ -361,21 +361,10 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
// We exit the safepoint inside DLRT_GetFfiCallbackMetadata in order to save
// code size on this shared stub.
{
__ mov(R0, Operand(R4));
// We also need to look up the entry point for the trampoline. This is
// returned using a pointer passed to the second arg of the C function
// below. We aim that pointer at a reserved stack slot.
__ sub(SP, SP, Operand(compiler::target::kWordSize));
__ mov(R1, Operand(SP));
// We also need to know if this is a sync or async callback. This is also
// returned by pointer.
__ sub(SP, SP, Operand(compiler::target::kWordSize));
__ mov(R2, Operand(SP));
__ EnterFrame(1 << FP, 0);
__ ReserveAlignedFrameSpace(0);
__ ReserveAlignedFrameSpace(3 * target::kWordSize);
__ mov(R0, Operand(R4));
__ mov(R1, Operand(SP));
GenerateLoadFfiCallbackMetadataRuntimeFunction(
FfiCallbackMetadata::kGetFfiCallbackMetadata, R4);
@@ -383,131 +372,54 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
__ blx(R4);
__ mov(THR, Operand(R0));
__ ldr(TMP, Address(SP, 0 * target::kWordSize)); // entry_point
__ ldr(R4, Address(SP, 1 * target::kWordSize)); // is_tail
__ ldr(R5, Address(SP, 2 * target::kWordSize)); // epilogue
__ LeaveFrame(1 << FP);
// The trampoline type is at the top of the stack. Pop it into R4.
__ Pop(R4);
// Entry point is now at the top of the stack. Pop it into R5.
__ Pop(R5);
}
__ PopRegisters(argument_registers);
Label async_callback;
Label sync_isolate_group_bound_callback;
Label sync_callback_isolate_ownership;
Label done;
Label tail;
__ cmp(R4, Operand(0));
__ b(&tail, NOT_ZERO);
// Check the trampoline type to see how the callback should be invoked.
__ cmp(
R4,
Operand(static_cast<uword>(FfiCallbackMetadata::TrampolineType::kAsync)));
__ b(&async_callback, EQ);
const RegisterSet return_registers(
(1 << CallingConventions::kReturnReg) |
(1 << CallingConventions::kSecondReturnReg),
1 << CallingConventions::kReturnFpuReg);
ASSERT(Utils::IsAligned(return_registers.SpillSize(), 8));
__ cmp(R4, Operand(static_cast<uword>(
FfiCallbackMetadata::TrampolineType::kSyncIsolateGroupBound)));
__ b(&sync_isolate_group_bound_callback, EQ);
__ tst(R4, Operand(FfiCallbackMetadata::kSyncCallbackIsolateOwnershipFlag));
__ b(&sync_callback_isolate_ownership, NE);
// Sync callback. The entry point contains the target function, so just call
// it. DLRT_GetThreadForNativeCallbackTrampoline exited the safepoint, so
// re-enter it afterwards.
// On entry to the function, there will be four extra slots on the stack:
// saved THR, R4, R5 and the return address. The target will know to skip
// them.
__ blx(R5);
// Clobbers R4, R5 and TMP, all saved or volatile.
__ EnterFullSafepoint(R4, R5);
__ b(&done);
__ Bind(&sync_callback_isolate_ownership);
__ blx(R5);
// Exit the target isolate.
{
__ EnterFrame(1 << FP, 0);
__ ReserveAlignedFrameSpace(0);
const RegisterSet return_registers(
(1 << CallingConventions::kReturnReg) |
(1 << CallingConventions::kSecondReturnReg),
1 << CallingConventions::kReturnFpuReg);
__ blx(TMP); // entry_point
__ PushRegisters(return_registers);
GenerateLoadFfiCallbackMetadataRuntimeFunction(
FfiCallbackMetadata::kExitSyncCallbackTargetIsolate, R4);
__ blx(R4);
__ mov(R0, Operand(THR));
__ blx(R5); // DLRT_ExitSyncCallback, etc
if (FLAG_target_memory_sanitizer) {
__ blx(R0); // dart_msan_unpoison_retval
}
__ PopRegisters(return_registers);
__ LeaveFrame(1 << FP);
// Returns.
RESTORES_LR_FROM_FRAME(
__ PopList((1 << PC) | (1 << THR) | (1 << R4) | (1 << R5)));
}
__ b(&done);
__ Bind(&sync_isolate_group_bound_callback);
__ blx(R5);
// Exit isolate group bound isolate.
{
__ EnterFrame(1 << FP, 0);
__ ReserveAlignedFrameSpace(0);
const RegisterSet return_registers(
(1 << CallingConventions::kReturnReg) |
(1 << CallingConventions::kSecondReturnReg),
1 << CallingConventions::kReturnFpuReg);
__ PushRegisters(return_registers);
GenerateLoadFfiCallbackMetadataRuntimeFunction(
FfiCallbackMetadata::kExitIsolateGroupBoundIsolate, R4);
__ blx(R4);
__ PopRegisters(return_registers);
__ LeaveFrame(1 << FP);
}
__ b(&done);
__ Bind(&async_callback);
// Async callback. The entrypoint marshals the arguments into a message and
// sends it over the send port. DLRT_GetThreadForNativeCallbackTrampoline
// entered a temporary isolate, so exit it afterwards.
// On entry to the function, there will be four extra slots on the stack:
// saved THR, R4, R5 and the return address. The target will know to skip
// them.
__ blx(R5);
// Exit the temporary isolate.
{
GenerateLoadFfiCallbackMetadataRuntimeFunction(
FfiCallbackMetadata::kExitTemporaryIsolate, R0);
CLOBBERS_LR(__ PopList((1 << LR) | (1 << THR) | (1 << R4) | (1 << R5)));
SPILLS_LR_TO_FRAME(); //...
__ Bind(&tail);
__ blx(TMP); // entry_point
__ mov(R0, Operand(THR));
__ mov(R1, Operand(R5));
RESTORES_LR_FROM_FRAME(
__ PopList((1 << LR) | (1 << THR) | (1 << R4) | (1 << R5)));
// Tail-call DLRT_ExitTemporaryIsolate. It is not safe to return to this
// stub, since it might be deleted once DLRT_ExitTemporaryIsolate proceeds
// enough for VM shutdown.
__ bx(R0);
__ bx(R1); // DLRT_ExitTemporaryIsolate.
__ Breakpoint();
}
__ Bind(&done);
// Returns.
__ PopList((1 << PC) | (1 << THR) | (1 << R4) | (1 << R5));
ASSERT_LESS_OR_EQUAL(__ CodeSize() - shared_stub_start,
FfiCallbackMetadata::kNativeCallbackSharedStubSize);
ASSERT_LESS_OR_EQUAL(__ CodeSize(), FfiCallbackMetadata::kPageSize);
runtime/vm/compiler/stub_code_compiler_arm64.cc
+39 -240
View File
@@ -497,9 +497,10 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
// Save THR (callee-saved) and LR on the real C stack (CSP). Keeps it
// aligned.
COMPILE_ASSERT(FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta == 2);
COMPILE_ASSERT(FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta == 4);
SPILLS_LR_TO_FRAME(__ stp(
THR, LR, Address(CSP, -2 * target::kWordSize, Address::PairPreIndex)));
__ stp(R20, R21, Address(CSP, -2 * target::kWordSize, Address::PairPreIndex));
COMPILE_ASSERT(!IsArgumentRegister(THR));
@@ -519,21 +520,10 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
// This saves too much: we only need the D half of Q registers.
__ PushRegisters(all_registers);
__ mov(R0, R9);
// We also need to look up the entry point for the trampoline. This is
// returned using a pointer passed to the second arg of the C function
// below. We aim that pointer at a reserved stack slot.
__ AddImmediate(SP, SP, -compiler::target::kWordSize);
__ mov(R1, SP);
// We also need to know if this is a sync or async callback. This is also
// returned by pointer.
__ AddImmediate(SP, SP, -compiler::target::kWordSize);
__ mov(R2, SP);
__ EnterFrame(0);
__ ReserveAlignedFrameSpace(0);
__ ReserveAlignedFrameSpace(3 * target::kWordSize);
__ mov(R0, R9);
__ mov(R1, SP);
#if defined(DART_TARGET_OS_FUCHSIA)
// TODO(https://dartbug.com/52579): Remove.
@@ -542,7 +532,7 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
R9);
} else {
Label call;
__ ldr(R4, compiler::Address::PC(2 * Instr::kInstrSize));
__ ldr(R4, Address::PC(2 * Instr::kInstrSize));
__ b(&call);
__ Emit64(reinterpret_cast<int64_t>(&DLRT_GetFfiCallbackMetadata));
__ Bind(&call);
@@ -553,253 +543,62 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
#endif // defined(DART_TARGET_OS_FUCHSIA)
__ mov(CSP, SP);
__ blr(R4);
__ blr(R4); // DLRT_GetFfiCallbackMetadata
__ mov(SP, CSP);
__ mov(THR, R0);
COMPILE_ASSERT(!IsCalleeSavedRegister(R10) && !IsArgumentRegister(R10));
__ ldr(R10, Address(CSP, 0 * target::kWordSize)); // entry_point
__ ldr(R20, Address(CSP, 1 * target::kWordSize)); // is_tail
__ ldr(R21, Address(CSP, 2 * target::kWordSize)); // epilogue
__ LeaveFrame();
// The trampoline type is at the top of the stack. Pop it into R9.
__ Pop(R9);
// Entry point is now at the top of the stack. Pop it into R10.
COMPILE_ASSERT(!IsCalleeSavedRegister(R10) && !IsArgumentRegister(R10));
__ Pop(R10);
__ PopRegisters(all_registers);
__ LeaveFrame();
__ RestoreCSP();
}
Label async_callback;
Label sync_isolate_group_bound_callback;
Label sync_callback_isolate_ownership;
Label done;
Label tail;
__ cbnz(&tail, R20);
// Check the trampoline type to see how the callback should be invoked.
__ cmp(
R9,
Operand(static_cast<uword>(FfiCallbackMetadata::TrampolineType::kAsync)));
__ b(&async_callback, EQ);
__ cmp(R9, Operand(static_cast<uword>(
FfiCallbackMetadata::TrampolineType::kSyncIsolateGroupBound)));
__ b(&sync_isolate_group_bound_callback, EQ);
__ tsti(R9,
Immediate(FfiCallbackMetadata::kSyncCallbackIsolateOwnershipFlag));
__ b(&sync_callback_isolate_ownership, NOT_ZERO);
// Sync callback. The entry point contains the target function, so just call
// it. DLRT_GetThreadForNativeCallbackTrampoline exited the safepoint, so
// re-enter it afterwards.
// Clobbers all volatile registers, including the callback ID in R9.
// Resets CSP and SP, important for EnterSafepoint below.
__ blr(R10);
// Clobbers TMP, TMP2 and R9 -- all volatile and not holding return values.
__ EnterFullSafepoint(/*scratch=*/R9);
if (FLAG_target_memory_sanitizer) {
__ SetupDartSP();
__ EnterFrame(0);
__ ReserveAlignedFrameSpace(0);
const RegisterSet return_registers(
(1 << CallingConventions::kReturnReg) |
(1 << CallingConventions::kSecondReturnReg),
(1 << CallingConventions::kReturnFpuReg) |
(1 << CallingConventions::kSecondReturnFpuReg) |
(1 << CallingConventions::kThirdReturnFpuReg) |
(1 << CallingConventions::kFourthReturnFpuReg));
// This saves too much: we only need the D half of Q registers.
__ PushRegisters(return_registers);
#if defined(DART_TARGET_OS_FUCHSIA)
// TODO(https://dartbug.com/52579): Remove.
if (FLAG_precompiled_mode) {
GenerateLoadBSSEntry(BSS::Relocation::DLRT_ExitSyncCallback, R4, R9);
} else {
Label call;
__ ldr(R4, compiler::Address::PC(2 * Instr::kInstrSize));
__ b(&call);
__ Emit64(reinterpret_cast<int64_t>(&DLRT_ExitSyncCallback));
__ Bind(&call);
}
#else
GenerateLoadFfiCallbackMetadataRuntimeFunction(
FfiCallbackMetadata::kExitSyncCallback, R4);
#endif
__ mov(CSP, SP);
__ blr(R4);
__ blr(R0); // dart_msan_unpoison_retval
__ mov(SP, CSP);
__ mov(THR, R0);
__ PopRegisters(return_registers);
__ LeaveFrame();
__ RestoreCSP();
}
__ b(&done);
__ Bind(&sync_callback_isolate_ownership);
__ blr(R10);
// Exit the target isolate.
{
__ SetupDartSP();
__ EnterFrame(0);
__ ReserveAlignedFrameSpace(0);
const RegisterSet return_registers(
(1 << CallingConventions::kReturnReg) |
(1 << CallingConventions::kSecondReturnReg),
(1 << CallingConventions::kReturnFpuReg) |
(1 << CallingConventions::kSecondReturnFpuReg) |
(1 << CallingConventions::kThirdReturnFpuReg) |
(1 << CallingConventions::kFourthReturnFpuReg));
// This saves too much: we only need the D half of Q registers.
__ PushRegisters(return_registers);
#if defined(DART_TARGET_OS_FUCHSIA)
// TODO(https://dartbug.com/52579): Remove.
if (FLAG_precompiled_mode) {
GenerateLoadBSSEntry(BSS::Relocation::DLRT_ExitSyncCallbackTargetIsolate,
R4, R9);
} else {
Label call;
__ ldr(R4, compiler::Address::PC(2 * Instr::kInstrSize));
__ b(&call);
__ Emit64(reinterpret_cast<int64_t>(&DLRT_ExitSyncCallbackTargetIsolate));
__ Bind(&call);
}
#else
GenerateLoadFfiCallbackMetadataRuntimeFunction(
FfiCallbackMetadata::kExitSyncCallbackTargetIsolate, R4);
#endif
__ mov(CSP, SP);
__ blr(R4);
__ blr(R10); // entry_point
__ stp(R0, R1, Address(CSP, -2 * target::kWordSize, Address::PairPreIndex));
__ fstp(V0, V1, Address(CSP, -2 * 8, Address::PairPreIndex), kDWord);
__ fstp(V2, V3, Address(CSP, -2 * 8, Address::PairPreIndex), kDWord);
__ mov(R0, THR);
__ blr(R21); // DLRT_ExitSyncCallback, etc
if (FLAG_target_memory_sanitizer) {
__ blr(R0); // dart_msan_unpoison_retval
}
__ mov(SP, CSP);
__ mov(THR, R0);
__ PopRegisters(return_registers);
__ LeaveFrame();
__ RestoreCSP();
}
__ b(&done);
__ Bind(&sync_isolate_group_bound_callback);
__ blr(R10);
// Exit isolate group bound isolate.
{
__ SetupDartSP();
__ EnterFrame(0);
__ ReserveAlignedFrameSpace(0);
const RegisterSet return_registers(
(1 << CallingConventions::kReturnReg) |
(1 << CallingConventions::kSecondReturnReg),
(1 << CallingConventions::kReturnFpuReg) |
(1 << CallingConventions::kSecondReturnFpuReg) |
(1 << CallingConventions::kThirdReturnFpuReg) |
(1 << CallingConventions::kFourthReturnFpuReg));
// This saves too much: we only need the D half of Q registers.
__ PushRegisters(return_registers);
#if defined(DART_TARGET_OS_FUCHSIA)
// TODO(https://dartbug.com/52579): Remove.
if (FLAG_precompiled_mode) {
GenerateLoadBSSEntry(BSS::Relocation::DLRT_ExitIsolateGroupBoundIsolate,
R4, R9);
} else {
Label call;
__ ldr(R4, compiler::Address::PC(2 * Instr::kInstrSize));
__ b(&call);
__ Emit64(reinterpret_cast<int64_t>(&DLRT_ExitIsolateGroupBoundIsolate));
__ Bind(&call);
}
#else
GenerateLoadFfiCallbackMetadataRuntimeFunction(
FfiCallbackMetadata::kExitIsolateGroupBoundIsolate, R4);
#endif
__ mov(CSP, SP);
__ blr(R4);
if (FLAG_target_memory_sanitizer) {
__ blr(R0); // dart_msan_unpoison_retval
}
__ mov(SP, CSP);
__ mov(THR, R0);
__ PopRegisters(return_registers);
__ LeaveFrame();
__ RestoreCSP();
}
__ b(&done);
__ Bind(&async_callback);
// Async callback. The entrypoint marshals the arguments into a message and
// sends it over the send port. DLRT_GetThreadForNativeCallbackTrampoline
// entered a temporary isolate, so exit it afterwards.
// Clobbers all volatile registers, including the callback ID in R9.
// Resets CSP and SP, important for EnterSafepoint below.
__ blr(R10);
// Exit the temporary isolate.
{
#if defined(DART_TARGET_OS_FUCHSIA)
// TODO(https://dartbug.com/52579): Remove.
if (FLAG_precompiled_mode) {
GenerateLoadBSSEntry(BSS::Relocation::DLRT_ExitTemporaryIsolate, R4, R9);
} else {
Label call;
__ ldr(R4, compiler::Address::PC(2 * Instr::kInstrSize));
__ b(&call);
__ Emit64(reinterpret_cast<int64_t>(&DLRT_ExitTemporaryIsolate));
__ Bind(&call);
}
#else
GenerateLoadFfiCallbackMetadataRuntimeFunction(
FfiCallbackMetadata::kExitTemporaryIsolate, R4);
#endif
// Pop LR and THR from the real stack (CSP).
CLOBBERS_LR(__ ldp(
__ fldp(V2, V3, Address(CSP, 2 * 8, Address::PairPostIndex), kDWord);
__ fldp(V0, V1, Address(CSP, 2 * 8, Address::PairPostIndex), kDWord);
__ ldp(R0, R1, Address(CSP, 2 * target::kWordSize, Address::PairPostIndex));
__ ldp(R20, R21,
Address(CSP, 2 * target::kWordSize, Address::PairPostIndex));
RESTORES_LR_FROM_FRAME(__ ldp(
THR, LR, Address(CSP, 2 * target::kWordSize, Address::PairPostIndex)));
__ ret();
}
{
SPILLS_LR_TO_FRAME(); //...
__ Bind(&tail);
__ blr(R10);
__ mov(R0, THR);
__ mov(R1, R21);
__ ldp(R20, R21,
Address(CSP, 2 * target::kWordSize, Address::PairPostIndex));
RESTORES_LR_FROM_FRAME(__ ldp(
THR, LR, Address(CSP, 2 * target::kWordSize, Address::PairPostIndex)));
// Tail-call DLRT_ExitTemporaryIsolate. It is not safe to return to this
// stub, since it might be deleted once DLRT_ExitTemporaryIsolate proceeds
// enough for VM shutdown.
__ br(R4);
__ br(R1); // DLRT_ExitTemporaryIsolate.
__ brk(0);
}
__ Bind(&done);
// Pop LR and THR from the real stack (CSP).
RESTORES_LR_FROM_FRAME(__ ldp(
THR, LR, Address(CSP, 2 * target::kWordSize, Address::PairPostIndex)));
__ ret();
ASSERT_LESS_OR_EQUAL(__ CodeSize() - shared_stub_start,
FfiCallbackMetadata::kNativeCallbackSharedStubSize);
ASSERT_LESS_OR_EQUAL(__ CodeSize(), FfiCallbackMetadata::kPageSize);
runtime/vm/compiler/stub_code_compiler_ia32.cc
+55 -153
View File
@@ -245,6 +245,7 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
// Save THR and EBX which are callee-saved.
__ pushl(THR);
__ pushl(EBX);
__ pushl(ECX);
// THR & return address
COMPILE_ASSERT(FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta == 4);
@@ -255,20 +256,16 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
// code size on this shared stub.
{
__ EnterFrame(0);
// entry_point, trampoline_type, &trampoline_type, &entry_point, trampoline
// ^------ GetFfiCallbackMetadata args ------^
__ ReserveAlignedFrameSpace(5 * target::kWordSize);
// SP[4] CallbackMetadata.epilogue
// SP[3] CallbackMetadata.is_tail
// SP[2] CallbackMetadata.entry_point
// SP[1] out -> SP[2]
// SP[0] trampoline
// Trampoline arg.
__ movl(Address(SPREG, 0 * target::kWordSize), EAX);
// Pointer to trampoline type stack slot.
__ movl(EAX, SPREG);
__ addl(EAX, Immediate(3 * target::kWordSize));
__ movl(Address(SPREG, 2 * target::kWordSize), EAX);
// Pointer to entry point stack slot.
__ addl(EAX, Immediate(target::kWordSize));
__ addl(EAX, Immediate(2 * target::kWordSize));
__ movl(Address(SPREG, 1 * target::kWordSize), EAX);
__ movl(EAX,
@@ -276,179 +273,84 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
__ call(EAX);
__ movl(THR, EAX);
// Save the trampoline type in EBX, and the entry point in ECX.
__ movl(EBX, Address(SPREG, 3 * target::kWordSize));
__ movl(ECX, Address(SPREG, 4 * target::kWordSize));
__ movl(EAX, Address(SPREG, 2 * target::kWordSize)); // entry_point
__ movl(ECX, Address(SPREG, 3 * target::kWordSize)); // is_tail
__ movl(EBX, Address(SPREG, 4 * target::kWordSize)); // epilogue
__ LeaveFrame();
// Save the trampoline type to the stack, because we'll need it after the
// call to decide whether to ret() or ret(4).
__ pushl(EBX);
}
COMPILE_ASSERT(!IsCalleeSavedRegister(ECX) && !IsArgumentRegister(ECX));
COMPILE_ASSERT(ECX != THR);
Label call, call_ret4, tail;
__ cmpl(ECX, Immediate(0));
__ j(EQUAL, &call);
__ cmpl(ECX, Immediate(1));
__ j(EQUAL, &tail);
__ cmpl(ECX, Immediate(2));
__ j(EQUAL, &call_ret4);
__ int3();
Label async_callback;
Label sync_isolate_group_bound_callback;
Label sync_callback_isolate_ownership;
Label done;
// Check the trampoline type to see how the callback should be invoked.
__ cmpl(EBX, Immediate(static_cast<uword>(
FfiCallbackMetadata::TrampolineType::kAsync)));
__ j(EQUAL, &async_callback);
__ cmpl(EBX,
Immediate(static_cast<uword>(
FfiCallbackMetadata::TrampolineType::kSyncIsolateGroupBound)));
__ j(EQUAL, &sync_isolate_group_bound_callback, Assembler::kNearJump);
__ testl(EBX,
Immediate(FfiCallbackMetadata::kSyncCallbackIsolateOwnershipFlag));
__ j(NOT_ZERO, &sync_callback_isolate_ownership, Assembler::kNearJump);
// Sync callback. The entry point contains the target function, so just call
// it. DLRT_GetThreadForNativeCallbackTrampoline exited the safepoint, so
// re-enter it afterwards.
// On entry to the function, there will be two extra slots on the stack:
// the saved THR and the return address. The target will know to skip them.
__ call(ECX);
// Takes care to not clobber *any* registers (besides scratch).
__ EnterFullSafepoint(/*scratch=*/ECX);
// Pop the trampoline type into ECX.
__ popl(ECX);
// Restore callee-saved registers.
__ popl(EBX);
__ popl(THR);
__ cmpl(ECX, Immediate(static_cast<uword>(
FfiCallbackMetadata::TrampolineType::kSync)));
__ j(NOT_EQUAL, &ret_4, Assembler::kNearJump);
__ ret();
__ Bind(&ret_4);
__ ret(Immediate(4));
__ Bind(&sync_callback_isolate_ownership);
__ call(ECX);
// Exit the target isolate.
{
__ Bind(&call);
__ call(EAX); // entry_point
__ pushl(CallingConventions::kReturnReg);
__ pushl(CallingConventions::kSecondReturnReg);
__ subl(ESP, Immediate(kFpuRegisterSize));
__ movups(Address(ESP, 0), CallingConventions::kReturnFpuReg);
__ subl(ESP, Immediate(8));
__ movsd(Address(ESP, 0), CallingConventions::kReturnFpuReg);
// 4 + 4 + 8 = 16 (stack alignment)
__ EnterFrame(0);
__ ReserveAlignedFrameSpace(0);
__ pushl(THR);
__ call(EBX); // DLRT_ExitSyncCallback, etc
__ popl(EAX);
__ movl(EAX, Immediate(reinterpret_cast<int64_t>(
DLRT_ExitSyncCallbackTargetIsolate)));
__ CallCFunction(EAX);
__ LeaveFrame();
__ movups(Address(ESP, 0), CallingConventions::kReturnFpuReg);
__ addl(ESP, Immediate(kFpuRegisterSize));
__ movsd(CallingConventions::kReturnFpuReg, Address(ESP, 0));
__ addl(ESP, Immediate(8));
__ popl(CallingConventions::kSecondReturnReg);
__ popl(CallingConventions::kReturnReg);
// Pop the trampoline type into ECX.
__ popl(ECX);
// Restore callee-saved registers.
__ popl(EBX);
__ popl(THR);
Label ownership_ret_4;
__ cmpl(ECX,
Immediate(
static_cast<uword>(FfiCallbackMetadata::TrampolineType::kSync) |
FfiCallbackMetadata::kSyncCallbackIsolateOwnershipFlag));
__ j(NOT_EQUAL, &ownership_ret_4, Assembler::kNearJump);
__ ret();
}
{
__ Bind(&call_ret4);
__ call(EAX); // entry_point
__ Bind(&ownership_ret_4);
__ pushl(CallingConventions::kReturnReg);
__ pushl(CallingConventions::kSecondReturnReg);
__ subl(ESP, Immediate(8));
__ movsd(Address(ESP, 0), CallingConventions::kReturnFpuReg);
// 4 + 4 + 8 = 16 (stack alignment)
__ pushl(THR);
__ call(EBX); // DLRT_ExitSyncCallback, etc
__ popl(EAX);
__ movsd(CallingConventions::kReturnFpuReg, Address(ESP, 0));
__ addl(ESP, Immediate(8));
__ popl(CallingConventions::kSecondReturnReg);
__ popl(CallingConventions::kReturnReg);
__ popl(ECX);
__ popl(EBX);
__ popl(THR);
__ ret(Immediate(4));
}
__ jmp(&done, Assembler::kNearJump);
__ Bind(&sync_isolate_group_bound_callback);
__ call(ECX);
// Exit isolate group bound isolate.
{
__ pushl(CallingConventions::kReturnReg);
__ pushl(CallingConventions::kSecondReturnReg);
__ subl(ESP, Immediate(kFpuRegisterSize));
__ movups(Address(ESP, 0), CallingConventions::kReturnFpuReg);
__ EnterFrame(0);
__ ReserveAlignedFrameSpace(0);
__ movl(EAX, Immediate(reinterpret_cast<int64_t>(
DLRT_ExitIsolateGroupBoundIsolate)));
__ CallCFunction(EAX);
__ LeaveFrame();
__ movups(Address(ESP, 0), CallingConventions::kReturnFpuReg);
__ addl(ESP, Immediate(kFpuRegisterSize));
__ popl(CallingConventions::kSecondReturnReg);
__ popl(CallingConventions::kReturnReg);
}
__ jmp(&done, Assembler::kNearJump);
__ Bind(&async_callback);
// Async callback. The entrypoint marshals the arguments into a message and
// sends it over the send port. DLRT_GetThreadForNativeCallbackTrampoline
// entered a temporary isolate, so exit it afterwards.
// On entry to the function, there will be two extra slots on the stack:
// the saved THR and the return address. The target will know to skip them.
__ call(ECX);
// Exit the temporary isolate.
{
// Pop the trampoline type into ECX.
__ Bind(&tail);
__ call(EAX); // entry_point
__ movl(EAX, EBX);
__ popl(ECX);
// Restore callee-saved registers.
__ popl(EBX);
__ popl(THR);
// Tail-call DLRT_ExitTemporaryIsolate. It is not safe to return to this
// stub, since it might be deleted once DLRT_ExitTemporaryIsolate proceeds
// enough for VM shutdown.
__ movl(EAX,
Immediate(reinterpret_cast<int64_t>(DLRT_ExitTemporaryIsolate)));
__ jmp(EAX);
__ jmp(EAX); // DLRT_ExitTemporaryIsolate
__ int3();
}
__ Bind(&done);
// Pop the trampoline type into ECX.
__ popl(ECX);
// Restore callee-saved registers.
__ popl(EBX);
__ popl(THR);
// Stack delta is always 0 for async callbacks.
__ ret();
// 'kNativeCallbackSharedStubSize' is an upper bound because the exact
// instruction size can vary slightly based on OS calling conventions.
ASSERT_LESS_OR_EQUAL(__ CodeSize() - shared_stub_start,
runtime/vm/compiler/stub_code_compiler_riscv.cc
+57 -257
View File
@@ -351,8 +351,9 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
__ Bind(&body);
// Save THR (callee-saved) and RA. Keeps stack aligned.
COMPILE_ASSERT(FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta == 2);
COMPILE_ASSERT(FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta == 4);
__ PushRegisterPair(RA, THR);
__ PushRegisterPair(S2, S3);
COMPILE_ASSERT(!IsArgumentRegister(THR));
// Load the thread, verify the callback ID and exit the safepoint.
@@ -361,29 +362,26 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
// code size on this shared stub.
{
// Push arguments and callback id.
__ subi(SP, SP, 9 * target::kWordSize);
__ sx(T1, Address(SP, 8 * target::kWordSize));
__ sx(A7, Address(SP, 7 * target::kWordSize));
__ sx(A6, Address(SP, 6 * target::kWordSize));
__ sx(A5, Address(SP, 5 * target::kWordSize));
__ sx(A4, Address(SP, 4 * target::kWordSize));
__ sx(A3, Address(SP, 3 * target::kWordSize));
__ sx(A2, Address(SP, 2 * target::kWordSize));
__ sx(A1, Address(SP, 1 * target::kWordSize));
__ sx(A0, Address(SP, 0 * target::kWordSize));
__ subi(SP, SP, 12 * target::kWordSize);
__ sx(A7, Address(SP, 11 * target::kWordSize));
__ sx(A6, Address(SP, 10 * target::kWordSize));
__ sx(A5, Address(SP, 9 * target::kWordSize));
__ sx(A4, Address(SP, 8 * target::kWordSize));
__ sx(A3, Address(SP, 7 * target::kWordSize));
__ sx(A2, Address(SP, 6 * target::kWordSize));
__ sx(A1, Address(SP, 5 * target::kWordSize));
__ sx(A0, Address(SP, 4 * target::kWordSize));
// 3 - alignment gap
// 2 - Out.epilogue
// 1 - Out.is_tail
// 0 - Out.entry_point
__ EnterFrame(0);
// Reserve one slot for the entry point and one for the tramp abi.
__ ReserveAlignedFrameSpace(2 * target::kWordSize);
__ mv(A0, T1);
__ mv(A1, SP);
// Since DLRT_GetFfiCallbackMetadata can theoretically be loaded anywhere,
// we use the same trick as before to ensure a predictable instruction
// sequence.
Label call;
__ mv(A0, T1); // trampoline
__ mv(A1, SPREG); // out_entry_point
__ addi(A2, SPREG, target::kWordSize); // out_trampoline_type
#if defined(DART_TARGET_OS_FUCHSIA)
// TODO(https://dartbug.com/52579): Remove.
if (FLAG_precompiled_mode) {
@@ -394,6 +392,7 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
intptr_t start = __ CodeSize();
__ auipc(T1, 0);
__ lx(T1, Address(T1, kPCRelativeLoadOffset));
Label call;
__ j(&call);
ASSERT_EQUAL(__ CodeSize() - start, kPCRelativeLoadOffset);
@@ -402,268 +401,69 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
#else
__ Emit64(reinterpret_cast<int64_t>(&DLRT_GetFfiCallbackMetadata));
#endif
__ Bind(&call);
}
#else
GenerateLoadFfiCallbackMetadataRuntimeFunction(
FfiCallbackMetadata::kGetFfiCallbackMetadata, T1);
#endif // defined(DART_TARGET_OS_FUCHSIA)
__ Bind(&call);
__ jalr(T1);
__ mv(THR, A0);
__ lx(T2, Address(SPREG, 0)); // entry_point
__ lx(T3, Address(SPREG, target::kWordSize)); // trampoline_type
__ lx(T2, Address(SP, 0 * target::kWordSize)); // entry_point
__ lx(S2, Address(SP, 1 * target::kWordSize)); // is_tail
__ lx(S3, Address(SP, 2 * target::kWordSize)); // epilogue
__ LeaveFrame();
// Restore arguments and callback id.
__ lx(A0, Address(SP, 0 * target::kWordSize));
__ lx(A1, Address(SP, 1 * target::kWordSize));
__ lx(A2, Address(SP, 2 * target::kWordSize));
__ lx(A3, Address(SP, 3 * target::kWordSize));
__ lx(A4, Address(SP, 4 * target::kWordSize));
__ lx(A5, Address(SP, 5 * target::kWordSize));
__ lx(A6, Address(SP, 6 * target::kWordSize));
__ lx(A7, Address(SP, 7 * target::kWordSize));
__ lx(T1, Address(SP, 8 * target::kWordSize));
__ addi(SP, SP, 9 * target::kWordSize);
__ lx(A0, Address(SP, 4 * target::kWordSize));
__ lx(A1, Address(SP, 5 * target::kWordSize));
__ lx(A2, Address(SP, 6 * target::kWordSize));
__ lx(A3, Address(SP, 7 * target::kWordSize));
__ lx(A4, Address(SP, 8 * target::kWordSize));
__ lx(A5, Address(SP, 9 * target::kWordSize));
__ lx(A6, Address(SP, 10 * target::kWordSize));
__ lx(A7, Address(SP, 11 * target::kWordSize));
__ addi(SP, SP, 12 * target::kWordSize);
}
COMPILE_ASSERT(!IsCalleeSavedRegister(T2) && !IsArgumentRegister(T2));
COMPILE_ASSERT(!IsCalleeSavedRegister(T3) && !IsArgumentRegister(T3));
Label tail;
__ bnez(S2, &tail, Assembler::kNearJump);
Label something_other_than_sync_callback;
Label async_callback;
Label sync_isolate_group_bound_callback;
Label done;
// Check the trampoline type to see how the callback should be invoked.
COMPILE_ASSERT(
static_cast<uword>(FfiCallbackMetadata::TrampolineType::kSync) == 0);
__ bnez(T3, &something_other_than_sync_callback);
// Sync callback. The entry point contains the target function, so just call
// it. DLRT_GetThreadForNativeCallbackTrampoline exited the safepoint, so
// re-enter it afterwards.
// Clobbers all volatile registers, including the callback ID in T1.
__ jalr(T2);
// Clobbers TMP, TMP2 and T1 -- all volatile and not holding return values.
__ EnterFullSafepoint(/*scratch=*/T1);
if (FLAG_target_memory_sanitizer) {
__ EnterFrame(0);
__ ReserveAlignedFrameSpace(0);
const RegisterSet return_registers(
(1 << CallingConventions::kReturnReg) |
(1 << CallingConventions::kSecondReturnReg),
(1 << CallingConventions::kReturnFpuReg) |
(1 << CallingConventions::kSecondReturnFpuReg));
__ PushRegisters(return_registers);
Label call;
#if defined(DART_TARGET_OS_FUCHSIA)
// TODO(https://dartbug.com/52579): Remove.
if (FLAG_precompiled_mode) {
GenerateLoadBSSEntry(BSS::Relocation::DRT_ExitSyncCallback, T1, T2);
} else {
const intptr_t kPCRelativeLoadOffset = 12;
intptr_t start = __ CodeSize();
__ auipc(T1, 0);
__ lx(T1, Address(T1, kPCRelativeLoadOffset));
__ j(&call);
ASSERT_EQUAL(__ CodeSize() - start, kPCRelativeLoadOffset);
#if XLEN == 32
__ Emit32(reinterpret_cast<int32_t>(&DLRT_ExitSyncCallback));
#else
__ Emit64(reinterpret_cast<int64_t>(&DLRT_ExitSyncCallback));
#endif
}
#else
GenerateLoadFfiCallbackMetadataRuntimeFunction(
FfiCallbackMetadata::kExitSyncCallbackTargetIsolate, T1);
#endif // defined(DART_TARGET_OS_FUCHSIA)
__ Bind(&call);
__ jalr(T1);
__ jalr(A0); // dart_msan_unpoison_retval
__ PopRegisters(return_registers);
__ LeaveFrame();
__ j(&done, Assembler::kNearJump);
}
__ j(&done);
__ Bind(&something_other_than_sync_callback);
__ li(T4, static_cast<uword>(FfiCallbackMetadata::TrampolineType::kAsync));
__ beq(T3, T4, &async_callback, Assembler::kNearJump);
__ li(T4, static_cast<uword>(
FfiCallbackMetadata::TrampolineType::kSyncIsolateGroupBound));
__ beq(T3, T4, &sync_isolate_group_bound_callback, Assembler::kNearJump);
// Sync callback that entered the target isolate.
__ jalr(T2);
// Exit the target isolate.
{
__ EnterFrame(0);
__ ReserveAlignedFrameSpace(0);
const RegisterSet return_registers(
(1 << CallingConventions::kReturnReg) |
(1 << CallingConventions::kSecondReturnReg),
(1 << CallingConventions::kReturnFpuReg) |
(1 << CallingConventions::kSecondReturnFpuReg));
__ PushRegisters(return_registers);
Label call;
#if defined(DART_TARGET_OS_FUCHSIA)
// TODO(https://dartbug.com/52579): Remove.
if (FLAG_precompiled_mode) {
GenerateLoadBSSEntry(BSS::Relocation::DRT_ExitSyncCallbackTargetIsolate,
T1, T2);
} else {
const intptr_t kPCRelativeLoadOffset = 12;
intptr_t start = __ CodeSize();
__ auipc(T1, 0);
__ lx(T1, Address(T1, kPCRelativeLoadOffset));
__ j(&call);
ASSERT_EQUAL(__ CodeSize() - start, kPCRelativeLoadOffset);
#if XLEN == 32
__ Emit32(reinterpret_cast<int32_t>(&DLRT_ExitSyncCallbackTargetIsolate));
#else
__ Emit64(reinterpret_cast<int64_t>(&DLRT_ExitSyncCallbackTargetIsolate));
#endif
}
#else
GenerateLoadFfiCallbackMetadataRuntimeFunction(
FfiCallbackMetadata::kExitSyncCallbackTargetIsolate, T1);
#endif // defined(DART_TARGET_OS_FUCHSIA)
__ Bind(&call);
__ jalr(T1);
__ jalr(T2); // entry_point
__ subi(SP, SP, 32); // Not using word size to keep RV32 stack aligned.
__ sx(A0, Address(SP, 24));
__ sx(A1, Address(SP, 16));
__ fsd(FA0, Address(SP, 8));
__ fsd(FA1, Address(SP, 0));
__ mv(A0, THR);
__ jalr(S3); // DLRT_ExitSyncCallback, etc
if (FLAG_target_memory_sanitizer) {
__ jalr(A0); // dart_msan_unpoison_retval
}
__ PopRegisters(return_registers);
__ LeaveFrame();
__ j(&done, Assembler::kNearJump);
}
__ Bind(&sync_isolate_group_bound_callback);
__ jalr(T2);
// Exit isolate group bound isolate.
{
__ EnterFrame(0);
__ ReserveAlignedFrameSpace(0);
const RegisterSet return_registers(
(1 << CallingConventions::kReturnReg) |
(1 << CallingConventions::kSecondReturnReg),
(1 << CallingConventions::kReturnFpuReg) |
(1 << CallingConventions::kSecondReturnFpuReg));
__ PushRegisters(return_registers);
Label call;
#if defined(DART_TARGET_OS_FUCHSIA)
// TODO(https://dartbug.com/52579): Remove.
if (FLAG_precompiled_mode) {
GenerateLoadBSSEntry(BSS::Relocation::DLRT_ExitIsolateGroupBoundIsolate,
T1, T2);
} else {
const intptr_t kPCRelativeLoadOffset = 12;
intptr_t start = __ CodeSize();
__ auipc(T1, 0);
__ lx(T1, Address(T1, kPCRelativeLoadOffset));
__ j(&call);
ASSERT_EQUAL(__ CodeSize() - start, kPCRelativeLoadOffset);
#if XLEN == 32
__ Emit32(reinterpret_cast<int32_t>(&DLRT_ExitIsolateGroupBoundIsolate));
#else
__ Emit64(reinterpret_cast<int64_t>(&DLRT_ExitIsolateGroupBoundIsolate));
#endif
}
#else
GenerateLoadFfiCallbackMetadataRuntimeFunction(
FfiCallbackMetadata::kExitIsolateGroupBoundIsolate, T1);
#endif // defined(DART_TARGET_OS_FUCHSIA)
__ Bind(&call);
__ jalr(T1);
if (FLAG_target_memory_sanitizer) {
__ jalr(A0); // dart_msan_unpoison_retval
}
__ PopRegisters(return_registers);
__ LeaveFrame();
__ j(&done, Assembler::kNearJump);
}
__ Bind(&async_callback);
// Async callback. The entrypoint marshals the arguments into a message and
// sends it over the send port. DLRT_GetThreadForNativeCallbackTrampoline
// entered a temporary isolate, so exit it afterwards.
// Clobbers all volatile registers, including the callback ID in T1.
__ jalr(T2);
// Exit the temporary isolate.
{
#if defined(DART_TARGET_OS_FUCHSIA)
// TODO(https://dartbug.com/52579): Remove.
if (FLAG_precompiled_mode) {
GenerateLoadBSSEntry(BSS::Relocation::DLRT_ExitTemporaryIsolate, T1, T2);
} else {
Label call;
const intptr_t kPCRelativeLoadOffset = 12;
intptr_t start = __ CodeSize();
__ auipc(T1, 0);
__ lx(T1, Address(T1, kPCRelativeLoadOffset));
__ j(&call);
ASSERT_EQUAL(__ CodeSize() - start, kPCRelativeLoadOffset);
#if XLEN == 32
__ Emit32(reinterpret_cast<int32_t>(&DLRT_ExitTemporaryIsolate));
#else
__ Emit64(reinterpret_cast<int64_t>(&DLRT_ExitTemporaryIsolate));
#endif
__ Bind(&call);
}
#else
GenerateLoadFfiCallbackMetadataRuntimeFunction(
FfiCallbackMetadata::kExitTemporaryIsolate, T1);
#endif // defined(DART_TARGET_OS_FUCHSIA)
__ fld(FA1, Address(SP, 0));
__ fld(FA0, Address(SP, 8));
__ lx(A1, Address(SP, 16));
__ lx(A0, Address(SP, 24));
__ addi(SP, SP, 32);
__ PopRegisterPair(S2, S3);
__ PopRegisterPair(RA, THR);
__ ret();
}
{
__ Bind(&tail);
__ jalr(T2); // entry_point
__ mv(A0, THR);
__ mv(A1, S3);
__ PopRegisterPair(S2, S3);
__ PopRegisterPair(RA, THR);
// Tail-call DLRT_ExitTemporaryIsolate. It is not safe to return to this
// stub, since it might be deleted once DLRT_ExitTemporaryIsolate proceeds
// enough for VM shutdown.
__ jr(T1);
__ jr(A1); // DLRT_ExitTemporaryIsolate.
__ ebreak();
}
__ Bind(&done);
__ PopRegisterPair(RA, THR);
__ ret();
ASSERT_LESS_OR_EQUAL(__ CodeSize() - shared_stub_start,
FfiCallbackMetadata::kNativeCallbackSharedStubSize);
ASSERT_LESS_OR_EQUAL(__ CodeSize(), FfiCallbackMetadata::kPageSize);
runtime/vm/compiler/stub_code_compiler_x64.cc
+36 -208
View File
@@ -537,9 +537,11 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
// Save THR which is callee-saved.
__ pushq(THR);
__ pushq(R12);
__ pushq(R13);
// 2 = THR & return address
COMPILE_ASSERT(2 == FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta);
// 4 = return address, THR, R12, R13
COMPILE_ASSERT(4 == FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta);
// Save all registers which might hold arguments.
__ PushRegisters(kArgumentRegisterSet);
@@ -552,19 +554,6 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
COMPILE_ASSERT(RAX != CallingConventions::kArg1Reg);
__ movq(CallingConventions::kArg1Reg, RAX);
// We also need to look up the entry point for the trampoline. This is
// returned using a pointer passed to the second arg of the C function
// below. We aim that pointer at a reserved stack slot.
COMPILE_ASSERT(RAX != CallingConventions::kArg2Reg);
__ pushq(Immediate(0)); // Reserve a stack slot for the entry point.
__ movq(CallingConventions::kArg2Reg, RSP);
// We also need to know if this is a sync or async callback. This is also
// returned by pointer.
COMPILE_ASSERT(RAX != CallingConventions::kArg3Reg);
__ pushq(Immediate(0)); // Reserve a stack slot for the trampoline type.
__ movq(CallingConventions::kArg3Reg, RSP);
#if defined(DART_TARGET_OS_FUCHSIA)
// TODO(https://dartbug.com/52579): Remove.
if (FLAG_precompiled_mode) {
@@ -580,223 +569,62 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() {
#endif // defined(DART_TARGET_OS_FUCHSIA)
__ EnterFrame(0);
__ ReserveAlignedFrameSpace(0);
__ ReserveAlignedFrameSpace(3 * target::kWordSize);
__ movq(CallingConventions::kArg2Reg, RSP);
__ CallCFunction(RAX);
__ CallCFunction(RAX, /*restore_rsp=*/true);
__ movq(THR, RAX);
__ movq(RAX, Address(RSP, 0 * target::kWordSize)); // entry_point
__ movq(R12, Address(RSP, 1 * target::kWordSize)); // is_tail
__ movq(R13, Address(RSP, 2 * target::kWordSize)); // epilogue
__ LeaveFrame();
// The trampoline type is at the top of the stack. Pop it into RAX.
__ popq(RAX);
// Entry point is now at the top of the stack. Pop it into TMP.
__ popq(TMP);
}
// Restore the arguments.
__ PopRegisters(kArgumentRegisterSet);
// Current state:
//
// Stack:
// <old stack (arguments)>
// <return address>
// <saved THR>
//
// Registers: Like entry, except TMP == target, RAX == abi, and THR == thread
// All argument registers are untouched.
Label tail;
__ cmpq(R12, Immediate(0));
__ j(NOT_ZERO, &tail);
Label async_callback;
Label sync_isolate_group_bound_callback;
Label sync_callback_isolate_ownership;
Label done;
const RegisterSet return_registers(
(1 << CallingConventions::kReturnReg) |
(1 << CallingConventions::kSecondReturnReg),
(1 << CallingConventions::kReturnFpuReg) |
(1 << CallingConventions::kSecondReturnFpuReg));
// Check the trampoline type to see how the callback should be invoked.
__ cmpq(RAX, Immediate(static_cast<uword>(
FfiCallbackMetadata::TrampolineType::kAsync)));
__ j(EQUAL, &async_callback);
__ cmpq(RAX,
Immediate(static_cast<uword>(
FfiCallbackMetadata::TrampolineType::kSyncIsolateGroupBound)));
__ j(EQUAL, &sync_isolate_group_bound_callback);
__ testq(RAX,
Immediate(FfiCallbackMetadata::kSyncCallbackIsolateOwnershipFlag));
__ j(NOT_ZERO, &sync_callback_isolate_ownership);
// Sync callback. The entry point contains the target function, so just call
// it. DLRT_GetThreadForNativeCallbackTrampoline exited the safepoint, so
// re-enter it afterwards.
// On entry to the function, there will be two extra slots on the stack:
// the saved THR and the return address. The target will know to skip them.
__ call(TMP);
// Takes care to not clobber *any* registers (besides TMP).
__ EnterFullSafepoint();
if (FLAG_target_memory_sanitizer) {
const RegisterSet return_registers(
(1 << CallingConventions::kReturnReg) |
(1 << CallingConventions::kSecondReturnReg),
(1 << CallingConventions::kReturnFpuReg) |
(1 << CallingConventions::kSecondReturnFpuReg));
__ PushRegisters(return_registers);
#if defined(DART_TARGET_OS_FUCHSIA)
// TODO(https://dartbug.com/52579): Remove.
if (FLAG_precompiled_mode) {
GenerateLoadBSSEntry(BSS::Relocation::DLRT_ExitSyncCallback, RAX, TMP);
} else {
__ movq(RAX, Immediate(reinterpret_cast<int64_t>(DLRT_ExitSyncCallback)));
}
#else
GenerateLoadFfiCallbackMetadataRuntimeFunction(
FfiCallbackMetadata::kExitSyncCallback, RAX);
#endif // defined(DART_TARGET_OS_FUCHSIA)
__ EnterFrame(0);
__ ReserveAlignedFrameSpace(0);
__ CallCFunction(RAX);
__ CallCFunction(RAX); // dart_msan_unpoison_retval
__ LeaveFrame();
__ PopRegisters(return_registers);
}
__ jmp(&done);
__ Bind(&sync_callback_isolate_ownership);
__ call(TMP);
// Exit the target isolate.
{
const RegisterSet return_registers(
(1 << CallingConventions::kReturnReg) |
(1 << CallingConventions::kSecondReturnReg),
(1 << CallingConventions::kReturnFpuReg) |
(1 << CallingConventions::kSecondReturnFpuReg));
__ call(RAX); // entry_point
__ PushRegisters(return_registers);
#if defined(DART_TARGET_OS_FUCHSIA)
// TODO(https://dartbug.com/52579): Remove.
if (FLAG_precompiled_mode) {
GenerateLoadBSSEntry(BSS::Relocation::DLRT_ExitSyncCallbackTargetIsolate,
RAX, TMP);
} else {
__ movq(RAX, Immediate(reinterpret_cast<int64_t>(
DLRT_ExitSyncCallbackTargetIsolate)));
}
#else
GenerateLoadFfiCallbackMetadataRuntimeFunction(
FfiCallbackMetadata::kExitSyncCallbackTargetIsolate, RAX);
#endif // defined(DART_TARGET_OS_FUCHSIA)
__ EnterFrame(0);
__ ReserveAlignedFrameSpace(0);
__ CallCFunction(RAX);
__ movq(CallingConventions::kArg1Reg, THR);
__ CallCFunction(R13, /*restore_rsp=*/true); // DLRT_ExitSyncCallback, etc
if (FLAG_target_memory_sanitizer) {
__ CallCFunction(RAX); // dart_msan_unpoison_retval
__ CallCFunction(RAX, /*restore_rsp=*/true); // dart_msan_unpoison_retval
}
__ LeaveFrame();
__ PopRegisters(return_registers);
}
__ jmp(&done, Assembler::kNearJump);
__ Bind(&sync_isolate_group_bound_callback);
__ call(TMP);
// Exit isolate group bound isolate.
{
const RegisterSet return_registers(
(1 << CallingConventions::kReturnReg) |
(1 << CallingConventions::kSecondReturnReg),
(1 << CallingConventions::kReturnFpuReg) |
(1 << CallingConventions::kSecondReturnFpuReg));
__ PushRegisters(return_registers);
#if defined(DART_TARGET_OS_FUCHSIA)
// TODO(https://dartbug.com/52579): Remove.
if (FLAG_precompiled_mode) {
GenerateLoadBSSEntry(BSS::Relocation::DLRT_ExitIsolateGroupBoundIsolate,
RAX, TMP);
} else {
__ movq(RAX, Immediate(reinterpret_cast<int64_t>(
DLRT_ExitIsolateGroupBoundIsolate)));
}
#else
GenerateLoadFfiCallbackMetadataRuntimeFunction(
FfiCallbackMetadata::kExitIsolateGroupBoundIsolate, RAX);
#endif // defined(DART_TARGET_OS_FUCHSIA)
__ EnterFrame(0);
__ ReserveAlignedFrameSpace(0);
__ CallCFunction(RAX);
if (FLAG_target_memory_sanitizer) {
__ CallCFunction(RAX); // dart_msan_unpoison_retval
}
__ LeaveFrame();
__ PopRegisters(return_registers);
}
__ jmp(&done, Assembler::kNearJump);
__ Bind(&async_callback);
// Async callback. The entrypoint marshals the arguments into a message and
// sends it over the send port. DLRT_GetThreadForNativeCallbackTrampoline
// entered a temporary isolate, so exit it afterwards.
// On entry to the function, there will be two extra slots on the stack:
// the saved THR and the return address. The target will know to skip them.
__ call(TMP);
// Exit the temporary isolate.
{
#if defined(DART_TARGET_OS_FUCHSIA)
// TODO(https://dartbug.com/52579): Remove.
if (FLAG_precompiled_mode) {
GenerateLoadBSSEntry(BSS::Relocation::DLRT_ExitTemporaryIsolate, RAX,
TMP);
} else {
__ movq(RAX,
Immediate(reinterpret_cast<int64_t>(DLRT_ExitTemporaryIsolate)));
}
#else
GenerateLoadFfiCallbackMetadataRuntimeFunction(
FfiCallbackMetadata::kExitTemporaryIsolate, RAX);
#endif // defined(DART_TARGET_OS_FUCHSIA)
// Restore THR (callee-saved).
__ popq(R13);
__ popq(R12);
__ popq(THR);
__ ret();
}
{
__ Bind(&tail);
__ call(RAX); // entry_point
__ movq(CallingConventions::kArg1Reg, THR);
__ movq(RAX, R13);
__ popq(R13);
__ popq(R12);
__ popq(THR);
// Tail-call DLRT_ExitTemporaryIsolate. It is not safe to return to this
// stub, since it might be deleted once DLRT_ExitTemporaryIsolate proceeds
// enough for VM shutdown.
__ jmp(RAX);
__ jmp(RAX); // DLRT_ExitTemporaryIsolate
__ int3();
}
__ Bind(&done);
// Restore THR (callee-saved).
__ popq(THR);
__ ret();
// 'kNativeCallbackSharedStubSize' is an upper bound because the exact
// instruction size can vary slightly based on OS calling conventions.
ASSERT_LESS_OR_EQUAL(__ CodeSize() - shared_stub_start,
runtime/vm/ffi_callback_metadata.cc
-10
View File
@@ -230,16 +230,6 @@ void FfiCallbackMetadata::EnsureFreeListNotEmptyLocked() {
// Fill in the runtime functions.
FillRuntimeFunction(new_page, kGetFfiCallbackMetadata,
reinterpret_cast<void*>(DLRT_GetFfiCallbackMetadata));
FillRuntimeFunction(new_page, kExitTemporaryIsolate,
reinterpret_cast<void*>(DLRT_ExitTemporaryIsolate));
FillRuntimeFunction(
new_page, kExitIsolateGroupBoundIsolate,
reinterpret_cast<void*>(DLRT_ExitIsolateGroupBoundIsolate));
FillRuntimeFunction(
new_page, kExitSyncCallbackTargetIsolate,
reinterpret_cast<void*>(DLRT_ExitSyncCallbackTargetIsolate));
FillRuntimeFunction(new_page, kExitSyncCallback,
reinterpret_cast<void*>(DLRT_ExitSyncCallback));
// Add all the trampolines to the free list.
const intptr_t trampolines_per_page = NumCallbackTrampolinesPerPage();
runtime/vm/ffi_callback_metadata.h
+10 -10
View File
@@ -329,28 +329,28 @@ class FfiCallbackMetadata {
#if defined(TARGET_ARCH_X64)
static constexpr intptr_t kNativeCallbackTrampolineSize = 12;
static constexpr intptr_t kNativeCallbackSharedStubSize = 487;
static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 2;
static constexpr intptr_t kNativeCallbackSharedStubSize = 257;
static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 4;
#elif defined(TARGET_ARCH_IA32)
static constexpr intptr_t kNativeCallbackTrampolineSize = 10;
static constexpr intptr_t kNativeCallbackSharedStubSize = 241;
static constexpr intptr_t kNativeCallbackSharedStubSize = 160;
static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 4;
#elif defined(TARGET_ARCH_ARM)
static constexpr intptr_t kNativeCallbackTrampolineSize = 8;
static constexpr intptr_t kNativeCallbackSharedStubSize = 400;
static constexpr intptr_t kNativeCallbackSharedStubSize = 168;
static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 4;
#elif defined(TARGET_ARCH_ARM64)
static constexpr intptr_t kNativeCallbackTrampolineSize = 8;
static constexpr intptr_t kNativeCallbackSharedStubSize = 596;
static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 2;
static constexpr intptr_t kNativeCallbackSharedStubSize = 268;
static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 4;
#elif defined(TARGET_ARCH_RISCV32)
static constexpr intptr_t kNativeCallbackTrampolineSize = 8;
static constexpr intptr_t kNativeCallbackSharedStubSize = 368;
static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 2;
static constexpr intptr_t kNativeCallbackSharedStubSize = 154;
static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 4;
#elif defined(TARGET_ARCH_RISCV64)
static constexpr intptr_t kNativeCallbackTrampolineSize = 8;
static constexpr intptr_t kNativeCallbackSharedStubSize = 368;
static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 2;
static constexpr intptr_t kNativeCallbackSharedStubSize = 154;
static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 4;
#else
#error What architecture?
#endif
runtime/vm/runtime_entry.cc
+51 -38
View File
@@ -4986,9 +4986,7 @@ DEFINE_LEAF_RUNTIME_ENTRY(ExitSafepoint,
DLRT_ExitSafepoint);
namespace {
Thread* HandleAsyncFfiCallback(FfiCallbackMetadata::Metadata metadata,
uword* out_entry_point,
uword* out_trampoline_type) {
Thread* HandleAsyncFfiCallback(FfiCallbackMetadata::Metadata metadata) {
// NOTE: This is only thread safe if the user is using the API correctly.
// Otherwise, the callback could have been deleted and replaced, in which case
// IsLive would still be true. Or it could have been deleted after we looked
@@ -4998,8 +4996,6 @@ Thread* HandleAsyncFfiCallback(FfiCallbackMetadata::Metadata metadata,
// after free errors. Trying to lock FfiCallbackMetadata::lock_, or any
// similar lock, leads to deadlocks.
*out_trampoline_type = static_cast<uword>(metadata.trampoline_type());
*out_entry_point = metadata.target_entry_point();
Isolate* target_isolate = metadata.target_isolate();
Isolate* current_isolate = nullptr;
@@ -5033,14 +5029,9 @@ Thread* HandleAsyncFfiCallback(FfiCallbackMetadata::Metadata metadata,
}
Thread* HandleIsolateGroupBoundSyncFfiCallback(
FfiCallbackMetadata::Metadata metadata,
uword* out_entry_point,
uword* out_trampoline_type) {
FfiCallbackMetadata::Metadata metadata) {
Thread* current_thread = Thread::Current();
*out_entry_point = metadata.target_entry_point();
*out_trampoline_type = static_cast<uword>(metadata.trampoline_type());
if (current_thread != nullptr) {
current_thread->ExitSafepointFromNative();
current_thread->set_execution_state(Thread::kThreadInVM);
@@ -5086,13 +5077,9 @@ void FfiCallbackThreadChecks(Thread* thread, Isolate* target_isolate) {
Thread* HandleIsolateBoundSyncFfiCallback(
FfiCallbackMetadata::Metadata metadata,
uword* out_entry_point,
uword* out_trampoline_type) {
CallbackMetadata* out) {
Thread* current_thread = Thread::Current();
*out_entry_point = metadata.target_entry_point();
*out_trampoline_type = static_cast<uword>(metadata.trampoline_type());
Isolate* target_isolate = metadata.target_isolate();
if (current_thread == nullptr) {
if (!PortMap::IsOwnedByCurrentThread(target_isolate->main_port())) {
@@ -5100,15 +5087,16 @@ Thread* HandleIsolateBoundSyncFfiCallback(
}
Thread::EnterIsolate(target_isolate);
current_thread = Thread::Current();
*out_trampoline_type |=
FfiCallbackMetadata::kSyncCallbackIsolateOwnershipFlag;
FfiCallbackThreadChecks(current_thread, target_isolate);
out->epilogue =
reinterpret_cast<uword>(&DLRT_ExitSyncCallbackTargetIsolate);
} else {
FfiCallbackThreadChecks(current_thread, target_isolate);
if (current_thread->execution_state() != Thread::kThreadInNative) {
FATAL("Cannot invoke native callback from a leaf call.");
}
current_thread->ExitSafepointFromNative();
out->epilogue = reinterpret_cast<uword>(&DLRT_ExitSyncCallback);
}
current_thread->set_execution_state(Thread::kThreadInVM);
@@ -5123,13 +5111,11 @@ Thread* HandleIsolateBoundSyncFfiCallback(
// a runtime entry because we can't use Thread to look it up.
extern "C" Thread* DLRT_GetFfiCallbackMetadata(
FfiCallbackMetadata::Trampoline trampoline,
uword* out_entry_point,
uword* out_trampoline_type) {
CallbackMetadata* out) {
CHECK_STACK_ALIGNMENT;
TRACE_RUNTIME_CALL("GetFfiCallbackMetadata %p",
reinterpret_cast<void*>(trampoline));
ASSERT(out_entry_point != nullptr);
ASSERT(out_trampoline_type != nullptr);
ASSERT(out != nullptr);
if (!Isolate::IsolateCreationEnabled()) {
FATAL("GetFfiCallbackMetadata called after shutdown %p",
@@ -5155,23 +5141,37 @@ extern "C" Thread* DLRT_GetFfiCallbackMetadata(
}
Thread* thread = nullptr;
out->entry_point = metadata.target_entry_point();
switch (metadata.trampoline_type()) {
default:
out->type = 0; // Call
break;
case FfiCallbackMetadata::TrampolineType::kAsync:
out->type = 1; // Tail call.
break;
#if defined(TARGET_ARCH_IA32)
case FfiCallbackMetadata::TrampolineType::kSyncStackDelta4:
case FfiCallbackMetadata::TrampolineType::kSyncIsolateGroupBoundStackDelta4:
out->type = 2; // Call, ret4.
break;
#endif
}
if (metadata.trampoline_type() ==
FfiCallbackMetadata::TrampolineType::kAsync) {
thread =
HandleAsyncFfiCallback(metadata, out_entry_point, out_trampoline_type);
thread = HandleAsyncFfiCallback(metadata);
out->epilogue = reinterpret_cast<uword>(&DLRT_ExitTemporaryIsolate);
} else if (metadata.is_isolate_group_bound()) {
thread = HandleIsolateGroupBoundSyncFfiCallback(metadata, out_entry_point,
out_trampoline_type);
thread = HandleIsolateGroupBoundSyncFfiCallback(metadata);
out->epilogue = reinterpret_cast<uword>(&DLRT_ExitIsolateGroupBoundIsolate);
} else {
thread = HandleIsolateBoundSyncFfiCallback(metadata, out_entry_point,
out_trampoline_type);
thread = HandleIsolateBoundSyncFfiCallback(metadata, out);
}
TRACE_RUNTIME_CALL("GetFfiCallbackMetadata thread %p", thread);
TRACE_RUNTIME_CALL("GetFfiCallbackMetadata entry_point %p",
(void*)*out_entry_point);
TRACE_RUNTIME_CALL("GetFfiCallbackMetadata trampoline_type %p",
(void*)*out_trampoline_type);
reinterpret_cast<void*>(out->entry_point));
return thread;
}
@@ -5187,10 +5187,10 @@ extern "C" LargestReturn dart_msan_unpoison_retval() {
}
#endif
extern "C" void* DLRT_ExitIsolateGroupBoundIsolate() {
extern "C" void* DLRT_ExitIsolateGroupBoundIsolate(Thread* thread) {
TRACE_RUNTIME_CALL("ExitIsolateGroupBoundIsolate%s", "");
Thread* thread = Thread::Current();
ASSERT(thread != nullptr);
ASSERT(thread == Thread::Current());
Isolate* source_isolate =
reinterpret_cast<Isolate*>(thread->unboxed_int64_runtime_second_arg());
// Need to accommodate ExitIsolateGroupAsHelper assumptions.
@@ -5207,10 +5207,10 @@ extern "C" void* DLRT_ExitIsolateGroupBoundIsolate() {
#endif
}
extern "C" void* DLRT_ExitSyncCallbackTargetIsolate() {
extern "C" void* DLRT_ExitSyncCallbackTargetIsolate(Thread* thread) {
TRACE_RUNTIME_CALL("ExitSyncCallbackTargetIsolate%s", "");
Thread* thread = Thread::Current();
ASSERT(thread != nullptr);
ASSERT(thread == Thread::Current());
thread->set_execution_state(Thread::kThreadInVM);
Thread::ExitIsolate(/*isolate_shutdown=*/false);
#if defined(USING_MEMORY_SANITIZER)
@@ -5220,7 +5220,12 @@ extern "C" void* DLRT_ExitSyncCallbackTargetIsolate() {
#endif
}
extern "C" void* DLRT_ExitSyncCallback() {
extern "C" void* DLRT_ExitSyncCallback(Thread* thread) {
ASSERT(thread != nullptr);
ASSERT(thread == Thread::Current());
thread->EnterSafepointToNative();
#if defined(USING_MEMORY_SANITIZER)
return reinterpret_cast<void*>(dart_msan_unpoison_retval);
#else
@@ -5228,10 +5233,17 @@ extern "C" void* DLRT_ExitSyncCallback() {
#endif
}
extern "C" void DLRT_ExitTemporaryIsolate() {
TRACE_RUNTIME_CALL("ExitTemporaryIsolate%s", "");
#if defined(HOST_ARCH_IA32)
// A function with arguments isn't compatible with the tail-call because on IA32
// the arguments are on the stack and caller pops.
extern "C" void* DLRT_ExitTemporaryIsolate() {
Thread* thread = Thread::Current();
#else
extern "C" void* DLRT_ExitTemporaryIsolate(Thread* thread) {
#endif
TRACE_RUNTIME_CALL("ExitTemporaryIsolate%s", "");
ASSERT(thread != nullptr);
ASSERT(thread == Thread::Current());
Isolate* source_isolate =
reinterpret_cast<Isolate*>(thread->unboxed_int64_runtime_second_arg());
@@ -5250,6 +5262,7 @@ extern "C" void DLRT_ExitTemporaryIsolate() {
thread->EnterSafepoint();
}
TRACE_RUNTIME_CALL("ExitTemporaryIsolate %s", "done");
return nullptr;
}
extern "C" ApiLocalScope* DLRT_EnterHandleScope(Thread* thread) {
runtime/vm/runtime_entry.h
+14 -6
View File
@@ -96,13 +96,21 @@ LEAF_RUNTIME_ENTRY_LIST(DECLARE_LEAF_RUNTIME_ENTRY)
#undef DECLARE_LEAF_RUNTIME_ENTRY
// See StubCode::GenerateFfiCallbackTrampolineStub.
struct CallbackMetadata {
uword entry_point;
uword type; // FfiCallbackMetadata::CallType
uword epilogue;
};
extern "C" Thread* DLRT_GetFfiCallbackMetadata(uword trampoline,
uword* out_entry_point,
uword* out_callback_kind);
extern "C" void DLRT_ExitTemporaryIsolate();
extern "C" void* DLRT_ExitIsolateGroupBoundIsolate();
extern "C" void* DLRT_ExitSyncCallbackTargetIsolate();
extern "C" void* DLRT_ExitSyncCallback();
CallbackMetadata* out);
#if defined(HOST_ARCH_IA32)
extern "C" void* DLRT_ExitTemporaryIsolate();
#else
extern "C" void* DLRT_ExitTemporaryIsolate(Thread*);
#endif
extern "C" void* DLRT_ExitIsolateGroupBoundIsolate(Thread*);
extern "C" void* DLRT_ExitSyncCallbackTargetIsolate(Thread*);
extern "C" void* DLRT_ExitSyncCallback(Thread*);
const char* DeoptReasonToCString(ICData::DeoptReasonId deopt_reason);
runtime/vm/simulator_arm64.cc
+15 -22
View File
@@ -1831,10 +1831,8 @@ struct CallbackContext {
extern "C" void DoRedirectedFfiCallback(CallbackContext* ctxt,
uword trampoline) {
uword entry_point;
uword trampoline_type;
Thread* thread =
DLRT_GetFfiCallbackMetadata(trampoline, &entry_point, &trampoline_type);
CallbackMetadata out;
Thread* thread = DLRT_GetFfiCallbackMetadata(trampoline, &out);
if (thread == nullptr) {
// If GetFfiCallbackMetadata returned a null thread, it means that the async
// callback was invoked after it was deleted. In this case, do nothing.
@@ -1843,14 +1841,13 @@ extern "C" void DoRedirectedFfiCallback(CallbackContext* ctxt,
Simulator* sim = Simulator::Current();
ASSERT(sim != nullptr);
sim->DoRedirectedFfiCallback(thread, ctxt, entry_point, trampoline_type);
sim->DoRedirectedFfiCallback(thread, ctxt, &out);
}
// Compare FfiCallbackTrampolineStub.
void Simulator::DoRedirectedFfiCallback(Thread* thread,
CallbackContext* ctxt,
uword entry_point,
uword trampoline_type) {
CallbackMetadata* out) {
// The C caller might not be using frame pointers, so we just hard-code a
// maximum frame size instead of using FP-SP like we do for callouts.
constexpr intptr_t kStackSlotsCopied = 128;
@@ -1863,11 +1860,13 @@ void Simulator::DoRedirectedFfiCallback(Thread* thread,
for (intptr_t i = kStackSlotsCopied - 1; i >= 0; i--) {
*--sp = sp_in[i];
}
*--sp = get_register(LR);
*--sp = get_register(THR);
*--sp = get_register(LR);
*--sp = get_register(R20);
*--sp = get_register(R21);
set_register(nullptr, R31, reinterpret_cast<uword>(sp));
COMPILE_ASSERT(FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta ==
2);
4);
}
set_register(nullptr, R0, ctxt->integer_arguments[0]);
@@ -1890,7 +1889,7 @@ void Simulator::DoRedirectedFfiCallback(Thread* thread,
set_register(nullptr, THR, reinterpret_cast<uword>(thread));
set_register(nullptr, LR, kEndSimulatingPC);
set_pc(entry_point);
set_pc(out->entry_point);
Execute();
ctxt->integer_arguments[0] = get_register(R0);
@@ -1904,24 +1903,18 @@ void Simulator::DoRedirectedFfiCallback(Thread* thread,
// ldp lr, thr, [sp], 16!
// <drop arguments>
uword* sp = reinterpret_cast<uword*>(get_register(R31, R31IsSP));
set_register(nullptr, THR, *sp++);
set_register(nullptr, R21, *sp++);
set_register(nullptr, R20, *sp++);
set_register(nullptr, LR, *sp++);
set_register(nullptr, THR, *sp++);
sp += kStackSlotsCopied;
set_register(nullptr, R31, reinterpret_cast<uword>(sp));
COMPILE_ASSERT(FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta ==
2);
4);
}
if (trampoline_type ==
static_cast<uword>(FfiCallbackMetadata::TrampolineType::kAsync)) {
DLRT_ExitTemporaryIsolate();
} else if ((trampoline_type &
FfiCallbackMetadata::kSyncCallbackIsolateOwnershipFlag) != 0) {
thread->set_execution_state(Thread::kThreadInVM);
Thread::ExitIsolate(/*isolate_shutdown=*/false);
} else {
thread->EnterSafepointToNative();
}
auto epilogue = reinterpret_cast<void* (*)(Thread*)>(out->epilogue);
epilogue(thread);
}
void Simulator::ClobberVolatileRegisters() {
runtime/vm/simulator_arm64.h
+2 -2
View File
@@ -27,6 +27,7 @@ class Mutex;
class SimulatorSetjmpBuffer;
class Thread;
struct CallbackContext;
struct CallbackMetadata;
typedef struct {
union {
@@ -102,8 +103,7 @@ class Simulator {
bool fp_args = false);
void DoRedirectedFfiCallback(Thread* thread,
CallbackContext* ctxt,
uword entry_point,
uword trampoline_type);
CallbackMetadata* out);
// Runtime and native call support.
enum CallKind {