diff --git a/runtime/vm/bss_relocs.cc b/runtime/vm/bss_relocs.cc index 53928a53323..39de35b886d 100644 --- a/runtime/vm/bss_relocs.cc +++ b/runtime/vm/bss_relocs.cc @@ -31,17 +31,6 @@ void BSS::Initialize(Thread* current, uword* bss_start, bool vm) { InitializeBSSEntry(Relocation::DLRT_GetFfiCallbackMetadata, reinterpret_cast(DLRT_GetFfiCallbackMetadata), bss_start); - InitializeBSSEntry(Relocation::DLRT_ExitTemporaryIsolate, - reinterpret_cast(DLRT_ExitTemporaryIsolate), - bss_start); - InitializeBSSEntry(Relocation::DLRT_ExitIsolateGroupBoundIsolate, - reinterpret_cast(DLRT_ExitIsolateGroupBoundIsolate), - bss_start); - InitializeBSSEntry( - Relocation::DLRT_ExitSyncCallbackTargetIsolate, - reinterpret_cast(DLRT_ExitSyncCallbackTargetIsolate), bss_start); - InitializeBSSEntry(Relocation::DLRT_ExitSyncCallback, - reinterpret_cast(DLRT_ExitSyncCallback), bss_start); } } // namespace dart diff --git a/runtime/vm/bss_relocs.h b/runtime/vm/bss_relocs.h index 345ed266eaf..36a370760cb 100644 --- a/runtime/vm/bss_relocs.h +++ b/runtime/vm/bss_relocs.h @@ -17,10 +17,6 @@ class BSS : public AllStatic { // stored at the index. enum class Relocation : intptr_t { DLRT_GetFfiCallbackMetadata, // TODO(https://dartbug.com/52579): Remove. - DLRT_ExitTemporaryIsolate, // TODO(https://dartbug.com/52579): Remove. - DLRT_ExitIsolateGroupBoundIsolate, // TODO(https://dartbug.com/52579) - DLRT_ExitSyncCallbackTargetIsolate, // TODO(https://dartbug.com/52579) - DLRT_ExitSyncCallback, // TODO(https://dartbug.com/52579) EndOfVmEntries, // We don't have any isolate group specific entries at the moment. diff --git a/runtime/vm/compiler/ffi/abi.h b/runtime/vm/compiler/ffi/abi.h index ff917bf55d1..1c2bd40526d 100644 --- a/runtime/vm/compiler/ffi/abi.h +++ b/runtime/vm/compiler/ffi/abi.h @@ -42,6 +42,8 @@ enum class Abi { kWindowsArm64, kWindowsIA32, kWindowsX64, + + kMacOSRiscv64 = kLinuxRiscv64, // To build simriscv64 on Mac. }; const int64_t num_abis = static_cast(Abi::kWindowsX64) + 1; diff --git a/runtime/vm/compiler/stub_code_compiler_arm.cc b/runtime/vm/compiler/stub_code_compiler_arm.cc index 772c38036c1..39819b48814 100644 --- a/runtime/vm/compiler/stub_code_compiler_arm.cc +++ b/runtime/vm/compiler/stub_code_compiler_arm.cc @@ -361,21 +361,10 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { // We exit the safepoint inside DLRT_GetFfiCallbackMetadata in order to save // code size on this shared stub. { - __ mov(R0, Operand(R4)); - - // We also need to look up the entry point for the trampoline. This is - // returned using a pointer passed to the second arg of the C function - // below. We aim that pointer at a reserved stack slot. - __ sub(SP, SP, Operand(compiler::target::kWordSize)); - __ mov(R1, Operand(SP)); - - // We also need to know if this is a sync or async callback. This is also - // returned by pointer. - __ sub(SP, SP, Operand(compiler::target::kWordSize)); - __ mov(R2, Operand(SP)); - __ EnterFrame(1 << FP, 0); - __ ReserveAlignedFrameSpace(0); + __ ReserveAlignedFrameSpace(3 * target::kWordSize); + __ mov(R0, Operand(R4)); + __ mov(R1, Operand(SP)); GenerateLoadFfiCallbackMetadataRuntimeFunction( FfiCallbackMetadata::kGetFfiCallbackMetadata, R4); @@ -383,131 +372,54 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { __ blx(R4); __ mov(THR, Operand(R0)); + __ ldr(TMP, Address(SP, 0 * target::kWordSize)); // entry_point + __ ldr(R4, Address(SP, 1 * target::kWordSize)); // is_tail + __ ldr(R5, Address(SP, 2 * target::kWordSize)); // epilogue + __ LeaveFrame(1 << FP); - - // The trampoline type is at the top of the stack. Pop it into R4. - __ Pop(R4); - - // Entry point is now at the top of the stack. Pop it into R5. - __ Pop(R5); } __ PopRegisters(argument_registers); - Label async_callback; - Label sync_isolate_group_bound_callback; - Label sync_callback_isolate_ownership; - Label done; + Label tail; + __ cmp(R4, Operand(0)); + __ b(&tail, NOT_ZERO); - // Check the trampoline type to see how the callback should be invoked. - __ cmp( - R4, - Operand(static_cast(FfiCallbackMetadata::TrampolineType::kAsync))); - __ b(&async_callback, EQ); + const RegisterSet return_registers( + (1 << CallingConventions::kReturnReg) | + (1 << CallingConventions::kSecondReturnReg), + 1 << CallingConventions::kReturnFpuReg); + ASSERT(Utils::IsAligned(return_registers.SpillSize(), 8)); - __ cmp(R4, Operand(static_cast( - FfiCallbackMetadata::TrampolineType::kSyncIsolateGroupBound))); - __ b(&sync_isolate_group_bound_callback, EQ); - - __ tst(R4, Operand(FfiCallbackMetadata::kSyncCallbackIsolateOwnershipFlag)); - __ b(&sync_callback_isolate_ownership, NE); - - // Sync callback. The entry point contains the target function, so just call - // it. DLRT_GetThreadForNativeCallbackTrampoline exited the safepoint, so - // re-enter it afterwards. - - // On entry to the function, there will be four extra slots on the stack: - // saved THR, R4, R5 and the return address. The target will know to skip - // them. - __ blx(R5); - - // Clobbers R4, R5 and TMP, all saved or volatile. - __ EnterFullSafepoint(R4, R5); - - __ b(&done); - - __ Bind(&sync_callback_isolate_ownership); - - __ blx(R5); - - // Exit the target isolate. { - __ EnterFrame(1 << FP, 0); - __ ReserveAlignedFrameSpace(0); - - const RegisterSet return_registers( - (1 << CallingConventions::kReturnReg) | - (1 << CallingConventions::kSecondReturnReg), - 1 << CallingConventions::kReturnFpuReg); + __ blx(TMP); // entry_point __ PushRegisters(return_registers); - - GenerateLoadFfiCallbackMetadataRuntimeFunction( - FfiCallbackMetadata::kExitSyncCallbackTargetIsolate, R4); - - __ blx(R4); - + __ mov(R0, Operand(THR)); + __ blx(R5); // DLRT_ExitSyncCallback, etc + if (FLAG_target_memory_sanitizer) { + __ blx(R0); // dart_msan_unpoison_retval + } __ PopRegisters(return_registers); - __ LeaveFrame(1 << FP); + // Returns. + RESTORES_LR_FROM_FRAME( + __ PopList((1 << PC) | (1 << THR) | (1 << R4) | (1 << R5))); } - __ b(&done); - - __ Bind(&sync_isolate_group_bound_callback); - - __ blx(R5); - - // Exit isolate group bound isolate. { - __ EnterFrame(1 << FP, 0); - __ ReserveAlignedFrameSpace(0); - - const RegisterSet return_registers( - (1 << CallingConventions::kReturnReg) | - (1 << CallingConventions::kSecondReturnReg), - 1 << CallingConventions::kReturnFpuReg); - __ PushRegisters(return_registers); - - GenerateLoadFfiCallbackMetadataRuntimeFunction( - FfiCallbackMetadata::kExitIsolateGroupBoundIsolate, R4); - - __ blx(R4); - - __ PopRegisters(return_registers); - __ LeaveFrame(1 << FP); - } - - __ b(&done); - - __ Bind(&async_callback); - - // Async callback. The entrypoint marshals the arguments into a message and - // sends it over the send port. DLRT_GetThreadForNativeCallbackTrampoline - // entered a temporary isolate, so exit it afterwards. - - // On entry to the function, there will be four extra slots on the stack: - // saved THR, R4, R5 and the return address. The target will know to skip - // them. - __ blx(R5); - - // Exit the temporary isolate. - { - GenerateLoadFfiCallbackMetadataRuntimeFunction( - FfiCallbackMetadata::kExitTemporaryIsolate, R0); - - CLOBBERS_LR(__ PopList((1 << LR) | (1 << THR) | (1 << R4) | (1 << R5))); - + SPILLS_LR_TO_FRAME(); //... + __ Bind(&tail); + __ blx(TMP); // entry_point + __ mov(R0, Operand(THR)); + __ mov(R1, Operand(R5)); + RESTORES_LR_FROM_FRAME( + __ PopList((1 << LR) | (1 << THR) | (1 << R4) | (1 << R5))); // Tail-call DLRT_ExitTemporaryIsolate. It is not safe to return to this // stub, since it might be deleted once DLRT_ExitTemporaryIsolate proceeds // enough for VM shutdown. - __ bx(R0); + __ bx(R1); // DLRT_ExitTemporaryIsolate. __ Breakpoint(); } - __ Bind(&done); - - // Returns. - __ PopList((1 << PC) | (1 << THR) | (1 << R4) | (1 << R5)); - ASSERT_LESS_OR_EQUAL(__ CodeSize() - shared_stub_start, FfiCallbackMetadata::kNativeCallbackSharedStubSize); ASSERT_LESS_OR_EQUAL(__ CodeSize(), FfiCallbackMetadata::kPageSize); diff --git a/runtime/vm/compiler/stub_code_compiler_arm64.cc b/runtime/vm/compiler/stub_code_compiler_arm64.cc index 820496d9ad7..43bf56b8073 100644 --- a/runtime/vm/compiler/stub_code_compiler_arm64.cc +++ b/runtime/vm/compiler/stub_code_compiler_arm64.cc @@ -497,9 +497,10 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { // Save THR (callee-saved) and LR on the real C stack (CSP). Keeps it // aligned. - COMPILE_ASSERT(FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta == 2); + COMPILE_ASSERT(FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta == 4); SPILLS_LR_TO_FRAME(__ stp( THR, LR, Address(CSP, -2 * target::kWordSize, Address::PairPreIndex))); + __ stp(R20, R21, Address(CSP, -2 * target::kWordSize, Address::PairPreIndex)); COMPILE_ASSERT(!IsArgumentRegister(THR)); @@ -519,21 +520,10 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { // This saves too much: we only need the D half of Q registers. __ PushRegisters(all_registers); - __ mov(R0, R9); - - // We also need to look up the entry point for the trampoline. This is - // returned using a pointer passed to the second arg of the C function - // below. We aim that pointer at a reserved stack slot. - __ AddImmediate(SP, SP, -compiler::target::kWordSize); - __ mov(R1, SP); - - // We also need to know if this is a sync or async callback. This is also - // returned by pointer. - __ AddImmediate(SP, SP, -compiler::target::kWordSize); - __ mov(R2, SP); - __ EnterFrame(0); - __ ReserveAlignedFrameSpace(0); + __ ReserveAlignedFrameSpace(3 * target::kWordSize); + __ mov(R0, R9); + __ mov(R1, SP); #if defined(DART_TARGET_OS_FUCHSIA) // TODO(https://dartbug.com/52579): Remove. @@ -542,7 +532,7 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { R9); } else { Label call; - __ ldr(R4, compiler::Address::PC(2 * Instr::kInstrSize)); + __ ldr(R4, Address::PC(2 * Instr::kInstrSize)); __ b(&call); __ Emit64(reinterpret_cast(&DLRT_GetFfiCallbackMetadata)); __ Bind(&call); @@ -553,253 +543,62 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { #endif // defined(DART_TARGET_OS_FUCHSIA) __ mov(CSP, SP); - __ blr(R4); + __ blr(R4); // DLRT_GetFfiCallbackMetadata __ mov(SP, CSP); + __ mov(THR, R0); + COMPILE_ASSERT(!IsCalleeSavedRegister(R10) && !IsArgumentRegister(R10)); + __ ldr(R10, Address(CSP, 0 * target::kWordSize)); // entry_point + __ ldr(R20, Address(CSP, 1 * target::kWordSize)); // is_tail + __ ldr(R21, Address(CSP, 2 * target::kWordSize)); // epilogue __ LeaveFrame(); - - // The trampoline type is at the top of the stack. Pop it into R9. - __ Pop(R9); - - // Entry point is now at the top of the stack. Pop it into R10. - COMPILE_ASSERT(!IsCalleeSavedRegister(R10) && !IsArgumentRegister(R10)); - __ Pop(R10); - __ PopRegisters(all_registers); __ LeaveFrame(); __ RestoreCSP(); } - Label async_callback; - Label sync_isolate_group_bound_callback; - Label sync_callback_isolate_ownership; - Label done; + Label tail; + __ cbnz(&tail, R20); - // Check the trampoline type to see how the callback should be invoked. - __ cmp( - R9, - Operand(static_cast(FfiCallbackMetadata::TrampolineType::kAsync))); - __ b(&async_callback, EQ); - - __ cmp(R9, Operand(static_cast( - FfiCallbackMetadata::TrampolineType::kSyncIsolateGroupBound))); - __ b(&sync_isolate_group_bound_callback, EQ); - - __ tsti(R9, - Immediate(FfiCallbackMetadata::kSyncCallbackIsolateOwnershipFlag)); - __ b(&sync_callback_isolate_ownership, NOT_ZERO); - - // Sync callback. The entry point contains the target function, so just call - // it. DLRT_GetThreadForNativeCallbackTrampoline exited the safepoint, so - // re-enter it afterwards. - - // Clobbers all volatile registers, including the callback ID in R9. - // Resets CSP and SP, important for EnterSafepoint below. - __ blr(R10); - - // Clobbers TMP, TMP2 and R9 -- all volatile and not holding return values. - __ EnterFullSafepoint(/*scratch=*/R9); - - if (FLAG_target_memory_sanitizer) { - __ SetupDartSP(); - __ EnterFrame(0); - __ ReserveAlignedFrameSpace(0); - - const RegisterSet return_registers( - (1 << CallingConventions::kReturnReg) | - (1 << CallingConventions::kSecondReturnReg), - (1 << CallingConventions::kReturnFpuReg) | - (1 << CallingConventions::kSecondReturnFpuReg) | - (1 << CallingConventions::kThirdReturnFpuReg) | - (1 << CallingConventions::kFourthReturnFpuReg)); - // This saves too much: we only need the D half of Q registers. - __ PushRegisters(return_registers); - -#if defined(DART_TARGET_OS_FUCHSIA) - // TODO(https://dartbug.com/52579): Remove. - if (FLAG_precompiled_mode) { - GenerateLoadBSSEntry(BSS::Relocation::DLRT_ExitSyncCallback, R4, R9); - } else { - Label call; - __ ldr(R4, compiler::Address::PC(2 * Instr::kInstrSize)); - __ b(&call); - __ Emit64(reinterpret_cast(&DLRT_ExitSyncCallback)); - __ Bind(&call); - } -#else - GenerateLoadFfiCallbackMetadataRuntimeFunction( - FfiCallbackMetadata::kExitSyncCallback, R4); -#endif - - __ mov(CSP, SP); - __ blr(R4); - __ blr(R0); // dart_msan_unpoison_retval - __ mov(SP, CSP); - __ mov(THR, R0); - - __ PopRegisters(return_registers); - - __ LeaveFrame(); - __ RestoreCSP(); - } - - __ b(&done); - - __ Bind(&sync_callback_isolate_ownership); - - __ blr(R10); - - // Exit the target isolate. { - __ SetupDartSP(); - __ EnterFrame(0); - __ ReserveAlignedFrameSpace(0); - - const RegisterSet return_registers( - (1 << CallingConventions::kReturnReg) | - (1 << CallingConventions::kSecondReturnReg), - (1 << CallingConventions::kReturnFpuReg) | - (1 << CallingConventions::kSecondReturnFpuReg) | - (1 << CallingConventions::kThirdReturnFpuReg) | - (1 << CallingConventions::kFourthReturnFpuReg)); - // This saves too much: we only need the D half of Q registers. - __ PushRegisters(return_registers); - -#if defined(DART_TARGET_OS_FUCHSIA) - // TODO(https://dartbug.com/52579): Remove. - if (FLAG_precompiled_mode) { - GenerateLoadBSSEntry(BSS::Relocation::DLRT_ExitSyncCallbackTargetIsolate, - R4, R9); - } else { - Label call; - __ ldr(R4, compiler::Address::PC(2 * Instr::kInstrSize)); - __ b(&call); - __ Emit64(reinterpret_cast(&DLRT_ExitSyncCallbackTargetIsolate)); - __ Bind(&call); - } -#else - GenerateLoadFfiCallbackMetadataRuntimeFunction( - FfiCallbackMetadata::kExitSyncCallbackTargetIsolate, R4); -#endif - - __ mov(CSP, SP); - __ blr(R4); + __ blr(R10); // entry_point + __ stp(R0, R1, Address(CSP, -2 * target::kWordSize, Address::PairPreIndex)); + __ fstp(V0, V1, Address(CSP, -2 * 8, Address::PairPreIndex), kDWord); + __ fstp(V2, V3, Address(CSP, -2 * 8, Address::PairPreIndex), kDWord); + __ mov(R0, THR); + __ blr(R21); // DLRT_ExitSyncCallback, etc if (FLAG_target_memory_sanitizer) { __ blr(R0); // dart_msan_unpoison_retval } - __ mov(SP, CSP); - __ mov(THR, R0); - - __ PopRegisters(return_registers); - - __ LeaveFrame(); - __ RestoreCSP(); - } - - __ b(&done); - - __ Bind(&sync_isolate_group_bound_callback); - - __ blr(R10); - - // Exit isolate group bound isolate. - { - __ SetupDartSP(); - __ EnterFrame(0); - __ ReserveAlignedFrameSpace(0); - - const RegisterSet return_registers( - (1 << CallingConventions::kReturnReg) | - (1 << CallingConventions::kSecondReturnReg), - (1 << CallingConventions::kReturnFpuReg) | - (1 << CallingConventions::kSecondReturnFpuReg) | - (1 << CallingConventions::kThirdReturnFpuReg) | - (1 << CallingConventions::kFourthReturnFpuReg)); - // This saves too much: we only need the D half of Q registers. - __ PushRegisters(return_registers); - -#if defined(DART_TARGET_OS_FUCHSIA) - // TODO(https://dartbug.com/52579): Remove. - if (FLAG_precompiled_mode) { - GenerateLoadBSSEntry(BSS::Relocation::DLRT_ExitIsolateGroupBoundIsolate, - R4, R9); - } else { - Label call; - __ ldr(R4, compiler::Address::PC(2 * Instr::kInstrSize)); - __ b(&call); - __ Emit64(reinterpret_cast(&DLRT_ExitIsolateGroupBoundIsolate)); - __ Bind(&call); - } -#else - GenerateLoadFfiCallbackMetadataRuntimeFunction( - FfiCallbackMetadata::kExitIsolateGroupBoundIsolate, R4); -#endif - - __ mov(CSP, SP); - __ blr(R4); - if (FLAG_target_memory_sanitizer) { - __ blr(R0); // dart_msan_unpoison_retval - } - __ mov(SP, CSP); - __ mov(THR, R0); - - __ PopRegisters(return_registers); - - __ LeaveFrame(); - __ RestoreCSP(); - } - - __ b(&done); - - __ Bind(&async_callback); - - // Async callback. The entrypoint marshals the arguments into a message and - // sends it over the send port. DLRT_GetThreadForNativeCallbackTrampoline - // entered a temporary isolate, so exit it afterwards. - - // Clobbers all volatile registers, including the callback ID in R9. - // Resets CSP and SP, important for EnterSafepoint below. - __ blr(R10); - - // Exit the temporary isolate. - { -#if defined(DART_TARGET_OS_FUCHSIA) - // TODO(https://dartbug.com/52579): Remove. - if (FLAG_precompiled_mode) { - GenerateLoadBSSEntry(BSS::Relocation::DLRT_ExitTemporaryIsolate, R4, R9); - } else { - Label call; - __ ldr(R4, compiler::Address::PC(2 * Instr::kInstrSize)); - __ b(&call); - __ Emit64(reinterpret_cast(&DLRT_ExitTemporaryIsolate)); - __ Bind(&call); - } -#else - GenerateLoadFfiCallbackMetadataRuntimeFunction( - FfiCallbackMetadata::kExitTemporaryIsolate, R4); -#endif - - // Pop LR and THR from the real stack (CSP). - CLOBBERS_LR(__ ldp( + __ fldp(V2, V3, Address(CSP, 2 * 8, Address::PairPostIndex), kDWord); + __ fldp(V0, V1, Address(CSP, 2 * 8, Address::PairPostIndex), kDWord); + __ ldp(R0, R1, Address(CSP, 2 * target::kWordSize, Address::PairPostIndex)); + __ ldp(R20, R21, + Address(CSP, 2 * target::kWordSize, Address::PairPostIndex)); + RESTORES_LR_FROM_FRAME(__ ldp( THR, LR, Address(CSP, 2 * target::kWordSize, Address::PairPostIndex))); + __ ret(); + } + { + SPILLS_LR_TO_FRAME(); //... + __ Bind(&tail); + __ blr(R10); + __ mov(R0, THR); + __ mov(R1, R21); + __ ldp(R20, R21, + Address(CSP, 2 * target::kWordSize, Address::PairPostIndex)); + RESTORES_LR_FROM_FRAME(__ ldp( + THR, LR, Address(CSP, 2 * target::kWordSize, Address::PairPostIndex))); // Tail-call DLRT_ExitTemporaryIsolate. It is not safe to return to this // stub, since it might be deleted once DLRT_ExitTemporaryIsolate proceeds // enough for VM shutdown. - __ br(R4); + __ br(R1); // DLRT_ExitTemporaryIsolate. __ brk(0); } - __ Bind(&done); - - // Pop LR and THR from the real stack (CSP). - RESTORES_LR_FROM_FRAME(__ ldp( - THR, LR, Address(CSP, 2 * target::kWordSize, Address::PairPostIndex))); - - __ ret(); - ASSERT_LESS_OR_EQUAL(__ CodeSize() - shared_stub_start, FfiCallbackMetadata::kNativeCallbackSharedStubSize); ASSERT_LESS_OR_EQUAL(__ CodeSize(), FfiCallbackMetadata::kPageSize); diff --git a/runtime/vm/compiler/stub_code_compiler_ia32.cc b/runtime/vm/compiler/stub_code_compiler_ia32.cc index 69e4c8b4f1c..61b8ea77164 100644 --- a/runtime/vm/compiler/stub_code_compiler_ia32.cc +++ b/runtime/vm/compiler/stub_code_compiler_ia32.cc @@ -245,6 +245,7 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { // Save THR and EBX which are callee-saved. __ pushl(THR); __ pushl(EBX); + __ pushl(ECX); // THR & return address COMPILE_ASSERT(FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta == 4); @@ -255,20 +256,16 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { // code size on this shared stub. { __ EnterFrame(0); - // entry_point, trampoline_type, &trampoline_type, &entry_point, trampoline - // ^------ GetFfiCallbackMetadata args ------^ __ ReserveAlignedFrameSpace(5 * target::kWordSize); + // SP[4] CallbackMetadata.epilogue + // SP[3] CallbackMetadata.is_tail + // SP[2] CallbackMetadata.entry_point + // SP[1] out -> SP[2] + // SP[0] trampoline - // Trampoline arg. __ movl(Address(SPREG, 0 * target::kWordSize), EAX); - - // Pointer to trampoline type stack slot. __ movl(EAX, SPREG); - __ addl(EAX, Immediate(3 * target::kWordSize)); - __ movl(Address(SPREG, 2 * target::kWordSize), EAX); - - // Pointer to entry point stack slot. - __ addl(EAX, Immediate(target::kWordSize)); + __ addl(EAX, Immediate(2 * target::kWordSize)); __ movl(Address(SPREG, 1 * target::kWordSize), EAX); __ movl(EAX, @@ -276,179 +273,84 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { __ call(EAX); __ movl(THR, EAX); - // Save the trampoline type in EBX, and the entry point in ECX. - __ movl(EBX, Address(SPREG, 3 * target::kWordSize)); - __ movl(ECX, Address(SPREG, 4 * target::kWordSize)); + __ movl(EAX, Address(SPREG, 2 * target::kWordSize)); // entry_point + __ movl(ECX, Address(SPREG, 3 * target::kWordSize)); // is_tail + __ movl(EBX, Address(SPREG, 4 * target::kWordSize)); // epilogue __ LeaveFrame(); - - // Save the trampoline type to the stack, because we'll need it after the - // call to decide whether to ret() or ret(4). - __ pushl(EBX); } - COMPILE_ASSERT(!IsCalleeSavedRegister(ECX) && !IsArgumentRegister(ECX)); - COMPILE_ASSERT(ECX != THR); + Label call, call_ret4, tail; + __ cmpl(ECX, Immediate(0)); + __ j(EQUAL, &call); + __ cmpl(ECX, Immediate(1)); + __ j(EQUAL, &tail); + __ cmpl(ECX, Immediate(2)); + __ j(EQUAL, &call_ret4); + __ int3(); - Label async_callback; - Label sync_isolate_group_bound_callback; - Label sync_callback_isolate_ownership; - Label done; - - // Check the trampoline type to see how the callback should be invoked. - __ cmpl(EBX, Immediate(static_cast( - FfiCallbackMetadata::TrampolineType::kAsync))); - __ j(EQUAL, &async_callback); - - __ cmpl(EBX, - Immediate(static_cast( - FfiCallbackMetadata::TrampolineType::kSyncIsolateGroupBound))); - __ j(EQUAL, &sync_isolate_group_bound_callback, Assembler::kNearJump); - - __ testl(EBX, - Immediate(FfiCallbackMetadata::kSyncCallbackIsolateOwnershipFlag)); - __ j(NOT_ZERO, &sync_callback_isolate_ownership, Assembler::kNearJump); - - // Sync callback. The entry point contains the target function, so just call - // it. DLRT_GetThreadForNativeCallbackTrampoline exited the safepoint, so - // re-enter it afterwards. - - // On entry to the function, there will be two extra slots on the stack: - // the saved THR and the return address. The target will know to skip them. - __ call(ECX); - - // Takes care to not clobber *any* registers (besides scratch). - __ EnterFullSafepoint(/*scratch=*/ECX); - - // Pop the trampoline type into ECX. - __ popl(ECX); - - // Restore callee-saved registers. - __ popl(EBX); - __ popl(THR); - - __ cmpl(ECX, Immediate(static_cast( - FfiCallbackMetadata::TrampolineType::kSync))); - __ j(NOT_EQUAL, &ret_4, Assembler::kNearJump); - __ ret(); - - __ Bind(&ret_4); - __ ret(Immediate(4)); - - __ Bind(&sync_callback_isolate_ownership); - - __ call(ECX); - - // Exit the target isolate. { + __ Bind(&call); + __ call(EAX); // entry_point + __ pushl(CallingConventions::kReturnReg); __ pushl(CallingConventions::kSecondReturnReg); - __ subl(ESP, Immediate(kFpuRegisterSize)); - __ movups(Address(ESP, 0), CallingConventions::kReturnFpuReg); + __ subl(ESP, Immediate(8)); + __ movsd(Address(ESP, 0), CallingConventions::kReturnFpuReg); + // 4 + 4 + 8 = 16 (stack alignment) - __ EnterFrame(0); - __ ReserveAlignedFrameSpace(0); + __ pushl(THR); + __ call(EBX); // DLRT_ExitSyncCallback, etc + __ popl(EAX); - __ movl(EAX, Immediate(reinterpret_cast( - DLRT_ExitSyncCallbackTargetIsolate))); - __ CallCFunction(EAX); - - __ LeaveFrame(); - - __ movups(Address(ESP, 0), CallingConventions::kReturnFpuReg); - __ addl(ESP, Immediate(kFpuRegisterSize)); + __ movsd(CallingConventions::kReturnFpuReg, Address(ESP, 0)); + __ addl(ESP, Immediate(8)); __ popl(CallingConventions::kSecondReturnReg); __ popl(CallingConventions::kReturnReg); - // Pop the trampoline type into ECX. __ popl(ECX); - - // Restore callee-saved registers. __ popl(EBX); __ popl(THR); - - Label ownership_ret_4; - __ cmpl(ECX, - Immediate( - static_cast(FfiCallbackMetadata::TrampolineType::kSync) | - FfiCallbackMetadata::kSyncCallbackIsolateOwnershipFlag)); - __ j(NOT_EQUAL, &ownership_ret_4, Assembler::kNearJump); __ ret(); + } + { + __ Bind(&call_ret4); + __ call(EAX); // entry_point - __ Bind(&ownership_ret_4); + __ pushl(CallingConventions::kReturnReg); + __ pushl(CallingConventions::kSecondReturnReg); + __ subl(ESP, Immediate(8)); + __ movsd(Address(ESP, 0), CallingConventions::kReturnFpuReg); + // 4 + 4 + 8 = 16 (stack alignment) + + __ pushl(THR); + __ call(EBX); // DLRT_ExitSyncCallback, etc + __ popl(EAX); + + __ movsd(CallingConventions::kReturnFpuReg, Address(ESP, 0)); + __ addl(ESP, Immediate(8)); + __ popl(CallingConventions::kSecondReturnReg); + __ popl(CallingConventions::kReturnReg); + + __ popl(ECX); + __ popl(EBX); + __ popl(THR); __ ret(Immediate(4)); } - - __ jmp(&done, Assembler::kNearJump); - - __ Bind(&sync_isolate_group_bound_callback); - - __ call(ECX); - - // Exit isolate group bound isolate. { - __ pushl(CallingConventions::kReturnReg); - __ pushl(CallingConventions::kSecondReturnReg); - __ subl(ESP, Immediate(kFpuRegisterSize)); - __ movups(Address(ESP, 0), CallingConventions::kReturnFpuReg); - - __ EnterFrame(0); - __ ReserveAlignedFrameSpace(0); - - __ movl(EAX, Immediate(reinterpret_cast( - DLRT_ExitIsolateGroupBoundIsolate))); - __ CallCFunction(EAX); - - __ LeaveFrame(); - - __ movups(Address(ESP, 0), CallingConventions::kReturnFpuReg); - __ addl(ESP, Immediate(kFpuRegisterSize)); - __ popl(CallingConventions::kSecondReturnReg); - __ popl(CallingConventions::kReturnReg); - } - - __ jmp(&done, Assembler::kNearJump); - - __ Bind(&async_callback); - - // Async callback. The entrypoint marshals the arguments into a message and - // sends it over the send port. DLRT_GetThreadForNativeCallbackTrampoline - // entered a temporary isolate, so exit it afterwards. - - // On entry to the function, there will be two extra slots on the stack: - // the saved THR and the return address. The target will know to skip them. - __ call(ECX); - - // Exit the temporary isolate. - { - // Pop the trampoline type into ECX. + __ Bind(&tail); + __ call(EAX); // entry_point + __ movl(EAX, EBX); __ popl(ECX); - - // Restore callee-saved registers. __ popl(EBX); __ popl(THR); - // Tail-call DLRT_ExitTemporaryIsolate. It is not safe to return to this // stub, since it might be deleted once DLRT_ExitTemporaryIsolate proceeds // enough for VM shutdown. - __ movl(EAX, - Immediate(reinterpret_cast(DLRT_ExitTemporaryIsolate))); - __ jmp(EAX); + __ jmp(EAX); // DLRT_ExitTemporaryIsolate __ int3(); } - __ Bind(&done); - - // Pop the trampoline type into ECX. - __ popl(ECX); - - // Restore callee-saved registers. - __ popl(EBX); - __ popl(THR); - - // Stack delta is always 0 for async callbacks. - __ ret(); - // 'kNativeCallbackSharedStubSize' is an upper bound because the exact // instruction size can vary slightly based on OS calling conventions. ASSERT_LESS_OR_EQUAL(__ CodeSize() - shared_stub_start, diff --git a/runtime/vm/compiler/stub_code_compiler_riscv.cc b/runtime/vm/compiler/stub_code_compiler_riscv.cc index a58cb2a8b7f..f0ac333ae50 100644 --- a/runtime/vm/compiler/stub_code_compiler_riscv.cc +++ b/runtime/vm/compiler/stub_code_compiler_riscv.cc @@ -351,8 +351,9 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { __ Bind(&body); // Save THR (callee-saved) and RA. Keeps stack aligned. - COMPILE_ASSERT(FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta == 2); + COMPILE_ASSERT(FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta == 4); __ PushRegisterPair(RA, THR); + __ PushRegisterPair(S2, S3); COMPILE_ASSERT(!IsArgumentRegister(THR)); // Load the thread, verify the callback ID and exit the safepoint. @@ -361,29 +362,26 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { // code size on this shared stub. { // Push arguments and callback id. - __ subi(SP, SP, 9 * target::kWordSize); - __ sx(T1, Address(SP, 8 * target::kWordSize)); - __ sx(A7, Address(SP, 7 * target::kWordSize)); - __ sx(A6, Address(SP, 6 * target::kWordSize)); - __ sx(A5, Address(SP, 5 * target::kWordSize)); - __ sx(A4, Address(SP, 4 * target::kWordSize)); - __ sx(A3, Address(SP, 3 * target::kWordSize)); - __ sx(A2, Address(SP, 2 * target::kWordSize)); - __ sx(A1, Address(SP, 1 * target::kWordSize)); - __ sx(A0, Address(SP, 0 * target::kWordSize)); + __ subi(SP, SP, 12 * target::kWordSize); + __ sx(A7, Address(SP, 11 * target::kWordSize)); + __ sx(A6, Address(SP, 10 * target::kWordSize)); + __ sx(A5, Address(SP, 9 * target::kWordSize)); + __ sx(A4, Address(SP, 8 * target::kWordSize)); + __ sx(A3, Address(SP, 7 * target::kWordSize)); + __ sx(A2, Address(SP, 6 * target::kWordSize)); + __ sx(A1, Address(SP, 5 * target::kWordSize)); + __ sx(A0, Address(SP, 4 * target::kWordSize)); + // 3 - alignment gap + // 2 - Out.epilogue + // 1 - Out.is_tail + // 0 - Out.entry_point - __ EnterFrame(0); - // Reserve one slot for the entry point and one for the tramp abi. - __ ReserveAlignedFrameSpace(2 * target::kWordSize); + __ mv(A0, T1); + __ mv(A1, SP); // Since DLRT_GetFfiCallbackMetadata can theoretically be loaded anywhere, // we use the same trick as before to ensure a predictable instruction // sequence. - Label call; - __ mv(A0, T1); // trampoline - __ mv(A1, SPREG); // out_entry_point - __ addi(A2, SPREG, target::kWordSize); // out_trampoline_type - #if defined(DART_TARGET_OS_FUCHSIA) // TODO(https://dartbug.com/52579): Remove. if (FLAG_precompiled_mode) { @@ -394,6 +392,7 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { intptr_t start = __ CodeSize(); __ auipc(T1, 0); __ lx(T1, Address(T1, kPCRelativeLoadOffset)); + Label call; __ j(&call); ASSERT_EQUAL(__ CodeSize() - start, kPCRelativeLoadOffset); @@ -402,268 +401,69 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { #else __ Emit64(reinterpret_cast(&DLRT_GetFfiCallbackMetadata)); #endif + __ Bind(&call); } #else GenerateLoadFfiCallbackMetadataRuntimeFunction( FfiCallbackMetadata::kGetFfiCallbackMetadata, T1); #endif // defined(DART_TARGET_OS_FUCHSIA) - __ Bind(&call); __ jalr(T1); __ mv(THR, A0); - __ lx(T2, Address(SPREG, 0)); // entry_point - __ lx(T3, Address(SPREG, target::kWordSize)); // trampoline_type + __ lx(T2, Address(SP, 0 * target::kWordSize)); // entry_point + __ lx(S2, Address(SP, 1 * target::kWordSize)); // is_tail + __ lx(S3, Address(SP, 2 * target::kWordSize)); // epilogue - __ LeaveFrame(); - - // Restore arguments and callback id. - __ lx(A0, Address(SP, 0 * target::kWordSize)); - __ lx(A1, Address(SP, 1 * target::kWordSize)); - __ lx(A2, Address(SP, 2 * target::kWordSize)); - __ lx(A3, Address(SP, 3 * target::kWordSize)); - __ lx(A4, Address(SP, 4 * target::kWordSize)); - __ lx(A5, Address(SP, 5 * target::kWordSize)); - __ lx(A6, Address(SP, 6 * target::kWordSize)); - __ lx(A7, Address(SP, 7 * target::kWordSize)); - __ lx(T1, Address(SP, 8 * target::kWordSize)); - __ addi(SP, SP, 9 * target::kWordSize); + __ lx(A0, Address(SP, 4 * target::kWordSize)); + __ lx(A1, Address(SP, 5 * target::kWordSize)); + __ lx(A2, Address(SP, 6 * target::kWordSize)); + __ lx(A3, Address(SP, 7 * target::kWordSize)); + __ lx(A4, Address(SP, 8 * target::kWordSize)); + __ lx(A5, Address(SP, 9 * target::kWordSize)); + __ lx(A6, Address(SP, 10 * target::kWordSize)); + __ lx(A7, Address(SP, 11 * target::kWordSize)); + __ addi(SP, SP, 12 * target::kWordSize); } - COMPILE_ASSERT(!IsCalleeSavedRegister(T2) && !IsArgumentRegister(T2)); - COMPILE_ASSERT(!IsCalleeSavedRegister(T3) && !IsArgumentRegister(T3)); + Label tail; + __ bnez(S2, &tail, Assembler::kNearJump); - Label something_other_than_sync_callback; - Label async_callback; - Label sync_isolate_group_bound_callback; - Label done; - - // Check the trampoline type to see how the callback should be invoked. - - COMPILE_ASSERT( - static_cast(FfiCallbackMetadata::TrampolineType::kSync) == 0); - __ bnez(T3, &something_other_than_sync_callback); - - // Sync callback. The entry point contains the target function, so just call - // it. DLRT_GetThreadForNativeCallbackTrampoline exited the safepoint, so - // re-enter it afterwards. - - // Clobbers all volatile registers, including the callback ID in T1. - __ jalr(T2); - - // Clobbers TMP, TMP2 and T1 -- all volatile and not holding return values. - __ EnterFullSafepoint(/*scratch=*/T1); - - if (FLAG_target_memory_sanitizer) { - __ EnterFrame(0); - __ ReserveAlignedFrameSpace(0); - - const RegisterSet return_registers( - (1 << CallingConventions::kReturnReg) | - (1 << CallingConventions::kSecondReturnReg), - (1 << CallingConventions::kReturnFpuReg) | - (1 << CallingConventions::kSecondReturnFpuReg)); - __ PushRegisters(return_registers); - - Label call; - -#if defined(DART_TARGET_OS_FUCHSIA) - // TODO(https://dartbug.com/52579): Remove. - if (FLAG_precompiled_mode) { - GenerateLoadBSSEntry(BSS::Relocation::DRT_ExitSyncCallback, T1, T2); - } else { - const intptr_t kPCRelativeLoadOffset = 12; - intptr_t start = __ CodeSize(); - __ auipc(T1, 0); - __ lx(T1, Address(T1, kPCRelativeLoadOffset)); - __ j(&call); - - ASSERT_EQUAL(__ CodeSize() - start, kPCRelativeLoadOffset); -#if XLEN == 32 - __ Emit32(reinterpret_cast(&DLRT_ExitSyncCallback)); -#else - __ Emit64(reinterpret_cast(&DLRT_ExitSyncCallback)); -#endif - } -#else - GenerateLoadFfiCallbackMetadataRuntimeFunction( - FfiCallbackMetadata::kExitSyncCallbackTargetIsolate, T1); -#endif // defined(DART_TARGET_OS_FUCHSIA) - - __ Bind(&call); - __ jalr(T1); - __ jalr(A0); // dart_msan_unpoison_retval - - __ PopRegisters(return_registers); - - __ LeaveFrame(); - __ j(&done, Assembler::kNearJump); - } - - __ j(&done); - - __ Bind(&something_other_than_sync_callback); - __ li(T4, static_cast(FfiCallbackMetadata::TrampolineType::kAsync)); - __ beq(T3, T4, &async_callback, Assembler::kNearJump); - - __ li(T4, static_cast( - FfiCallbackMetadata::TrampolineType::kSyncIsolateGroupBound)); - __ beq(T3, T4, &sync_isolate_group_bound_callback, Assembler::kNearJump); - - // Sync callback that entered the target isolate. - __ jalr(T2); - - // Exit the target isolate. { - __ EnterFrame(0); - __ ReserveAlignedFrameSpace(0); - - const RegisterSet return_registers( - (1 << CallingConventions::kReturnReg) | - (1 << CallingConventions::kSecondReturnReg), - (1 << CallingConventions::kReturnFpuReg) | - (1 << CallingConventions::kSecondReturnFpuReg)); - __ PushRegisters(return_registers); - - Label call; - -#if defined(DART_TARGET_OS_FUCHSIA) - // TODO(https://dartbug.com/52579): Remove. - if (FLAG_precompiled_mode) { - GenerateLoadBSSEntry(BSS::Relocation::DRT_ExitSyncCallbackTargetIsolate, - T1, T2); - } else { - const intptr_t kPCRelativeLoadOffset = 12; - intptr_t start = __ CodeSize(); - __ auipc(T1, 0); - __ lx(T1, Address(T1, kPCRelativeLoadOffset)); - __ j(&call); - - ASSERT_EQUAL(__ CodeSize() - start, kPCRelativeLoadOffset); -#if XLEN == 32 - __ Emit32(reinterpret_cast(&DLRT_ExitSyncCallbackTargetIsolate)); -#else - __ Emit64(reinterpret_cast(&DLRT_ExitSyncCallbackTargetIsolate)); -#endif - } -#else - GenerateLoadFfiCallbackMetadataRuntimeFunction( - FfiCallbackMetadata::kExitSyncCallbackTargetIsolate, T1); -#endif // defined(DART_TARGET_OS_FUCHSIA) - - __ Bind(&call); - __ jalr(T1); + __ jalr(T2); // entry_point + __ subi(SP, SP, 32); // Not using word size to keep RV32 stack aligned. + __ sx(A0, Address(SP, 24)); + __ sx(A1, Address(SP, 16)); + __ fsd(FA0, Address(SP, 8)); + __ fsd(FA1, Address(SP, 0)); + __ mv(A0, THR); + __ jalr(S3); // DLRT_ExitSyncCallback, etc if (FLAG_target_memory_sanitizer) { __ jalr(A0); // dart_msan_unpoison_retval } - - __ PopRegisters(return_registers); - - __ LeaveFrame(); - __ j(&done, Assembler::kNearJump); - } - - __ Bind(&sync_isolate_group_bound_callback); - __ jalr(T2); - - // Exit isolate group bound isolate. - { - __ EnterFrame(0); - __ ReserveAlignedFrameSpace(0); - - const RegisterSet return_registers( - (1 << CallingConventions::kReturnReg) | - (1 << CallingConventions::kSecondReturnReg), - (1 << CallingConventions::kReturnFpuReg) | - (1 << CallingConventions::kSecondReturnFpuReg)); - __ PushRegisters(return_registers); - - Label call; - -#if defined(DART_TARGET_OS_FUCHSIA) - // TODO(https://dartbug.com/52579): Remove. - if (FLAG_precompiled_mode) { - GenerateLoadBSSEntry(BSS::Relocation::DLRT_ExitIsolateGroupBoundIsolate, - T1, T2); - } else { - const intptr_t kPCRelativeLoadOffset = 12; - intptr_t start = __ CodeSize(); - __ auipc(T1, 0); - __ lx(T1, Address(T1, kPCRelativeLoadOffset)); - __ j(&call); - - ASSERT_EQUAL(__ CodeSize() - start, kPCRelativeLoadOffset); -#if XLEN == 32 - __ Emit32(reinterpret_cast(&DLRT_ExitIsolateGroupBoundIsolate)); -#else - __ Emit64(reinterpret_cast(&DLRT_ExitIsolateGroupBoundIsolate)); -#endif - } -#else - GenerateLoadFfiCallbackMetadataRuntimeFunction( - FfiCallbackMetadata::kExitIsolateGroupBoundIsolate, T1); -#endif // defined(DART_TARGET_OS_FUCHSIA) - - __ Bind(&call); - __ jalr(T1); - if (FLAG_target_memory_sanitizer) { - __ jalr(A0); // dart_msan_unpoison_retval - } - - __ PopRegisters(return_registers); - - __ LeaveFrame(); - __ j(&done, Assembler::kNearJump); - } - - __ Bind(&async_callback); - - // Async callback. The entrypoint marshals the arguments into a message and - // sends it over the send port. DLRT_GetThreadForNativeCallbackTrampoline - // entered a temporary isolate, so exit it afterwards. - - // Clobbers all volatile registers, including the callback ID in T1. - __ jalr(T2); - - // Exit the temporary isolate. - { -#if defined(DART_TARGET_OS_FUCHSIA) - // TODO(https://dartbug.com/52579): Remove. - if (FLAG_precompiled_mode) { - GenerateLoadBSSEntry(BSS::Relocation::DLRT_ExitTemporaryIsolate, T1, T2); - } else { - Label call; - const intptr_t kPCRelativeLoadOffset = 12; - intptr_t start = __ CodeSize(); - __ auipc(T1, 0); - __ lx(T1, Address(T1, kPCRelativeLoadOffset)); - __ j(&call); - - ASSERT_EQUAL(__ CodeSize() - start, kPCRelativeLoadOffset); -#if XLEN == 32 - __ Emit32(reinterpret_cast(&DLRT_ExitTemporaryIsolate)); -#else - __ Emit64(reinterpret_cast(&DLRT_ExitTemporaryIsolate)); -#endif - __ Bind(&call); - } -#else - GenerateLoadFfiCallbackMetadataRuntimeFunction( - FfiCallbackMetadata::kExitTemporaryIsolate, T1); -#endif // defined(DART_TARGET_OS_FUCHSIA) - + __ fld(FA1, Address(SP, 0)); + __ fld(FA0, Address(SP, 8)); + __ lx(A1, Address(SP, 16)); + __ lx(A0, Address(SP, 24)); + __ addi(SP, SP, 32); + __ PopRegisterPair(S2, S3); __ PopRegisterPair(RA, THR); + __ ret(); + } + { + __ Bind(&tail); + __ jalr(T2); // entry_point + __ mv(A0, THR); + __ mv(A1, S3); + __ PopRegisterPair(S2, S3); + __ PopRegisterPair(RA, THR); // Tail-call DLRT_ExitTemporaryIsolate. It is not safe to return to this // stub, since it might be deleted once DLRT_ExitTemporaryIsolate proceeds // enough for VM shutdown. - __ jr(T1); + __ jr(A1); // DLRT_ExitTemporaryIsolate. __ ebreak(); } - __ Bind(&done); - __ PopRegisterPair(RA, THR); - __ ret(); - ASSERT_LESS_OR_EQUAL(__ CodeSize() - shared_stub_start, FfiCallbackMetadata::kNativeCallbackSharedStubSize); ASSERT_LESS_OR_EQUAL(__ CodeSize(), FfiCallbackMetadata::kPageSize); diff --git a/runtime/vm/compiler/stub_code_compiler_x64.cc b/runtime/vm/compiler/stub_code_compiler_x64.cc index 854022cac60..3efe9700302 100644 --- a/runtime/vm/compiler/stub_code_compiler_x64.cc +++ b/runtime/vm/compiler/stub_code_compiler_x64.cc @@ -537,9 +537,11 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { // Save THR which is callee-saved. __ pushq(THR); + __ pushq(R12); + __ pushq(R13); - // 2 = THR & return address - COMPILE_ASSERT(2 == FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta); + // 4 = return address, THR, R12, R13 + COMPILE_ASSERT(4 == FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta); // Save all registers which might hold arguments. __ PushRegisters(kArgumentRegisterSet); @@ -552,19 +554,6 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { COMPILE_ASSERT(RAX != CallingConventions::kArg1Reg); __ movq(CallingConventions::kArg1Reg, RAX); - // We also need to look up the entry point for the trampoline. This is - // returned using a pointer passed to the second arg of the C function - // below. We aim that pointer at a reserved stack slot. - COMPILE_ASSERT(RAX != CallingConventions::kArg2Reg); - __ pushq(Immediate(0)); // Reserve a stack slot for the entry point. - __ movq(CallingConventions::kArg2Reg, RSP); - - // We also need to know if this is a sync or async callback. This is also - // returned by pointer. - COMPILE_ASSERT(RAX != CallingConventions::kArg3Reg); - __ pushq(Immediate(0)); // Reserve a stack slot for the trampoline type. - __ movq(CallingConventions::kArg3Reg, RSP); - #if defined(DART_TARGET_OS_FUCHSIA) // TODO(https://dartbug.com/52579): Remove. if (FLAG_precompiled_mode) { @@ -580,223 +569,62 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { #endif // defined(DART_TARGET_OS_FUCHSIA) __ EnterFrame(0); - __ ReserveAlignedFrameSpace(0); + __ ReserveAlignedFrameSpace(3 * target::kWordSize); + __ movq(CallingConventions::kArg2Reg, RSP); - __ CallCFunction(RAX); + __ CallCFunction(RAX, /*restore_rsp=*/true); __ movq(THR, RAX); + __ movq(RAX, Address(RSP, 0 * target::kWordSize)); // entry_point + __ movq(R12, Address(RSP, 1 * target::kWordSize)); // is_tail + __ movq(R13, Address(RSP, 2 * target::kWordSize)); // epilogue + __ LeaveFrame(); - - // The trampoline type is at the top of the stack. Pop it into RAX. - __ popq(RAX); - - // Entry point is now at the top of the stack. Pop it into TMP. - __ popq(TMP); } // Restore the arguments. __ PopRegisters(kArgumentRegisterSet); - // Current state: - // - // Stack: - // - // - // - // - // Registers: Like entry, except TMP == target, RAX == abi, and THR == thread - // All argument registers are untouched. + Label tail; + __ cmpq(R12, Immediate(0)); + __ j(NOT_ZERO, &tail); - Label async_callback; - Label sync_isolate_group_bound_callback; - Label sync_callback_isolate_ownership; - Label done; + const RegisterSet return_registers( + (1 << CallingConventions::kReturnReg) | + (1 << CallingConventions::kSecondReturnReg), + (1 << CallingConventions::kReturnFpuReg) | + (1 << CallingConventions::kSecondReturnFpuReg)); - // Check the trampoline type to see how the callback should be invoked. - __ cmpq(RAX, Immediate(static_cast( - FfiCallbackMetadata::TrampolineType::kAsync))); - __ j(EQUAL, &async_callback); - - __ cmpq(RAX, - Immediate(static_cast( - FfiCallbackMetadata::TrampolineType::kSyncIsolateGroupBound))); - __ j(EQUAL, &sync_isolate_group_bound_callback); - - __ testq(RAX, - Immediate(FfiCallbackMetadata::kSyncCallbackIsolateOwnershipFlag)); - __ j(NOT_ZERO, &sync_callback_isolate_ownership); - - // Sync callback. The entry point contains the target function, so just call - // it. DLRT_GetThreadForNativeCallbackTrampoline exited the safepoint, so - // re-enter it afterwards. - - // On entry to the function, there will be two extra slots on the stack: - // the saved THR and the return address. The target will know to skip them. - __ call(TMP); - - // Takes care to not clobber *any* registers (besides TMP). - __ EnterFullSafepoint(); - - if (FLAG_target_memory_sanitizer) { - const RegisterSet return_registers( - (1 << CallingConventions::kReturnReg) | - (1 << CallingConventions::kSecondReturnReg), - (1 << CallingConventions::kReturnFpuReg) | - (1 << CallingConventions::kSecondReturnFpuReg)); - __ PushRegisters(return_registers); - -#if defined(DART_TARGET_OS_FUCHSIA) - // TODO(https://dartbug.com/52579): Remove. - if (FLAG_precompiled_mode) { - GenerateLoadBSSEntry(BSS::Relocation::DLRT_ExitSyncCallback, RAX, TMP); - } else { - __ movq(RAX, Immediate(reinterpret_cast(DLRT_ExitSyncCallback))); - } -#else - GenerateLoadFfiCallbackMetadataRuntimeFunction( - FfiCallbackMetadata::kExitSyncCallback, RAX); -#endif // defined(DART_TARGET_OS_FUCHSIA) - - __ EnterFrame(0); - __ ReserveAlignedFrameSpace(0); - - __ CallCFunction(RAX); - __ CallCFunction(RAX); // dart_msan_unpoison_retval - - __ LeaveFrame(); - - __ PopRegisters(return_registers); - } - - __ jmp(&done); - - __ Bind(&sync_callback_isolate_ownership); - - __ call(TMP); - - // Exit the target isolate. { - const RegisterSet return_registers( - (1 << CallingConventions::kReturnReg) | - (1 << CallingConventions::kSecondReturnReg), - (1 << CallingConventions::kReturnFpuReg) | - (1 << CallingConventions::kSecondReturnFpuReg)); + __ call(RAX); // entry_point __ PushRegisters(return_registers); - -#if defined(DART_TARGET_OS_FUCHSIA) - // TODO(https://dartbug.com/52579): Remove. - if (FLAG_precompiled_mode) { - GenerateLoadBSSEntry(BSS::Relocation::DLRT_ExitSyncCallbackTargetIsolate, - RAX, TMP); - } else { - __ movq(RAX, Immediate(reinterpret_cast( - DLRT_ExitSyncCallbackTargetIsolate))); - } -#else - GenerateLoadFfiCallbackMetadataRuntimeFunction( - FfiCallbackMetadata::kExitSyncCallbackTargetIsolate, RAX); -#endif // defined(DART_TARGET_OS_FUCHSIA) - - __ EnterFrame(0); - __ ReserveAlignedFrameSpace(0); - - __ CallCFunction(RAX); + __ movq(CallingConventions::kArg1Reg, THR); + __ CallCFunction(R13, /*restore_rsp=*/true); // DLRT_ExitSyncCallback, etc if (FLAG_target_memory_sanitizer) { - __ CallCFunction(RAX); // dart_msan_unpoison_retval + __ CallCFunction(RAX, /*restore_rsp=*/true); // dart_msan_unpoison_retval } - - __ LeaveFrame(); - __ PopRegisters(return_registers); - } - - __ jmp(&done, Assembler::kNearJump); - - __ Bind(&sync_isolate_group_bound_callback); - - __ call(TMP); - - // Exit isolate group bound isolate. - { - const RegisterSet return_registers( - (1 << CallingConventions::kReturnReg) | - (1 << CallingConventions::kSecondReturnReg), - (1 << CallingConventions::kReturnFpuReg) | - (1 << CallingConventions::kSecondReturnFpuReg)); - __ PushRegisters(return_registers); - -#if defined(DART_TARGET_OS_FUCHSIA) - // TODO(https://dartbug.com/52579): Remove. - if (FLAG_precompiled_mode) { - GenerateLoadBSSEntry(BSS::Relocation::DLRT_ExitIsolateGroupBoundIsolate, - RAX, TMP); - } else { - __ movq(RAX, Immediate(reinterpret_cast( - DLRT_ExitIsolateGroupBoundIsolate))); - } -#else - GenerateLoadFfiCallbackMetadataRuntimeFunction( - FfiCallbackMetadata::kExitIsolateGroupBoundIsolate, RAX); -#endif // defined(DART_TARGET_OS_FUCHSIA) - - __ EnterFrame(0); - __ ReserveAlignedFrameSpace(0); - - __ CallCFunction(RAX); - if (FLAG_target_memory_sanitizer) { - __ CallCFunction(RAX); // dart_msan_unpoison_retval - } - - __ LeaveFrame(); - - __ PopRegisters(return_registers); - } - - __ jmp(&done, Assembler::kNearJump); - - __ Bind(&async_callback); - - // Async callback. The entrypoint marshals the arguments into a message and - // sends it over the send port. DLRT_GetThreadForNativeCallbackTrampoline - // entered a temporary isolate, so exit it afterwards. - - // On entry to the function, there will be two extra slots on the stack: - // the saved THR and the return address. The target will know to skip them. - __ call(TMP); - - // Exit the temporary isolate. - { -#if defined(DART_TARGET_OS_FUCHSIA) - // TODO(https://dartbug.com/52579): Remove. - if (FLAG_precompiled_mode) { - GenerateLoadBSSEntry(BSS::Relocation::DLRT_ExitTemporaryIsolate, RAX, - TMP); - } else { - __ movq(RAX, - Immediate(reinterpret_cast(DLRT_ExitTemporaryIsolate))); - } -#else - GenerateLoadFfiCallbackMetadataRuntimeFunction( - FfiCallbackMetadata::kExitTemporaryIsolate, RAX); -#endif // defined(DART_TARGET_OS_FUCHSIA) - - // Restore THR (callee-saved). + __ popq(R13); + __ popq(R12); __ popq(THR); + __ ret(); + } + { + __ Bind(&tail); + __ call(RAX); // entry_point + __ movq(CallingConventions::kArg1Reg, THR); + __ movq(RAX, R13); + __ popq(R13); + __ popq(R12); + __ popq(THR); // Tail-call DLRT_ExitTemporaryIsolate. It is not safe to return to this // stub, since it might be deleted once DLRT_ExitTemporaryIsolate proceeds // enough for VM shutdown. - __ jmp(RAX); + __ jmp(RAX); // DLRT_ExitTemporaryIsolate __ int3(); } - __ Bind(&done); - - // Restore THR (callee-saved). - __ popq(THR); - - __ ret(); - // 'kNativeCallbackSharedStubSize' is an upper bound because the exact // instruction size can vary slightly based on OS calling conventions. ASSERT_LESS_OR_EQUAL(__ CodeSize() - shared_stub_start, diff --git a/runtime/vm/ffi_callback_metadata.cc b/runtime/vm/ffi_callback_metadata.cc index d9e4269bfa7..a7eee874eea 100644 --- a/runtime/vm/ffi_callback_metadata.cc +++ b/runtime/vm/ffi_callback_metadata.cc @@ -230,16 +230,6 @@ void FfiCallbackMetadata::EnsureFreeListNotEmptyLocked() { // Fill in the runtime functions. FillRuntimeFunction(new_page, kGetFfiCallbackMetadata, reinterpret_cast(DLRT_GetFfiCallbackMetadata)); - FillRuntimeFunction(new_page, kExitTemporaryIsolate, - reinterpret_cast(DLRT_ExitTemporaryIsolate)); - FillRuntimeFunction( - new_page, kExitIsolateGroupBoundIsolate, - reinterpret_cast(DLRT_ExitIsolateGroupBoundIsolate)); - FillRuntimeFunction( - new_page, kExitSyncCallbackTargetIsolate, - reinterpret_cast(DLRT_ExitSyncCallbackTargetIsolate)); - FillRuntimeFunction(new_page, kExitSyncCallback, - reinterpret_cast(DLRT_ExitSyncCallback)); // Add all the trampolines to the free list. const intptr_t trampolines_per_page = NumCallbackTrampolinesPerPage(); diff --git a/runtime/vm/ffi_callback_metadata.h b/runtime/vm/ffi_callback_metadata.h index a83c6c7e12e..c6bc417fa71 100644 --- a/runtime/vm/ffi_callback_metadata.h +++ b/runtime/vm/ffi_callback_metadata.h @@ -329,28 +329,28 @@ class FfiCallbackMetadata { #if defined(TARGET_ARCH_X64) static constexpr intptr_t kNativeCallbackTrampolineSize = 12; - static constexpr intptr_t kNativeCallbackSharedStubSize = 487; - static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 2; + static constexpr intptr_t kNativeCallbackSharedStubSize = 257; + static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 4; #elif defined(TARGET_ARCH_IA32) static constexpr intptr_t kNativeCallbackTrampolineSize = 10; - static constexpr intptr_t kNativeCallbackSharedStubSize = 241; + static constexpr intptr_t kNativeCallbackSharedStubSize = 160; static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 4; #elif defined(TARGET_ARCH_ARM) static constexpr intptr_t kNativeCallbackTrampolineSize = 8; - static constexpr intptr_t kNativeCallbackSharedStubSize = 400; + static constexpr intptr_t kNativeCallbackSharedStubSize = 168; static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 4; #elif defined(TARGET_ARCH_ARM64) static constexpr intptr_t kNativeCallbackTrampolineSize = 8; - static constexpr intptr_t kNativeCallbackSharedStubSize = 596; - static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 2; + static constexpr intptr_t kNativeCallbackSharedStubSize = 268; + static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 4; #elif defined(TARGET_ARCH_RISCV32) static constexpr intptr_t kNativeCallbackTrampolineSize = 8; - static constexpr intptr_t kNativeCallbackSharedStubSize = 368; - static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 2; + static constexpr intptr_t kNativeCallbackSharedStubSize = 154; + static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 4; #elif defined(TARGET_ARCH_RISCV64) static constexpr intptr_t kNativeCallbackTrampolineSize = 8; - static constexpr intptr_t kNativeCallbackSharedStubSize = 368; - static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 2; + static constexpr intptr_t kNativeCallbackSharedStubSize = 154; + static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 4; #else #error What architecture? #endif diff --git a/runtime/vm/runtime_entry.cc b/runtime/vm/runtime_entry.cc index 303c0581085..b10d001dbdf 100644 --- a/runtime/vm/runtime_entry.cc +++ b/runtime/vm/runtime_entry.cc @@ -4986,9 +4986,7 @@ DEFINE_LEAF_RUNTIME_ENTRY(ExitSafepoint, DLRT_ExitSafepoint); namespace { -Thread* HandleAsyncFfiCallback(FfiCallbackMetadata::Metadata metadata, - uword* out_entry_point, - uword* out_trampoline_type) { +Thread* HandleAsyncFfiCallback(FfiCallbackMetadata::Metadata metadata) { // NOTE: This is only thread safe if the user is using the API correctly. // Otherwise, the callback could have been deleted and replaced, in which case // IsLive would still be true. Or it could have been deleted after we looked @@ -4998,8 +4996,6 @@ Thread* HandleAsyncFfiCallback(FfiCallbackMetadata::Metadata metadata, // after free errors. Trying to lock FfiCallbackMetadata::lock_, or any // similar lock, leads to deadlocks. - *out_trampoline_type = static_cast(metadata.trampoline_type()); - *out_entry_point = metadata.target_entry_point(); Isolate* target_isolate = metadata.target_isolate(); Isolate* current_isolate = nullptr; @@ -5033,14 +5029,9 @@ Thread* HandleAsyncFfiCallback(FfiCallbackMetadata::Metadata metadata, } Thread* HandleIsolateGroupBoundSyncFfiCallback( - FfiCallbackMetadata::Metadata metadata, - uword* out_entry_point, - uword* out_trampoline_type) { + FfiCallbackMetadata::Metadata metadata) { Thread* current_thread = Thread::Current(); - *out_entry_point = metadata.target_entry_point(); - *out_trampoline_type = static_cast(metadata.trampoline_type()); - if (current_thread != nullptr) { current_thread->ExitSafepointFromNative(); current_thread->set_execution_state(Thread::kThreadInVM); @@ -5086,13 +5077,9 @@ void FfiCallbackThreadChecks(Thread* thread, Isolate* target_isolate) { Thread* HandleIsolateBoundSyncFfiCallback( FfiCallbackMetadata::Metadata metadata, - uword* out_entry_point, - uword* out_trampoline_type) { + CallbackMetadata* out) { Thread* current_thread = Thread::Current(); - *out_entry_point = metadata.target_entry_point(); - *out_trampoline_type = static_cast(metadata.trampoline_type()); - Isolate* target_isolate = metadata.target_isolate(); if (current_thread == nullptr) { if (!PortMap::IsOwnedByCurrentThread(target_isolate->main_port())) { @@ -5100,15 +5087,16 @@ Thread* HandleIsolateBoundSyncFfiCallback( } Thread::EnterIsolate(target_isolate); current_thread = Thread::Current(); - *out_trampoline_type |= - FfiCallbackMetadata::kSyncCallbackIsolateOwnershipFlag; FfiCallbackThreadChecks(current_thread, target_isolate); + out->epilogue = + reinterpret_cast(&DLRT_ExitSyncCallbackTargetIsolate); } else { FfiCallbackThreadChecks(current_thread, target_isolate); if (current_thread->execution_state() != Thread::kThreadInNative) { FATAL("Cannot invoke native callback from a leaf call."); } current_thread->ExitSafepointFromNative(); + out->epilogue = reinterpret_cast(&DLRT_ExitSyncCallback); } current_thread->set_execution_state(Thread::kThreadInVM); @@ -5123,13 +5111,11 @@ Thread* HandleIsolateBoundSyncFfiCallback( // a runtime entry because we can't use Thread to look it up. extern "C" Thread* DLRT_GetFfiCallbackMetadata( FfiCallbackMetadata::Trampoline trampoline, - uword* out_entry_point, - uword* out_trampoline_type) { + CallbackMetadata* out) { CHECK_STACK_ALIGNMENT; TRACE_RUNTIME_CALL("GetFfiCallbackMetadata %p", reinterpret_cast(trampoline)); - ASSERT(out_entry_point != nullptr); - ASSERT(out_trampoline_type != nullptr); + ASSERT(out != nullptr); if (!Isolate::IsolateCreationEnabled()) { FATAL("GetFfiCallbackMetadata called after shutdown %p", @@ -5155,23 +5141,37 @@ extern "C" Thread* DLRT_GetFfiCallbackMetadata( } Thread* thread = nullptr; + + out->entry_point = metadata.target_entry_point(); + switch (metadata.trampoline_type()) { + default: + out->type = 0; // Call + break; + case FfiCallbackMetadata::TrampolineType::kAsync: + out->type = 1; // Tail call. + break; +#if defined(TARGET_ARCH_IA32) + case FfiCallbackMetadata::TrampolineType::kSyncStackDelta4: + case FfiCallbackMetadata::TrampolineType::kSyncIsolateGroupBoundStackDelta4: + out->type = 2; // Call, ret4. + break; +#endif + } + if (metadata.trampoline_type() == FfiCallbackMetadata::TrampolineType::kAsync) { - thread = - HandleAsyncFfiCallback(metadata, out_entry_point, out_trampoline_type); + thread = HandleAsyncFfiCallback(metadata); + out->epilogue = reinterpret_cast(&DLRT_ExitTemporaryIsolate); } else if (metadata.is_isolate_group_bound()) { - thread = HandleIsolateGroupBoundSyncFfiCallback(metadata, out_entry_point, - out_trampoline_type); + thread = HandleIsolateGroupBoundSyncFfiCallback(metadata); + out->epilogue = reinterpret_cast(&DLRT_ExitIsolateGroupBoundIsolate); } else { - thread = HandleIsolateBoundSyncFfiCallback(metadata, out_entry_point, - out_trampoline_type); + thread = HandleIsolateBoundSyncFfiCallback(metadata, out); } TRACE_RUNTIME_CALL("GetFfiCallbackMetadata thread %p", thread); TRACE_RUNTIME_CALL("GetFfiCallbackMetadata entry_point %p", - (void*)*out_entry_point); - TRACE_RUNTIME_CALL("GetFfiCallbackMetadata trampoline_type %p", - (void*)*out_trampoline_type); + reinterpret_cast(out->entry_point)); return thread; } @@ -5187,10 +5187,10 @@ extern "C" LargestReturn dart_msan_unpoison_retval() { } #endif -extern "C" void* DLRT_ExitIsolateGroupBoundIsolate() { +extern "C" void* DLRT_ExitIsolateGroupBoundIsolate(Thread* thread) { TRACE_RUNTIME_CALL("ExitIsolateGroupBoundIsolate%s", ""); - Thread* thread = Thread::Current(); ASSERT(thread != nullptr); + ASSERT(thread == Thread::Current()); Isolate* source_isolate = reinterpret_cast(thread->unboxed_int64_runtime_second_arg()); // Need to accommodate ExitIsolateGroupAsHelper assumptions. @@ -5207,10 +5207,10 @@ extern "C" void* DLRT_ExitIsolateGroupBoundIsolate() { #endif } -extern "C" void* DLRT_ExitSyncCallbackTargetIsolate() { +extern "C" void* DLRT_ExitSyncCallbackTargetIsolate(Thread* thread) { TRACE_RUNTIME_CALL("ExitSyncCallbackTargetIsolate%s", ""); - Thread* thread = Thread::Current(); ASSERT(thread != nullptr); + ASSERT(thread == Thread::Current()); thread->set_execution_state(Thread::kThreadInVM); Thread::ExitIsolate(/*isolate_shutdown=*/false); #if defined(USING_MEMORY_SANITIZER) @@ -5220,7 +5220,12 @@ extern "C" void* DLRT_ExitSyncCallbackTargetIsolate() { #endif } -extern "C" void* DLRT_ExitSyncCallback() { +extern "C" void* DLRT_ExitSyncCallback(Thread* thread) { + ASSERT(thread != nullptr); + ASSERT(thread == Thread::Current()); + + thread->EnterSafepointToNative(); + #if defined(USING_MEMORY_SANITIZER) return reinterpret_cast(dart_msan_unpoison_retval); #else @@ -5228,10 +5233,17 @@ extern "C" void* DLRT_ExitSyncCallback() { #endif } -extern "C" void DLRT_ExitTemporaryIsolate() { - TRACE_RUNTIME_CALL("ExitTemporaryIsolate%s", ""); +#if defined(HOST_ARCH_IA32) +// A function with arguments isn't compatible with the tail-call because on IA32 +// the arguments are on the stack and caller pops. +extern "C" void* DLRT_ExitTemporaryIsolate() { Thread* thread = Thread::Current(); +#else +extern "C" void* DLRT_ExitTemporaryIsolate(Thread* thread) { +#endif + TRACE_RUNTIME_CALL("ExitTemporaryIsolate%s", ""); ASSERT(thread != nullptr); + ASSERT(thread == Thread::Current()); Isolate* source_isolate = reinterpret_cast(thread->unboxed_int64_runtime_second_arg()); @@ -5250,6 +5262,7 @@ extern "C" void DLRT_ExitTemporaryIsolate() { thread->EnterSafepoint(); } TRACE_RUNTIME_CALL("ExitTemporaryIsolate %s", "done"); + return nullptr; } extern "C" ApiLocalScope* DLRT_EnterHandleScope(Thread* thread) { diff --git a/runtime/vm/runtime_entry.h b/runtime/vm/runtime_entry.h index a9fdec9b6e5..f03eda52d4b 100644 --- a/runtime/vm/runtime_entry.h +++ b/runtime/vm/runtime_entry.h @@ -96,13 +96,21 @@ LEAF_RUNTIME_ENTRY_LIST(DECLARE_LEAF_RUNTIME_ENTRY) #undef DECLARE_LEAF_RUNTIME_ENTRY // See StubCode::GenerateFfiCallbackTrampolineStub. +struct CallbackMetadata { + uword entry_point; + uword type; // FfiCallbackMetadata::CallType + uword epilogue; +}; extern "C" Thread* DLRT_GetFfiCallbackMetadata(uword trampoline, - uword* out_entry_point, - uword* out_callback_kind); -extern "C" void DLRT_ExitTemporaryIsolate(); -extern "C" void* DLRT_ExitIsolateGroupBoundIsolate(); -extern "C" void* DLRT_ExitSyncCallbackTargetIsolate(); -extern "C" void* DLRT_ExitSyncCallback(); + CallbackMetadata* out); +#if defined(HOST_ARCH_IA32) +extern "C" void* DLRT_ExitTemporaryIsolate(); +#else +extern "C" void* DLRT_ExitTemporaryIsolate(Thread*); +#endif +extern "C" void* DLRT_ExitIsolateGroupBoundIsolate(Thread*); +extern "C" void* DLRT_ExitSyncCallbackTargetIsolate(Thread*); +extern "C" void* DLRT_ExitSyncCallback(Thread*); const char* DeoptReasonToCString(ICData::DeoptReasonId deopt_reason); diff --git a/runtime/vm/simulator_arm64.cc b/runtime/vm/simulator_arm64.cc index afadc2fb59b..b8134100d32 100644 --- a/runtime/vm/simulator_arm64.cc +++ b/runtime/vm/simulator_arm64.cc @@ -1831,10 +1831,8 @@ struct CallbackContext { extern "C" void DoRedirectedFfiCallback(CallbackContext* ctxt, uword trampoline) { - uword entry_point; - uword trampoline_type; - Thread* thread = - DLRT_GetFfiCallbackMetadata(trampoline, &entry_point, &trampoline_type); + CallbackMetadata out; + Thread* thread = DLRT_GetFfiCallbackMetadata(trampoline, &out); if (thread == nullptr) { // If GetFfiCallbackMetadata returned a null thread, it means that the async // callback was invoked after it was deleted. In this case, do nothing. @@ -1843,14 +1841,13 @@ extern "C" void DoRedirectedFfiCallback(CallbackContext* ctxt, Simulator* sim = Simulator::Current(); ASSERT(sim != nullptr); - sim->DoRedirectedFfiCallback(thread, ctxt, entry_point, trampoline_type); + sim->DoRedirectedFfiCallback(thread, ctxt, &out); } // Compare FfiCallbackTrampolineStub. void Simulator::DoRedirectedFfiCallback(Thread* thread, CallbackContext* ctxt, - uword entry_point, - uword trampoline_type) { + CallbackMetadata* out) { // The C caller might not be using frame pointers, so we just hard-code a // maximum frame size instead of using FP-SP like we do for callouts. constexpr intptr_t kStackSlotsCopied = 128; @@ -1863,11 +1860,13 @@ void Simulator::DoRedirectedFfiCallback(Thread* thread, for (intptr_t i = kStackSlotsCopied - 1; i >= 0; i--) { *--sp = sp_in[i]; } - *--sp = get_register(LR); *--sp = get_register(THR); + *--sp = get_register(LR); + *--sp = get_register(R20); + *--sp = get_register(R21); set_register(nullptr, R31, reinterpret_cast(sp)); COMPILE_ASSERT(FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta == - 2); + 4); } set_register(nullptr, R0, ctxt->integer_arguments[0]); @@ -1890,7 +1889,7 @@ void Simulator::DoRedirectedFfiCallback(Thread* thread, set_register(nullptr, THR, reinterpret_cast(thread)); set_register(nullptr, LR, kEndSimulatingPC); - set_pc(entry_point); + set_pc(out->entry_point); Execute(); ctxt->integer_arguments[0] = get_register(R0); @@ -1904,24 +1903,18 @@ void Simulator::DoRedirectedFfiCallback(Thread* thread, // ldp lr, thr, [sp], 16! // uword* sp = reinterpret_cast(get_register(R31, R31IsSP)); - set_register(nullptr, THR, *sp++); + set_register(nullptr, R21, *sp++); + set_register(nullptr, R20, *sp++); set_register(nullptr, LR, *sp++); + set_register(nullptr, THR, *sp++); sp += kStackSlotsCopied; set_register(nullptr, R31, reinterpret_cast(sp)); COMPILE_ASSERT(FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta == - 2); + 4); } - if (trampoline_type == - static_cast(FfiCallbackMetadata::TrampolineType::kAsync)) { - DLRT_ExitTemporaryIsolate(); - } else if ((trampoline_type & - FfiCallbackMetadata::kSyncCallbackIsolateOwnershipFlag) != 0) { - thread->set_execution_state(Thread::kThreadInVM); - Thread::ExitIsolate(/*isolate_shutdown=*/false); - } else { - thread->EnterSafepointToNative(); - } + auto epilogue = reinterpret_cast(out->epilogue); + epilogue(thread); } void Simulator::ClobberVolatileRegisters() { diff --git a/runtime/vm/simulator_arm64.h b/runtime/vm/simulator_arm64.h index 9ef60d657dd..20ea871498e 100644 --- a/runtime/vm/simulator_arm64.h +++ b/runtime/vm/simulator_arm64.h @@ -27,6 +27,7 @@ class Mutex; class SimulatorSetjmpBuffer; class Thread; struct CallbackContext; +struct CallbackMetadata; typedef struct { union { @@ -102,8 +103,7 @@ class Simulator { bool fp_args = false); void DoRedirectedFfiCallback(Thread* thread, CallbackContext* ctxt, - uword entry_point, - uword trampoline_type); + CallbackMetadata* out); // Runtime and native call support. enum CallKind {