diff --git a/runtime/vm/compiler/assembler/assembler_arm.cc b/runtime/vm/compiler/assembler/assembler_arm.cc index cce91ac27eb..73ef837a9a5 100644 --- a/runtime/vm/compiler/assembler/assembler_arm.cc +++ b/runtime/vm/compiler/assembler/assembler_arm.cc @@ -2445,6 +2445,27 @@ void Assembler::PopRegisters(const RegisterSet& regs) { } } +void Assembler::PushRegistersAligned(const RegisterSet& register_set, + intptr_t space) { + PushRegisters(register_set); + intptr_t aligned_space = Utils::RoundUp(register_set.SpillSize() + space, + OS::ActivationFrameAlignment()) - + register_set.SpillSize(); + if (aligned_space != 0) { + sub(SP, SP, Operand(aligned_space)); + } +} +void Assembler::PopRegistersAligned(const RegisterSet& register_set, + intptr_t space) { + intptr_t aligned_space = Utils::RoundUp(register_set.SpillSize() + space, + OS::ActivationFrameAlignment()) - + register_set.SpillSize(); + if (aligned_space != 0) { + add(SP, SP, Operand(aligned_space)); + } + PopRegisters(register_set); +} + void Assembler::PushRegistersInOrder(std::initializer_list regs) { // Collect the longest descending sequences of registers and // push them with a single STMDB instruction. diff --git a/runtime/vm/compiler/assembler/assembler_arm.h b/runtime/vm/compiler/assembler/assembler_arm.h index 7d2bcf4a042..649ce565f07 100644 --- a/runtime/vm/compiler/assembler/assembler_arm.h +++ b/runtime/vm/compiler/assembler/assembler_arm.h @@ -1280,6 +1280,8 @@ class Assembler : public AssemblerBase { void PushRegisters(const RegisterSet& regs); void PopRegisters(const RegisterSet& regs); + void PushRegistersAligned(const RegisterSet& registers, intptr_t space); + void PopRegistersAligned(const RegisterSet& registers, intptr_t space); void PushRegistersInOrder(std::initializer_list regs); diff --git a/runtime/vm/compiler/assembler/assembler_arm64.cc b/runtime/vm/compiler/assembler/assembler_arm64.cc index 9f85fa61848..dc1616fb6be 100644 --- a/runtime/vm/compiler/assembler/assembler_arm64.cc +++ b/runtime/vm/compiler/assembler/assembler_arm64.cc @@ -2406,6 +2406,27 @@ void Assembler::PopRegisters(const RegisterSet& regs) { ASSERT(vprev == kNoVRegister); } +void Assembler::PushRegistersAligned(const RegisterSet& register_set, + intptr_t space) { + PushRegisters(register_set); + intptr_t aligned_space = Utils::RoundUp(register_set.SpillSize() + space, + OS::ActivationFrameAlignment()) - + register_set.SpillSize(); + if (aligned_space != 0) { + sub(SP, SP, Operand(aligned_space)); + } +} +void Assembler::PopRegistersAligned(const RegisterSet& register_set, + intptr_t space) { + intptr_t aligned_space = Utils::RoundUp(register_set.SpillSize() + space, + OS::ActivationFrameAlignment()) - + register_set.SpillSize(); + if (aligned_space != 0) { + add(SP, SP, Operand(aligned_space)); + } + PopRegisters(register_set); +} + void Assembler::PushRegistersInOrder(std::initializer_list regs) { // Use STP to push registers in pairs. Register pending_reg = kNoRegister; diff --git a/runtime/vm/compiler/assembler/assembler_arm64.h b/runtime/vm/compiler/assembler/assembler_arm64.h index ddc2bf389b1..2f22f61c609 100644 --- a/runtime/vm/compiler/assembler/assembler_arm64.h +++ b/runtime/vm/compiler/assembler/assembler_arm64.h @@ -459,6 +459,8 @@ class Assembler : public AssemblerBase { void PushRegisters(const RegisterSet& registers); void PopRegisters(const RegisterSet& registers); + void PushRegistersAligned(const RegisterSet& registers, intptr_t space); + void PopRegistersAligned(const RegisterSet& registers, intptr_t space); void PushRegistersInOrder(std::initializer_list regs); diff --git a/runtime/vm/compiler/assembler/assembler_riscv.cc b/runtime/vm/compiler/assembler/assembler_riscv.cc index 58f447f32b1..618c40cec08 100644 --- a/runtime/vm/compiler/assembler/assembler_riscv.cc +++ b/runtime/vm/compiler/assembler/assembler_riscv.cc @@ -3147,6 +3147,27 @@ void Assembler::PopRegisters(const RegisterSet& regs) { addi(SP, SP, size); } +void Assembler::PushRegistersAligned(const RegisterSet& register_set, + intptr_t space) { + PushRegisters(register_set); + intptr_t aligned_space = Utils::RoundUp(register_set.SpillSize() + space, + OS::ActivationFrameAlignment()) - + register_set.SpillSize(); + if (aligned_space != 0) { + subi(SP, SP, aligned_space); + } +} +void Assembler::PopRegistersAligned(const RegisterSet& register_set, + intptr_t space) { + intptr_t aligned_space = Utils::RoundUp(register_set.SpillSize() + space, + OS::ActivationFrameAlignment()) - + register_set.SpillSize(); + if (aligned_space != 0) { + addi(SP, SP, aligned_space); + } + PopRegisters(register_set); +} + void Assembler::PushRegistersInOrder(std::initializer_list regs) { intptr_t offset = regs.size() * target::kWordSize; subi(SP, SP, offset); diff --git a/runtime/vm/compiler/assembler/assembler_riscv.h b/runtime/vm/compiler/assembler/assembler_riscv.h index c45880efdfe..f92039d6db5 100644 --- a/runtime/vm/compiler/assembler/assembler_riscv.h +++ b/runtime/vm/compiler/assembler/assembler_riscv.h @@ -1030,6 +1030,8 @@ class Assembler : public MicroAssembler { void PushRegisters(const RegisterSet& registers); void PopRegisters(const RegisterSet& registers); + void PushRegistersAligned(const RegisterSet& registers, intptr_t space); + void PopRegistersAligned(const RegisterSet& registers, intptr_t space); void PushRegistersInOrder(std::initializer_list regs); diff --git a/runtime/vm/compiler/assembler/assembler_x64.cc b/runtime/vm/compiler/assembler/assembler_x64.cc index f0b1a1d962d..1b3d9ed61dd 100644 --- a/runtime/vm/compiler/assembler/assembler_x64.cc +++ b/runtime/vm/compiler/assembler/assembler_x64.cc @@ -2022,6 +2022,27 @@ void Assembler::PopRegisters(const RegisterSet& register_set) { } } +void Assembler::PushRegistersAligned(const RegisterSet& register_set, + intptr_t space) { + PushRegisters(register_set); + intptr_t aligned_space = Utils::RoundUp(register_set.SpillSize() + space, + OS::ActivationFrameAlignment()) - + register_set.SpillSize(); + if (aligned_space != 0) { + subq(RSP, Immediate(aligned_space)); + } +} +void Assembler::PopRegistersAligned(const RegisterSet& register_set, + intptr_t space) { + intptr_t aligned_space = Utils::RoundUp(register_set.SpillSize() + space, + OS::ActivationFrameAlignment()) - + register_set.SpillSize(); + if (aligned_space != 0) { + addq(RSP, Immediate(aligned_space)); + } + PopRegisters(register_set); +} + void Assembler::PushRegistersInOrder(std::initializer_list regs) { for (Register reg : regs) { PushRegister(reg); diff --git a/runtime/vm/compiler/assembler/assembler_x64.h b/runtime/vm/compiler/assembler/assembler_x64.h index 22e014e49e9..129761813e5 100644 --- a/runtime/vm/compiler/assembler/assembler_x64.h +++ b/runtime/vm/compiler/assembler/assembler_x64.h @@ -951,6 +951,8 @@ class Assembler : public AssemblerBase { void PushRegisters(const RegisterSet& registers); void PopRegisters(const RegisterSet& registers); + void PushRegistersAligned(const RegisterSet& registers, intptr_t space); + void PopRegistersAligned(const RegisterSet& registers, intptr_t space); void PushRegistersInOrder(std::initializer_list regs); diff --git a/runtime/vm/compiler/stub_code_compiler_arm.cc b/runtime/vm/compiler/stub_code_compiler_arm.cc index 39819b48814..a7afdc82000 100644 --- a/runtime/vm/compiler/stub_code_compiler_arm.cc +++ b/runtime/vm/compiler/stub_code_compiler_arm.cc @@ -326,9 +326,8 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { for (intptr_t i = 0; i < FfiCallbackMetadata::NumCallbackTrampolinesPerPage(); ++i) { // The FfiCallbackMetadata table is keyed by the trampoline entry point. So - // look up the current PC, then jump to the shared section. The PC is offset - // by Instr::kPCReadOffset, which is subtracted below. - __ mov(TMP, Operand(PC)); + // look up the current PC, then jump to the shared section. + __ sub(TMP, PC, Operand(Instr::kPCReadOffset)); __ b(&body); } @@ -340,30 +339,24 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { const intptr_t shared_stub_start = __ CodeSize(); - // Save THR (callee-saved), R4 & R5 (temporaries, callee-saved), and LR. + // Save LR, FP, THR (callee-saved) & R4 (temporaries, callee-saved). COMPILE_ASSERT(FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta == 4); - SPILLS_LR_TO_FRAME( - __ PushList((1 << LR) | (1 << THR) | (1 << R4) | (1 << R5))); - - // The PC is in TMP, but is offset by kPCReadOffset. To get the actual - // trampoline entry point we need to subtract that. - __ sub(R4, TMP, Operand(Instr::kPCReadOffset)); + SPILLS_LR_TO_FRAME(__ EnterFrame((1 << FP) | (1 << LR), 0)); + __ PushList((1 << THR) | (1 << R4)); COMPILE_ASSERT(IsCalleeSavedRegister(R4)); COMPILE_ASSERT(!IsArgumentRegister(THR)); RegisterSet argument_registers; argument_registers.AddAllArgumentRegisters(); - __ PushRegisters(argument_registers); // Load the thread, verify the callback ID and exit the safepoint. // // We exit the safepoint inside DLRT_GetFfiCallbackMetadata in order to save // code size on this shared stub. { - __ EnterFrame(1 << FP, 0); - __ ReserveAlignedFrameSpace(3 * target::kWordSize); - __ mov(R0, Operand(R4)); + __ PushRegistersAligned(argument_registers, 3 * target::kWordSize); + __ mov(R0, Operand(TMP)); __ mov(R1, Operand(SP)); GenerateLoadFfiCallbackMetadataRuntimeFunction( @@ -372,37 +365,35 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { __ blx(R4); __ mov(THR, Operand(R0)); - __ ldr(TMP, Address(SP, 0 * target::kWordSize)); // entry_point - __ ldr(R4, Address(SP, 1 * target::kWordSize)); // is_tail - __ ldr(R5, Address(SP, 2 * target::kWordSize)); // epilogue + __ ldr(TMP, Address(SP, 0 * target::kWordSize)); // entry_point + CLOBBERS_LR(__ ldr(LR, Address(SP, 1 * target::kWordSize))); // is_tail + __ ldr(R4, Address(SP, 2 * target::kWordSize)); // epilogue - __ LeaveFrame(1 << FP); + __ PopRegistersAligned(argument_registers, 3 * target::kWordSize); } - __ PopRegisters(argument_registers); - Label tail; - __ cmp(R4, Operand(0)); + CLOBBERS_LR(__ cmp(LR, Operand(0))); __ b(&tail, NOT_ZERO); const RegisterSet return_registers( (1 << CallingConventions::kReturnReg) | (1 << CallingConventions::kSecondReturnReg), 1 << CallingConventions::kReturnFpuReg); - ASSERT(Utils::IsAligned(return_registers.SpillSize(), 8)); { __ blx(TMP); // entry_point - __ PushRegisters(return_registers); + __ PushRegistersAligned(return_registers, 0); __ mov(R0, Operand(THR)); - __ blx(R5); // DLRT_ExitSyncCallback, etc + __ blx(R4); // DLRT_ExitSyncCallback, etc if (FLAG_target_memory_sanitizer) { __ blx(R0); // dart_msan_unpoison_retval } - __ PopRegisters(return_registers); + __ PopRegistersAligned(return_registers, 0); + __ PopList((1 << THR) | (1 << R4)); // Returns. - RESTORES_LR_FROM_FRAME( - __ PopList((1 << PC) | (1 << THR) | (1 << R4) | (1 << R5))); + RESTORES_LR_FROM_FRAME(__ PopList((1 << PC) | (1 << FP))); + __ Breakpoint(); } { @@ -410,9 +401,9 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { __ Bind(&tail); __ blx(TMP); // entry_point __ mov(R0, Operand(THR)); - __ mov(R1, Operand(R5)); - RESTORES_LR_FROM_FRAME( - __ PopList((1 << LR) | (1 << THR) | (1 << R4) | (1 << R5))); + __ mov(R1, Operand(R4)); + __ PopList((1 << THR) | (1 << R4)); + RESTORES_LR_FROM_FRAME(__ PopList((1 << LR) | (1 << FP))); // Tail-call DLRT_ExitTemporaryIsolate. It is not safe to return to this // stub, since it might be deleted once DLRT_ExitTemporaryIsolate proceeds // enough for VM shutdown. diff --git a/runtime/vm/compiler/stub_code_compiler_arm64.cc b/runtime/vm/compiler/stub_code_compiler_arm64.cc index 43bf56b8073..975cddda4a3 100644 --- a/runtime/vm/compiler/stub_code_compiler_arm64.cc +++ b/runtime/vm/compiler/stub_code_compiler_arm64.cc @@ -499,14 +499,15 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { // aligned. COMPILE_ASSERT(FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta == 4); SPILLS_LR_TO_FRAME(__ stp( - THR, LR, Address(CSP, -2 * target::kWordSize, Address::PairPreIndex))); - __ stp(R20, R21, Address(CSP, -2 * target::kWordSize, Address::PairPreIndex)); + FP, LR, Address(CSP, -2 * target::kWordSize, Address::PairPreIndex))); + __ mov(FP, CSP); + __ stp(R20, THR, Address(CSP, -2 * target::kWordSize, Address::PairPreIndex)); COMPILE_ASSERT(!IsArgumentRegister(THR)); - RegisterSet all_registers; - all_registers.AddAllArgumentRegisters(); - all_registers.Add(Location::RegisterLocation( + RegisterSet argument_registers; + argument_registers.AddAllArgumentRegisters(); + argument_registers.Add(Location::RegisterLocation( CallingConventions::kPointerToReturnStructRegisterCall)); // Load the thread, verify the callback ID and exit the safepoint. @@ -514,14 +515,9 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { // We exit the safepoint inside DLRT_GetFfiCallbackMetadata in order to save // code size on this shared stub. { - __ SetupDartSP(); - - __ EnterFrame(0); + __ mov(SP, CSP); // This saves too much: we only need the D half of Q registers. - __ PushRegisters(all_registers); - - __ EnterFrame(0); - __ ReserveAlignedFrameSpace(3 * target::kWordSize); + __ PushRegistersAligned(argument_registers, 3 * target::kWordSize); __ mov(R0, R9); __ mov(R1, SP); @@ -549,18 +545,17 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { __ mov(THR, R0); COMPILE_ASSERT(!IsCalleeSavedRegister(R10) && !IsArgumentRegister(R10)); __ ldr(R10, Address(CSP, 0 * target::kWordSize)); // entry_point - __ ldr(R20, Address(CSP, 1 * target::kWordSize)); // is_tail - __ ldr(R21, Address(CSP, 2 * target::kWordSize)); // epilogue + COMPILE_ASSERT(!IsCalleeSavedRegister(R11) && !IsArgumentRegister(R11)); + __ ldr(R11, Address(CSP, 1 * target::kWordSize)); // is_tail + COMPILE_ASSERT(IsCalleeSavedRegister(R20)); + __ ldr(R20, Address(CSP, 2 * target::kWordSize)); // epilogue - __ LeaveFrame(); - __ PopRegisters(all_registers); - __ LeaveFrame(); - - __ RestoreCSP(); + __ PopRegistersAligned(argument_registers, 3 * target::kWordSize); + __ mov(CSP, SP); } Label tail; - __ cbnz(&tail, R20); + __ cbnz(&tail, R11); { __ blr(R10); // entry_point @@ -568,17 +563,17 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { __ fstp(V0, V1, Address(CSP, -2 * 8, Address::PairPreIndex), kDWord); __ fstp(V2, V3, Address(CSP, -2 * 8, Address::PairPreIndex), kDWord); __ mov(R0, THR); - __ blr(R21); // DLRT_ExitSyncCallback, etc + __ blr(R20); // DLRT_ExitSyncCallback, etc if (FLAG_target_memory_sanitizer) { __ blr(R0); // dart_msan_unpoison_retval } __ fldp(V2, V3, Address(CSP, 2 * 8, Address::PairPostIndex), kDWord); __ fldp(V0, V1, Address(CSP, 2 * 8, Address::PairPostIndex), kDWord); __ ldp(R0, R1, Address(CSP, 2 * target::kWordSize, Address::PairPostIndex)); - __ ldp(R20, R21, + __ ldp(R20, THR, Address(CSP, 2 * target::kWordSize, Address::PairPostIndex)); RESTORES_LR_FROM_FRAME(__ ldp( - THR, LR, Address(CSP, 2 * target::kWordSize, Address::PairPostIndex))); + FP, LR, Address(CSP, 2 * target::kWordSize, Address::PairPostIndex))); __ ret(); } @@ -587,11 +582,11 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { __ Bind(&tail); __ blr(R10); __ mov(R0, THR); - __ mov(R1, R21); - __ ldp(R20, R21, + __ mov(R1, R20); + __ ldp(R20, THR, Address(CSP, 2 * target::kWordSize, Address::PairPostIndex)); RESTORES_LR_FROM_FRAME(__ ldp( - THR, LR, Address(CSP, 2 * target::kWordSize, Address::PairPostIndex))); + FP, LR, Address(CSP, 2 * target::kWordSize, Address::PairPostIndex))); // Tail-call DLRT_ExitTemporaryIsolate. It is not safe to return to this // stub, since it might be deleted once DLRT_ExitTemporaryIsolate proceeds // enough for VM shutdown. diff --git a/runtime/vm/compiler/stub_code_compiler_riscv.cc b/runtime/vm/compiler/stub_code_compiler_riscv.cc index f0ac333ae50..d8ad381f73d 100644 --- a/runtime/vm/compiler/stub_code_compiler_riscv.cc +++ b/runtime/vm/compiler/stub_code_compiler_riscv.cc @@ -325,6 +325,15 @@ void StubCodeCompiler::GenerateLoadFfiCallbackMetadataRuntimeFunction( FfiCallbackMetadata::RuntimeFunctionOffset(function_index)); } +static const RegisterSet kArgumentRegisterSet( + CallingConventions::kArgumentRegisters, + CallingConventions::kFpuArgumentRegisters); +static const RegisterSet kReturnRegisterSet( + (1 << CallingConventions::kReturnReg) | + (1 << CallingConventions::kSecondReturnReg), + (1 << CallingConventions::kReturnFpuReg) | + (1 << CallingConventions::kSecondReturnFpuReg)); + void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { #if defined(DART_INCLUDE_SIMULATOR) && !defined(DART_PRECOMPILER) // TODO(37299): FFI is not supported in SIMRISCV32/64. @@ -350,10 +359,13 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { __ Bind(&body); - // Save THR (callee-saved) and RA. Keeps stack aligned. COMPILE_ASSERT(FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta == 4); - __ PushRegisterPair(RA, THR); - __ PushRegisterPair(S2, S3); + __ subi(SP, SP, 4 * target::kWordSize); + __ sx(RA, Address(SP, 3 * target::kWordSize)); + __ sx(FP, Address(SP, 2 * target::kWordSize)); + __ sx(THR, Address(SP, 1 * target::kWordSize)); + __ sx(S2, Address(SP, 0 * target::kWordSize)); + __ addi(FP, SP, 4 * target::kWordSize); COMPILE_ASSERT(!IsArgumentRegister(THR)); // Load the thread, verify the callback ID and exit the safepoint. @@ -361,21 +373,7 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { // We exit the safepoint inside DLRT_GetFfiCallbackMetadata in order to save // code size on this shared stub. { - // Push arguments and callback id. - __ subi(SP, SP, 12 * target::kWordSize); - __ sx(A7, Address(SP, 11 * target::kWordSize)); - __ sx(A6, Address(SP, 10 * target::kWordSize)); - __ sx(A5, Address(SP, 9 * target::kWordSize)); - __ sx(A4, Address(SP, 8 * target::kWordSize)); - __ sx(A3, Address(SP, 7 * target::kWordSize)); - __ sx(A2, Address(SP, 6 * target::kWordSize)); - __ sx(A1, Address(SP, 5 * target::kWordSize)); - __ sx(A0, Address(SP, 4 * target::kWordSize)); - // 3 - alignment gap - // 2 - Out.epilogue - // 1 - Out.is_tail - // 0 - Out.entry_point - + __ PushRegistersAligned(kArgumentRegisterSet, 3 * target::kWordSize); __ mv(A0, T1); __ mv(A1, SP); @@ -411,42 +409,29 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { __ jalr(T1); __ mv(THR, A0); __ lx(T2, Address(SP, 0 * target::kWordSize)); // entry_point - __ lx(S2, Address(SP, 1 * target::kWordSize)); // is_tail - __ lx(S3, Address(SP, 2 * target::kWordSize)); // epilogue + __ lx(T3, Address(SP, 1 * target::kWordSize)); // is_tail + __ lx(S2, Address(SP, 2 * target::kWordSize)); // epilogue - __ lx(A0, Address(SP, 4 * target::kWordSize)); - __ lx(A1, Address(SP, 5 * target::kWordSize)); - __ lx(A2, Address(SP, 6 * target::kWordSize)); - __ lx(A3, Address(SP, 7 * target::kWordSize)); - __ lx(A4, Address(SP, 8 * target::kWordSize)); - __ lx(A5, Address(SP, 9 * target::kWordSize)); - __ lx(A6, Address(SP, 10 * target::kWordSize)); - __ lx(A7, Address(SP, 11 * target::kWordSize)); - __ addi(SP, SP, 12 * target::kWordSize); + __ PopRegistersAligned(kArgumentRegisterSet, 3 * target::kWordSize); } Label tail; - __ bnez(S2, &tail, Assembler::kNearJump); + __ bnez(T3, &tail, Assembler::kNearJump); { - __ jalr(T2); // entry_point - __ subi(SP, SP, 32); // Not using word size to keep RV32 stack aligned. - __ sx(A0, Address(SP, 24)); - __ sx(A1, Address(SP, 16)); - __ fsd(FA0, Address(SP, 8)); - __ fsd(FA1, Address(SP, 0)); + __ jalr(T2); // entry_point + __ PushRegistersAligned(kReturnRegisterSet, 0); __ mv(A0, THR); - __ jalr(S3); // DLRT_ExitSyncCallback, etc + __ jalr(S2); // DLRT_ExitSyncCallback, etc if (FLAG_target_memory_sanitizer) { __ jalr(A0); // dart_msan_unpoison_retval } - __ fld(FA1, Address(SP, 0)); - __ fld(FA0, Address(SP, 8)); - __ lx(A1, Address(SP, 16)); - __ lx(A0, Address(SP, 24)); - __ addi(SP, SP, 32); - __ PopRegisterPair(S2, S3); - __ PopRegisterPair(RA, THR); + __ PopRegistersAligned(kReturnRegisterSet, 0); + __ lx(S2, Address(SP, 0 * target::kWordSize)); + __ lx(THR, Address(SP, 1 * target::kWordSize)); + __ lx(FP, Address(SP, 2 * target::kWordSize)); + __ lx(RA, Address(SP, 3 * target::kWordSize)); + __ addi(SP, SP, 4 * target::kWordSize); __ ret(); } @@ -454,9 +439,12 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { __ Bind(&tail); __ jalr(T2); // entry_point __ mv(A0, THR); - __ mv(A1, S3); - __ PopRegisterPair(S2, S3); - __ PopRegisterPair(RA, THR); + __ mv(A1, S2); + __ lx(S2, Address(SP, 0 * target::kWordSize)); + __ lx(THR, Address(SP, 1 * target::kWordSize)); + __ lx(FP, Address(SP, 2 * target::kWordSize)); + __ lx(RA, Address(SP, 3 * target::kWordSize)); + __ addi(SP, SP, 4 * target::kWordSize); // Tail-call DLRT_ExitTemporaryIsolate. It is not safe to return to this // stub, since it might be deleted once DLRT_ExitTemporaryIsolate proceeds // enough for VM shutdown. diff --git a/runtime/vm/compiler/stub_code_compiler_x64.cc b/runtime/vm/compiler/stub_code_compiler_x64.cc index 3efe9700302..0d3a1bee795 100644 --- a/runtime/vm/compiler/stub_code_compiler_x64.cc +++ b/runtime/vm/compiler/stub_code_compiler_x64.cc @@ -536,23 +536,22 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { const intptr_t shared_stub_start = __ CodeSize(); // Save THR which is callee-saved. + __ EnterFrame(0); __ pushq(THR); __ pushq(R12); - __ pushq(R13); - - // 4 = return address, THR, R12, R13 + // 4 = return address, FP, THR, R12 COMPILE_ASSERT(4 == FfiCallbackMetadata::kNativeCallbackTrampolineStackDelta); - // Save all registers which might hold arguments. - __ PushRegisters(kArgumentRegisterSet); - // Load the thread, verify the callback ID and exit the safepoint. // // We exit the safepoint inside DLRT_GetFfiCallbackMetadata in order to save // code size of this shared stub. { + // Save all registers which might hold arguments. + __ PushRegistersAligned(kArgumentRegisterSet, 3 * target::kWordSize); COMPILE_ASSERT(RAX != CallingConventions::kArg1Reg); __ movq(CallingConventions::kArg1Reg, RAX); + __ movq(CallingConventions::kArg2Reg, RSP); #if defined(DART_TARGET_OS_FUCHSIA) // TODO(https://dartbug.com/52579): Remove. @@ -568,25 +567,19 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { FfiCallbackMetadata::kGetFfiCallbackMetadata, RAX); #endif // defined(DART_TARGET_OS_FUCHSIA) - __ EnterFrame(0); - __ ReserveAlignedFrameSpace(3 * target::kWordSize); - __ movq(CallingConventions::kArg2Reg, RSP); - __ CallCFunction(RAX, /*restore_rsp=*/true); __ movq(THR, RAX); __ movq(RAX, Address(RSP, 0 * target::kWordSize)); // entry_point - __ movq(R12, Address(RSP, 1 * target::kWordSize)); // is_tail - __ movq(R13, Address(RSP, 2 * target::kWordSize)); // epilogue + __ movq(TMP, Address(RSP, 1 * target::kWordSize)); // is_tail + __ movq(R12, Address(RSP, 2 * target::kWordSize)); // epilogue - __ LeaveFrame(); + // Restore the arguments. + __ PopRegistersAligned(kArgumentRegisterSet, 3 * target::kWordSize); } - // Restore the arguments. - __ PopRegisters(kArgumentRegisterSet); - Label tail; - __ cmpq(R12, Immediate(0)); + __ cmpq(TMP, Immediate(0)); __ j(NOT_ZERO, &tail); const RegisterSet return_registers( @@ -597,16 +590,16 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { { __ call(RAX); // entry_point - __ PushRegisters(return_registers); + __ PushRegistersAligned(return_registers, 0); __ movq(CallingConventions::kArg1Reg, THR); - __ CallCFunction(R13, /*restore_rsp=*/true); // DLRT_ExitSyncCallback, etc + __ CallCFunction(R12, /*restore_rsp=*/true); // DLRT_ExitSyncCallback, etc if (FLAG_target_memory_sanitizer) { __ CallCFunction(RAX, /*restore_rsp=*/true); // dart_msan_unpoison_retval } - __ PopRegisters(return_registers); - __ popq(R13); + __ PopRegistersAligned(return_registers, 0); __ popq(R12); __ popq(THR); + __ LeaveFrame(); __ ret(); } @@ -614,10 +607,10 @@ void StubCodeCompiler::GenerateFfiCallbackTrampolineStub() { __ Bind(&tail); __ call(RAX); // entry_point __ movq(CallingConventions::kArg1Reg, THR); - __ movq(RAX, R13); - __ popq(R13); + __ movq(RAX, R12); __ popq(R12); __ popq(THR); + __ LeaveFrame(); // Tail-call DLRT_ExitTemporaryIsolate. It is not safe to return to this // stub, since it might be deleted once DLRT_ExitTemporaryIsolate proceeds // enough for VM shutdown. diff --git a/runtime/vm/ffi_callback_metadata.h b/runtime/vm/ffi_callback_metadata.h index c6bc417fa71..22df057191f 100644 --- a/runtime/vm/ffi_callback_metadata.h +++ b/runtime/vm/ffi_callback_metadata.h @@ -341,15 +341,15 @@ class FfiCallbackMetadata { static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 4; #elif defined(TARGET_ARCH_ARM64) static constexpr intptr_t kNativeCallbackTrampolineSize = 8; - static constexpr intptr_t kNativeCallbackSharedStubSize = 268; + static constexpr intptr_t kNativeCallbackSharedStubSize = 232; static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 4; #elif defined(TARGET_ARCH_RISCV32) static constexpr intptr_t kNativeCallbackTrampolineSize = 8; - static constexpr intptr_t kNativeCallbackSharedStubSize = 154; + static constexpr intptr_t kNativeCallbackSharedStubSize = 186; static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 4; #elif defined(TARGET_ARCH_RISCV64) static constexpr intptr_t kNativeCallbackTrampolineSize = 8; - static constexpr intptr_t kNativeCallbackSharedStubSize = 154; + static constexpr intptr_t kNativeCallbackSharedStubSize = 186; static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 4; #else #error What architecture?