diff --git a/runtime/lib/ffi.cc b/runtime/lib/ffi.cc index 16c6b23f22a..5dd43672a72 100644 --- a/runtime/lib/ffi.cc +++ b/runtime/lib/ffi.cc @@ -552,12 +552,11 @@ static uword CompileNativeCallback(const Function& c_signature, const Function& dart_target) { #if defined(DART_PRECOMPILED_RUNTIME) || defined(DART_PRECOMPILER) UNREACHABLE(); -#elif !defined(TARGET_ARCH_X64) && !defined(TARGET_ARCH_IA32) && \ - !defined(TARGET_ARCH_ARM64) +#elif defined(TARGET_ARCH_DBC) // https://github.com/dart-lang/sdk/issues/35774 // FFI is supported, but callbacks are not. Exceptions::ThrowUnsupportedError( - "FFI callbacks are currently supported on Intel and 64-bit ARM only."); + "FFI callbacks are not yet supported on DBC."); #else Thread* const thread = Thread::Current(); const int32_t callback_id = thread->AllocateFfiCallbackId(); diff --git a/runtime/vm/compiler/assembler/assembler_arm.cc b/runtime/vm/compiler/assembler/assembler_arm.cc index 9a406331bc0..dbfc95e6c57 100644 --- a/runtime/vm/compiler/assembler/assembler_arm.cc +++ b/runtime/vm/compiler/assembler/assembler_arm.cc @@ -548,10 +548,12 @@ void Assembler::strex(Register rd, Register rt, Register rn, Condition cond) { } void Assembler::TransitionGeneratedToNative(Register destination_address, + Register exit_frame_fp, Register addr, Register state) { // Save exit frame information to enable stack walking. - StoreToOffset(kWord, FP, THR, Thread::top_exit_frame_info_offset()); + StoreToOffset(kWord, exit_frame_fp, THR, + Thread::top_exit_frame_info_offset()); // Mark that the thread is executing native code. StoreToOffset(kWord, destination_address, THR, Thread::vm_tag_offset()); @@ -2496,6 +2498,33 @@ void Assembler::PopRegisters(const RegisterSet& regs) { } } +void Assembler::PushNativeCalleeSavedRegisters() { + // Save new context and C++ ABI callee-saved registers. + PushList(kAbiPreservedCpuRegs); + + const DRegister firstd = EvenDRegisterOf(kAbiFirstPreservedFpuReg); + if (TargetCPUFeatures::vfp_supported()) { + ASSERT(2 * kAbiPreservedFpuRegCount < 16); + // Save FPU registers. 2 D registers per Q register. + vstmd(DB_W, SP, firstd, 2 * kAbiPreservedFpuRegCount); + } else { + sub(SP, SP, Operand(kAbiPreservedFpuRegCount * kFpuRegisterSize)); + } +} + +void Assembler::PopNativeCalleeSavedRegisters() { + const DRegister firstd = EvenDRegisterOf(kAbiFirstPreservedFpuReg); + // Restore C++ ABI callee-saved registers. + if (TargetCPUFeatures::vfp_supported()) { + // Restore FPU registers. 2 D registers per Q register. + vldmd(IA_W, SP, firstd, 2 * kAbiPreservedFpuRegCount); + } else { + AddImmediate(SP, kAbiPreservedFpuRegCount * kFpuRegisterSize); + } + // Restore CPU registers. + PopList(kAbiPreservedCpuRegs); +} + void Assembler::MoveRegister(Register rd, Register rm, Condition cond) { if (rd != rm) { mov(rd, Operand(rm), cond); @@ -3215,6 +3244,23 @@ void Assembler::ReserveAlignedFrameSpace(intptr_t frame_space) { } } +void Assembler::EmitEntryFrameVerification(Register scratch) { +#if defined(DEBUG) + Label done; + ASSERT(!constant_pool_allowed()); + LoadImmediate(scratch, + compiler::target::frame_layout.exit_link_slot_from_entry_fp * + compiler::target::kWordSize); + add(scratch, scratch, Operand(FPREG)); + cmp(scratch, Operand(SPREG)); + b(&done, EQ); + + Breakpoint(); + + Bind(&done); +#endif +} + void Assembler::EnterCallRuntimeFrame(intptr_t frame_space) { Comment("EnterCallRuntimeFrame"); // Preserve volatile CPU registers and PP. diff --git a/runtime/vm/compiler/assembler/assembler_arm.h b/runtime/vm/compiler/assembler/assembler_arm.h index 7d2e40c7368..ceb18208ffa 100644 --- a/runtime/vm/compiler/assembler/assembler_arm.h +++ b/runtime/vm/compiler/assembler/assembler_arm.h @@ -510,9 +510,12 @@ class Assembler : public AssemblerBase { void ldrex(Register rd, Register rn, Condition cond = AL); void strex(Register rd, Register rt, Register rn, Condition cond = AL); - // Requires two temporary registers 'scratch0' and 'scratch1' (in addition to - // TMP). + // Emit code to transition between generated and native modes. + // + // These require that CSP and SP are equal and aligned and require two scratch + // registers (in addition to TMP). void TransitionGeneratedToNative(Register destination_address, + Register exit_frame_fp, Register scratch0, Register scratch1); void TransitionNativeToGenerated(Register scratch0, Register scratch1); @@ -917,6 +920,12 @@ class Assembler : public AssemblerBase { void PushRegisters(const RegisterSet& regs); void PopRegisters(const RegisterSet& regs); + // Push all registers which are callee-saved according to the ARM ABI. + void PushNativeCalleeSavedRegisters(); + + // Pop all registers which are callee-saved according to the ARM ABI. + void PopNativeCalleeSavedRegisters(); + void CompareRegisters(Register rn, Register rm) { cmp(rn, Operand(rm)); } void BranchIf(Condition condition, Label* label) { b(label, condition); } @@ -1003,6 +1012,13 @@ class Assembler : public AssemblerBase { void Ret(); void ReserveAlignedFrameSpace(intptr_t frame_space); + // In debug mode, this generates code to check that: + // FP + kExitLinkSlotFromEntryFp == SP + // or triggers breakpoint otherwise. + // + // Requires a scratch register in addition to the assembler temporary. + void EmitEntryFrameVerification(Register scratch); + // Create a frame for calling into runtime that preserves all volatile // registers. Frame's SP is guaranteed to be correctly aligned and // frame_space bytes are reserved under it. diff --git a/runtime/vm/compiler/backend/il.cc b/runtime/vm/compiler/backend/il.cc index 341d902519d..170af1ed633 100644 --- a/runtime/vm/compiler/backend/il.cc +++ b/runtime/vm/compiler/backend/il.cc @@ -3960,17 +3960,6 @@ LocationSummary* NativeEntryInstr::MakeLocationSummary(Zone* zone, UNREACHABLE(); } -#if !defined(TARGET_ARCH_X64) && !defined(TARGET_ARCH_IA32) && \ - !defined(TARGET_ARCH_ARM64) -void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) { - UNREACHABLE(); -} - -void NativeReturnInstr::EmitNativeCode(FlowGraphCompiler* compiler) { - UNREACHABLE(); -} -#endif - LocationSummary* OsrEntryInstr::MakeLocationSummary(Zone* zone, bool optimizing) const { UNREACHABLE(); @@ -5533,11 +5522,6 @@ LocationSummary* FfiCallInstr::MakeLocationSummary(Zone* zone, return summary; } -LocationSummary* NativeReturnInstr::MakeLocationSummary(Zone* zone, - bool opt) const { - UNREACHABLE(); -} - #endif // !defined(TARGET_ARCH_DBC) Representation FfiCallInstr::representation() const { diff --git a/runtime/vm/compiler/backend/il_arm.cc b/runtime/vm/compiler/backend/il_arm.cc index 5a34ee3ae84..e8331598069 100644 --- a/runtime/vm/compiler/backend/il_arm.cc +++ b/runtime/vm/compiler/backend/il_arm.cc @@ -1023,7 +1023,8 @@ void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { RawPcDescriptors::Kind::kOther, locs()); // Update information in the thread object and enter a safepoint. - __ TransitionGeneratedToNative(branch, saved_fp, locs()->temp(1).reg()); + __ TransitionGeneratedToNative(branch, FPREG, saved_fp, + locs()->temp(1).reg()); __ blx(branch); @@ -1044,6 +1045,174 @@ void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { __ PopRegister(TMP); } +void NativeReturnInstr::EmitNativeCode(FlowGraphCompiler* compiler) { + __ LeaveDartFrame(); + + // The dummy return address is in LR, no need to pop it as on Intel. + + // These can be anything besides the return registers (R0 and R1) and THR + // (R10). + const Register vm_tag_reg = R2, old_exit_frame_reg = R3, tmp = R4, tmp1 = R5; + + __ Pop(old_exit_frame_reg); + + // Restore top_resource. + __ Pop(tmp); + __ StoreToOffset(kWord, tmp, THR, + compiler::target::Thread::top_resource_offset()); + + __ Pop(vm_tag_reg); + + // Reset the exit frame info to + // old_exit_frame_reg *before* entering the safepoint. + __ TransitionGeneratedToNative(vm_tag_reg, old_exit_frame_reg, tmp, tmp1); + + __ PopNativeCalleeSavedRegisters(); + + // Leave the entry frame. + __ LeaveFrame(1 << LR | 1 << FP); + + // Leave the dummy frame holding the pushed arguments. + __ LeaveFrame(1 << LR | 1 << FP); + + __ Ret(); + + // For following blocks. + __ set_constant_pool_allowed(true); +} + +void NativeEntryInstr::SaveArgument(FlowGraphCompiler* compiler, + Location loc) const { + if (loc.IsPairLocation()) { + // Save higher-order component first, so bytes are in little-endian layout + // overall. + for (intptr_t i : {1, 0}) { + SaveArgument(compiler, loc.Component(i)); + } + return; + } + + if (loc.HasStackIndex()) return; + + if (loc.IsRegister()) { + __ Push(loc.reg()); + } else if (loc.IsFpuRegister()) { + const DRegister src = EvenDRegisterOf(loc.fpu_reg()); + __ SubImmediateSetFlags(SPREG, SPREG, 8, AL); + __ StoreDToOffset(src, SPREG, 0); + } else { + UNREACHABLE(); + } +} + +void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) { + if (FLAG_precompiled_mode) { + UNREACHABLE(); + } + + // Constant pool cannot be used until we enter the actual Dart frame. + __ set_constant_pool_allowed(false); + + __ Bind(compiler->GetJumpLabel(this)); + + // Create a dummy frame holding the pushed arguments. This simplifies + // NativeReturnInstr::EmitNativeCode. + __ EnterFrame((1 << FP) | (1 << LR), 0); + + // Save the argument registers, in reverse order. + for (intptr_t i = argument_locations_->length(); i-- > 0;) { + SaveArgument(compiler, argument_locations_->At(i)); + } + + // Enter the entry frame. + __ EnterFrame((1 << FP) | (1 << LR), 0); + + // Save a space for the code object. + __ PushImmediate(0); + + __ PushNativeCalleeSavedRegisters(); + + // Load the thread object. + // TODO(35765): Fix linking issue on AOT. + // TOOD(35934): Exclude native callbacks from snapshots. + // + // Create another frame to align the frame before continuing in "native" code. + { + __ EnterFrame(1 << FP, 0); + __ ReserveAlignedFrameSpace(0); + + __ LoadImmediate( + R0, reinterpret_cast(DLRT_GetThreadForNativeCallback)); + __ blx(R0); + __ mov(THR, Operand(R0)); + + __ LeaveFrame(1 << FP); + } + + // Save the current VMTag on the stack. + __ LoadFromOffset(kWord, R0, THR, compiler::target::Thread::vm_tag_offset()); + __ Push(R0); + + // Save top resource. + const intptr_t top_resource_offset = + compiler::target::Thread::top_resource_offset(); + __ LoadFromOffset(kWord, R0, THR, top_resource_offset); + __ Push(R0); + __ LoadImmediate(R0, 0); + __ StoreToOffset(kWord, R0, THR, top_resource_offset); + + // Save top exit frame info. Don't set it to 0 yet -- + // TransitionNativeToGenerated will handle that *after* leaving the safepoint. + __ LoadFromOffset(kWord, R0, THR, + compiler::target::Thread::top_exit_frame_info_offset()); + __ Push(R0); + + __ EmitEntryFrameVerification(R0); + + __ TransitionNativeToGenerated(/*scratch0=*/R0, /*scratch1=*/R1); + + // Now that the safepoint has ended, we can touch Dart objects without + // handles. + + // Otherwise we'll clobber the argument sent from the caller. + ASSERT(CallingConventions::ArgumentRegisters[0] != TMP && + CallingConventions::ArgumentRegisters[0] != TMP2 && + CallingConventions::ArgumentRegisters[0] != R1); + __ LoadImmediate(CallingConventions::ArgumentRegisters[0], callback_id_); + __ LoadFromOffset( + kWord, R1, THR, + compiler::target::Thread::verify_callback_isolate_entry_point_offset()); + __ blx(R1); + + // Load the code object. + __ LoadFromOffset(kWord, R0, THR, + compiler::target::Thread::callback_code_offset()); + __ LoadFieldFromOffset(kWord, R0, R0, + compiler::target::GrowableObjectArray::data_offset()); + __ LoadFieldFromOffset(kWord, CODE_REG, R0, + compiler::target::Array::data_offset() + + callback_id_ * compiler::target::kWordSize); + + // Put the code object in the reserved slot. + __ StoreToOffset(kWord, CODE_REG, FPREG, + kPcMarkerSlotFromFp * compiler::target::kWordSize); + if (FLAG_precompiled_mode && FLAG_use_bare_instructions) { + __ ldr(PP, + Address(THR, compiler::target::Thread::global_object_pool_offset())); + } else { + __ LoadImmediate(PP, 0); // GC safe value into PP. + } + + // Load a dummy return address which suggests that we are inside of + // InvokeDartCodeStub. This is how the stack walker detects an entry frame. + __ LoadFromOffset(kWord, LR, THR, + compiler::target::Thread::invoke_dart_code_stub_offset()); + __ LoadFieldFromOffset(kWord, LR, LR, + compiler::target::Code::entry_point_offset()); + + FunctionEntryInstr::EmitNativeCode(compiler); +} + LocationSummary* OneByteStringFromCharCodeInstr::MakeLocationSummary( Zone* zone, bool opt) const { diff --git a/runtime/vm/compiler/backend/il_arm64.cc b/runtime/vm/compiler/backend/il_arm64.cc index 7bbc81fd21c..15ae1deeb23 100644 --- a/runtime/vm/compiler/backend/il_arm64.cc +++ b/runtime/vm/compiler/backend/il_arm64.cc @@ -949,7 +949,7 @@ void NativeReturnInstr::EmitNativeCode(FlowGraphCompiler* compiler) { // The dummy return address is in LR, no need to pop it as on Intel. - // These can be anything besides the return register (R0). + // These can be anything besides the return register (R0) and THR (R26). const Register vm_tag_reg = R1, old_exit_frame_reg = R2, tmp = R3; __ Pop(old_exit_frame_reg); diff --git a/runtime/vm/compiler/backend/il_dbc.cc b/runtime/vm/compiler/backend/il_dbc.cc index 3f993b36de5..c869600a696 100644 --- a/runtime/vm/compiler/backend/il_dbc.cc +++ b/runtime/vm/compiler/backend/il_dbc.cc @@ -64,7 +64,8 @@ DECLARE_FLAG(int, optimization_counter_threshold); M(UnaryInt64Op) \ M(CheckedSmiOp) \ M(CheckedSmiComparison) \ - M(SimdOp) + M(SimdOp) \ + M(NativeReturn) // Location summaries actually are not used by the unoptimizing DBC compiler // because we don't allocate any registers. @@ -2168,6 +2169,10 @@ EMIT_NATIVE_CODE(CheckArrayBound, 2) { (licm_hoisted_ ? ICData::kHoisted : 0)); } +void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) { + UNREACHABLE(); +} + } // namespace dart #endif // defined TARGET_ARCH_DBC diff --git a/runtime/vm/compiler/stub_code_compiler_arm.cc b/runtime/vm/compiler/stub_code_compiler_arm.cc index 07ef14790ed..7f4f9e2ce69 100644 --- a/runtime/vm/compiler/stub_code_compiler_arm.cc +++ b/runtime/vm/compiler/stub_code_compiler_arm.cc @@ -301,7 +301,21 @@ void StubCodeCompiler::GenerateExitSafepointStub(Assembler* assembler) { } void StubCodeCompiler::GenerateVerifyCallbackStub(Assembler* assembler) { - __ Breakpoint(); + __ EnterFrame(1 << FP | 1 << LR, 0); + __ ReserveAlignedFrameSpace(0); + + // First argument is already set up by the caller. + // + // Second argument is the return address of the caller. + __ mov(CallingConventions::ArgumentRegisters[1], Operand(LR)); + ASSERT(R2 != CallingConventions::ArgumentRegisters[0] && + R2 != CallingConventions::ArgumentRegisters[1]); + __ LoadFromOffset(kWord, R2, THR, + kVerifyCallbackIsolateRuntimeEntry.OffsetFromThread()); + __ blx(R2); + + __ LeaveFrame(1 << FP | 1 << LR); + __ Ret(); } void StubCodeCompiler::GenerateNullErrorSharedWithoutFPURegsStub( @@ -1045,17 +1059,7 @@ void StubCodeCompiler::GenerateInvokeDartCodeStub(Assembler* assembler) { __ ldr(IP, Address(R3, target::Thread::invoke_dart_code_stub_offset())); __ Push(IP); - // Save new context and C++ ABI callee-saved registers. - __ PushList(kAbiPreservedCpuRegs); - - const DRegister firstd = EvenDRegisterOf(kAbiFirstPreservedFpuReg); - if (TargetCPUFeatures::vfp_supported()) { - ASSERT(2 * kAbiPreservedFpuRegCount < 16); - // Save FPU registers. 2 D registers per Q register. - __ vstmd(DB_W, SP, firstd, 2 * kAbiPreservedFpuRegCount); - } else { - __ sub(SP, SP, Operand(kAbiPreservedFpuRegCount * kFpuRegisterSize)); - } + __ PushNativeCalleeSavedRegisters(); // Set up THR, which caches the current thread in Dart code. if (THR != R3) { @@ -1086,6 +1090,8 @@ void StubCodeCompiler::GenerateInvokeDartCodeStub(Assembler* assembler) { #endif __ Push(R9); + __ EmitEntryFrameVerification(R9); + // Mark that the thread is executing Dart code. Do this after initializing the // exit link for the profiler. __ LoadImmediate(R9, VMTag::kDartCompiledTagId); @@ -1149,15 +1155,8 @@ void StubCodeCompiler::GenerateInvokeDartCodeStub(Assembler* assembler) { __ Pop(R4); __ StoreToOffset(kWord, R4, THR, target::Thread::vm_tag_offset()); - // Restore C++ ABI callee-saved registers. - if (TargetCPUFeatures::vfp_supported()) { - // Restore FPU registers. 2 D registers per Q register. - __ vldmd(IA_W, SP, firstd, 2 * kAbiPreservedFpuRegCount); - } else { - __ AddImmediate(SP, kAbiPreservedFpuRegCount * kFpuRegisterSize); - } - // Restore CPU registers. - __ PopList(kAbiPreservedCpuRegs); + __ PopNativeCalleeSavedRegisters(); + __ set_constant_pool_allowed(false); // Restore the frame pointer and return. diff --git a/runtime/vm/message.cc b/runtime/vm/message.cc index 542ccc09413..0bfca6d985c 100644 --- a/runtime/vm/message.cc +++ b/runtime/vm/message.cc @@ -4,6 +4,8 @@ #include "vm/message.h" +#include + #include "vm/dart_entry.h" #include "vm/json_stream.h" #include "vm/object.h" @@ -11,7 +13,7 @@ namespace dart { -const Dart_Port Message::kIllegalPort; +const Dart_Port Message::kIllegalPort = 0; Message::Message(Dart_Port dest_port, uint8_t* snapshot, diff --git a/runtime/vm/message.h b/runtime/vm/message.h index 4bf23034f22..03f30742055 100644 --- a/runtime/vm/message.h +++ b/runtime/vm/message.h @@ -6,6 +6,7 @@ #define RUNTIME_VM_MESSAGE_H_ #include +#include #include "platform/assert.h" #include "vm/allocation.h" @@ -42,7 +43,7 @@ class Message { } OOBMsgTag; // A port number which is never used. - static const Dart_Port kIllegalPort = 0; + static const Dart_Port kIllegalPort; // A new message to be sent between two isolates. The data handed to this // message will be disposed by calling free() once the message object is diff --git a/runtime/vm/stack_frame.h b/runtime/vm/stack_frame.h index cd3b5a11b09..7753a4f4726 100644 --- a/runtime/vm/stack_frame.h +++ b/runtime/vm/stack_frame.h @@ -470,14 +470,6 @@ DART_FORCE_INLINE static uword LocalVarAddress(uword fp, intptr_t index) { return fp + LocalVarIndex(0, index) * kWordSize; } -#if !defined(TARGET_ARCH_X64) && !defined(TARGET_ARCH_IA32) && \ - !defined(TARGET_ARCH_ARM64) -// For FFI native -> Dart callbacks, the number of stack slots between arguments -// passed on stack and arguments saved in callback prologue. This placeholder -// here is for unsupported architectures. -constexpr intptr_t kCallbackSlotsBeforeSavedArguments = -1; -#endif - } // namespace dart #endif // RUNTIME_VM_STACK_FRAME_H_ diff --git a/runtime/vm/stack_frame_arm.h b/runtime/vm/stack_frame_arm.h index 2b5dc3ea62a..cf95a6703b2 100644 --- a/runtime/vm/stack_frame_arm.h +++ b/runtime/vm/stack_frame_arm.h @@ -55,6 +55,12 @@ COMPILE_ASSERT(kAbiPreservedCpuRegCount == 7); COMPILE_ASSERT(kAbiPreservedFpuRegCount == 4); #endif +// For FFI native -> Dart callbacks, the number of stack slots between arguments +// passed on stack and arguments saved in callback prologue. +// +// 2 = return adddress (1) + saved frame pointer (1). +constexpr intptr_t kCallbackSlotsBeforeSavedArguments = 2; + } // namespace dart #endif // RUNTIME_VM_STACK_FRAME_ARM_H_ diff --git a/runtime/vm/stack_frame_dbc.h b/runtime/vm/stack_frame_dbc.h index 632f68c4581..e3d8395904d 100644 --- a/runtime/vm/stack_frame_dbc.h +++ b/runtime/vm/stack_frame_dbc.h @@ -81,6 +81,10 @@ static const int kExitLinkSlotFromEntryFp = 0; // on all other architectures. static const uword kInterruptStackLimit = 0; +// TODO(37140): For FFI native -> Dart callbacks, the number of stack slots +// between arguments passed on stack and arguments saved in callback prologue. +constexpr intptr_t kCallbackSlotsBeforeSavedArguments = -1; + } // namespace dart #endif // RUNTIME_VM_STACK_FRAME_DBC_H_ diff --git a/tests/ffi/ffi.status b/tests/ffi/ffi.status index 194f8f2de56..0e982e40481 100644 --- a/tests/ffi/ffi.status +++ b/tests/ffi/ffi.status @@ -24,9 +24,6 @@ negative_function_test: Skip [ $arch == x64 || $arch == arm64 || $arch == simdbc64 ] enable_structs_test: SkipByDesign # Tests that structs don't work on 32-bit systems. -[ $arch != x64 && $arch != ia32 && $arch != arm64 ] -function_callbacks_test: Skip # Issue 35761 - [ $runtime == dart_precompiled ] *: Skip # AOT is not yet supported: dartbug.com/35765 @@ -44,3 +41,6 @@ function_callbacks_test: Skip # Issue 35761 [ $system != android && $arch == arm ] *: Skip # "hardfp" calling convention is not yet supported (iOS is also supported but not tested): dartbug.com/36309 + +[ $arch == simdbc64 ] +function_callbacks_test: Skip # Issue 37140