diff --git a/runtime/vm/compiler/assembler/assembler_ia32.cc b/runtime/vm/compiler/assembler/assembler_ia32.cc index a6d5523a7b6..de73c0f8f9d 100644 --- a/runtime/vm/compiler/assembler/assembler_ia32.cc +++ b/runtime/vm/compiler/assembler/assembler_ia32.cc @@ -2180,7 +2180,7 @@ void Assembler::IncrementSmiField(const Address& dest, int32_t increment) { addl(dest, inc_imm); } -void Assembler::LoadDoubleConstant(XmmRegister dst, double value) { +void Assembler::LoadDImmediate(XmmRegister dst, double value) { // TODO(5410843): Need to have a code constants table. int64_t constant = bit_cast(value); pushl(Immediate(Utils::High32Bits(constant))); diff --git a/runtime/vm/compiler/assembler/assembler_ia32.h b/runtime/vm/compiler/assembler/assembler_ia32.h index f059fb2c148..8cee67df228 100644 --- a/runtime/vm/compiler/assembler/assembler_ia32.h +++ b/runtime/vm/compiler/assembler/assembler_ia32.h @@ -713,6 +713,8 @@ class Assembler : public AssemblerBase { } } + void LoadDImmediate(XmmRegister dst, double value); + void Drop(intptr_t stack_elements); void LoadIsolate(Register dst); @@ -732,7 +734,6 @@ class Assembler : public AssemblerBase { void PushObject(const Object& object); void CompareObject(Register reg, const Object& object); - void LoadDoubleConstant(XmmRegister dst, double value); void LoadCompressed(Register dest, const Address& slot) { movl(dest, slot); } diff --git a/runtime/vm/compiler/assembler/assembler_ia32_test.cc b/runtime/vm/compiler/assembler/assembler_ia32_test.cc index 43c481880e1..e89073d497f 100644 --- a/runtime/vm/compiler/assembler/assembler_ia32_test.cc +++ b/runtime/vm/compiler/assembler/assembler_ia32_test.cc @@ -4558,8 +4558,8 @@ ASSEMBLER_TEST_RUN(ConditionalMovesCompare, test) { "ret\n"); } -ASSEMBLER_TEST_GENERATE(TestLoadDoubleConstant, assembler) { - __ LoadDoubleConstant(XMM3, -12.34); +ASSEMBLER_TEST_GENERATE(TestLoadDImmediate, assembler) { + __ LoadDImmediate(XMM3, -12.34); __ pushl(EAX); __ pushl(EAX); __ movsd(Address(ESP, 0), XMM3); @@ -4569,9 +4569,9 @@ ASSEMBLER_TEST_GENERATE(TestLoadDoubleConstant, assembler) { __ ret(); } -ASSEMBLER_TEST_RUN(TestLoadDoubleConstant, test) { - typedef double (*TestLoadDoubleConstantCode)(); - double res = reinterpret_cast(test->entry())(); +ASSEMBLER_TEST_RUN(TestLoadDImmediate, test) { + typedef double (*TestLoadDImmediateCode)(); + double res = reinterpret_cast(test->entry())(); EXPECT_FLOAT_EQ(-12.34, res, 0.0001); EXPECT_DISASSEMBLY( "push 0xc028ae14\n" diff --git a/runtime/vm/compiler/backend/flow_graph_compiler.cc b/runtime/vm/compiler/backend/flow_graph_compiler.cc index 08da16e6b4a..3ec06a6d0b4 100644 --- a/runtime/vm/compiler/backend/flow_graph_compiler.cc +++ b/runtime/vm/compiler/backend/flow_graph_compiler.cc @@ -1796,9 +1796,6 @@ void FlowGraphCompiler::AllocateRegistersLocally(Instruction* instr) { result_location = Location::FpuRegisterLocation( AllocateFreeFpuRegister(blocked_fpu_registers)); break; - case Location::kRequiresStackSlot: - UNREACHABLE(); - break; } locs->set_out(0, result_location); } diff --git a/runtime/vm/compiler/backend/il.cc b/runtime/vm/compiler/backend/il.cc index f9e6f0ee67f..30f6bec8400 100644 --- a/runtime/vm/compiler/backend/il.cc +++ b/runtime/vm/compiler/backend/il.cc @@ -6494,26 +6494,21 @@ Representation FfiCallInstr::RequiredInputRepresentation(intptr_t idx) const { LocationSummary* FfiCallInstr::MakeLocationSummaryInternal( Zone* zone, bool is_optimizing, - const Register temp) const { - // The temporary register needs to be callee-saved and not an argument - // register. - ASSERT(((1 << CallingConventions::kFfiAnyNonAbiRegister) & - CallingConventions::kArgumentRegisters) == 0); + const RegList temps) const { + auto contains_call = + is_leaf_ ? LocationSummary::kNativeLeafCall : LocationSummary::kCall; - // TODO(dartbug.com/45468): Investigate whether we can avoid spilling - // registers across ffi leaf calls by not using `kCall` here. LocationSummary* summary = new (zone) LocationSummary( zone, /*num_inputs=*/InputCount(), - /*num_temps=*/temp == kNoRegister ? 2 : 3, LocationSummary::kCall); + /*num_temps=*/Utils::CountOneBitsWord(temps), contains_call); - const Register temp0 = CallingConventions::kFfiAnyNonAbiRegister; - const Register temp1 = CallingConventions::kSecondNonArgumentRegister; - ASSERT(temp0 != temp1); - summary->set_temp(0, Location::RegisterLocation(temp0)); - summary->set_temp(1, Location::RegisterLocation(temp1)); - - if (temp != kNoRegister) { - summary->set_temp(2, Location::RegisterLocation(temp)); + intptr_t reg_i = 0; + for (intptr_t reg = 0; reg < kNumberOfCpuRegisters; reg++) { + if ((temps & (1 << reg)) != 0) { + summary->set_temp(reg_i, + Location::RegisterLocation(static_cast(reg))); + reg_i++; + } } summary->set_in(TargetAddressIndex(), @@ -6524,10 +6519,7 @@ LocationSummary* FfiCallInstr::MakeLocationSummaryInternal( } if (marshaller_.PassTypedData()) { - // The register allocator already preserves this value across the call on - // a stack slot, so we'll use the spilled value directly. - summary->set_in(TypedDataIndex(), Location::RequiresStackSlot()); - + summary->set_in(TypedDataIndex(), Location::Any()); // We don't care about return location, but we need to pass a register. summary->set_out( 0, Location::RegisterLocation(CallingConventions::kReturnReg)); @@ -6654,14 +6646,14 @@ void FfiCallInstr::EmitParamMoves(FlowGraphCompiler* compiler, void FfiCallInstr::EmitReturnMoves(FlowGraphCompiler* compiler, const Register temp0, const Register temp1) { - __ Comment("EmitReturnMoves"); - const auto& returnLocation = marshaller_.Location(compiler::ffi::kResultIndex); if (returnLocation.payload_type().IsVoid()) { return; } + __ Comment("EmitReturnMoves"); + NoTemporaryAllocator no_temp; if (returnLocation.IsRegisters() || returnLocation.IsFpuRegisters()) { const auto& src = returnLocation; @@ -6675,15 +6667,20 @@ void FfiCallInstr::EmitReturnMoves(FlowGraphCompiler* compiler, // Get the typed data pointer which we have pinned to a stack slot. const Location typed_data_loc = locs()->in(TypedDataIndex()); - ASSERT(typed_data_loc.IsStackSlot()); - ASSERT(typed_data_loc.base_reg() == FPREG); - // If this is a leaf call there is no extra call frame to step through. - if (is_leaf_) { - __ LoadMemoryValue(temp0, FPREG, typed_data_loc.ToStackSlotOffset()); + if (typed_data_loc.IsStackSlot()) { + ASSERT(typed_data_loc.base_reg() == FPREG); + // If this is a leaf call there is no extra call frame to step through. + if (is_leaf_) { + __ LoadMemoryValue(temp0, FPREG, typed_data_loc.ToStackSlotOffset()); + } else { + __ LoadMemoryValue( + temp0, FPREG, + kSavedCallerFpSlotFromFp * compiler::target::kWordSize); + __ LoadMemoryValue(temp0, temp0, typed_data_loc.ToStackSlotOffset()); + } } else { - __ LoadMemoryValue( - temp0, FPREG, kSavedCallerFpSlotFromFp * compiler::target::kWordSize); - __ LoadMemoryValue(temp0, temp0, typed_data_loc.ToStackSlotOffset()); + compiler->EmitMove(Location::RegisterLocation(temp0), typed_data_loc, + &no_temp); } __ LoadField(temp0, compiler::FieldAddress( diff --git a/runtime/vm/compiler/backend/il.h b/runtime/vm/compiler/backend/il.h index 65a9afbaa56..a7dd7aa2cda 100644 --- a/runtime/vm/compiler/backend/il.h +++ b/runtime/vm/compiler/backend/il.h @@ -5305,7 +5305,7 @@ class FfiCallInstr : public Definition { LocationSummary* MakeLocationSummaryInternal(Zone* zone, bool is_optimizing, - const Register temp) const; + const RegList temps) const; // Clobbers both given registers. // `saved_fp` is used as the frame base to rebase off of. diff --git a/runtime/vm/compiler/backend/il_arm.cc b/runtime/vm/compiler/backend/il_arm.cc index 411d58b5ed6..3dab29ace0e 100644 --- a/runtime/vm/compiler/backend/il_arm.cc +++ b/runtime/vm/compiler/backend/il_arm.cc @@ -1402,27 +1402,34 @@ void NativeCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { __ Drop(ArgumentCount()); // Drop the arguments. } +#define R(r) (1 << r) + LocationSummary* FfiCallInstr::MakeLocationSummary(Zone* zone, bool is_optimizing) const { - return MakeLocationSummaryInternal(zone, is_optimizing, R0); + return MakeLocationSummaryInternal( + zone, is_optimizing, + (R(R0) | R(CallingConventions::kFfiAnyNonAbiRegister) | + R(CallingConventions::kSecondNonArgumentRegister))); } +#undef R + void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { + const Register branch = locs()->in(TargetAddressIndex()).reg(); + + // The temps are indexed according to their register number. + const Register temp2 = locs()->temp(0).reg(); // For regular calls, this holds the FP for rebasing the original locations // during EmitParamMoves. // For leaf calls, this holds the SP used to restore the pre-aligned SP after // the call. - const Register saved_fp_or_sp = locs()->temp(0).reg(); - RELEASE_ASSERT((CallingConventions::kCalleeSaveCpuRegisters & - (1 << saved_fp_or_sp)) != 0); - const Register temp1 = locs()->temp(1).reg(); - const Register temp2 = locs()->temp(2).reg(); - const Register branch = locs()->in(TargetAddressIndex()).reg(); + const Register saved_fp_or_sp = locs()->temp(1).reg(); + const Register temp1 = locs()->temp(2).reg(); // Ensure these are callee-saved register and are preserved across the call. ASSERT((CallingConventions::kCalleeSaveCpuRegisters & (1 << saved_fp_or_sp)) != 0); - // temp doesn't need to be preserved. + // Other temps don't need to be preserved. __ mov(saved_fp_or_sp, is_leaf_ ? compiler::Operand(SPREG) : compiler::Operand(FPREG)); diff --git a/runtime/vm/compiler/backend/il_arm64.cc b/runtime/vm/compiler/backend/il_arm64.cc index 410cb0e127d..7ba86030160 100644 --- a/runtime/vm/compiler/backend/il_arm64.cc +++ b/runtime/vm/compiler/backend/il_arm64.cc @@ -1227,27 +1227,36 @@ void NativeCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { __ Drop(ArgumentCount()); // Drop the arguments. } +#define R(r) (1 << r) + LocationSummary* FfiCallInstr::MakeLocationSummary(Zone* zone, bool is_optimizing) const { - return MakeLocationSummaryInternal(zone, is_optimizing, R11); + return MakeLocationSummaryInternal( + zone, is_optimizing, + (R(CallingConventions::kSecondNonArgumentRegister) | R(R11) | + R(CallingConventions::kFfiAnyNonAbiRegister) | R(R25))); } +#undef R + void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { + const Register branch = locs()->in(TargetAddressIndex()).reg(); + + // The temps are indexed according to their register number. + const Register temp1 = locs()->temp(0).reg(); + const Register temp2 = locs()->temp(1).reg(); // For regular calls, this holds the FP for rebasing the original locations // during EmitParamMoves. // For leaf calls, this holds the SP used to restore the pre-aligned SP after // the call. - const Register saved_fp_or_sp = locs()->temp(0).reg(); - RELEASE_ASSERT((CallingConventions::kCalleeSaveCpuRegisters & - (1 << saved_fp_or_sp)) != 0); - const Register temp1 = locs()->temp(1).reg(); - const Register temp2 = locs()->temp(2).reg(); - const Register branch = locs()->in(TargetAddressIndex()).reg(); + const Register saved_fp_or_sp = locs()->temp(2).reg(); + const Register temp_csp = locs()->temp(3).reg(); // Ensure these are callee-saved register and are preserved across the call. ASSERT((CallingConventions::kCalleeSaveCpuRegisters & (1 << saved_fp_or_sp)) != 0); - // temps don't need to be preserved. + ASSERT((CallingConventions::kCalleeSaveCpuRegisters & (1 << temp_csp)) != 0); + // Other temps don't need to be preserved. __ mov(saved_fp_or_sp, is_leaf_ ? SPREG : FPREG); @@ -1279,14 +1288,14 @@ void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { // We are entering runtime code, so the C stack pointer must be restored // from the stack limit to the top of the stack. - __ mov(R25, CSP); + __ mov(temp_csp, CSP); __ mov(CSP, SP); __ blr(branch); // Restore the Dart stack pointer. __ mov(SP, CSP); - __ mov(CSP, R25); + __ mov(CSP, temp_csp); #if !defined(PRODUCT) __ LoadImmediate(temp1, compiler::target::Thread::vm_tag_dart_id()); @@ -1315,14 +1324,14 @@ void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { // We are entering runtime code, so the C stack pointer must be restored // from the stack limit to the top of the stack. - __ mov(R25, CSP); + __ mov(temp_csp, CSP); __ mov(CSP, SP); __ blr(branch); // Restore the Dart stack pointer. __ mov(SP, CSP); - __ mov(CSP, R25); + __ mov(CSP, temp_csp); // Update information in the thread object and leave the safepoint. __ TransitionNativeToGenerated(temp1, /*leave_safepoint=*/true); diff --git a/runtime/vm/compiler/backend/il_ia32.cc b/runtime/vm/compiler/backend/il_ia32.cc index ce4c3688701..d6086370e94 100644 --- a/runtime/vm/compiler/backend/il_ia32.cc +++ b/runtime/vm/compiler/backend/il_ia32.cc @@ -998,24 +998,33 @@ void NativeCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { __ Drop(ArgumentCount()); // Drop the arguments. } +#define R(r) (1 << r) + LocationSummary* FfiCallInstr::MakeLocationSummary(Zone* zone, bool is_optimizing) const { - return MakeLocationSummaryInternal(zone, is_optimizing, kNoRegister); + return MakeLocationSummaryInternal( + zone, is_optimizing, + (R(CallingConventions::kSecondNonArgumentRegister) | + R(CallingConventions::kFfiAnyNonAbiRegister))); } +#undef R + void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { + const Register branch = locs()->in(TargetAddressIndex()).reg(); + + // The temps are indexed according to their register number. + const Register temp = locs()->temp(0).reg(); // For regular calls, this holds the FP for rebasing the original locations // during EmitParamMoves. // For leaf calls, this holds the SP used to restore the pre-aligned SP after // the call. - const Register saved_fp_or_sp = locs()->temp(0).reg(); - const Register temp = locs()->temp(1).reg(); - const Register branch = locs()->in(TargetAddressIndex()).reg(); + const Register saved_fp_or_sp = locs()->temp(1).reg(); // Ensure these are callee-saved register and are preserved across the call. ASSERT((CallingConventions::kCalleeSaveCpuRegisters & (1 << saved_fp_or_sp)) != 0); - // temp doesn't need to be preserved. + // Other temps don't need to be preserved. __ movl(saved_fp_or_sp, is_leaf_ ? SPREG : FPREG); diff --git a/runtime/vm/compiler/backend/il_riscv.cc b/runtime/vm/compiler/backend/il_riscv.cc index 9068abbc383..9f576c937e8 100644 --- a/runtime/vm/compiler/backend/il_riscv.cc +++ b/runtime/vm/compiler/backend/il_riscv.cc @@ -1374,10 +1374,14 @@ void NativeCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { __ Drop(ArgumentCount() + 1); // Drop the arguments and result. } +#define R(r) (1 << r) + LocationSummary* FfiCallInstr::MakeLocationSummary(Zone* zone, bool is_optimizing) const { - LocationSummary* summary = - MakeLocationSummaryInternal(zone, is_optimizing, CALLEE_SAVED_TEMP2); + LocationSummary* summary = MakeLocationSummaryInternal( + zone, is_optimizing, + (R(CallingConventions::kSecondNonArgumentRegister) | + R(CallingConventions::kFfiAnyNonAbiRegister) | R(CALLEE_SAVED_TEMP2))); // A3/A4/A5 are blocked during Dart register allocation because they are // assigned to TMP/TMP2/PP. This assignment is important for reducing code // size. To work around this for FFI calls, the FFI argument definitions are @@ -1400,17 +1404,20 @@ LocationSummary* FfiCallInstr::MakeLocationSummary(Zone* zone, return summary; } +#undef R + void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { + const Register target = locs()->in(TargetAddressIndex()).reg(); + + // The temps are indexed according to their register number. + const Register temp1 = locs()->temp(0).reg(); // For regular calls, this holds the FP for rebasing the original locations // during EmitParamMoves. // For leaf calls, this holds the SP used to restore the pre-aligned SP after // the call. - const Register saved_fp_or_sp = locs()->temp(0).reg(); - RELEASE_ASSERT((CallingConventions::kCalleeSaveCpuRegisters & - (1 << saved_fp_or_sp)) != 0); - const Register temp1 = locs()->temp(1).reg(); + const Register saved_fp_or_sp = locs()->temp(1).reg(); const Register temp2 = locs()->temp(2).reg(); - const Register target = locs()->in(TargetAddressIndex()).reg(); + ASSERT(temp1 != target); ASSERT(temp2 != target); ASSERT(temp1 != saved_fp_or_sp); @@ -1420,7 +1427,7 @@ void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { // Ensure these are callee-saved register and are preserved across the call. ASSERT((CallingConventions::kCalleeSaveCpuRegisters & (1 << saved_fp_or_sp)) != 0); - // temps don't need to be preserved. + // Other temps don't need to be preserved. __ mv(saved_fp_or_sp, is_leaf_ ? SPREG : FPREG); diff --git a/runtime/vm/compiler/backend/il_test.cc b/runtime/vm/compiler/backend/il_test.cc index a03d0e86174..95f8ac0b470 100644 --- a/runtime/vm/compiler/backend/il_test.cc +++ b/runtime/vm/compiler/backend/il_test.cc @@ -952,6 +952,227 @@ ISOLATE_UNIT_TEST_CASE(IRTest_LoadThread) { EXPECT_EQ(reinterpret_cast(thread), result_int); } +// Helper to set up an inlined FfiCall by replacing a StaticCall. +FlowGraph* SetupFfiFlowgraph(TestPipeline* pipeline, + Zone* zone, + const compiler::ffi::CallMarshaller& marshaller, + uword native_entry, + bool is_leaf) { + FlowGraph* flow_graph = pipeline->RunPasses({CompilerPass::kComputeSSA}); + + // Make an FfiCall based on ffi_trampoline that calls our native function. + auto ffi_call = new FfiCallInstr(zone, DeoptId::kNone, marshaller, is_leaf); + RELEASE_ASSERT(ffi_call->InputCount() == 1); + // TargetAddress is the function pointer called. + const Representation address_repr = + compiler::target::kWordSize == 4 ? kUnboxedUint32 : kUnboxedInt64; + ffi_call->SetInputAt( + ffi_call->TargetAddressIndex(), + new Value(flow_graph->GetConstant( + Integer::Handle(Integer::NewCanonical(native_entry)), address_repr))); + + // Replace the placeholder StaticCall with an FfiCall to our native function. + { + StaticCallInstr* static_call = nullptr; + { + ILMatcher cursor(flow_graph, flow_graph->graph_entry()->normal_entry(), + /*trace=*/false); + cursor.TryMatch({kMoveGlob, {kMatchStaticCall, &static_call}}); + } + RELEASE_ASSERT(static_call != nullptr); + + flow_graph->InsertBefore(static_call, ffi_call, /*env=*/nullptr, + FlowGraph::kEffect); + static_call->RemoveFromGraph(/*return_previous=*/false); + } + + // Run remaining relevant compiler passes. + pipeline->RunAdditionalPasses({ + CompilerPass::kApplyICData, + CompilerPass::kTryOptimizePatterns, + CompilerPass::kSetOuterInliningId, + CompilerPass::kTypePropagation, + // Skipping passes that don't seem to do anything for this test. + CompilerPass::kWidenSmiToInt32, + CompilerPass::kSelectRepresentations, + // Skipping passes that don't seem to do anything for this test. + CompilerPass::kTypePropagation, + CompilerPass::kRangeAnalysis, + // Skipping passes that don't seem to do anything for this test. + CompilerPass::kFinalizeGraph, + CompilerPass::kCanonicalize, + CompilerPass::kAllocateRegisters, + CompilerPass::kReorderBlocks, + }); + + return flow_graph; +} + +// Test that FFI calls spill all live values to the stack, and that FFI leaf +// calls are free to use available ABI callee-save registers to avoid spilling. +// Additionally test that register allocation is done correctly by clobbering +// all volatile registers in the native function being called. +ISOLATE_UNIT_TEST_CASE(IRTest_FfiCallInstrLeafDoesntSpill) { + SetFlagScope sfs(&FLAG_sound_null_safety, kNullSafetyOptionStrong); + + const char* kScript = R"( + import 'dart:ffi'; + + // This is purely a placeholder and is never called. + void placeholder() {} + + // Will call the "doFfiCall" and exercise its code. + bool invokeDoFfiCall() { + final double result = doFfiCall(1, 2, 3, 1.0, 2.0, 3.0); + if (result != (2 + 3 + 4 + 2.0 + 3.0 + 4.0)) { + throw 'Failed. Result was $result.'; + } + return true; + } + + // Will perform a "C" call while having live values in registers + // across the FfiCall. + double doFfiCall(int a, int b, int c, double x, double y, double z) { + // Ensure there is at least one live value in a register. + a += 1; + b += 1; + c += 1; + x += 1.0; + y += 1.0; + z += 1.0; + // We'll replace this StaticCall with an FfiCall. + placeholder(); + // Use the live value. + return (a + b + c + x + y + z); + } + + // FFI trampoline function. + typedef NT = Void Function(); + typedef DT = void Function(); + Pointer> ptr = Pointer.fromAddress(0); + DT getFfiTrampolineClosure() => ptr.asFunction(isLeaf:true); + )"; + + const auto& root_library = Library::Handle(LoadTestScript(kScript)); + + // Build a "C" function that we can actually invoke. + auto& c_function = Instructions::Handle( + BuildInstructions([](compiler::Assembler* assembler) { + // Clobber all volatile registers to make sure caller doesn't rely on + // any non-callee-save register. + for (intptr_t reg = 0; reg < kNumberOfFpuRegisters; reg++) { + if ((kAbiVolatileFpuRegs & (1 << reg)) != 0) { +#if defined(TARGET_ARCH_ARM) + // On ARM we need an extra scratch register for LoadDImmediate. + assembler->LoadDImmediate(static_cast(reg), 0.0, R3); +#else + assembler->LoadDImmediate(static_cast(reg), 0.0); +#endif + } + } + for (intptr_t reg = 0; reg < kNumberOfCpuRegisters; reg++) { + if ((kDartVolatileCpuRegs & (1 << reg)) != 0) { + assembler->LoadImmediate(static_cast(reg), 0xDEADBEEF); + } + } + assembler->Ret(); + })); + uword native_entry = c_function.EntryPoint(); + + // Get initial compilation done. + Invoke(root_library, "invokeDoFfiCall"); + + const Function& do_ffi_call = + Function::Handle(GetFunction(root_library, "doFfiCall")); + RELEASE_ASSERT(!do_ffi_call.IsNull()); + + const auto& value = Closure::Handle( + Closure::RawCast(Invoke(root_library, "getFfiTrampolineClosure"))); + RELEASE_ASSERT(value.IsClosure()); + const auto& ffi_trampoline = + Function::ZoneHandle(Closure::Cast(value).function()); + RELEASE_ASSERT(!ffi_trampoline.IsNull()); + + // Construct the FFICallInstr from the trampoline matching our native + // function. + const char* error = nullptr; + const auto marshaller_ptr = compiler::ffi::CallMarshaller::FromFunction( + thread->zone(), ffi_trampoline, &error); + RELEASE_ASSERT(error == nullptr); + RELEASE_ASSERT(marshaller_ptr != nullptr); + const auto& marshaller = *marshaller_ptr; + + const auto& compile_and_run = + [&](bool is_leaf, std::function verify) { + // Build the SSA graph for "doFfiCall" + TestPipeline pipeline(do_ffi_call, CompilerPass::kJIT); + FlowGraph* flow_graph = SetupFfiFlowgraph( + &pipeline, thread->zone(), marshaller, native_entry, is_leaf); + + { + ParallelMoveInstr* parallel_move = nullptr; + ILMatcher cursor(flow_graph, + flow_graph->graph_entry()->normal_entry(), + /*trace=*/false); + while (cursor.TryMatch( + {kMoveGlob, {kMatchAndMoveParallelMove, ¶llel_move}})) { + verify(parallel_move); + } + } + + // Finish the compilation and attach code so we can run it. + pipeline.CompileGraphAndAttachFunction(); + + // Ensure we can successfully invoke the FFI call. + auto& result = Object::Handle(Invoke(root_library, "invokeDoFfiCall")); + RELEASE_ASSERT(result.IsBool()); + EXPECT(Bool::Cast(result).value()); + }; + + intptr_t num_cpu_reg_to_stack_nonleaf = 0; + intptr_t num_cpu_reg_to_stack_leaf = 0; + intptr_t num_fpu_reg_to_stack_nonleaf = 0; + intptr_t num_fpu_reg_to_stack_leaf = 0; + + // Test non-leaf spills live values. + compile_and_run(/*is_leaf=*/false, [&](ParallelMoveInstr* parallel_move) { + // TargetAddress is passed in register, live values are all spilled. + for (int i = 0; i < parallel_move->NumMoves(); i++) { + auto move = parallel_move->moves()[i]; + if (move->src_slot()->IsRegister() && move->dest_slot()->IsStackSlot()) { + num_cpu_reg_to_stack_nonleaf++; + } else if (move->src_slot()->IsFpuRegister() && + move->dest_slot()->IsDoubleStackSlot()) { + num_fpu_reg_to_stack_nonleaf++; + } + } + }); + + // Test leaf calls do not cause spills of live values. + compile_and_run(/*is_leaf=*/true, [&](ParallelMoveInstr* parallel_move) { + // TargetAddress is passed in registers, live values are not spilled and + // remains in callee-save registers. + for (int i = 0; i < parallel_move->NumMoves(); i++) { + auto move = parallel_move->moves()[i]; + if (move->src_slot()->IsRegister() && move->dest_slot()->IsStackSlot()) { + num_cpu_reg_to_stack_leaf++; + } else if (move->src_slot()->IsFpuRegister() && + move->dest_slot()->IsDoubleStackSlot()) { + num_fpu_reg_to_stack_leaf++; + } + } + }); + + // We should have less moves to the stack (i.e. spilling) in leaf calls. + EXPECT_LT(num_cpu_reg_to_stack_leaf, num_cpu_reg_to_stack_nonleaf); + // We don't have volatile FPU registers on all platforms. + const bool has_callee_save_fpu_regs = + Utils::CountOneBitsWord(kAbiVolatileFpuRegs) < + Utils::CountOneBitsWord(kAllFpuRegistersList); + EXPECT(!has_callee_save_fpu_regs || + num_fpu_reg_to_stack_leaf < num_fpu_reg_to_stack_nonleaf); +} + static void TestConstantFoldToSmi(const Library& root_library, const char* function_name, CompilerPass::PipelineMode mode, diff --git a/runtime/vm/compiler/backend/il_x64.cc b/runtime/vm/compiler/backend/il_x64.cc index 33b66eea300..9a12a656b28 100644 --- a/runtime/vm/compiler/backend/il_x64.cc +++ b/runtime/vm/compiler/backend/il_x64.cc @@ -1175,29 +1175,38 @@ void NativeCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { __ Drop(ArgumentCount()); // Drop the arguments. } +#define R(r) (1 << r) + LocationSummary* FfiCallInstr::MakeLocationSummary(Zone* zone, bool is_optimizing) const { - // Use R10 as a temp register. We can't use RDI, RSI, RDX, R8, R9 as they are - // arg registers, and R11 is TMP. - return MakeLocationSummaryInternal(zone, is_optimizing, R10); + // Use R10 as a temp. register. We can't use RDI, RSI, RDX, R8, R9 as they are + // argument registers, and R11 is TMP. + return MakeLocationSummaryInternal( + zone, is_optimizing, + (R(CallingConventions::kSecondNonArgumentRegister) | R(R10) | + R(CallingConventions::kFfiAnyNonAbiRegister))); } +#undef R + void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { + const Register target_address = locs()->in(TargetAddressIndex()).reg(); + + // The temps are indexed according to their register number. + // For regular calls, this holds the FP for rebasing the original locations + // during EmitParamMoves. + const Register saved_fp = locs()->temp(0).reg(); + const Register temp = locs()->temp(1).reg(); // For leaf calls, this holds the SP used to restore the pre-aligned SP after // the call. // Note: R12 doubles as CODE_REG, which gets clobbered during frame setup in // regular calls. - const Register saved_sp = locs()->temp(0).reg(); - // For regular calls, this holds the FP for rebasing the original locations - // during EmitParamMoves. - const Register saved_fp = locs()->temp(1).reg(); - const Register temp = locs()->temp(2).reg(); - const Register target_address = locs()->in(TargetAddressIndex()).reg(); + const Register saved_sp = locs()->temp(2).reg(); // Ensure these are callee-saved register and are preserved across the call. ASSERT((CallingConventions::kCalleeSaveCpuRegisters & (1 << saved_sp)) != 0); ASSERT((CallingConventions::kCalleeSaveCpuRegisters & (1 << saved_fp)) != 0); - // temp doesn't need to be preserved. + // Other temps don't need to be preserved. if (is_leaf_) { __ movq(saved_sp, SPREG); diff --git a/runtime/vm/compiler/backend/linearscan.cc b/runtime/vm/compiler/backend/linearscan.cc index 19191d1542e..266b48541a0 100644 --- a/runtime/vm/compiler/backend/linearscan.cc +++ b/runtime/vm/compiler/backend/linearscan.cc @@ -465,6 +465,28 @@ void FlowGraphAllocator::BlockLocation(Location loc, } } +void FlowGraphAllocator::BlockCpuRegisters(intptr_t registers, + intptr_t from, + intptr_t to) { + for (intptr_t r = 0; r < kNumberOfCpuRegisters; r++) { + if ((registers & (1 << r)) != 0) { + BlockLocation(Location::RegisterLocation(static_cast(r)), from, + to); + } + } +} + +void FlowGraphAllocator::BlockFpuRegisters(intptr_t fpu_registers, + intptr_t from, + intptr_t to) { + for (intptr_t r = 0; r < kNumberOfFpuRegisters; r++) { + if ((fpu_registers & (1 << r)) != 0) { + BlockLocation(Location::FpuRegisterLocation(static_cast(r)), + from, to); + } + } +} + void LiveRange::Print() { if (first_use_interval() == NULL) { return; @@ -1389,6 +1411,21 @@ void FlowGraphAllocator::ProcessOneInstruction(BlockEntryInstr* block, } } + // Block all volatile (i.e. not native ABI callee-save) registers. + if (locs->native_leaf_call()) { + BlockCpuRegisters(kDartVolatileCpuRegs, pos, pos + 1); + BlockFpuRegisters(kAbiVolatileFpuRegs, pos, pos + 1); +#if defined(TARGET_ARCH_ARM) + // We do not yet have a way to say that we only want FPU registers that + // overlap S registers. + // Block all Q/D FPU registers above the 8/16 that have S registers in + // VFPv3-D32. + // This way we avoid ending up trying to do single-word operations on + // registers that don't support it. + BlockFpuRegisters(kFpuRegistersWithoutSOverlap, pos, pos + 1); +#endif + } + // Block all allocatable registers for calls. if (locs->always_calls() && !locs->callee_safe_call()) { // Expected shape of live range: @@ -1397,16 +1434,9 @@ void FlowGraphAllocator::ProcessOneInstruction(BlockEntryInstr* block, // [--) // // The stack bitmap describes the position i. - for (intptr_t reg = 0; reg < kNumberOfCpuRegisters; reg++) { - BlockLocation(Location::RegisterLocation(static_cast(reg)), pos, - pos + 1); - } + BlockCpuRegisters(kAllCpuRegistersList, pos, pos + 1); - for (intptr_t reg = 0; reg < kNumberOfFpuRegisters; reg++) { - BlockLocation( - Location::FpuRegisterLocation(static_cast(reg)), pos, - pos + 1); - } + BlockFpuRegisters(kAllFpuRegistersList, pos, pos + 1); #if defined(DEBUG) // Verify that temps, inputs and output were specified as fixed @@ -1426,8 +1456,7 @@ void FlowGraphAllocator::ProcessOneInstruction(BlockEntryInstr* block, pair->At(1).policy() == Location::kAny); } else { ASSERT(!locs->in(j).IsUnallocated() || - locs->in(j).policy() == Location::kAny || - locs->in(j).policy() == Location::kRequiresStackSlot); + locs->in(j).policy() == Location::kAny); } } @@ -1441,7 +1470,7 @@ void FlowGraphAllocator::ProcessOneInstruction(BlockEntryInstr* block, #endif } - if (locs->can_call()) { + if (locs->can_call() && !locs->native_leaf_call()) { safepoints_.Add(current); } diff --git a/runtime/vm/compiler/backend/linearscan.h b/runtime/vm/compiler/backend/linearscan.h index 2d6e91965b8..bc26c637767 100644 --- a/runtime/vm/compiler/backend/linearscan.h +++ b/runtime/vm/compiler/backend/linearscan.h @@ -156,6 +156,10 @@ class FlowGraphAllocator : public ValueObject { bool* blocked_registers, LiveRange** blocking_ranges); + void BlockCpuRegisters(intptr_t registers, intptr_t from, intptr_t to); + + void BlockFpuRegisters(intptr_t fpu_registers, intptr_t from, intptr_t to); + intptr_t NumberOfRegisters() const { return number_of_registers_; } // Find all safepoints that are covered by this live range. diff --git a/runtime/vm/compiler/backend/locations.cc b/runtime/vm/compiler/backend/locations.cc index 56640f80152..9b5b1664753 100644 --- a/runtime/vm/compiler/backend/locations.cc +++ b/runtime/vm/compiler/backend/locations.cc @@ -185,8 +185,7 @@ void LocationSummary::set_in(intptr_t index, Location loc) { // restrictions. if (always_calls()) { if (loc.IsUnallocated()) { - ASSERT(loc.policy() == Location::kAny || - loc.policy() == Location::kRequiresStackSlot); + ASSERT(loc.policy() == Location::kAny); } else if (loc.IsPairLocation()) { ASSERT(!loc.AsPairLocation()->At(0).IsUnallocated() || loc.AsPairLocation()->At(0).policy() == Location::kAny); @@ -311,8 +310,6 @@ const char* Location::Name() const { return "R"; case kRequiresFpuRegister: return "DR"; - case kRequiresStackSlot: - return "RS"; case kWritableRegister: return "WR"; case kSameAsFirstInput: diff --git a/runtime/vm/compiler/backend/locations.h b/runtime/vm/compiler/backend/locations.h index b9988948a26..754852e0ebf 100644 --- a/runtime/vm/compiler/backend/locations.h +++ b/runtime/vm/compiler/backend/locations.h @@ -257,7 +257,6 @@ class Location : public ValueObject { kPrefersRegister, kRequiresRegister, kRequiresFpuRegister, - kRequiresStackSlot, kWritableRegister, kSameAsFirstInput, }; @@ -285,10 +284,6 @@ class Location : public ValueObject { return UnallocatedLocation(kRequiresFpuRegister); } - static Location RequiresStackSlot() { - return UnallocatedLocation(kRequiresStackSlot); - } - static Location WritableRegister() { return UnallocatedLocation(kWritableRegister); } @@ -719,12 +714,19 @@ class RegisterSet : public ValueObject { class LocationSummary : public ZoneAllocated { public: enum ContainsCall { - kNoCall, // Used registers must be reserved as tmp. - kCall, // Registers have been saved and can be used without reservation. - kCallCalleeSafe, // Registers will be saved by the callee. - kCallOnSlowPath, // Used registers must be reserved as tmp. - kCallOnSharedSlowPath // Registers used to invoke shared stub must be - // reserved as tmp. + // Used registers must be reserved as tmp. + kNoCall, + // Registers have been saved and can be used without reservation. + kCall, + // Registers will be saved by the callee. + kCallCalleeSafe, + // Used registers must be reserved as tmp. + kCallOnSlowPath, + // Registers used to invoke shared stub must be reserved as tmp. + kCallOnSharedSlowPath, + // Location is a native leaf call so any register not in the native ABI + // callee-save (or input/output/tmp) set might get clobbered. + kNativeLeafCall }; LocationSummary(Zone* zone, @@ -800,6 +802,8 @@ class LocationSummary : public ZoneAllocated { return contains_call_ == kCallOnSharedSlowPath; } + bool native_leaf_call() const { return contains_call_ == kNativeLeafCall; } + void PrintTo(BaseTextBuffer* f) const; static LocationSummary* Make(Zone* zone, diff --git a/runtime/vm/constants_arm.h b/runtime/vm/constants_arm.h index 7b438127207..7406b47a00e 100644 --- a/runtime/vm/constants_arm.h +++ b/runtime/vm/constants_arm.h @@ -30,6 +30,8 @@ namespace dart { // to compare LR register code. #define LR LR_DO_NOT_USE_DIRECTLY +#define R(reg) (static_cast(1) << (reg)) + // We support both VFPv3-D16 and VFPv3-D32 profiles, but currently only one at // a time. #if defined(__ARM_ARCH_7A__) @@ -209,6 +211,9 @@ enum DRegister { D31 = 31, kNumberOfDRegisters = 32, #endif + // Number of D registers that overlap S registers. + // One D register overlaps two S registers, so regardless of the numbers of D + // registers, there are only 32 S registers that are overlapped. kNumberOfOverlappingDRegisters = 16, }; @@ -243,6 +248,10 @@ enum QRegister { Q15 = 15, kNumberOfQRegisters = 16, #endif + // Number of Q registers that overlap S registers. + // One Q register overlaps four S registers, so regardless of the numbers of Q + // registers, there are only 32 S registers that are overlapped. + kNumberOfOverlappingQRegisters = 8, }; static inline DRegister EvenDRegisterOf(QRegister q) { @@ -256,7 +265,7 @@ static inline DRegister OddDRegisterOf(QRegister q) { static inline SRegister EvenSRegisterOf(DRegister d) { #if defined(VFPv3_D32) // When we have 32 D registers, the S registers only overlap the first 16. - // That is, there are only 32 S registers. + // That is, there are only ever 32 S registers in any extension. ASSERT(d < D16); #endif return static_cast(d * 2); @@ -535,6 +544,7 @@ struct DispatchTableNullErrorABI { // List of registers used in load/store multiple. typedef uint16_t RegList; const RegList kAllCpuRegistersList = 0xFFFF; +const RegList kAllFpuRegistersList = (1 << kNumberOfFpuRegisters) - 1; // C++ ABI call registers. const RegList kAbiArgumentCpuRegs = @@ -572,7 +582,11 @@ const int kDartVolatileCpuRegCount = 6; const int kDartVolatileCpuRegCount = 5; #endif -#define R(reg) (static_cast(1) << (reg)) +const RegList kAbiVolatileFpuRegs = R(Q0) | R(Q1) | R(Q2) | R(Q3); + +const RegList kFpuRegistersWithoutSOverlap = + kAllFpuRegistersList & + ~((1 << QRegister::kNumberOfOverlappingQRegisters) - 1); class CallingConventions { public: diff --git a/runtime/vm/constants_arm64.h b/runtime/vm/constants_arm64.h index a7f00d94b72..5be6e984825 100644 --- a/runtime/vm/constants_arm64.h +++ b/runtime/vm/constants_arm64.h @@ -81,7 +81,9 @@ enum Register { }; enum VRegister { + // v0 Volatile; Parameter/scratch register, result register. V0 = 0, + // v1-v7 Volatile; Parameter/scratch register. V1 = 1, V2 = 2, V3 = 3, @@ -89,6 +91,8 @@ enum VRegister { V5 = 5, V6 = 6, V7 = 7, + // v8-v15 Non-volatile; Scratch registers + // Only the bottom 64 bits are non-volatile! [ARM IHI 0055B, 5.1.2] V8 = 8, V9 = 9, V10 = 10, @@ -97,6 +101,7 @@ enum VRegister { V13 = 13, V14 = 14, V15 = 15, + // v16-v31 Volatile; Scratch registers. V16 = 16, V17 = 17, V18 = 18, @@ -104,7 +109,7 @@ enum VRegister { V20 = 20, V21 = 21, V22 = 22, - V23 = 24, + V23 = 23, V24 = 24, V25 = 25, V26 = 26, @@ -430,6 +435,11 @@ const Register kDartLastVolatileCpuReg = R14; const int kDartVolatileCpuRegCount = 15; const int kDartVolatileFpuRegCount = 24; +const RegList kAbiVolatileFpuRegs = + R(V0) | R(V1) | R(V2) | R(V3) | R(V4) | R(V5) | R(V6) | R(V7) | R(V16) | + R(V17) | R(V18) | R(V19) | R(V20) | R(V21) | R(V22) | R(V23) | R(V24) | + R(V25) | R(V26) | R(V27) | R(V28) | R(V29) | R(V30) | R(V31); + constexpr int kStoreBufferWrapperSize = 32; class CallingConventions { diff --git a/runtime/vm/constants_ia32.h b/runtime/vm/constants_ia32.h index 061771de24d..733dd8c39a2 100644 --- a/runtime/vm/constants_ia32.h +++ b/runtime/vm/constants_ia32.h @@ -16,6 +16,8 @@ namespace dart { +#define R(reg) (1 << (reg)) + enum Register { EAX = 0, ECX = 1, @@ -29,6 +31,8 @@ enum Register { kNoRegister = -1, // Signals an illegal register. }; +// Low and high bytes registers of the the first four general purpose registers. +// The other four general purpose registers do not have byte registers. enum ByteRegister { AL = 0, CL = 1, @@ -42,6 +46,10 @@ enum ByteRegister { }; inline ByteRegister ByteRegisterOf(Register reg) { + // This only works for EAX, ECX, EDX, EBX. + // Remaining Register values map to high byte of the above registers. + RELEASE_ASSERT(reg == Register::EAX || reg == Register::ECX || + reg == Register::EDX || reg == Register::EBX); return static_cast(reg); } @@ -263,12 +271,23 @@ struct DispatchTableNullErrorABI { typedef uint32_t RegList; const RegList kAllCpuRegistersList = 0xFF; +const RegList kAllFpuRegistersList = (1 << kNumberOfFpuRegisters) - 1; const intptr_t kReservedCpuRegisters = (1 << SPREG) | (1 << FPREG) | (1 << THR); // CPU registers available to Dart allocator. const RegList kDartAvailableCpuRegs = kAllCpuRegistersList & ~kReservedCpuRegisters; +const RegList kAbiPreservedCpuRegs = (1 << EDI) | (1 << ESI) | (1 << EBX); + +// Registers available to Dart that are not preserved by runtime calls. +const RegList kDartVolatileCpuRegs = + kDartAvailableCpuRegs & ~kAbiPreservedCpuRegs; + +const RegList kAbiVolatileFpuRegs = kAllFpuRegistersList; + +#undef R + enum ScaleFactor { TIMES_1 = 0, TIMES_2 = 1, @@ -332,8 +351,7 @@ class CallingConventions { static const intptr_t kXmmArgumentRegisters = 0; static const intptr_t kNumFpuArgRegs = 0; - static constexpr intptr_t kCalleeSaveCpuRegisters = - (1 << EDI) | (1 << ESI) | (1 << EBX); + static constexpr intptr_t kCalleeSaveCpuRegisters = kAbiPreservedCpuRegs; static const bool kArgumentIntRegXorFpuReg = false; diff --git a/runtime/vm/constants_riscv.h b/runtime/vm/constants_riscv.h index 30a720ff4b6..f8430df8071 100644 --- a/runtime/vm/constants_riscv.h +++ b/runtime/vm/constants_riscv.h @@ -387,6 +387,7 @@ struct DispatchTableNullErrorABI { typedef uint32_t RegList; const RegList kAllCpuRegistersList = 0xFFFFFFFF; +const RegList kAllFpuRegistersList = 0xFFFFFFFF; #define R(reg) (static_cast(1) << (reg)) diff --git a/runtime/vm/constants_x64.h b/runtime/vm/constants_x64.h index b8bc0e716d8..930f5889aa7 100644 --- a/runtime/vm/constants_x64.h +++ b/runtime/vm/constants_x64.h @@ -16,6 +16,8 @@ namespace dart { +#define R(reg) (static_cast(1) << (reg)) + enum Register { RAX = 0, RCX = 1, @@ -345,7 +347,7 @@ const RegList kAllCpuRegistersList = 0xFFFF; const RegList kAllFpuRegistersList = 0xFFFF; const RegList kReservedCpuRegisters = - (1 << SPREG) | (1 << FPREG) | (1 << TMP) | (1 << PP) | (1 << THR); + R(SPREG) | R(FPREG) | R(TMP) | R(PP) | R(THR); constexpr intptr_t kNumberOfReservedCpuRegisters = 5; // CPU registers available to Dart allocator. const RegList kDartAvailableCpuRegs = @@ -354,6 +356,20 @@ constexpr int kNumberOfDartAvailableCpuRegs = kNumberOfCpuRegisters - kNumberOfReservedCpuRegisters; constexpr int kStoreBufferWrapperSize = 13; +#if defined(DART_TARGET_OS_WINDOWS) +const RegList kAbiPreservedCpuRegs = + R(RBX) | R(RSI) | R(RDI) | R(R12) | R(R13) | R(R14) | R(R15); +const RegList kAbiVolatileFpuRegs = + R(XMM0) | R(XMM1) | R(XMM2) | R(XMM3) | R(XMM4) | R(XMM5); +#else +const RegList kAbiPreservedCpuRegs = R(RBX) | R(R12) | R(R13) | R(R14) | R(R15); +const RegList kAbiVolatileFpuRegs = kAllFpuRegistersList; +#endif + +// Registers available to Dart that are not preserved by runtime calls. +const RegList kDartVolatileCpuRegs = + kDartAvailableCpuRegs & ~kAbiPreservedCpuRegs; + enum ScaleFactor { TIMES_1 = 0, TIMES_2 = 1, @@ -383,8 +399,6 @@ enum ScaleFactor { TIMES_COMPRESSED_HALF_WORD_SIZE = TIMES_COMPRESSED_WORD_SIZE - 1, }; -#define R(reg) (static_cast(1) << (reg)) - class CallingConventions { public: #if defined(DART_TARGET_OS_WINDOWS) @@ -422,11 +436,7 @@ class CallingConventions { static const intptr_t kVolatileCpuRegisters = R(RAX) | R(RCX) | R(RDX) | R(R8) | R(R9) | R(R10) | R(R11); - static const intptr_t kVolatileXmmRegisters = - R(XMM0) | R(XMM1) | R(XMM2) | R(XMM3) | R(XMM4) | R(XMM5); - - static const intptr_t kCalleeSaveCpuRegisters = - R(RBX) | R(RSI) | R(RDI) | R(R12) | R(R13) | R(R14) | R(R15); + static const RegList kVolatileXmmRegisters = kAbiVolatileFpuRegs; static const intptr_t kCalleeSaveXmmRegisters = R(XMM6) | R(XMM7) | R(XMM8) | R(XMM9) | R(XMM10) | R(XMM11) | R(XMM12) | @@ -490,13 +500,7 @@ class CallingConventions { R(RSI) | R(RDI) | R(R8) | R(R9) | R(R10) | R(R11); - static const intptr_t kVolatileXmmRegisters = - R(XMM0) | R(XMM1) | R(XMM2) | R(XMM3) | R(XMM4) | R(XMM5) | R(XMM6) | - R(XMM7) | R(XMM8) | R(XMM9) | R(XMM10) | R(XMM11) | R(XMM12) | R(XMM13) | - R(XMM14) | R(XMM15); - - static const intptr_t kCalleeSaveCpuRegisters = - R(RBX) | R(R12) | R(R13) | R(R14) | R(R15); + static const RegList kVolatileXmmRegisters = kAbiVolatileFpuRegs; static const intptr_t kCalleeSaveXmmRegisters = 0; @@ -530,6 +534,8 @@ class CallingConventions { #endif + static const intptr_t kCalleeSaveCpuRegisters = kAbiPreservedCpuRegs; + COMPILE_ASSERT((kArgumentRegisters & kReservedCpuRegisters) == 0); static constexpr Register kFfiAnyNonAbiRegister = R12; @@ -544,9 +550,6 @@ class CallingConventions { (kArgumentRegisters | R(kPointerToReturnStructRegisterCall))) == 0); }; -constexpr intptr_t kAbiPreservedCpuRegs = - CallingConventions::kCalleeSaveCpuRegisters; - #undef R class Instr {