From ba341d740a255a508c0977e1ca94b5546de02937 Mon Sep 17 00:00:00 2001 From: Clement Skau Date: Wed, 30 Mar 2022 10:57:26 +0000 Subject: [PATCH] [vm] Avoid spills for most FFI leaf calls. Adds an additional `contains_call` mode specifying that the register allocator should block all non-callee-save ABI registers. As opposed to `kCall`, this frees the use of callee-save registers across FfiCallInstr, meaning that live values do not always have to be spilled. For architectures with sufficient free callee-save registers this should result in less overhead for each FfiCall. This change also removes RequiresStackSlot as it was only being used for FfiCall's TypedData, and wasn't implemented in the Register Allocator. Since FfiCall no longer blocks all registers TypedData can now live in a register, so the previous assumptions about it always being on stack have to be relaxed. This change also works around an issue where the register allocator might allocate an FPU register that does not have an associated S register. This causes problems when FFI tries to pass 32 bit floats in said register. As a work around we block all FPU regs. above the 8 that have S regs. for FFI calls in VFPv3-D32 mode. This change also adds an additional temp registers which was previous used but not being reserved in FfiCall IL on Arm64. Some additional prereq. fixes were branched out into: https://dart-review.googlesource.com/c/sdk/+/237690 TEST=Adds IRTest_FfiCallInstrLeafDoesntSpill. Bug: https://github.com/dart-lang/sdk/issues/45468 Change-Id: Icdeccb0de77e46f5bc34dd5bd4e308d27bc1ef99 Cq-Do-Not-Cancel-Tryjobs: true Cq-Include-Trybots: luci.dart.try:vm-kernel-precomp-win-release-x64-try, vm-precomp-ffi-qemu-linux-release-arm-try, vm-kernel-mac-release-arm64-try, vm-kernel-nnbd-mac-debug-arm64-try, vm-kernel-nnbd-mac-release-arm64-try Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/221624 Reviewed-by: Martin Kustermann Commit-Queue: Clement Skau --- .../vm/compiler/assembler/assembler_ia32.cc | 2 +- .../vm/compiler/assembler/assembler_ia32.h | 3 +- .../compiler/assembler/assembler_ia32_test.cc | 10 +- .../compiler/backend/flow_graph_compiler.cc | 3 - runtime/vm/compiler/backend/il.cc | 57 +++-- runtime/vm/compiler/backend/il.h | 2 +- runtime/vm/compiler/backend/il_arm.cc | 23 +- runtime/vm/compiler/backend/il_arm64.cc | 33 ++- runtime/vm/compiler/backend/il_ia32.cc | 19 +- runtime/vm/compiler/backend/il_riscv.cc | 23 +- runtime/vm/compiler/backend/il_test.cc | 221 ++++++++++++++++++ runtime/vm/compiler/backend/il_x64.cc | 29 ++- runtime/vm/compiler/backend/linearscan.cc | 53 ++++- runtime/vm/compiler/backend/linearscan.h | 4 + runtime/vm/compiler/backend/locations.cc | 5 +- runtime/vm/compiler/backend/locations.h | 26 ++- runtime/vm/constants_arm.h | 18 +- runtime/vm/constants_arm64.h | 12 +- runtime/vm/constants_ia32.h | 22 +- runtime/vm/constants_riscv.h | 1 + runtime/vm/constants_x64.h | 39 ++-- 21 files changed, 471 insertions(+), 134 deletions(-) diff --git a/runtime/vm/compiler/assembler/assembler_ia32.cc b/runtime/vm/compiler/assembler/assembler_ia32.cc index a6d5523a7b6..de73c0f8f9d 100644 --- a/runtime/vm/compiler/assembler/assembler_ia32.cc +++ b/runtime/vm/compiler/assembler/assembler_ia32.cc @@ -2180,7 +2180,7 @@ void Assembler::IncrementSmiField(const Address& dest, int32_t increment) { addl(dest, inc_imm); } -void Assembler::LoadDoubleConstant(XmmRegister dst, double value) { +void Assembler::LoadDImmediate(XmmRegister dst, double value) { // TODO(5410843): Need to have a code constants table. int64_t constant = bit_cast(value); pushl(Immediate(Utils::High32Bits(constant))); diff --git a/runtime/vm/compiler/assembler/assembler_ia32.h b/runtime/vm/compiler/assembler/assembler_ia32.h index f059fb2c148..8cee67df228 100644 --- a/runtime/vm/compiler/assembler/assembler_ia32.h +++ b/runtime/vm/compiler/assembler/assembler_ia32.h @@ -713,6 +713,8 @@ class Assembler : public AssemblerBase { } } + void LoadDImmediate(XmmRegister dst, double value); + void Drop(intptr_t stack_elements); void LoadIsolate(Register dst); @@ -732,7 +734,6 @@ class Assembler : public AssemblerBase { void PushObject(const Object& object); void CompareObject(Register reg, const Object& object); - void LoadDoubleConstant(XmmRegister dst, double value); void LoadCompressed(Register dest, const Address& slot) { movl(dest, slot); } diff --git a/runtime/vm/compiler/assembler/assembler_ia32_test.cc b/runtime/vm/compiler/assembler/assembler_ia32_test.cc index 43c481880e1..e89073d497f 100644 --- a/runtime/vm/compiler/assembler/assembler_ia32_test.cc +++ b/runtime/vm/compiler/assembler/assembler_ia32_test.cc @@ -4558,8 +4558,8 @@ ASSEMBLER_TEST_RUN(ConditionalMovesCompare, test) { "ret\n"); } -ASSEMBLER_TEST_GENERATE(TestLoadDoubleConstant, assembler) { - __ LoadDoubleConstant(XMM3, -12.34); +ASSEMBLER_TEST_GENERATE(TestLoadDImmediate, assembler) { + __ LoadDImmediate(XMM3, -12.34); __ pushl(EAX); __ pushl(EAX); __ movsd(Address(ESP, 0), XMM3); @@ -4569,9 +4569,9 @@ ASSEMBLER_TEST_GENERATE(TestLoadDoubleConstant, assembler) { __ ret(); } -ASSEMBLER_TEST_RUN(TestLoadDoubleConstant, test) { - typedef double (*TestLoadDoubleConstantCode)(); - double res = reinterpret_cast(test->entry())(); +ASSEMBLER_TEST_RUN(TestLoadDImmediate, test) { + typedef double (*TestLoadDImmediateCode)(); + double res = reinterpret_cast(test->entry())(); EXPECT_FLOAT_EQ(-12.34, res, 0.0001); EXPECT_DISASSEMBLY( "push 0xc028ae14\n" diff --git a/runtime/vm/compiler/backend/flow_graph_compiler.cc b/runtime/vm/compiler/backend/flow_graph_compiler.cc index 08da16e6b4a..3ec06a6d0b4 100644 --- a/runtime/vm/compiler/backend/flow_graph_compiler.cc +++ b/runtime/vm/compiler/backend/flow_graph_compiler.cc @@ -1796,9 +1796,6 @@ void FlowGraphCompiler::AllocateRegistersLocally(Instruction* instr) { result_location = Location::FpuRegisterLocation( AllocateFreeFpuRegister(blocked_fpu_registers)); break; - case Location::kRequiresStackSlot: - UNREACHABLE(); - break; } locs->set_out(0, result_location); } diff --git a/runtime/vm/compiler/backend/il.cc b/runtime/vm/compiler/backend/il.cc index f9e6f0ee67f..30f6bec8400 100644 --- a/runtime/vm/compiler/backend/il.cc +++ b/runtime/vm/compiler/backend/il.cc @@ -6494,26 +6494,21 @@ Representation FfiCallInstr::RequiredInputRepresentation(intptr_t idx) const { LocationSummary* FfiCallInstr::MakeLocationSummaryInternal( Zone* zone, bool is_optimizing, - const Register temp) const { - // The temporary register needs to be callee-saved and not an argument - // register. - ASSERT(((1 << CallingConventions::kFfiAnyNonAbiRegister) & - CallingConventions::kArgumentRegisters) == 0); + const RegList temps) const { + auto contains_call = + is_leaf_ ? LocationSummary::kNativeLeafCall : LocationSummary::kCall; - // TODO(dartbug.com/45468): Investigate whether we can avoid spilling - // registers across ffi leaf calls by not using `kCall` here. LocationSummary* summary = new (zone) LocationSummary( zone, /*num_inputs=*/InputCount(), - /*num_temps=*/temp == kNoRegister ? 2 : 3, LocationSummary::kCall); + /*num_temps=*/Utils::CountOneBitsWord(temps), contains_call); - const Register temp0 = CallingConventions::kFfiAnyNonAbiRegister; - const Register temp1 = CallingConventions::kSecondNonArgumentRegister; - ASSERT(temp0 != temp1); - summary->set_temp(0, Location::RegisterLocation(temp0)); - summary->set_temp(1, Location::RegisterLocation(temp1)); - - if (temp != kNoRegister) { - summary->set_temp(2, Location::RegisterLocation(temp)); + intptr_t reg_i = 0; + for (intptr_t reg = 0; reg < kNumberOfCpuRegisters; reg++) { + if ((temps & (1 << reg)) != 0) { + summary->set_temp(reg_i, + Location::RegisterLocation(static_cast(reg))); + reg_i++; + } } summary->set_in(TargetAddressIndex(), @@ -6524,10 +6519,7 @@ LocationSummary* FfiCallInstr::MakeLocationSummaryInternal( } if (marshaller_.PassTypedData()) { - // The register allocator already preserves this value across the call on - // a stack slot, so we'll use the spilled value directly. - summary->set_in(TypedDataIndex(), Location::RequiresStackSlot()); - + summary->set_in(TypedDataIndex(), Location::Any()); // We don't care about return location, but we need to pass a register. summary->set_out( 0, Location::RegisterLocation(CallingConventions::kReturnReg)); @@ -6654,14 +6646,14 @@ void FfiCallInstr::EmitParamMoves(FlowGraphCompiler* compiler, void FfiCallInstr::EmitReturnMoves(FlowGraphCompiler* compiler, const Register temp0, const Register temp1) { - __ Comment("EmitReturnMoves"); - const auto& returnLocation = marshaller_.Location(compiler::ffi::kResultIndex); if (returnLocation.payload_type().IsVoid()) { return; } + __ Comment("EmitReturnMoves"); + NoTemporaryAllocator no_temp; if (returnLocation.IsRegisters() || returnLocation.IsFpuRegisters()) { const auto& src = returnLocation; @@ -6675,15 +6667,20 @@ void FfiCallInstr::EmitReturnMoves(FlowGraphCompiler* compiler, // Get the typed data pointer which we have pinned to a stack slot. const Location typed_data_loc = locs()->in(TypedDataIndex()); - ASSERT(typed_data_loc.IsStackSlot()); - ASSERT(typed_data_loc.base_reg() == FPREG); - // If this is a leaf call there is no extra call frame to step through. - if (is_leaf_) { - __ LoadMemoryValue(temp0, FPREG, typed_data_loc.ToStackSlotOffset()); + if (typed_data_loc.IsStackSlot()) { + ASSERT(typed_data_loc.base_reg() == FPREG); + // If this is a leaf call there is no extra call frame to step through. + if (is_leaf_) { + __ LoadMemoryValue(temp0, FPREG, typed_data_loc.ToStackSlotOffset()); + } else { + __ LoadMemoryValue( + temp0, FPREG, + kSavedCallerFpSlotFromFp * compiler::target::kWordSize); + __ LoadMemoryValue(temp0, temp0, typed_data_loc.ToStackSlotOffset()); + } } else { - __ LoadMemoryValue( - temp0, FPREG, kSavedCallerFpSlotFromFp * compiler::target::kWordSize); - __ LoadMemoryValue(temp0, temp0, typed_data_loc.ToStackSlotOffset()); + compiler->EmitMove(Location::RegisterLocation(temp0), typed_data_loc, + &no_temp); } __ LoadField(temp0, compiler::FieldAddress( diff --git a/runtime/vm/compiler/backend/il.h b/runtime/vm/compiler/backend/il.h index 65a9afbaa56..a7dd7aa2cda 100644 --- a/runtime/vm/compiler/backend/il.h +++ b/runtime/vm/compiler/backend/il.h @@ -5305,7 +5305,7 @@ class FfiCallInstr : public Definition { LocationSummary* MakeLocationSummaryInternal(Zone* zone, bool is_optimizing, - const Register temp) const; + const RegList temps) const; // Clobbers both given registers. // `saved_fp` is used as the frame base to rebase off of. diff --git a/runtime/vm/compiler/backend/il_arm.cc b/runtime/vm/compiler/backend/il_arm.cc index 411d58b5ed6..3dab29ace0e 100644 --- a/runtime/vm/compiler/backend/il_arm.cc +++ b/runtime/vm/compiler/backend/il_arm.cc @@ -1402,27 +1402,34 @@ void NativeCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { __ Drop(ArgumentCount()); // Drop the arguments. } +#define R(r) (1 << r) + LocationSummary* FfiCallInstr::MakeLocationSummary(Zone* zone, bool is_optimizing) const { - return MakeLocationSummaryInternal(zone, is_optimizing, R0); + return MakeLocationSummaryInternal( + zone, is_optimizing, + (R(R0) | R(CallingConventions::kFfiAnyNonAbiRegister) | + R(CallingConventions::kSecondNonArgumentRegister))); } +#undef R + void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { + const Register branch = locs()->in(TargetAddressIndex()).reg(); + + // The temps are indexed according to their register number. + const Register temp2 = locs()->temp(0).reg(); // For regular calls, this holds the FP for rebasing the original locations // during EmitParamMoves. // For leaf calls, this holds the SP used to restore the pre-aligned SP after // the call. - const Register saved_fp_or_sp = locs()->temp(0).reg(); - RELEASE_ASSERT((CallingConventions::kCalleeSaveCpuRegisters & - (1 << saved_fp_or_sp)) != 0); - const Register temp1 = locs()->temp(1).reg(); - const Register temp2 = locs()->temp(2).reg(); - const Register branch = locs()->in(TargetAddressIndex()).reg(); + const Register saved_fp_or_sp = locs()->temp(1).reg(); + const Register temp1 = locs()->temp(2).reg(); // Ensure these are callee-saved register and are preserved across the call. ASSERT((CallingConventions::kCalleeSaveCpuRegisters & (1 << saved_fp_or_sp)) != 0); - // temp doesn't need to be preserved. + // Other temps don't need to be preserved. __ mov(saved_fp_or_sp, is_leaf_ ? compiler::Operand(SPREG) : compiler::Operand(FPREG)); diff --git a/runtime/vm/compiler/backend/il_arm64.cc b/runtime/vm/compiler/backend/il_arm64.cc index 410cb0e127d..7ba86030160 100644 --- a/runtime/vm/compiler/backend/il_arm64.cc +++ b/runtime/vm/compiler/backend/il_arm64.cc @@ -1227,27 +1227,36 @@ void NativeCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { __ Drop(ArgumentCount()); // Drop the arguments. } +#define R(r) (1 << r) + LocationSummary* FfiCallInstr::MakeLocationSummary(Zone* zone, bool is_optimizing) const { - return MakeLocationSummaryInternal(zone, is_optimizing, R11); + return MakeLocationSummaryInternal( + zone, is_optimizing, + (R(CallingConventions::kSecondNonArgumentRegister) | R(R11) | + R(CallingConventions::kFfiAnyNonAbiRegister) | R(R25))); } +#undef R + void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { + const Register branch = locs()->in(TargetAddressIndex()).reg(); + + // The temps are indexed according to their register number. + const Register temp1 = locs()->temp(0).reg(); + const Register temp2 = locs()->temp(1).reg(); // For regular calls, this holds the FP for rebasing the original locations // during EmitParamMoves. // For leaf calls, this holds the SP used to restore the pre-aligned SP after // the call. - const Register saved_fp_or_sp = locs()->temp(0).reg(); - RELEASE_ASSERT((CallingConventions::kCalleeSaveCpuRegisters & - (1 << saved_fp_or_sp)) != 0); - const Register temp1 = locs()->temp(1).reg(); - const Register temp2 = locs()->temp(2).reg(); - const Register branch = locs()->in(TargetAddressIndex()).reg(); + const Register saved_fp_or_sp = locs()->temp(2).reg(); + const Register temp_csp = locs()->temp(3).reg(); // Ensure these are callee-saved register and are preserved across the call. ASSERT((CallingConventions::kCalleeSaveCpuRegisters & (1 << saved_fp_or_sp)) != 0); - // temps don't need to be preserved. + ASSERT((CallingConventions::kCalleeSaveCpuRegisters & (1 << temp_csp)) != 0); + // Other temps don't need to be preserved. __ mov(saved_fp_or_sp, is_leaf_ ? SPREG : FPREG); @@ -1279,14 +1288,14 @@ void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { // We are entering runtime code, so the C stack pointer must be restored // from the stack limit to the top of the stack. - __ mov(R25, CSP); + __ mov(temp_csp, CSP); __ mov(CSP, SP); __ blr(branch); // Restore the Dart stack pointer. __ mov(SP, CSP); - __ mov(CSP, R25); + __ mov(CSP, temp_csp); #if !defined(PRODUCT) __ LoadImmediate(temp1, compiler::target::Thread::vm_tag_dart_id()); @@ -1315,14 +1324,14 @@ void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { // We are entering runtime code, so the C stack pointer must be restored // from the stack limit to the top of the stack. - __ mov(R25, CSP); + __ mov(temp_csp, CSP); __ mov(CSP, SP); __ blr(branch); // Restore the Dart stack pointer. __ mov(SP, CSP); - __ mov(CSP, R25); + __ mov(CSP, temp_csp); // Update information in the thread object and leave the safepoint. __ TransitionNativeToGenerated(temp1, /*leave_safepoint=*/true); diff --git a/runtime/vm/compiler/backend/il_ia32.cc b/runtime/vm/compiler/backend/il_ia32.cc index ce4c3688701..d6086370e94 100644 --- a/runtime/vm/compiler/backend/il_ia32.cc +++ b/runtime/vm/compiler/backend/il_ia32.cc @@ -998,24 +998,33 @@ void NativeCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { __ Drop(ArgumentCount()); // Drop the arguments. } +#define R(r) (1 << r) + LocationSummary* FfiCallInstr::MakeLocationSummary(Zone* zone, bool is_optimizing) const { - return MakeLocationSummaryInternal(zone, is_optimizing, kNoRegister); + return MakeLocationSummaryInternal( + zone, is_optimizing, + (R(CallingConventions::kSecondNonArgumentRegister) | + R(CallingConventions::kFfiAnyNonAbiRegister))); } +#undef R + void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { + const Register branch = locs()->in(TargetAddressIndex()).reg(); + + // The temps are indexed according to their register number. + const Register temp = locs()->temp(0).reg(); // For regular calls, this holds the FP for rebasing the original locations // during EmitParamMoves. // For leaf calls, this holds the SP used to restore the pre-aligned SP after // the call. - const Register saved_fp_or_sp = locs()->temp(0).reg(); - const Register temp = locs()->temp(1).reg(); - const Register branch = locs()->in(TargetAddressIndex()).reg(); + const Register saved_fp_or_sp = locs()->temp(1).reg(); // Ensure these are callee-saved register and are preserved across the call. ASSERT((CallingConventions::kCalleeSaveCpuRegisters & (1 << saved_fp_or_sp)) != 0); - // temp doesn't need to be preserved. + // Other temps don't need to be preserved. __ movl(saved_fp_or_sp, is_leaf_ ? SPREG : FPREG); diff --git a/runtime/vm/compiler/backend/il_riscv.cc b/runtime/vm/compiler/backend/il_riscv.cc index 9068abbc383..9f576c937e8 100644 --- a/runtime/vm/compiler/backend/il_riscv.cc +++ b/runtime/vm/compiler/backend/il_riscv.cc @@ -1374,10 +1374,14 @@ void NativeCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { __ Drop(ArgumentCount() + 1); // Drop the arguments and result. } +#define R(r) (1 << r) + LocationSummary* FfiCallInstr::MakeLocationSummary(Zone* zone, bool is_optimizing) const { - LocationSummary* summary = - MakeLocationSummaryInternal(zone, is_optimizing, CALLEE_SAVED_TEMP2); + LocationSummary* summary = MakeLocationSummaryInternal( + zone, is_optimizing, + (R(CallingConventions::kSecondNonArgumentRegister) | + R(CallingConventions::kFfiAnyNonAbiRegister) | R(CALLEE_SAVED_TEMP2))); // A3/A4/A5 are blocked during Dart register allocation because they are // assigned to TMP/TMP2/PP. This assignment is important for reducing code // size. To work around this for FFI calls, the FFI argument definitions are @@ -1400,17 +1404,20 @@ LocationSummary* FfiCallInstr::MakeLocationSummary(Zone* zone, return summary; } +#undef R + void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { + const Register target = locs()->in(TargetAddressIndex()).reg(); + + // The temps are indexed according to their register number. + const Register temp1 = locs()->temp(0).reg(); // For regular calls, this holds the FP for rebasing the original locations // during EmitParamMoves. // For leaf calls, this holds the SP used to restore the pre-aligned SP after // the call. - const Register saved_fp_or_sp = locs()->temp(0).reg(); - RELEASE_ASSERT((CallingConventions::kCalleeSaveCpuRegisters & - (1 << saved_fp_or_sp)) != 0); - const Register temp1 = locs()->temp(1).reg(); + const Register saved_fp_or_sp = locs()->temp(1).reg(); const Register temp2 = locs()->temp(2).reg(); - const Register target = locs()->in(TargetAddressIndex()).reg(); + ASSERT(temp1 != target); ASSERT(temp2 != target); ASSERT(temp1 != saved_fp_or_sp); @@ -1420,7 +1427,7 @@ void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { // Ensure these are callee-saved register and are preserved across the call. ASSERT((CallingConventions::kCalleeSaveCpuRegisters & (1 << saved_fp_or_sp)) != 0); - // temps don't need to be preserved. + // Other temps don't need to be preserved. __ mv(saved_fp_or_sp, is_leaf_ ? SPREG : FPREG); diff --git a/runtime/vm/compiler/backend/il_test.cc b/runtime/vm/compiler/backend/il_test.cc index a03d0e86174..95f8ac0b470 100644 --- a/runtime/vm/compiler/backend/il_test.cc +++ b/runtime/vm/compiler/backend/il_test.cc @@ -952,6 +952,227 @@ ISOLATE_UNIT_TEST_CASE(IRTest_LoadThread) { EXPECT_EQ(reinterpret_cast(thread), result_int); } +// Helper to set up an inlined FfiCall by replacing a StaticCall. +FlowGraph* SetupFfiFlowgraph(TestPipeline* pipeline, + Zone* zone, + const compiler::ffi::CallMarshaller& marshaller, + uword native_entry, + bool is_leaf) { + FlowGraph* flow_graph = pipeline->RunPasses({CompilerPass::kComputeSSA}); + + // Make an FfiCall based on ffi_trampoline that calls our native function. + auto ffi_call = new FfiCallInstr(zone, DeoptId::kNone, marshaller, is_leaf); + RELEASE_ASSERT(ffi_call->InputCount() == 1); + // TargetAddress is the function pointer called. + const Representation address_repr = + compiler::target::kWordSize == 4 ? kUnboxedUint32 : kUnboxedInt64; + ffi_call->SetInputAt( + ffi_call->TargetAddressIndex(), + new Value(flow_graph->GetConstant( + Integer::Handle(Integer::NewCanonical(native_entry)), address_repr))); + + // Replace the placeholder StaticCall with an FfiCall to our native function. + { + StaticCallInstr* static_call = nullptr; + { + ILMatcher cursor(flow_graph, flow_graph->graph_entry()->normal_entry(), + /*trace=*/false); + cursor.TryMatch({kMoveGlob, {kMatchStaticCall, &static_call}}); + } + RELEASE_ASSERT(static_call != nullptr); + + flow_graph->InsertBefore(static_call, ffi_call, /*env=*/nullptr, + FlowGraph::kEffect); + static_call->RemoveFromGraph(/*return_previous=*/false); + } + + // Run remaining relevant compiler passes. + pipeline->RunAdditionalPasses({ + CompilerPass::kApplyICData, + CompilerPass::kTryOptimizePatterns, + CompilerPass::kSetOuterInliningId, + CompilerPass::kTypePropagation, + // Skipping passes that don't seem to do anything for this test. + CompilerPass::kWidenSmiToInt32, + CompilerPass::kSelectRepresentations, + // Skipping passes that don't seem to do anything for this test. + CompilerPass::kTypePropagation, + CompilerPass::kRangeAnalysis, + // Skipping passes that don't seem to do anything for this test. + CompilerPass::kFinalizeGraph, + CompilerPass::kCanonicalize, + CompilerPass::kAllocateRegisters, + CompilerPass::kReorderBlocks, + }); + + return flow_graph; +} + +// Test that FFI calls spill all live values to the stack, and that FFI leaf +// calls are free to use available ABI callee-save registers to avoid spilling. +// Additionally test that register allocation is done correctly by clobbering +// all volatile registers in the native function being called. +ISOLATE_UNIT_TEST_CASE(IRTest_FfiCallInstrLeafDoesntSpill) { + SetFlagScope sfs(&FLAG_sound_null_safety, kNullSafetyOptionStrong); + + const char* kScript = R"( + import 'dart:ffi'; + + // This is purely a placeholder and is never called. + void placeholder() {} + + // Will call the "doFfiCall" and exercise its code. + bool invokeDoFfiCall() { + final double result = doFfiCall(1, 2, 3, 1.0, 2.0, 3.0); + if (result != (2 + 3 + 4 + 2.0 + 3.0 + 4.0)) { + throw 'Failed. Result was $result.'; + } + return true; + } + + // Will perform a "C" call while having live values in registers + // across the FfiCall. + double doFfiCall(int a, int b, int c, double x, double y, double z) { + // Ensure there is at least one live value in a register. + a += 1; + b += 1; + c += 1; + x += 1.0; + y += 1.0; + z += 1.0; + // We'll replace this StaticCall with an FfiCall. + placeholder(); + // Use the live value. + return (a + b + c + x + y + z); + } + + // FFI trampoline function. + typedef NT = Void Function(); + typedef DT = void Function(); + Pointer> ptr = Pointer.fromAddress(0); + DT getFfiTrampolineClosure() => ptr.asFunction(isLeaf:true); + )"; + + const auto& root_library = Library::Handle(LoadTestScript(kScript)); + + // Build a "C" function that we can actually invoke. + auto& c_function = Instructions::Handle( + BuildInstructions([](compiler::Assembler* assembler) { + // Clobber all volatile registers to make sure caller doesn't rely on + // any non-callee-save register. + for (intptr_t reg = 0; reg < kNumberOfFpuRegisters; reg++) { + if ((kAbiVolatileFpuRegs & (1 << reg)) != 0) { +#if defined(TARGET_ARCH_ARM) + // On ARM we need an extra scratch register for LoadDImmediate. + assembler->LoadDImmediate(static_cast(reg), 0.0, R3); +#else + assembler->LoadDImmediate(static_cast(reg), 0.0); +#endif + } + } + for (intptr_t reg = 0; reg < kNumberOfCpuRegisters; reg++) { + if ((kDartVolatileCpuRegs & (1 << reg)) != 0) { + assembler->LoadImmediate(static_cast(reg), 0xDEADBEEF); + } + } + assembler->Ret(); + })); + uword native_entry = c_function.EntryPoint(); + + // Get initial compilation done. + Invoke(root_library, "invokeDoFfiCall"); + + const Function& do_ffi_call = + Function::Handle(GetFunction(root_library, "doFfiCall")); + RELEASE_ASSERT(!do_ffi_call.IsNull()); + + const auto& value = Closure::Handle( + Closure::RawCast(Invoke(root_library, "getFfiTrampolineClosure"))); + RELEASE_ASSERT(value.IsClosure()); + const auto& ffi_trampoline = + Function::ZoneHandle(Closure::Cast(value).function()); + RELEASE_ASSERT(!ffi_trampoline.IsNull()); + + // Construct the FFICallInstr from the trampoline matching our native + // function. + const char* error = nullptr; + const auto marshaller_ptr = compiler::ffi::CallMarshaller::FromFunction( + thread->zone(), ffi_trampoline, &error); + RELEASE_ASSERT(error == nullptr); + RELEASE_ASSERT(marshaller_ptr != nullptr); + const auto& marshaller = *marshaller_ptr; + + const auto& compile_and_run = + [&](bool is_leaf, std::function verify) { + // Build the SSA graph for "doFfiCall" + TestPipeline pipeline(do_ffi_call, CompilerPass::kJIT); + FlowGraph* flow_graph = SetupFfiFlowgraph( + &pipeline, thread->zone(), marshaller, native_entry, is_leaf); + + { + ParallelMoveInstr* parallel_move = nullptr; + ILMatcher cursor(flow_graph, + flow_graph->graph_entry()->normal_entry(), + /*trace=*/false); + while (cursor.TryMatch( + {kMoveGlob, {kMatchAndMoveParallelMove, ¶llel_move}})) { + verify(parallel_move); + } + } + + // Finish the compilation and attach code so we can run it. + pipeline.CompileGraphAndAttachFunction(); + + // Ensure we can successfully invoke the FFI call. + auto& result = Object::Handle(Invoke(root_library, "invokeDoFfiCall")); + RELEASE_ASSERT(result.IsBool()); + EXPECT(Bool::Cast(result).value()); + }; + + intptr_t num_cpu_reg_to_stack_nonleaf = 0; + intptr_t num_cpu_reg_to_stack_leaf = 0; + intptr_t num_fpu_reg_to_stack_nonleaf = 0; + intptr_t num_fpu_reg_to_stack_leaf = 0; + + // Test non-leaf spills live values. + compile_and_run(/*is_leaf=*/false, [&](ParallelMoveInstr* parallel_move) { + // TargetAddress is passed in register, live values are all spilled. + for (int i = 0; i < parallel_move->NumMoves(); i++) { + auto move = parallel_move->moves()[i]; + if (move->src_slot()->IsRegister() && move->dest_slot()->IsStackSlot()) { + num_cpu_reg_to_stack_nonleaf++; + } else if (move->src_slot()->IsFpuRegister() && + move->dest_slot()->IsDoubleStackSlot()) { + num_fpu_reg_to_stack_nonleaf++; + } + } + }); + + // Test leaf calls do not cause spills of live values. + compile_and_run(/*is_leaf=*/true, [&](ParallelMoveInstr* parallel_move) { + // TargetAddress is passed in registers, live values are not spilled and + // remains in callee-save registers. + for (int i = 0; i < parallel_move->NumMoves(); i++) { + auto move = parallel_move->moves()[i]; + if (move->src_slot()->IsRegister() && move->dest_slot()->IsStackSlot()) { + num_cpu_reg_to_stack_leaf++; + } else if (move->src_slot()->IsFpuRegister() && + move->dest_slot()->IsDoubleStackSlot()) { + num_fpu_reg_to_stack_leaf++; + } + } + }); + + // We should have less moves to the stack (i.e. spilling) in leaf calls. + EXPECT_LT(num_cpu_reg_to_stack_leaf, num_cpu_reg_to_stack_nonleaf); + // We don't have volatile FPU registers on all platforms. + const bool has_callee_save_fpu_regs = + Utils::CountOneBitsWord(kAbiVolatileFpuRegs) < + Utils::CountOneBitsWord(kAllFpuRegistersList); + EXPECT(!has_callee_save_fpu_regs || + num_fpu_reg_to_stack_leaf < num_fpu_reg_to_stack_nonleaf); +} + static void TestConstantFoldToSmi(const Library& root_library, const char* function_name, CompilerPass::PipelineMode mode, diff --git a/runtime/vm/compiler/backend/il_x64.cc b/runtime/vm/compiler/backend/il_x64.cc index 33b66eea300..9a12a656b28 100644 --- a/runtime/vm/compiler/backend/il_x64.cc +++ b/runtime/vm/compiler/backend/il_x64.cc @@ -1175,29 +1175,38 @@ void NativeCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { __ Drop(ArgumentCount()); // Drop the arguments. } +#define R(r) (1 << r) + LocationSummary* FfiCallInstr::MakeLocationSummary(Zone* zone, bool is_optimizing) const { - // Use R10 as a temp register. We can't use RDI, RSI, RDX, R8, R9 as they are - // arg registers, and R11 is TMP. - return MakeLocationSummaryInternal(zone, is_optimizing, R10); + // Use R10 as a temp. register. We can't use RDI, RSI, RDX, R8, R9 as they are + // argument registers, and R11 is TMP. + return MakeLocationSummaryInternal( + zone, is_optimizing, + (R(CallingConventions::kSecondNonArgumentRegister) | R(R10) | + R(CallingConventions::kFfiAnyNonAbiRegister))); } +#undef R + void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { + const Register target_address = locs()->in(TargetAddressIndex()).reg(); + + // The temps are indexed according to their register number. + // For regular calls, this holds the FP for rebasing the original locations + // during EmitParamMoves. + const Register saved_fp = locs()->temp(0).reg(); + const Register temp = locs()->temp(1).reg(); // For leaf calls, this holds the SP used to restore the pre-aligned SP after // the call. // Note: R12 doubles as CODE_REG, which gets clobbered during frame setup in // regular calls. - const Register saved_sp = locs()->temp(0).reg(); - // For regular calls, this holds the FP for rebasing the original locations - // during EmitParamMoves. - const Register saved_fp = locs()->temp(1).reg(); - const Register temp = locs()->temp(2).reg(); - const Register target_address = locs()->in(TargetAddressIndex()).reg(); + const Register saved_sp = locs()->temp(2).reg(); // Ensure these are callee-saved register and are preserved across the call. ASSERT((CallingConventions::kCalleeSaveCpuRegisters & (1 << saved_sp)) != 0); ASSERT((CallingConventions::kCalleeSaveCpuRegisters & (1 << saved_fp)) != 0); - // temp doesn't need to be preserved. + // Other temps don't need to be preserved. if (is_leaf_) { __ movq(saved_sp, SPREG); diff --git a/runtime/vm/compiler/backend/linearscan.cc b/runtime/vm/compiler/backend/linearscan.cc index 19191d1542e..266b48541a0 100644 --- a/runtime/vm/compiler/backend/linearscan.cc +++ b/runtime/vm/compiler/backend/linearscan.cc @@ -465,6 +465,28 @@ void FlowGraphAllocator::BlockLocation(Location loc, } } +void FlowGraphAllocator::BlockCpuRegisters(intptr_t registers, + intptr_t from, + intptr_t to) { + for (intptr_t r = 0; r < kNumberOfCpuRegisters; r++) { + if ((registers & (1 << r)) != 0) { + BlockLocation(Location::RegisterLocation(static_cast(r)), from, + to); + } + } +} + +void FlowGraphAllocator::BlockFpuRegisters(intptr_t fpu_registers, + intptr_t from, + intptr_t to) { + for (intptr_t r = 0; r < kNumberOfFpuRegisters; r++) { + if ((fpu_registers & (1 << r)) != 0) { + BlockLocation(Location::FpuRegisterLocation(static_cast(r)), + from, to); + } + } +} + void LiveRange::Print() { if (first_use_interval() == NULL) { return; @@ -1389,6 +1411,21 @@ void FlowGraphAllocator::ProcessOneInstruction(BlockEntryInstr* block, } } + // Block all volatile (i.e. not native ABI callee-save) registers. + if (locs->native_leaf_call()) { + BlockCpuRegisters(kDartVolatileCpuRegs, pos, pos + 1); + BlockFpuRegisters(kAbiVolatileFpuRegs, pos, pos + 1); +#if defined(TARGET_ARCH_ARM) + // We do not yet have a way to say that we only want FPU registers that + // overlap S registers. + // Block all Q/D FPU registers above the 8/16 that have S registers in + // VFPv3-D32. + // This way we avoid ending up trying to do single-word operations on + // registers that don't support it. + BlockFpuRegisters(kFpuRegistersWithoutSOverlap, pos, pos + 1); +#endif + } + // Block all allocatable registers for calls. if (locs->always_calls() && !locs->callee_safe_call()) { // Expected shape of live range: @@ -1397,16 +1434,9 @@ void FlowGraphAllocator::ProcessOneInstruction(BlockEntryInstr* block, // [--) // // The stack bitmap describes the position i. - for (intptr_t reg = 0; reg < kNumberOfCpuRegisters; reg++) { - BlockLocation(Location::RegisterLocation(static_cast(reg)), pos, - pos + 1); - } + BlockCpuRegisters(kAllCpuRegistersList, pos, pos + 1); - for (intptr_t reg = 0; reg < kNumberOfFpuRegisters; reg++) { - BlockLocation( - Location::FpuRegisterLocation(static_cast(reg)), pos, - pos + 1); - } + BlockFpuRegisters(kAllFpuRegistersList, pos, pos + 1); #if defined(DEBUG) // Verify that temps, inputs and output were specified as fixed @@ -1426,8 +1456,7 @@ void FlowGraphAllocator::ProcessOneInstruction(BlockEntryInstr* block, pair->At(1).policy() == Location::kAny); } else { ASSERT(!locs->in(j).IsUnallocated() || - locs->in(j).policy() == Location::kAny || - locs->in(j).policy() == Location::kRequiresStackSlot); + locs->in(j).policy() == Location::kAny); } } @@ -1441,7 +1470,7 @@ void FlowGraphAllocator::ProcessOneInstruction(BlockEntryInstr* block, #endif } - if (locs->can_call()) { + if (locs->can_call() && !locs->native_leaf_call()) { safepoints_.Add(current); } diff --git a/runtime/vm/compiler/backend/linearscan.h b/runtime/vm/compiler/backend/linearscan.h index 2d6e91965b8..bc26c637767 100644 --- a/runtime/vm/compiler/backend/linearscan.h +++ b/runtime/vm/compiler/backend/linearscan.h @@ -156,6 +156,10 @@ class FlowGraphAllocator : public ValueObject { bool* blocked_registers, LiveRange** blocking_ranges); + void BlockCpuRegisters(intptr_t registers, intptr_t from, intptr_t to); + + void BlockFpuRegisters(intptr_t fpu_registers, intptr_t from, intptr_t to); + intptr_t NumberOfRegisters() const { return number_of_registers_; } // Find all safepoints that are covered by this live range. diff --git a/runtime/vm/compiler/backend/locations.cc b/runtime/vm/compiler/backend/locations.cc index 56640f80152..9b5b1664753 100644 --- a/runtime/vm/compiler/backend/locations.cc +++ b/runtime/vm/compiler/backend/locations.cc @@ -185,8 +185,7 @@ void LocationSummary::set_in(intptr_t index, Location loc) { // restrictions. if (always_calls()) { if (loc.IsUnallocated()) { - ASSERT(loc.policy() == Location::kAny || - loc.policy() == Location::kRequiresStackSlot); + ASSERT(loc.policy() == Location::kAny); } else if (loc.IsPairLocation()) { ASSERT(!loc.AsPairLocation()->At(0).IsUnallocated() || loc.AsPairLocation()->At(0).policy() == Location::kAny); @@ -311,8 +310,6 @@ const char* Location::Name() const { return "R"; case kRequiresFpuRegister: return "DR"; - case kRequiresStackSlot: - return "RS"; case kWritableRegister: return "WR"; case kSameAsFirstInput: diff --git a/runtime/vm/compiler/backend/locations.h b/runtime/vm/compiler/backend/locations.h index b9988948a26..754852e0ebf 100644 --- a/runtime/vm/compiler/backend/locations.h +++ b/runtime/vm/compiler/backend/locations.h @@ -257,7 +257,6 @@ class Location : public ValueObject { kPrefersRegister, kRequiresRegister, kRequiresFpuRegister, - kRequiresStackSlot, kWritableRegister, kSameAsFirstInput, }; @@ -285,10 +284,6 @@ class Location : public ValueObject { return UnallocatedLocation(kRequiresFpuRegister); } - static Location RequiresStackSlot() { - return UnallocatedLocation(kRequiresStackSlot); - } - static Location WritableRegister() { return UnallocatedLocation(kWritableRegister); } @@ -719,12 +714,19 @@ class RegisterSet : public ValueObject { class LocationSummary : public ZoneAllocated { public: enum ContainsCall { - kNoCall, // Used registers must be reserved as tmp. - kCall, // Registers have been saved and can be used without reservation. - kCallCalleeSafe, // Registers will be saved by the callee. - kCallOnSlowPath, // Used registers must be reserved as tmp. - kCallOnSharedSlowPath // Registers used to invoke shared stub must be - // reserved as tmp. + // Used registers must be reserved as tmp. + kNoCall, + // Registers have been saved and can be used without reservation. + kCall, + // Registers will be saved by the callee. + kCallCalleeSafe, + // Used registers must be reserved as tmp. + kCallOnSlowPath, + // Registers used to invoke shared stub must be reserved as tmp. + kCallOnSharedSlowPath, + // Location is a native leaf call so any register not in the native ABI + // callee-save (or input/output/tmp) set might get clobbered. + kNativeLeafCall }; LocationSummary(Zone* zone, @@ -800,6 +802,8 @@ class LocationSummary : public ZoneAllocated { return contains_call_ == kCallOnSharedSlowPath; } + bool native_leaf_call() const { return contains_call_ == kNativeLeafCall; } + void PrintTo(BaseTextBuffer* f) const; static LocationSummary* Make(Zone* zone, diff --git a/runtime/vm/constants_arm.h b/runtime/vm/constants_arm.h index 7b438127207..7406b47a00e 100644 --- a/runtime/vm/constants_arm.h +++ b/runtime/vm/constants_arm.h @@ -30,6 +30,8 @@ namespace dart { // to compare LR register code. #define LR LR_DO_NOT_USE_DIRECTLY +#define R(reg) (static_cast(1) << (reg)) + // We support both VFPv3-D16 and VFPv3-D32 profiles, but currently only one at // a time. #if defined(__ARM_ARCH_7A__) @@ -209,6 +211,9 @@ enum DRegister { D31 = 31, kNumberOfDRegisters = 32, #endif + // Number of D registers that overlap S registers. + // One D register overlaps two S registers, so regardless of the numbers of D + // registers, there are only 32 S registers that are overlapped. kNumberOfOverlappingDRegisters = 16, }; @@ -243,6 +248,10 @@ enum QRegister { Q15 = 15, kNumberOfQRegisters = 16, #endif + // Number of Q registers that overlap S registers. + // One Q register overlaps four S registers, so regardless of the numbers of Q + // registers, there are only 32 S registers that are overlapped. + kNumberOfOverlappingQRegisters = 8, }; static inline DRegister EvenDRegisterOf(QRegister q) { @@ -256,7 +265,7 @@ static inline DRegister OddDRegisterOf(QRegister q) { static inline SRegister EvenSRegisterOf(DRegister d) { #if defined(VFPv3_D32) // When we have 32 D registers, the S registers only overlap the first 16. - // That is, there are only 32 S registers. + // That is, there are only ever 32 S registers in any extension. ASSERT(d < D16); #endif return static_cast(d * 2); @@ -535,6 +544,7 @@ struct DispatchTableNullErrorABI { // List of registers used in load/store multiple. typedef uint16_t RegList; const RegList kAllCpuRegistersList = 0xFFFF; +const RegList kAllFpuRegistersList = (1 << kNumberOfFpuRegisters) - 1; // C++ ABI call registers. const RegList kAbiArgumentCpuRegs = @@ -572,7 +582,11 @@ const int kDartVolatileCpuRegCount = 6; const int kDartVolatileCpuRegCount = 5; #endif -#define R(reg) (static_cast(1) << (reg)) +const RegList kAbiVolatileFpuRegs = R(Q0) | R(Q1) | R(Q2) | R(Q3); + +const RegList kFpuRegistersWithoutSOverlap = + kAllFpuRegistersList & + ~((1 << QRegister::kNumberOfOverlappingQRegisters) - 1); class CallingConventions { public: diff --git a/runtime/vm/constants_arm64.h b/runtime/vm/constants_arm64.h index a7f00d94b72..5be6e984825 100644 --- a/runtime/vm/constants_arm64.h +++ b/runtime/vm/constants_arm64.h @@ -81,7 +81,9 @@ enum Register { }; enum VRegister { + // v0 Volatile; Parameter/scratch register, result register. V0 = 0, + // v1-v7 Volatile; Parameter/scratch register. V1 = 1, V2 = 2, V3 = 3, @@ -89,6 +91,8 @@ enum VRegister { V5 = 5, V6 = 6, V7 = 7, + // v8-v15 Non-volatile; Scratch registers + // Only the bottom 64 bits are non-volatile! [ARM IHI 0055B, 5.1.2] V8 = 8, V9 = 9, V10 = 10, @@ -97,6 +101,7 @@ enum VRegister { V13 = 13, V14 = 14, V15 = 15, + // v16-v31 Volatile; Scratch registers. V16 = 16, V17 = 17, V18 = 18, @@ -104,7 +109,7 @@ enum VRegister { V20 = 20, V21 = 21, V22 = 22, - V23 = 24, + V23 = 23, V24 = 24, V25 = 25, V26 = 26, @@ -430,6 +435,11 @@ const Register kDartLastVolatileCpuReg = R14; const int kDartVolatileCpuRegCount = 15; const int kDartVolatileFpuRegCount = 24; +const RegList kAbiVolatileFpuRegs = + R(V0) | R(V1) | R(V2) | R(V3) | R(V4) | R(V5) | R(V6) | R(V7) | R(V16) | + R(V17) | R(V18) | R(V19) | R(V20) | R(V21) | R(V22) | R(V23) | R(V24) | + R(V25) | R(V26) | R(V27) | R(V28) | R(V29) | R(V30) | R(V31); + constexpr int kStoreBufferWrapperSize = 32; class CallingConventions { diff --git a/runtime/vm/constants_ia32.h b/runtime/vm/constants_ia32.h index 061771de24d..733dd8c39a2 100644 --- a/runtime/vm/constants_ia32.h +++ b/runtime/vm/constants_ia32.h @@ -16,6 +16,8 @@ namespace dart { +#define R(reg) (1 << (reg)) + enum Register { EAX = 0, ECX = 1, @@ -29,6 +31,8 @@ enum Register { kNoRegister = -1, // Signals an illegal register. }; +// Low and high bytes registers of the the first four general purpose registers. +// The other four general purpose registers do not have byte registers. enum ByteRegister { AL = 0, CL = 1, @@ -42,6 +46,10 @@ enum ByteRegister { }; inline ByteRegister ByteRegisterOf(Register reg) { + // This only works for EAX, ECX, EDX, EBX. + // Remaining Register values map to high byte of the above registers. + RELEASE_ASSERT(reg == Register::EAX || reg == Register::ECX || + reg == Register::EDX || reg == Register::EBX); return static_cast(reg); } @@ -263,12 +271,23 @@ struct DispatchTableNullErrorABI { typedef uint32_t RegList; const RegList kAllCpuRegistersList = 0xFF; +const RegList kAllFpuRegistersList = (1 << kNumberOfFpuRegisters) - 1; const intptr_t kReservedCpuRegisters = (1 << SPREG) | (1 << FPREG) | (1 << THR); // CPU registers available to Dart allocator. const RegList kDartAvailableCpuRegs = kAllCpuRegistersList & ~kReservedCpuRegisters; +const RegList kAbiPreservedCpuRegs = (1 << EDI) | (1 << ESI) | (1 << EBX); + +// Registers available to Dart that are not preserved by runtime calls. +const RegList kDartVolatileCpuRegs = + kDartAvailableCpuRegs & ~kAbiPreservedCpuRegs; + +const RegList kAbiVolatileFpuRegs = kAllFpuRegistersList; + +#undef R + enum ScaleFactor { TIMES_1 = 0, TIMES_2 = 1, @@ -332,8 +351,7 @@ class CallingConventions { static const intptr_t kXmmArgumentRegisters = 0; static const intptr_t kNumFpuArgRegs = 0; - static constexpr intptr_t kCalleeSaveCpuRegisters = - (1 << EDI) | (1 << ESI) | (1 << EBX); + static constexpr intptr_t kCalleeSaveCpuRegisters = kAbiPreservedCpuRegs; static const bool kArgumentIntRegXorFpuReg = false; diff --git a/runtime/vm/constants_riscv.h b/runtime/vm/constants_riscv.h index 30a720ff4b6..f8430df8071 100644 --- a/runtime/vm/constants_riscv.h +++ b/runtime/vm/constants_riscv.h @@ -387,6 +387,7 @@ struct DispatchTableNullErrorABI { typedef uint32_t RegList; const RegList kAllCpuRegistersList = 0xFFFFFFFF; +const RegList kAllFpuRegistersList = 0xFFFFFFFF; #define R(reg) (static_cast(1) << (reg)) diff --git a/runtime/vm/constants_x64.h b/runtime/vm/constants_x64.h index b8bc0e716d8..930f5889aa7 100644 --- a/runtime/vm/constants_x64.h +++ b/runtime/vm/constants_x64.h @@ -16,6 +16,8 @@ namespace dart { +#define R(reg) (static_cast(1) << (reg)) + enum Register { RAX = 0, RCX = 1, @@ -345,7 +347,7 @@ const RegList kAllCpuRegistersList = 0xFFFF; const RegList kAllFpuRegistersList = 0xFFFF; const RegList kReservedCpuRegisters = - (1 << SPREG) | (1 << FPREG) | (1 << TMP) | (1 << PP) | (1 << THR); + R(SPREG) | R(FPREG) | R(TMP) | R(PP) | R(THR); constexpr intptr_t kNumberOfReservedCpuRegisters = 5; // CPU registers available to Dart allocator. const RegList kDartAvailableCpuRegs = @@ -354,6 +356,20 @@ constexpr int kNumberOfDartAvailableCpuRegs = kNumberOfCpuRegisters - kNumberOfReservedCpuRegisters; constexpr int kStoreBufferWrapperSize = 13; +#if defined(DART_TARGET_OS_WINDOWS) +const RegList kAbiPreservedCpuRegs = + R(RBX) | R(RSI) | R(RDI) | R(R12) | R(R13) | R(R14) | R(R15); +const RegList kAbiVolatileFpuRegs = + R(XMM0) | R(XMM1) | R(XMM2) | R(XMM3) | R(XMM4) | R(XMM5); +#else +const RegList kAbiPreservedCpuRegs = R(RBX) | R(R12) | R(R13) | R(R14) | R(R15); +const RegList kAbiVolatileFpuRegs = kAllFpuRegistersList; +#endif + +// Registers available to Dart that are not preserved by runtime calls. +const RegList kDartVolatileCpuRegs = + kDartAvailableCpuRegs & ~kAbiPreservedCpuRegs; + enum ScaleFactor { TIMES_1 = 0, TIMES_2 = 1, @@ -383,8 +399,6 @@ enum ScaleFactor { TIMES_COMPRESSED_HALF_WORD_SIZE = TIMES_COMPRESSED_WORD_SIZE - 1, }; -#define R(reg) (static_cast(1) << (reg)) - class CallingConventions { public: #if defined(DART_TARGET_OS_WINDOWS) @@ -422,11 +436,7 @@ class CallingConventions { static const intptr_t kVolatileCpuRegisters = R(RAX) | R(RCX) | R(RDX) | R(R8) | R(R9) | R(R10) | R(R11); - static const intptr_t kVolatileXmmRegisters = - R(XMM0) | R(XMM1) | R(XMM2) | R(XMM3) | R(XMM4) | R(XMM5); - - static const intptr_t kCalleeSaveCpuRegisters = - R(RBX) | R(RSI) | R(RDI) | R(R12) | R(R13) | R(R14) | R(R15); + static const RegList kVolatileXmmRegisters = kAbiVolatileFpuRegs; static const intptr_t kCalleeSaveXmmRegisters = R(XMM6) | R(XMM7) | R(XMM8) | R(XMM9) | R(XMM10) | R(XMM11) | R(XMM12) | @@ -490,13 +500,7 @@ class CallingConventions { R(RSI) | R(RDI) | R(R8) | R(R9) | R(R10) | R(R11); - static const intptr_t kVolatileXmmRegisters = - R(XMM0) | R(XMM1) | R(XMM2) | R(XMM3) | R(XMM4) | R(XMM5) | R(XMM6) | - R(XMM7) | R(XMM8) | R(XMM9) | R(XMM10) | R(XMM11) | R(XMM12) | R(XMM13) | - R(XMM14) | R(XMM15); - - static const intptr_t kCalleeSaveCpuRegisters = - R(RBX) | R(R12) | R(R13) | R(R14) | R(R15); + static const RegList kVolatileXmmRegisters = kAbiVolatileFpuRegs; static const intptr_t kCalleeSaveXmmRegisters = 0; @@ -530,6 +534,8 @@ class CallingConventions { #endif + static const intptr_t kCalleeSaveCpuRegisters = kAbiPreservedCpuRegs; + COMPILE_ASSERT((kArgumentRegisters & kReservedCpuRegisters) == 0); static constexpr Register kFfiAnyNonAbiRegister = R12; @@ -544,9 +550,6 @@ class CallingConventions { (kArgumentRegisters | R(kPointerToReturnStructRegisterCall))) == 0); }; -constexpr intptr_t kAbiPreservedCpuRegs = - CallingConventions::kCalleeSaveCpuRegisters; - #undef R class Instr {