From 854dc204163ef6be6df30de9e2ecf0e077d5b391 Mon Sep 17 00:00:00 2001 From: Ryan Macnak Date: Mon, 27 Feb 2023 22:26:36 +0000 Subject: [PATCH] [vm, compiler] Use ARMv8.1 AMOs in the write barrier stub on Mac. Cf. 1af37bd92a9b08ac2d88d26c55feb1dd43382416. TEST=ci Change-Id: I61186646ef2aab69d8ffac36073829d5cc865408 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/284547 Reviewed-by: Alexander Markov Commit-Queue: Ryan Macnak --- .../vm/compiler/assembler/assembler_arm64.h | 35 ++++++++++ .../assembler/assembler_arm64_test.cc | 44 ++++++++++++ .../compiler/assembler/disassembler_arm64.cc | 18 ++++- .../vm/compiler/stub_code_compiler_arm64.cc | 45 ++++++++----- runtime/vm/constants_arm64.h | 8 +++ runtime/vm/cpu_arm64.h | 7 ++ runtime/vm/simulator_arm64.cc | 67 ++++++++++++++++++- 7 files changed, 205 insertions(+), 19 deletions(-) diff --git a/runtime/vm/compiler/assembler/assembler_arm64.h b/runtime/vm/compiler/assembler/assembler_arm64.h index 1c348e366ea..a6aca04d6df 100644 --- a/runtime/vm/compiler/assembler/assembler_arm64.h +++ b/runtime/vm/compiler/assembler/assembler_arm64.h @@ -1170,6 +1170,25 @@ class Assembler : public AssemblerBase { EmitLoadStoreExclusive(STLR, R31, rn, rt, sz); } + void ldclr(Register rs, + Register rt, + Register rn, + OperandSize sz = kEightBytes) { + // rs = value in + // rt = value out + // rn = address + EmitAtomicMemory(LDCLR, rs, rn, rt, sz); + } + void ldset(Register rs, + Register rt, + Register rn, + OperandSize sz = kEightBytes) { + // rs = value in + // rt = value out + // rn = address + EmitAtomicMemory(LDSET, rs, rn, rt, sz); + } + // Conditional select. void csel(Register rd, Register rn, Register rm, Condition cond) { EmitConditionalSelect(CSEL, rd, rn, rm, cond, kEightBytes); @@ -2818,7 +2837,23 @@ class Assembler : public AssemblerBase { const int32_t encoding = op | size | Arm64Encode::Rs(rs) | Arm64Encode::Rt2(R31) | Arm64Encode::Rn(rn) | Arm64Encode::Rt(rt); + Emit(encoding); + } + void EmitAtomicMemory(AtomicMemoryOp op, + Register rs, + Register rn, + Register rt, + OperandSize sz = kEightBytes) { + ASSERT(sz == kEightBytes || sz == kFourBytes); + const int32_t size = B31 | (sz == kEightBytes ? B30 : 0); + + ASSERT((rs != kNoRegister) && (rs != CSP)); + ASSERT((rn != kNoRegister) && (rn != ZR)); + ASSERT((rt != kNoRegister) && (rt != CSP)); + + const int32_t encoding = op | size | Arm64Encode::Rs(rs) | + Arm64Encode::Rn(rn) | Arm64Encode::Rt(rt); Emit(encoding); } diff --git a/runtime/vm/compiler/assembler/assembler_arm64_test.cc b/runtime/vm/compiler/assembler/assembler_arm64_test.cc index df7c4a1ccdb..24cdc4fbfd1 100644 --- a/runtime/vm/compiler/assembler/assembler_arm64_test.cc +++ b/runtime/vm/compiler/assembler/assembler_arm64_test.cc @@ -1157,6 +1157,50 @@ ASSEMBLER_TEST_RUN(FailedSemaphore32, test) { "ret\n"); } +ASSEMBLER_TEST_GENERATE(AtomicLoadClear, assembler) { + __ mov(R1, R0); + __ LoadImmediate(R2, 2); + __ ldclr(/*value in*/ R2, /*value out*/ R0, /*address*/ R1); + __ ret(); +} + +ASSEMBLER_TEST_RUN(AtomicLoadClear, test) { + typedef intptr_t (*AtomicLoadClear)(intptr_t) DART_UNUSED; + intptr_t x = 42; + EXPECT_EQ(42, + EXECUTE_TEST_CODE_INTPTR_INTPTR(AtomicLoadClear, test->entry(), + reinterpret_cast(&x))); + EXPECT_EQ(40, x); + + EXPECT_DISASSEMBLY( + "mov r1, r0\n" + "movz r2, #0x2\n" + "ldclr r2, r0, [r1]\n" + "ret\n"); +} + +ASSEMBLER_TEST_GENERATE(AtomicLoadSet, assembler) { + __ mov(R1, R0); + __ LoadImmediate(R2, 1); + __ ldset(/*value in*/ R2, /*value out*/ R0, /*address*/ R1); + __ ret(); +} + +ASSEMBLER_TEST_RUN(AtomicLoadSet, test) { + typedef intptr_t (*AtomicLoadSet)(intptr_t) DART_UNUSED; + int64_t x = 42; + EXPECT_EQ(42, + EXECUTE_TEST_CODE_INTPTR_INTPTR(AtomicLoadSet, test->entry(), + reinterpret_cast(&x))); + EXPECT_EQ(43, x); + + EXPECT_DISASSEMBLY( + "mov r1, r0\n" + "movz r2, #0x1\n" + "ldset r2, r0, [r1]\n" + "ret\n"); +} + ASSEMBLER_TEST_GENERATE(LoadAcquireStoreRelease, assembler) { // We cannot really test that ldar/stlr have the barrier behavior, but at // least we can test that the load/store behavior is correct. diff --git a/runtime/vm/compiler/assembler/disassembler_arm64.cc b/runtime/vm/compiler/assembler/disassembler_arm64.cc index d3d7b2cfafb..76c6ded1936 100644 --- a/runtime/vm/compiler/assembler/disassembler_arm64.cc +++ b/runtime/vm/compiler/assembler/disassembler_arm64.cc @@ -816,6 +816,20 @@ void ARM64Decoder::DecodeLoadStoreExclusive(Instr* instr) { } } +void ARM64Decoder::DecodeAtomicMemory(Instr* instr) { + switch (instr->Bits(12, 3)) { + case 1: + Format(instr, "ldclr'sz 'rs, 'rt, ['rn]"); + break; + case 3: + Format(instr, "ldset'sz 'rs, 'rt, ['rn]"); + break; + default: + Unknown(instr); + break; + } +} + void ARM64Decoder::DecodeAddSubImm(Instr* instr) { switch (instr->Bit(30)) { case 0: { @@ -1069,7 +1083,9 @@ void ARM64Decoder::DecodeCompareBranch(Instr* instr) { } void ARM64Decoder::DecodeLoadStore(Instr* instr) { - if (instr->IsLoadStoreRegOp()) { + if (instr->IsAtomicMemoryOp()) { + DecodeAtomicMemory(instr); + } else if (instr->IsLoadStoreRegOp()) { DecodeLoadStoreReg(instr); } else if (instr->IsLoadStoreRegPairOp()) { DecodeLoadStoreRegPair(instr); diff --git a/runtime/vm/compiler/stub_code_compiler_arm64.cc b/runtime/vm/compiler/stub_code_compiler_arm64.cc index 4e6e37a7350..ba3f7a69992 100644 --- a/runtime/vm/compiler/stub_code_compiler_arm64.cc +++ b/runtime/vm/compiler/stub_code_compiler_arm64.cc @@ -1926,15 +1926,21 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler, // Atomically clear kOldAndNotRememberedBit. ASSERT(target::Object::tags_offset() == 0); __ sub(R3, R1, Operand(kHeapObjectTag)); - // R3: Untagged address of header word (ldxr/stxr do not support offsets). - Label retry; - __ Bind(&retry); - __ ldxr(R2, R3, kEightBytes); - __ tbz(&lost_race, R2, target::UntaggedObject::kOldAndNotRememberedBit); - __ AndImmediate(R2, R2, - ~(1 << target::UntaggedObject::kOldAndNotRememberedBit)); - __ stxr(R4, R2, R3, kEightBytes); - __ cbnz(&retry, R4); + // R3: Untagged address of header word (atomics do not support offsets). + if (TargetCPUFeatures::atomic_memory_supported()) { + __ LoadImmediate(TMP, 1 << target::UntaggedObject::kOldAndNotRememberedBit); + __ ldclr(TMP, TMP, R3); + __ tbz(&lost_race, TMP, target::UntaggedObject::kOldAndNotRememberedBit); + } else { + Label retry; + __ Bind(&retry); + __ ldxr(R2, R3, kEightBytes); + __ tbz(&lost_race, R2, target::UntaggedObject::kOldAndNotRememberedBit); + __ AndImmediate(R2, R2, + ~(1 << target::UntaggedObject::kOldAndNotRememberedBit)); + __ stxr(R4, R2, R3, kEightBytes); + __ cbnz(&retry, R4); + } // Load the StoreBuffer block out of the thread. Then load top_ out of the // StoreBufferBlock and add the address to the pointers_. @@ -1979,13 +1985,20 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler, Label marking_retry, marking_overflow; ASSERT(target::Object::tags_offset() == 0); __ sub(R3, R0, Operand(kHeapObjectTag)); - // R3: Untagged address of header word (ldxr/stxr do not support offsets). - __ Bind(&marking_retry); - __ ldxr(R2, R3, kEightBytes); - __ tbz(&lost_race, R2, target::UntaggedObject::kOldAndNotMarkedBit); - __ AndImmediate(R2, R2, ~(1 << target::UntaggedObject::kOldAndNotMarkedBit)); - __ stxr(R4, R2, R3, kEightBytes); - __ cbnz(&marking_retry, R4); + // R3: Untagged address of header word (atomics do not support offsets). + if (TargetCPUFeatures::atomic_memory_supported()) { + __ LoadImmediate(TMP, 1 << target::UntaggedObject::kOldAndNotMarkedBit); + __ ldclr(TMP, TMP, R3); + __ tbz(&lost_race, TMP, target::UntaggedObject::kOldAndNotMarkedBit); + } else { + __ Bind(&marking_retry); + __ ldxr(R2, R3, kEightBytes); + __ tbz(&lost_race, R2, target::UntaggedObject::kOldAndNotMarkedBit); + __ AndImmediate(R2, R2, + ~(1 << target::UntaggedObject::kOldAndNotMarkedBit)); + __ stxr(R4, R2, R3, kEightBytes); + __ cbnz(&marking_retry, R4); + } __ LoadFromOffset(R4, THR, target::Thread::marking_stack_block_offset()); __ LoadFromOffset(R2, R4, target::MarkingStackBlock::top_offset(), diff --git a/runtime/vm/constants_arm64.h b/runtime/vm/constants_arm64.h index 884c985ad4c..d9a668caa6a 100644 --- a/runtime/vm/constants_arm64.h +++ b/runtime/vm/constants_arm64.h @@ -802,6 +802,13 @@ enum LoadStoreExclusiveOp { STLR = LoadStoreExclusiveFixed | B23 | B15, }; +enum AtomicMemoryOp { + AtomicMemoryMask = 0x3f200c00, + AtomicMemoryFixed = B29 | B28 | B27 | B21, + LDCLR = AtomicMemoryFixed | B12, + LDSET = AtomicMemoryFixed | B13 | B12, +}; + // C3.3.7-10 enum LoadStoreRegOp { LoadStoreRegMask = 0x3a000000, @@ -1077,6 +1084,7 @@ enum FPIntCvtOp { _V(LoadStoreRegPair) \ _V(LoadRegLiteral) \ _V(LoadStoreExclusive) \ + _V(AtomicMemory) \ _V(AddSubImm) \ _V(Bitfield) \ _V(LogicalImm) \ diff --git a/runtime/vm/cpu_arm64.h b/runtime/vm/cpu_arm64.h index 3e793ef6471..ff72423faf8 100644 --- a/runtime/vm/cpu_arm64.h +++ b/runtime/vm/cpu_arm64.h @@ -45,6 +45,13 @@ class TargetCPUFeatures : public AllStatic { static void Cleanup() { HostCPUFeatures::Cleanup(); } static const char* hardware() { return HostCPUFeatures::hardware(); } static bool double_truncate_round_supported() { return false; } + static bool atomic_memory_supported() { +#if defined(DART_TARGET_OS_MACOS) && !defined(DART_TARGET_OS_MACOS_IOS) + return true; // Mac ARM64 starts at ARMv8.5. +#else + return false; // All else start at ARMv8.0. +#endif + } }; } // namespace dart diff --git a/runtime/vm/simulator_arm64.cc b/runtime/vm/simulator_arm64.cc index a0facf2706e..c33e65bc507 100644 --- a/runtime/vm/simulator_arm64.cc +++ b/runtime/vm/simulator_arm64.cc @@ -2340,8 +2340,70 @@ void Simulator::DecodeLoadStoreExclusive(Instr* instr) { } } +void Simulator::DecodeAtomicMemory(Instr* instr) { + const int32_t size = instr->Bits(30, 2); + std::memory_order order; + switch (instr->Bits(22, 2)) { + case 3: + order = std::memory_order_acq_rel; + break; + case 2: + order = std::memory_order_acquire; + break; + case 1: + order = std::memory_order_release; + break; + case 0: + order = std::memory_order_relaxed; + break; + } + const Register rs = instr->RsField(); + const Register rn = instr->RnField(); + const Register rt = instr->RtField(); + const int32_t opc = instr->Bits(12, 3); + + if (size == 3) { + uint64_t in = get_register(rs, R31IsZR); + auto addr = + reinterpret_cast*>(get_register(rn, R31IsSP)); + uint64_t out; + switch (opc) { + case 1: + out = addr->fetch_and(~in, order); + break; + case 3: + out = addr->fetch_or(in, order); + break; + default: + UNIMPLEMENTED(); + } + set_register(instr, rt, out, R31IsZR); + } else if (size == 2) { + ASSERT(size == 2); + uint32_t in = get_wregister(rs, R31IsZR); + auto addr = + reinterpret_cast*>(get_register(rn, R31IsSP)); + uint32_t out; + switch (opc) { + case 1: + out = addr->fetch_and(~in, order); + break; + case 3: + out = addr->fetch_or(in, order); + break; + default: + UNIMPLEMENTED(); + } + set_wregister(rt, out, R31IsZR); + } else { + UNIMPLEMENTED(); + } +} + void Simulator::DecodeLoadStore(Instr* instr) { - if (instr->IsLoadStoreRegOp()) { + if (instr->IsAtomicMemoryOp()) { + DecodeAtomicMemory(instr); + } else if (instr->IsLoadStoreRegOp()) { DecodeLoadStoreReg(instr); } else if (instr->IsLoadStoreRegPairOp()) { DecodeLoadStoreRegPair(instr); @@ -2679,7 +2741,8 @@ void Simulator::DecodeMiscDP2Source(Instr* instr) { // Format(instr, "sdiv'sf 'rd, 'rn, 'rm"); const bool is_signed = instr->Bit(10) == 1; if (instr->SFField() == 1) { - set_register(instr, rd, divide64(rn_val64, rm_val64, is_signed), R31IsZR); + set_register(instr, rd, divide64(rn_val64, rm_val64, is_signed), + R31IsZR); } else { set_wregister(rd, divide32(rn_val32, rm_val32, is_signed), R31IsZR); }