dart-lang/sdk
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[vm, compiler] Use ARMv8.1 AMOs in the write barrier stub on Mac.

Browse Source 
Cf. 1af37bd92a.

TEST=ci
Change-Id: I61186646ef2aab69d8ffac36073829d5cc865408
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/284547
Reviewed-by: Alexander Markov <alexmarkov@google.com>
Commit-Queue: Ryan Macnak <rmacnak@google.com>
This commit is contained in:
Ryan Macnak
2023-02-27 22:26:36 +00:00
committed by Commit Queue
parent 4487154430
commit 854dc20416
7 changed files with 205 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/vm/compiler/assembler/assembler_arm64.h
+35
View File
@@ -1170,6 +1170,25 @@ class Assembler : public AssemblerBase {
EmitLoadStoreExclusive(STLR, R31, rn, rt, sz);
}
void ldclr(Register rs,
Register rt,
Register rn,
OperandSize sz = kEightBytes) {
// rs = value in
// rt = value out
// rn = address
EmitAtomicMemory(LDCLR, rs, rn, rt, sz);
}
void ldset(Register rs,
Register rt,
Register rn,
OperandSize sz = kEightBytes) {
// rs = value in
// rt = value out
// rn = address
EmitAtomicMemory(LDSET, rs, rn, rt, sz);
}
// Conditional select.
void csel(Register rd, Register rn, Register rm, Condition cond) {
EmitConditionalSelect(CSEL, rd, rn, rm, cond, kEightBytes);
@@ -2818,7 +2837,23 @@ class Assembler : public AssemblerBase {
const int32_t encoding = op | size | Arm64Encode::Rs(rs) |
Arm64Encode::Rt2(R31) | Arm64Encode::Rn(rn) |
Arm64Encode::Rt(rt);
Emit(encoding);
}
void EmitAtomicMemory(AtomicMemoryOp op,
Register rs,
Register rn,
Register rt,
OperandSize sz = kEightBytes) {
ASSERT(sz == kEightBytes || sz == kFourBytes);
const int32_t size = B31 | (sz == kEightBytes ? B30 : 0);
ASSERT((rs != kNoRegister) && (rs != CSP));
ASSERT((rn != kNoRegister) && (rn != ZR));
ASSERT((rt != kNoRegister) && (rt != CSP));
const int32_t encoding = op | size | Arm64Encode::Rs(rs) |
Arm64Encode::Rn(rn) | Arm64Encode::Rt(rt);
Emit(encoding);
}
runtime/vm/compiler/assembler/assembler_arm64_test.cc
+44
View File
@@ -1157,6 +1157,50 @@ ASSEMBLER_TEST_RUN(FailedSemaphore32, test) {
"ret\n");
}
ASSEMBLER_TEST_GENERATE(AtomicLoadClear, assembler) {
__ mov(R1, R0);
__ LoadImmediate(R2, 2);
__ ldclr(/*value in*/ R2, /*value out*/ R0, /*address*/ R1);
__ ret();
}
ASSEMBLER_TEST_RUN(AtomicLoadClear, test) {
typedef intptr_t (*AtomicLoadClear)(intptr_t) DART_UNUSED;
intptr_t x = 42;
EXPECT_EQ(42,
EXECUTE_TEST_CODE_INTPTR_INTPTR(AtomicLoadClear, test->entry(),
reinterpret_cast<intptr_t>(&x)));
EXPECT_EQ(40, x);
EXPECT_DISASSEMBLY(
"mov r1, r0\n"
"movz r2, #0x2\n"
"ldclr r2, r0, [r1]\n"
"ret\n");
}
ASSEMBLER_TEST_GENERATE(AtomicLoadSet, assembler) {
__ mov(R1, R0);
__ LoadImmediate(R2, 1);
__ ldset(/*value in*/ R2, /*value out*/ R0, /*address*/ R1);
__ ret();
}
ASSEMBLER_TEST_RUN(AtomicLoadSet, test) {
typedef intptr_t (*AtomicLoadSet)(intptr_t) DART_UNUSED;
int64_t x = 42;
EXPECT_EQ(42,
EXECUTE_TEST_CODE_INTPTR_INTPTR(AtomicLoadSet, test->entry(),
reinterpret_cast<intptr_t>(&x)));
EXPECT_EQ(43, x);
EXPECT_DISASSEMBLY(
"mov r1, r0\n"
"movz r2, #0x1\n"
"ldset r2, r0, [r1]\n"
"ret\n");
}
ASSEMBLER_TEST_GENERATE(LoadAcquireStoreRelease, assembler) {
// We cannot really test that ldar/stlr have the barrier behavior, but at
// least we can test that the load/store behavior is correct.
runtime/vm/compiler/assembler/disassembler_arm64.cc
+17 -1
View File
@@ -816,6 +816,20 @@ void ARM64Decoder::DecodeLoadStoreExclusive(Instr* instr) {
}
}
void ARM64Decoder::DecodeAtomicMemory(Instr* instr) {
switch (instr->Bits(12, 3)) {
case 1:
Format(instr, "ldclr'sz 'rs, 'rt, ['rn]");
break;
case 3:
Format(instr, "ldset'sz 'rs, 'rt, ['rn]");
break;
default:
Unknown(instr);
break;
}
}
void ARM64Decoder::DecodeAddSubImm(Instr* instr) {
switch (instr->Bit(30)) {
case 0: {
@@ -1069,7 +1083,9 @@ void ARM64Decoder::DecodeCompareBranch(Instr* instr) {
}
void ARM64Decoder::DecodeLoadStore(Instr* instr) {
if (instr->IsLoadStoreRegOp()) {
if (instr->IsAtomicMemoryOp()) {
DecodeAtomicMemory(instr);
} else if (instr->IsLoadStoreRegOp()) {
DecodeLoadStoreReg(instr);
} else if (instr->IsLoadStoreRegPairOp()) {
DecodeLoadStoreRegPair(instr);
runtime/vm/compiler/stub_code_compiler_arm64.cc
+29 -16
View File
@@ -1926,15 +1926,21 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler,
// Atomically clear kOldAndNotRememberedBit.
ASSERT(target::Object::tags_offset() == 0);
__ sub(R3, R1, Operand(kHeapObjectTag));
// R3: Untagged address of header word (ldxr/stxr do not support offsets).
Label retry;
__ Bind(&retry);
__ ldxr(R2, R3, kEightBytes);
__ tbz(&lost_race, R2, target::UntaggedObject::kOldAndNotRememberedBit);
__ AndImmediate(R2, R2,
~(1 << target::UntaggedObject::kOldAndNotRememberedBit));
__ stxr(R4, R2, R3, kEightBytes);
__ cbnz(&retry, R4);
// R3: Untagged address of header word (atomics do not support offsets).
if (TargetCPUFeatures::atomic_memory_supported()) {
__ LoadImmediate(TMP, 1 << target::UntaggedObject::kOldAndNotRememberedBit);
__ ldclr(TMP, TMP, R3);
__ tbz(&lost_race, TMP, target::UntaggedObject::kOldAndNotRememberedBit);
} else {
Label retry;
__ Bind(&retry);
__ ldxr(R2, R3, kEightBytes);
__ tbz(&lost_race, R2, target::UntaggedObject::kOldAndNotRememberedBit);
__ AndImmediate(R2, R2,
~(1 << target::UntaggedObject::kOldAndNotRememberedBit));
__ stxr(R4, R2, R3, kEightBytes);
__ cbnz(&retry, R4);
}
// Load the StoreBuffer block out of the thread. Then load top_ out of the
// StoreBufferBlock and add the address to the pointers_.
@@ -1979,13 +1985,20 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler,
Label marking_retry, marking_overflow;
ASSERT(target::Object::tags_offset() == 0);
__ sub(R3, R0, Operand(kHeapObjectTag));
// R3: Untagged address of header word (ldxr/stxr do not support offsets).
__ Bind(&marking_retry);
__ ldxr(R2, R3, kEightBytes);
__ tbz(&lost_race, R2, target::UntaggedObject::kOldAndNotMarkedBit);
__ AndImmediate(R2, R2, ~(1 << target::UntaggedObject::kOldAndNotMarkedBit));
__ stxr(R4, R2, R3, kEightBytes);
__ cbnz(&marking_retry, R4);
// R3: Untagged address of header word (atomics do not support offsets).
if (TargetCPUFeatures::atomic_memory_supported()) {
__ LoadImmediate(TMP, 1 << target::UntaggedObject::kOldAndNotMarkedBit);
__ ldclr(TMP, TMP, R3);
__ tbz(&lost_race, TMP, target::UntaggedObject::kOldAndNotMarkedBit);
} else {
__ Bind(&marking_retry);
__ ldxr(R2, R3, kEightBytes);
__ tbz(&lost_race, R2, target::UntaggedObject::kOldAndNotMarkedBit);
__ AndImmediate(R2, R2,
~(1 << target::UntaggedObject::kOldAndNotMarkedBit));
__ stxr(R4, R2, R3, kEightBytes);
__ cbnz(&marking_retry, R4);
}
__ LoadFromOffset(R4, THR, target::Thread::marking_stack_block_offset());
__ LoadFromOffset(R2, R4, target::MarkingStackBlock::top_offset(),
runtime/vm/constants_arm64.h
+8
View File
@@ -802,6 +802,13 @@ enum LoadStoreExclusiveOp {
STLR = LoadStoreExclusiveFixed | B23 | B15,
};
enum AtomicMemoryOp {
AtomicMemoryMask = 0x3f200c00,
AtomicMemoryFixed = B29 | B28 | B27 | B21,
LDCLR = AtomicMemoryFixed | B12,
LDSET = AtomicMemoryFixed | B13 | B12,
};
// C3.3.7-10
enum LoadStoreRegOp {
LoadStoreRegMask = 0x3a000000,
@@ -1077,6 +1084,7 @@ enum FPIntCvtOp {
_V(LoadStoreRegPair) \
_V(LoadRegLiteral) \
_V(LoadStoreExclusive) \
_V(AtomicMemory) \
_V(AddSubImm) \
_V(Bitfield) \
_V(LogicalImm) \
runtime/vm/cpu_arm64.h
+7
View File
@@ -45,6 +45,13 @@ class TargetCPUFeatures : public AllStatic {
static void Cleanup() { HostCPUFeatures::Cleanup(); }
static const char* hardware() { return HostCPUFeatures::hardware(); }
static bool double_truncate_round_supported() { return false; }
static bool atomic_memory_supported() {
#if defined(DART_TARGET_OS_MACOS) && !defined(DART_TARGET_OS_MACOS_IOS)
return true; // Mac ARM64 starts at ARMv8.5.
#else
return false; // All else start at ARMv8.0.
#endif
}
};
} // namespace dart
runtime/vm/simulator_arm64.cc
+65 -2
View File
@@ -2340,8 +2340,70 @@ void Simulator::DecodeLoadStoreExclusive(Instr* instr) {
}
}
void Simulator::DecodeAtomicMemory(Instr* instr) {
const int32_t size = instr->Bits(30, 2);
std::memory_order order;
switch (instr->Bits(22, 2)) {
case 3:
order = std::memory_order_acq_rel;
break;
case 2:
order = std::memory_order_acquire;
break;
case 1:
order = std::memory_order_release;
break;
case 0:
order = std::memory_order_relaxed;
break;
}
const Register rs = instr->RsField();
const Register rn = instr->RnField();
const Register rt = instr->RtField();
const int32_t opc = instr->Bits(12, 3);
if (size == 3) {
uint64_t in = get_register(rs, R31IsZR);
auto addr =
reinterpret_cast<std::atomic<uint64_t>*>(get_register(rn, R31IsSP));
uint64_t out;
switch (opc) {
case 1:
out = addr->fetch_and(~in, order);
break;
case 3:
out = addr->fetch_or(in, order);
break;
default:
UNIMPLEMENTED();
}
set_register(instr, rt, out, R31IsZR);
} else if (size == 2) {
ASSERT(size == 2);
uint32_t in = get_wregister(rs, R31IsZR);
auto addr =
reinterpret_cast<std::atomic<uint32_t>*>(get_register(rn, R31IsSP));
uint32_t out;
switch (opc) {
case 1:
out = addr->fetch_and(~in, order);
break;
case 3:
out = addr->fetch_or(in, order);
break;
default:
UNIMPLEMENTED();
}
set_wregister(rt, out, R31IsZR);
} else {
UNIMPLEMENTED();
}
}
void Simulator::DecodeLoadStore(Instr* instr) {
if (instr->IsLoadStoreRegOp()) {
if (instr->IsAtomicMemoryOp()) {
DecodeAtomicMemory(instr);
} else if (instr->IsLoadStoreRegOp()) {
DecodeLoadStoreReg(instr);
} else if (instr->IsLoadStoreRegPairOp()) {
DecodeLoadStoreRegPair(instr);
@@ -2679,7 +2741,8 @@ void Simulator::DecodeMiscDP2Source(Instr* instr) {
// Format(instr, "sdiv'sf 'rd, 'rn, 'rm");
const bool is_signed = instr->Bit(10) == 1;
if (instr->SFField() == 1) {
set_register(instr, rd, divide64(rn_val64, rm_val64, is_signed), R31IsZR);
set_register(instr, rd, divide64(rn_val64, rm_val64, is_signed),
R31IsZR);
} else {
set_wregister(rd, divide32(rn_val32, rm_val32, is_signed), R31IsZR);
}