dart-lang/sdk
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[vm] Use dmb ishst instead of dmb ish for store-store fence.

Browse Source 
TEST=ci
Change-Id: Id7fb33bb4d9626ad420fd5aac2df86b47bd5c7bf
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/459662
Commit-Queue: Ryan Macnak <rmacnak@google.com>
Reviewed-by: Alexander Aprelev <aam@google.com>
This commit is contained in:
Ryan Macnak
2025-11-05 13:29:00 -08:00
committed by Commit Queue
parent c1fff5ea8f
commit 8c73b534f5
16 changed files with 71 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/platform/atomic.h
+22
View File
@@ -153,6 +153,28 @@ static inline T LoadRelaxed(const T* ptr) {
std::memory_order_relaxed);
}
inline void StoreStoreFence() {
#if defined(HOST_ARCH_ARM) && defined(_MSC_VER)
__dmb(_ARM_BARRIER_ISHST);
#elif defined(HOST_ARCH_ARM64) && defined(_MSC_VER)
__dmb(_ARM64_BARRIER_ISHST);
#elif defined(HOST_ARCH_ARM64) && defined(__GNUC__)
__asm__ __volatile__("dmb ishst" : : : "memory");
#elif defined(HOST_ARCH_ARM) && defined(__GNUC__)
__asm__ __volatile__("dmb ishst" : : : "memory");
#elif defined(HOST_ARCH_RISCV32) && defined(__GNUC__)
__asm__ __volatile__("fence w,w" : : : "memory");
#elif defined(HOST_ARCH_RISCV64) && defined(__GNUC__)
__asm__ __volatile__("fence w,w" : : : "memory");
#else
// GCC warns that TSAN doesn't understand thread fences.
#if defined(__GNUC__) && !defined(__clang__)
#pragma GCC diagnostic ignored "-Wtsan"
#endif
std::atomic_thread_fence(std::memory_order_release);
#endif
}
} // namespace dart
#endif // RUNTIME_PLATFORM_ATOMIC_H_
runtime/vm/compiler/assembler/assembler_arm.cc
-5
View File
@@ -527,11 +527,6 @@ void Assembler::strex(Register rd, Register rt, Register rn, Condition cond) {
Emit(encoding);
}
void Assembler::dmb() {
// Emit a `dmb ish` instruction.
Emit(kDataMemoryBarrier);
}
static int32_t BitFieldExtractEncoding(bool sign_extend,
Register rd,
Register rn,
runtime/vm/compiler/assembler/assembler_arm.h
+8 -5
View File
@@ -438,12 +438,12 @@ class Assembler : public AssemblerBase {
const Address& address,
OperandSize size = kFourBytes) override {
Load(dst, address, size);
dmb();
dmb_ish();
}
void StoreRelease(Register src,
const Address& address,
OperandSize size = kFourBytes) override {
dmb();
dmb_ish();
Store(src, address, size);
}
@@ -469,6 +469,8 @@ class Assembler : public AssemblerBase {
// Debugging and bringup support.
void Breakpoint() override { bkpt(0); }
void StoreStoreFence() override { dmb_ishst(); }
// Data-processing instructions.
void and_(Register rd, Register rn, Operand o, Condition cond = AL);
void ands(Register rd, Register rn, Operand o, Condition cond = AL);
@@ -601,7 +603,8 @@ class Assembler : public AssemblerBase {
void ldrex(Register rd, Register rn, Condition cond = AL);
void strex(Register rd, Register rt, Register rn, Condition cond = AL);
void dmb();
void dmb_ish() { Emit(kDMB_ISH); }
void dmb_ishst() { Emit(kDMB_ISHST); }
// Media instructions.
void sbfx(Register rd,
@@ -1041,13 +1044,13 @@ class Assembler : public AssemblerBase {
void InitializeHeader(Register tags, Register object) {
str(tags, FieldAddress(object, target::Object::tags_offset()));
#if defined(TARGET_HAS_FAST_WRITE_WRITE_FENCE)
dmb();
StoreStoreFence();
#endif
}
void InitializeHeaderUntagged(Register tags, Register object) {
str(tags, Address(object, target::Object::tags_offset()));
#if defined(TARGET_HAS_FAST_WRITE_WRITE_FENCE)
dmb();
StoreStoreFence();
#endif
}
runtime/vm/compiler/assembler/assembler_arm64.h
+6 -3
View File
@@ -564,6 +564,8 @@ class Assembler : public AssemblerBase {
// Debugging and bringup support.
void Breakpoint() override { brk(0); }
void StoreStoreFence() override { dmb_ishst(); }
void SetPrologueOffset() {
if (prologue_offset_ == -1) {
prologue_offset_ = CodeSize();
@@ -1242,7 +1244,8 @@ class Assembler : public AssemblerBase {
// Breakpoint.
void brk(uint16_t imm) { EmitExceptionGenOp(BRK, imm); }
void dmb() { Emit(kDataMemoryBarrier); }
void dmb_ish() { Emit(kDMB_ISH); }
void dmb_ishst() { Emit(kDMB_ISHST); }
// Double floating point.
bool fmovdi(VRegister vd, double immd) {
@@ -1963,13 +1966,13 @@ class Assembler : public AssemblerBase {
void InitializeHeader(Register header, Register object) {
str(header, FieldAddress(object, target::Object::tags_offset()));
#if defined(TARGET_HAS_FAST_WRITE_WRITE_FENCE)
dmb();
StoreStoreFence();
#endif
}
void InitializeHeaderUntagged(Register header, Register object) {
str(header, Address(object, target::Object::tags_offset()));
#if defined(TARGET_HAS_FAST_WRITE_WRITE_FENCE)
dmb();
StoreStoreFence();
#endif
}
runtime/vm/compiler/assembler/assembler_base.h
+2
View File
@@ -630,6 +630,8 @@ class AssemblerBase : public StackResource {
virtual void Breakpoint() = 0;
virtual void StoreStoreFence() = 0;
virtual void SmiTag(Register r) = 0;
virtual void Bind(Label* label) = 0;
runtime/vm/compiler/assembler/assembler_ia32.h
+2
View File
@@ -1199,6 +1199,8 @@ class Assembler : public AssemblerBase {
// Debugging and bringup support.
void Breakpoint() override { int3(); }
void StoreStoreFence() override {}
// Check if the given value is an integer value that can be directly
// embedded into the code without additional XORing with jit_cookie.
// We consider 16-bit integers, powers of two and corresponding masks
runtime/vm/compiler/assembler/assembler_riscv.h
+2
View File
@@ -1027,6 +1027,8 @@ class Assembler : public MicroAssembler {
// Debugging and bringup support.
void Breakpoint() override { trap(); }
void StoreStoreFence() override { fence(kWrite, kWrite); }
void SetPrologueOffset() {
if (prologue_offset_ == -1) {
prologue_offset_ = CodeSize();
runtime/vm/compiler/assembler/assembler_x64.h
+2
View File
@@ -1345,6 +1345,8 @@ class Assembler : public AssemblerBase {
// Debugging and bringup support.
void Breakpoint() override { int3(); }
void StoreStoreFence() override {}
static bool AddressCanHoldConstantIndex(const Object& constant,
bool is_external,
intptr_t cid,
runtime/vm/compiler/assembler/disassembler_arm.cc
+3 -2
View File
@@ -1444,9 +1444,10 @@ void ARMDecoder::InstructionDecode(uword pc) {
if (instr->ConditionField() == kSpecialCondition) {
if (instr->InstructionBits() == static_cast<int32_t>(0xf57ff01f)) {
Format(instr, "clrex");
} else if (instr->InstructionBits() ==
static_cast<int32_t>(kDataMemoryBarrier)) {
} else if (instr->InstructionBits() == static_cast<int32_t>(kDMB_ISH)) {
Format(instr, "dmb ish");
} else if (instr->InstructionBits() == static_cast<int32_t>(kDMB_ISHST)) {
Format(instr, "dmb ishst");
} else {
if (instr->IsSIMDDataProcessing()) {
DecodeSIMDDataProcessing(instr);
runtime/vm/compiler/assembler/disassembler_arm64.cc
+5 -1
View File
@@ -994,10 +994,14 @@ void ARM64Decoder::DecodeSystem(Instr* instr) {
return;
}
if (instr->InstructionBits() == kDataMemoryBarrier) {
if (instr->InstructionBits() == kDMB_ISH) {
Format(instr, "dmb ish");
return;
}
if (instr->InstructionBits() == kDMB_ISHST) {
Format(instr, "dmb ishst");
return;
}
if ((instr->Bits(0, 8) == 0x1f) && (instr->Bits(12, 4) == 2) &&
(instr->Bits(16, 3) == 3) && (instr->Bits(19, 2) == 0) &&
runtime/vm/constants_arm.h
+3 -2
View File
@@ -1318,8 +1318,9 @@ float ReciprocalStep(float op1, float op2);
float ReciprocalSqrtEstimate(float op);
float ReciprocalSqrtStep(float op1, float op2);
constexpr uword kBreakInstructionFiller = 0xE1200070; // bkpt #0
constexpr uword kDataMemoryBarrier = 0xf57ff050 | 0xb; // dmb ish
constexpr uword kBreakInstructionFiller = 0xE1200070; // bkpt #0
constexpr uword kDMB_ISH = 0xf57ff05b; // dmb ish
constexpr uword kDMB_ISHST = 0xf57ff05a; // dmb ishst
struct LinkRegister {
const int32_t code = LR;
runtime/vm/constants_arm64.h
+2 -1
View File
@@ -1633,7 +1633,8 @@ class Instr {
};
const uint64_t kBreakInstructionFiller = 0xD4200000D4200000L; // brk #0; brk #0
constexpr int32_t kDataMemoryBarrier = 0xD5033BBF; // dmb ish
constexpr int32_t kDMB_ISH = 0xD5033BBF; // dmb ish
constexpr int32_t kDMB_ISHST = 0xD5033ABF; // dmb ishst
struct LinkRegister {};
runtime/vm/heap/heap_test.cc
+1 -1
View File
@@ -1555,7 +1555,7 @@ VM_UNIT_TEST_CASE(MutatorMarkerRace_Fence) {
for (size_t i = 0; i < kExistingObjectSlotCount; i++) {
NewObject* new_object = &new_page->objects[i];
new_object->header.store(2u, std::memory_order_relaxed);
std::atomic_thread_fence(std::memory_order_release);
StoreStoreFence();
for (size_t j = 0; j < kNewObjectSlotCount; j++) {
new_object->slots[j].store(existing_object,
std::memory_order_relaxed);
runtime/vm/object.cc
+1 -5
View File
@@ -2846,11 +2846,7 @@ void Object::InitializeObject(uword address,
reinterpret_cast<UntaggedObject*>(address)->tags_ = tags;
#if defined(HOST_HAS_FAST_WRITE_WRITE_FENCE)
// GCC warns that TSAN doesn't understand thread fences.
#if defined(__GNUC__) && !defined(__clang__)
#pragma GCC diagnostic ignored "-Wtsan"
#endif
std::atomic_thread_fence(std::memory_order_release);
StoreStoreFence();
#endif
}
runtime/vm/simulator_arm.cc
+5 -2
View File
@@ -3413,11 +3413,14 @@ DART_FORCE_INLINE void Simulator::InstructionDecodeImpl(Instr* instr) {
if (instr->InstructionBits() == static_cast<int32_t>(0xf57ff01f)) {
// Format(instr, "clrex");
ClearExclusive();
} else if (instr->InstructionBits() ==
static_cast<int32_t>(kDataMemoryBarrier)) {
} else if (instr->InstructionBits() == static_cast<int32_t>(kDMB_ISH)) {
// Format(instr, "dmb ish");
memory_.FlushAll();
std::atomic_thread_fence(std::memory_order_seq_cst);
} else if (instr->InstructionBits() == static_cast<int32_t>(kDMB_ISHST)) {
// Format(instr, "dmb ishst");
memory_.FlushAll();
StoreStoreFence();
} else {
if (instr->IsSIMDDataProcessing()) {
DecodeSIMDDataProcessing(instr);
runtime/vm/simulator_arm64.cc
+7 -1
View File
@@ -1982,12 +1982,18 @@ void Simulator::DecodeSystem(Instr* instr) {
return;
}
if (instr->InstructionBits() == kDataMemoryBarrier) {
if (instr->InstructionBits() == kDMB_ISH) {
// Format(instr, "dmb ish");
memory_.FlushAll();
std::atomic_thread_fence(std::memory_order_seq_cst);
return;
}
if (instr->InstructionBits() == kDMB_ISHST) {
// Format(instr, "dmb ishst");
memory_.FlushAll();
StoreStoreFence();
return;
}
if ((instr->Bits(0, 8) == 0x1f) && (instr->Bits(12, 4) == 2) &&
(instr->Bits(16, 3) == 3) && (instr->Bits(19, 2) == 0) &&