dart-lang/sdk
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[vm, gc] Allocate card tables up front.

Browse Source 
This avoids data races when multiple mutators compete to install the card table, and allows removing the initialization check from the write barrier.

TEST=ci, tsan
Bug: https://github.com/dart-lang/sdk/issues/56895
Change-Id: I30fe4687d3ca771f6c00dfa59e8dadec23d557a3
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/390023
Reviewed-by: Siva Annamalai <asiva@google.com>
Commit-Queue: Ryan Macnak <rmacnak@google.com>
This commit is contained in:
Ryan Macnak
2024-10-16 21:25:17 +00:00
committed by Commit Queue
parent 40c039ad03
commit da8f17762c
12 changed files with 1054 additions and 1143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/vm/app_snapshot.cc
+2
View File
@@ -1414,6 +1414,7 @@ class CanonicalSetDeserializationCluster : public DeserializationCluster {
if ((SetType::Storage::ArrayCid == kArrayCid) &&
Array::UseCardMarkingForAllocation(length)) {
table->untag()->SetCardRememberedBitUnsynchronized();
Page::Of(table)->AllocateCardTable();
}
InitTypeArgsOrNext(table);
table->untag()->length_ = Smi::New(length);
@@ -6508,6 +6509,7 @@ class ArrayDeserializationCluster
stamp_canonical);
if (Array::UseCardMarkingForAllocation(length)) {
array->untag()->SetCardRememberedBitUnsynchronized();
Page::Of(array)->AllocateCardTable();
}
array->untag()->type_arguments_ =
static_cast<TypeArgumentsPtr>(d.ReadRef());
runtime/vm/compiler/runtime_offsets_extracted.h
+1034 -1034
View File
File diff suppressed because it is too large Load Diff
runtime/vm/compiler/stub_code_compiler_arm.cc
+1 -14
View File
@@ -1859,15 +1859,13 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler, bool cards) {
__ Ret();
}
if (cards) {
Label remember_card_slow, retry;
Label retry;
// Get card table.
__ Bind(&remember_card);
__ AndImmediate(TMP, R1, target::kPageMask); // Page.
__ ldr(TMP,
Address(TMP, target::Page::card_table_offset())); // Card table.
__ cmp(TMP, Operand(0));
__ b(&remember_card_slow, EQ);
// Atomically dirty the card.
__ PushList((1 << R0) | (1 << R1) | (1 << R2));
@@ -1890,17 +1888,6 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler, bool cards) {
__ b(&retry, EQ);
__ PopList((1 << R0) | (1 << R1) | (1 << R2));
__ Ret();
// Card table not yet allocated.
__ Bind(&remember_card_slow);
{
LeafRuntimeScope rt(assembler, /*frame_size=*/0,
/*preserve_registers=*/true);
__ mov(R0, Operand(R1)); // Arg0 = Object
__ mov(R1, Operand(R9)); // Arg1 = Slot
rt.Call(kRememberCardRuntimeEntry, 2);
}
__ Ret();
}
}
runtime/vm/compiler/stub_code_compiler_arm64.cc
+1 -14
View File
@@ -2242,14 +2242,13 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler, bool cards) {
__ ret();
}
if (cards) {
Label remember_card_slow, retry;
Label retry;
// Get card table.
__ Bind(&remember_card);
__ AndImmediate(TMP, R1, target::kPageMask); // Page.
__ ldr(TMP2,
Address(TMP, target::Page::card_table_offset())); // Card table.
__ cbz(&remember_card_slow, TMP2);
// Atomically dirty the card.
__ sub(R25, R25, Operand(TMP)); // Offset in page.
@@ -2271,18 +2270,6 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler, bool cards) {
__ PopRegister(R0);
}
__ ret();
// Card table not yet allocated.
__ Bind(&remember_card_slow);
{
LeafRuntimeScope rt(assembler,
/*frame_size=*/0,
/*preserve_registers=*/true);
__ mov(R0, R1); // Arg0 = Object
__ mov(R1, R25); // Arg1 = Slot
rt.Call(kRememberCardRuntimeEntry, /*argument_count=*/2);
}
__ ret();
}
}
runtime/vm/compiler/stub_code_compiler_ia32.cc
-19
View File
@@ -1675,14 +1675,10 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler, bool cards) {
__ ret();
}
if (cards) {
Label remember_card_slow;
// Get card table.
__ Bind(&remember_card);
__ movl(EAX, EDX); // Object.
__ andl(EAX, Immediate(target::kPageMask)); // Page.
__ cmpl(Address(EAX, target::Page::card_table_offset()), Immediate(0));
__ j(EQUAL, &remember_card_slow, Assembler::kNearJump);
// Atomically dirty the card.
__ pushl(EBX);
@@ -1702,21 +1698,6 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler, bool cards) {
__ popl(ECX);
__ popl(EAX);
__ ret();
// Card table not yet allocated.
__ Bind(&remember_card_slow);
{
LeafRuntimeScope rt(assembler,
/*frame_size=*/2 * target::kWordSize,
/*preserve_registers=*/true);
__ movl(Address(ESP, 0 * target::kWordSize), EDX); // Object
__ movl(Address(ESP, 1 * target::kWordSize), EDI); // Slot
rt.Call(kRememberCardRuntimeEntry, 2);
}
__ popl(ECX);
__ popl(EAX);
__ ret();
}
}
runtime/vm/compiler/stub_code_compiler_riscv.cc
-14
View File
@@ -1841,14 +1841,11 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler, bool cards) {
__ ret();
}
if (cards) {
Label remember_card_slow;
// Get card table.
__ Bind(&remember_card);
__ AndImmediate(TMP, A0, target::kPageMask); // Page.
__ lx(TMP2,
Address(TMP, target::Page::card_table_offset())); // Card table.
__ beqz(TMP2, &remember_card_slow);
// Atomically dirty the card.
__ sub(A6, A6, TMP); // Offset in page.
@@ -1864,17 +1861,6 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler, bool cards) {
__ amoord(ZR, TMP, Address(TMP2, 0));
#endif
__ ret();
// Card table not yet allocated.
__ Bind(&remember_card_slow);
{
LeafRuntimeScope rt(assembler, /*frame_size=*/0,
/*preserve_registers=*/true);
__ mv(A0, A0); // Arg0 = Object
__ mv(A1, A6); // Arg1 = Slot
rt.Call(kRememberCardRuntimeEntry, /*argument_count=*/2);
}
__ ret();
}
}
runtime/vm/compiler/stub_code_compiler_x64.cc
-16
View File
@@ -2158,14 +2158,10 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler, bool cards) {
}
if (cards) {
Label remember_card_slow;
// Get card table.
__ Bind(&remember_card);
__ movq(TMP, RDX); // Object.
__ andq(TMP, Immediate(target::kPageMask)); // Page.
__ cmpq(Address(TMP, target::Page::card_table_offset()), Immediate(0));
__ j(EQUAL, &remember_card_slow, Assembler::kNearJump);
// Atomically dirty the card.
__ pushq(RAX);
@@ -2183,18 +2179,6 @@ static void GenerateWriteBarrierStubHelper(Assembler* assembler, bool cards) {
__ popq(RCX);
__ popq(RAX);
__ ret();
// Card table not yet allocated.
__ Bind(&remember_card_slow);
{
LeafRuntimeScope rt(assembler,
/*frame_size=*/0,
/*preserve_registers=*/true);
__ movq(CallingConventions::kArg1Reg, RDX);
__ movq(CallingConventions::kArg2Reg, R13);
rt.Call(kRememberCardRuntimeEntry, 2);
}
__ ret();
}
}
runtime/vm/heap/page.h
+14 -19
View File
@@ -271,32 +271,27 @@ class Page {
bool IsSurvivor(uword raw_addr) const { return raw_addr < survivor_end_; }
bool IsResolved() const { return top_ == resolved_top_; }
void AllocateCardTable() {
ASSERT(card_table_ == nullptr);
ASSERT(is_large());
size_t size_in_bits = card_table_size();
size_t size_in_bytes =
Utils::RoundUp(size_in_bits, kBitsPerWord) >> kBitsPerByteLog2;
card_table_ = reinterpret_cast<RelaxedAtomic<uword>*>(
calloc(size_in_bytes, sizeof(uint8_t)));
}
private:
void RememberCard(uword slot) {
ASSERT(Contains(slot));
uword* card_table = card_table_.load();
if (card_table == nullptr) {
size_t size_in_bits = card_table_size();
size_t size_in_bytes =
Utils::RoundUp(size_in_bits, kBitsPerWord) >> kBitsPerByteLog2;
uword* new_card_table =
reinterpret_cast<uword*>(calloc(size_in_bytes, sizeof(uint8_t)));
if (card_table_.compare_exchange_strong(card_table, new_card_table)) {
card_table = new_card_table;
} else {
// Lost race.
ASSERT(card_table != nullptr);
free(new_card_table);
}
}
ASSERT(card_table_ != nullptr);
intptr_t offset = slot - reinterpret_cast<uword>(this);
intptr_t index = offset >> kBytesPerCardLog2;
ASSERT((index >= 0) && (index < card_table_size()));
intptr_t word_offset = index >> kBitsPerWordLog2;
intptr_t bit_offset = index & (kBitsPerWord - 1);
uword bit_mask = static_cast<uword>(1) << bit_offset;
reinterpret_cast<std::atomic<uword>*>(&card_table[word_offset])
->fetch_or(bit_mask, std::memory_order_relaxed);
card_table_[word_offset].fetch_or(bit_mask);
}
bool IsCardRemembered(uword slot) {
ASSERT(Contains(slot));
@@ -309,7 +304,7 @@ class Page {
intptr_t word_offset = index >> kBitsPerWordLog2;
intptr_t bit_offset = index & (kBitsPerWord - 1);
uword bit_mask = static_cast<uword>(1) << bit_offset;
return (card_table_[word_offset] & bit_mask) != 0;
return (card_table_[word_offset].load() & bit_mask) != 0;
}
void set_object_end(uword value) {
@@ -328,7 +323,7 @@ class Page {
VirtualMemory* memory_;
Page* next_;
ForwardingPage* forwarding_page_;
RelaxedAtomic<uword*> card_table_; // Remembered set, not marking.
RelaxedAtomic<uword>* card_table_; // Remembered set, not marking.
RelaxedAtomic<intptr_t> progress_bar_;
// The thread using this page for allocation, otherwise nullptr.
runtime/vm/object.cc
+1
View File
@@ -25248,6 +25248,7 @@ ArrayPtr Array::NewUninitialized(intptr_t class_id,
if (UseCardMarkingForAllocation(len)) {
ASSERT(raw->IsOldObject());
raw->untag()->SetCardRememberedBitUnsynchronized();
Page::Of(raw)->AllocateCardTable();
}
return raw;
}
runtime/vm/object_graph_copy.cc
+1
View File
@@ -1607,6 +1607,7 @@ class SlowObjectCopyBase : public ObjectCopyBase {
if ((cid == kArrayCid || cid == kImmutableArrayCid) &&
!IsAllocatableInNewSpace(size)) {
to.untag()->SetCardRememberedBitUnsynchronized();
Page::Of(to)->AllocateCardTable();
}
if (IsExternalTypedDataClassId(cid)) {
const auto& external_to = slow_forward_map_.AddExternalTypedData(
runtime/vm/raw_object.cc
-12
View File
@@ -768,18 +768,6 @@ void UntaggedObject::RememberCard(CompressedObjectPtr const* slot) {
}
#endif
DEFINE_LEAF_RUNTIME_ENTRY(void,
RememberCard,
2,
uword /*ObjectPtr*/ object_in,
ObjectPtr* slot) {
ObjectPtr object = static_cast<ObjectPtr>(object_in);
ASSERT(object->IsOldObject());
ASSERT(object->untag()->IsCardRemembered());
Page::Of(object)->RememberCard(slot);
}
END_LEAF_RUNTIME_ENTRY
const char* UntaggedPcDescriptors::KindToCString(Kind k) {
switch (k) {
#define ENUM_CASE(name, init) \
runtime/vm/runtime_entry_list.h
-1
View File
@@ -91,7 +91,6 @@ namespace dart {
V(void, StoreBufferBlockProcess, Thread*) \
V(void, OldMarkingStackBlockProcess, Thread*) \
V(void, NewMarkingStackBlockProcess, Thread*) \
V(void, RememberCard, uword /*ObjectPtr*/, ObjectPtr*) \
V(uword /*ObjectPtr*/, EnsureRememberedAndMarkingDeferred, \
uword /*ObjectPtr*/ object, Thread* thread) \
V(double, LibcPow, double, double) \