diff --git a/runtime/docs/gc.md b/runtime/docs/gc.md new file mode 100644 index 00000000000..22aebee3904 --- /dev/null +++ b/runtime/docs/gc.md @@ -0,0 +1,98 @@ +# Garbage Collection + +## Object representation + +## Scavenge + +## Mark-Sweep + +## Mark-Compact + +## Concurrent Marking + +To reduce the time the mutator is paused for old-space GCs, we allow the mutator to continue running during most of the marking work. + +### Barrier + +With the mutator and marker running concurrently, the mutator could write a pointer to an object that has not been marked (TARGET) into an object that has already been marked and visited (SOURCE), leading to incorrect collection of TARGET. To prevent this, the write barrier checks if a store creates a pointer from an old-space object to an old-space object that is not marked, and marks the target object for such stores. We ignore pointers from new-space objects because we treat new-space objects as roots and will revisit them to finalize marking. We ignore the marking state of the source object to avoid expensive memory barriers required to ensure reordering of accesses to the header and slots can't lead skipped marking, and on the assumption that objects accessed during marking are likely to remain live when marking finishes. + +The barrier is equivalent to + +``` +StorePoint(RawObject* source, RawObject** slot, RawObject* target) { + *slot = target; + if (target->IsSmi()) return; + if (source->IsOldObject() && !source->IsRemembered() && target->IsNewObject()) { + source->SetRemembered(); + AddToRememberedSet(source); + } else if (source->IsOldObject() && target->IsOldObject() && !target->IsMarked() && Thread::Current()->IsMarking()) { + if (target->TryAcquireMarkBit()) { + AddToMarkList(target); + } + } +} +``` + +But we combine the generational and incremental checks with a shift-and-mask. + +``` +enum HeaderBits { + ... + kOldAndNotMarkedBit, // Incremental barrier target. + kNewBit, // Generational barrier target. + kOldBit, // Incremental barrier source. + kOldAndNotRememberedBit, // Generational barrier source. + ... +}; + +static const intptr_t kGenerationalBarrierMask = 1 << kNewBit; +static const intptr_t kIncrementalBarrierMask = 1 << kOldAndNotMarkedBit; +static const intptr_t kBarrierOverlapShift = 2; +COMPILE_ASSERT(kOldAndNotMarkedBit + kBarrierOverlapShift == kOldBit); +COMPILE_ASSERT(kNewBit + kBarrierOverlapShift == kOldAndNotRememberedBit); + +StorePointer(RawObject* source, RawObject** slot, RawObject* target) { + *slot = target; + if (target->IsSmi()) return; + if ((source->header() >> kBarrierOverlapShift) && + (target->header()) && + Thread::Current()->barrier_mask()) { + if (target->IsNewObject()) { + source->SetRemembered(); + AddToRememberedSet(source); + } else { + if (target->TryAcquireMarkBit()) { + AddToMarkList(target); + } + } + } +} + +StoreIntoObject(object, value, offset) + str value, object#offset + tbnz value, kSmiTagShift, done + lbu tmp, value#headerOffset + lbu tmp2, object#headerOffset + and tmp, tmp2 LSR kBarrierOverlapShift + tst tmp, BARRIER_MASK + bz done + mov tmp2, value + lw tmp, THR#writeBarrierEntryPointOffset + blr tmp +done: + +``` + +### Data races + +Operations on headers and slots use (relaxed ordering)[https://en.cppreference.com/w/cpp/atomic/memory_order] and do not provide synchronization. + +The concurrent marker starts with an acquire-release operation, so all writes by the mutator up to the time that marking starts are visible to the marker. + +For old-space objects created before marking started, in each slot the marker can see either its value at the time marking started or any subsequent value sorted in the slot. Any slot that contained a pointer continues to continue a valid pointer for the object's lifetime, so no matter which value the marker sees, it won't interpret a non-pointer as a pointer. (The one interesting case here is array truncation, where some slot in the array will become the header of a filler object. We ensure this is safe for concurrent marking by ensuring the header for the filler object looks like a Smi.) If the marker sees an old value, we may lose some precision and retain a dead object, but we remain correct because the new value has been marked by the mutator. + +For old-space objects created after marking started, the marker may see uninitialized values because operations on slots are not synchronized. To prevent this, during marking we allocate old-space objects black (marked) so the marker will not visit them. + +New-space objects and roots are only visited during a safepoint, and safepoints establish synchronization. + +When the mutator's mark block becomes full, it transfered to the marker by an acquire-release operation, so the marker will see the stores into the block. diff --git a/runtime/vm/compiler/assembler/assembler_arm.cc b/runtime/vm/compiler/assembler/assembler_arm.cc index 9f077c8280a..bb413602591 100644 --- a/runtime/vm/compiler/assembler/assembler_arm.cc +++ b/runtime/vm/compiler/assembler/assembler_arm.cc @@ -1592,12 +1592,40 @@ void Assembler::StoreIntoObject(Register object, ASSERT(object != LR); ASSERT(value != LR); +#if defined(CONCURRENT_MARKING) + ASSERT(object != TMP); + ASSERT(value != TMP); + + str(value, dest); + + // In parallel, test whether + // - object is old and not remembered and value is new, or + // - object is old and value is old and not marked and concurrent marking is + // in progress + // If so, call the WriteBarrier stub, which will either add object to the + // store buffer (case 1) or add value to the marking stack (case 2). + // Compare RawObject::StorePointer. + Label done; + if (can_be_smi == kValueCanBeSmi) { + BranchIfSmi(value, &done); + } + if (!lr_reserved) Push(LR); + ldrb(TMP, FieldAddress(object, Object::tags_offset())); + ldrb(LR, FieldAddress(value, Object::tags_offset())); + and_(TMP, LR, Operand(TMP, LSR, RawObject::kBarrierOverlapShift)); + ldr(LR, Address(THR, Thread::write_barrier_mask_offset())); + tst(TMP, Operand(LR)); + mov(TMP, Operand(value), NE); + ldr(LR, Address(THR, Thread::write_barrier_wrappers_offset(object)), NE); + blx(LR, NE); + if (!lr_reserved) Pop(LR); + Bind(&done); +#else str(value, dest); // A store buffer update is required. if (lr_reserved) { StoreIntoObjectFilter(object, value, nullptr, can_be_smi, kNoJump); - ldr(LR, Address(THR, Thread::update_store_buffer_wrappers_offset(object)), - NE); + ldr(LR, Address(THR, Thread::write_barrier_wrappers_offset(object)), NE); blx(LR, NE); } else { Label done; @@ -1611,11 +1639,12 @@ void Assembler::StoreIntoObject(Register object, if (object != R0) { mov(R0, Operand(object)); } - ldr(LR, Address(THR, Thread::update_store_buffer_entry_point_offset())); + ldr(LR, Address(THR, Thread::write_barrier_entry_point_offset())); blx(LR); PopList(regs); Bind(&done); } +#endif } void Assembler::StoreIntoObjectOffset(Register object, diff --git a/runtime/vm/compiler/assembler/assembler_arm.h b/runtime/vm/compiler/assembler/assembler_arm.h index f98b47ff1fc..e78943db958 100644 --- a/runtime/vm/compiler/assembler/assembler_arm.h +++ b/runtime/vm/compiler/assembler/assembler_arm.h @@ -808,6 +808,11 @@ class Assembler : public ValueObject { kValueCanBeSmi, }; + // Store into a heap object and apply the generational and incremental write + // barriers. All stores into heap objects must pass through this function or, + // if the value can be proven either Smi or old-and-premarked, its NoBarrier + // variants. + // Preserves object and value registers. void StoreIntoObject(Register object, // Object we are storing into. const Address& dest, // Where we are storing into. Register value, // Value we are storing. diff --git a/runtime/vm/compiler/assembler/assembler_arm64.cc b/runtime/vm/compiler/assembler/assembler_arm64.cc index 1d3afa65f64..0f44233dd14 100644 --- a/runtime/vm/compiler/assembler/assembler_arm64.cc +++ b/runtime/vm/compiler/assembler/assembler_arm64.cc @@ -981,14 +981,51 @@ void Assembler::StoreIntoObject(Register object, ASSERT(object != LR); ASSERT(value != LR); +#if defined(CONCURRENT_MARKING) + ASSERT(object != TMP); + ASSERT(object != TMP2); + ASSERT(value != TMP); + ASSERT(value != TMP2); + + str(value, dest); + + // In parallel, test whether + // - object is old and not remembered and value is new, or + // - object is old and value is old and not marked and concurrent marking is + // in progress + // If so, call the WriteBarrier stub, which will either add object to the + // store buffer (case 1) or add value to the marking stack (case 2). + // Compare RawObject::StorePointer. + Label done; + if (can_be_smi == kValueCanBeSmi) { + BranchIfSmi(value, &done); + } + ldr(TMP, FieldAddress(object, Object::tags_offset()), kUnsignedByte); + ldr(TMP2, FieldAddress(value, Object::tags_offset()), kUnsignedByte); + and_(TMP, TMP2, Operand(TMP, LSR, RawObject::kBarrierOverlapShift)); + tst(TMP, Operand(BARRIER_MASK)); + b(&done, EQ); + + if (!lr_reserved) Push(LR); + mov(TMP2, value); + ldr(LR, Address(THR, Thread::write_barrier_wrappers_offset(object))); + blr(LR); + if (!lr_reserved) Pop(LR); + Bind(&done); +#else + ASSERT(object != value); + ASSERT(object != LR); + ASSERT(value != LR); + str(value, dest); Label done; StoreIntoObjectFilter(object, value, &done, can_be_smi, kJumpToNoUpdate); if (!lr_reserved) Push(LR); - ldr(LR, Address(THR, Thread::update_store_buffer_wrappers_offset(object))); + ldr(LR, Address(THR, Thread::write_barrier_wrappers_offset(object))); blr(LR); if (!lr_reserved) Pop(LR); Bind(&done); +#endif } void Assembler::StoreIntoObjectNoBarrier(Register object, diff --git a/runtime/vm/compiler/assembler/assembler_arm64.h b/runtime/vm/compiler/assembler/assembler_arm64.h index 06f67fa6e07..785b0ac02fe 100644 --- a/runtime/vm/compiler/assembler/assembler_arm64.h +++ b/runtime/vm/compiler/assembler/assembler_arm64.h @@ -1467,7 +1467,11 @@ class Assembler : public ValueObject { kValueCanBeSmi, }; - // Storing into an object. + // Store into a heap object and apply the generational and incremental write + // barriers. All stores into heap objects must pass through this function or, + // if the value can be proven either Smi or old-and-premarked, its NoBarrier + // variants. + // Preserves object and value registers. void StoreIntoObject(Register object, const Address& dest, Register value, diff --git a/runtime/vm/compiler/assembler/assembler_arm64_test.cc b/runtime/vm/compiler/assembler/assembler_arm64_test.cc index bf690bcde54..e3625d06bb7 100644 --- a/runtime/vm/compiler/assembler/assembler_arm64_test.cc +++ b/runtime/vm/compiler/assembler/assembler_arm64_test.cc @@ -2058,14 +2058,17 @@ static void EnterTestFrame(Assembler* assembler) { __ EnterFrame(0); __ Push(CODE_REG); __ Push(THR); + __ Push(BARRIER_MASK); __ TagAndPushPP(); __ ldr(CODE_REG, Address(R0, VMHandles::kOffsetOfRawPtrInHandle)); __ mov(THR, R1); + __ ldr(BARRIER_MASK, Address(THR, Thread::write_barrier_mask_offset())); __ LoadPoolPointer(PP); } static void LeaveTestFrame(Assembler* assembler) { __ PopAndUntagPP(); + __ Pop(BARRIER_MASK); __ Pop(THR); __ Pop(CODE_REG); __ LeaveFrame(); @@ -4090,11 +4093,14 @@ ASSEMBLER_TEST_GENERATE(StoreIntoObject, assembler) { __ SetupDartSP(); __ Push(CODE_REG); __ Push(THR); + __ Push(BARRIER_MASK); __ Push(LR); __ mov(THR, R2); + __ ldr(BARRIER_MASK, Address(THR, Thread::write_barrier_mask_offset())); __ StoreIntoObject(R1, FieldAddress(R1, GrowableObjectArray::data_offset()), R0); __ Pop(LR); + __ Pop(BARRIER_MASK); __ Pop(THR); __ Pop(CODE_REG); __ RestoreCSP(); diff --git a/runtime/vm/compiler/assembler/assembler_ia32.cc b/runtime/vm/compiler/assembler/assembler_ia32.cc index 50d692ece66..082f20fce1d 100644 --- a/runtime/vm/compiler/assembler/assembler_ia32.cc +++ b/runtime/vm/compiler/assembler/assembler_ia32.cc @@ -1870,7 +1870,6 @@ void Assembler::StoreIntoObjectFilter(Register object, j(condition, label, distance); } -// Destroys the value register. void Assembler::StoreIntoObject(Register object, const Address& dest, Register value, @@ -1888,7 +1887,7 @@ void Assembler::StoreIntoObject(Register object, if (object != EDX) { movl(EDX, object); } - call(Address(THR, Thread::update_store_buffer_entry_point_offset())); + call(Address(THR, Thread::write_barrier_entry_point_offset())); if (value != EDX) { popl(EDX); // Restore EDX. } diff --git a/runtime/vm/compiler/assembler/assembler_ia32.h b/runtime/vm/compiler/assembler/assembler_ia32.h index 1684bb660d6..c3b8492f56c 100644 --- a/runtime/vm/compiler/assembler/assembler_ia32.h +++ b/runtime/vm/compiler/assembler/assembler_ia32.h @@ -605,6 +605,12 @@ class Assembler : public ValueObject { kValueCanBeSmi, }; + // Store into a heap object and apply the generational write barrier. (Unlike + // the other architectures, this does not apply the incremental write barrier, + // and so concurrent marking is not enabled for now on IA32.) All stores into + // heap objects must pass through this function or, if the value can be proven + // either Smi or old-and-premarked, its NoBarrier variants. + // Destroys the value register. void StoreIntoObject(Register object, // Object we are storing into. const Address& dest, // Where we are storing into. Register value, // Value we are storing. diff --git a/runtime/vm/compiler/assembler/assembler_x64.cc b/runtime/vm/compiler/assembler/assembler_x64.cc index 4ff49021b3d..24384cab41f 100644 --- a/runtime/vm/compiler/assembler/assembler_x64.cc +++ b/runtime/vm/compiler/assembler/assembler_x64.cc @@ -1275,12 +1275,40 @@ void Assembler::StoreIntoObject(Register object, // x.slot = x. Barrier should have be removed at the IL level. ASSERT(object != value); +#if defined(CONCURRENT_MARKING) + ASSERT(object != TMP); + ASSERT(value != TMP); + + movq(dest, value); + + // In parallel, test whether + // - object is old and not remembered and value is new, or + // - object is old and value is old and not marked and concurrent marking is + // in progress + // If so, call the WriteBarrier stub, which will either add object to the + // store buffer (case 1) or add value to the marking stack (case 2). + // Compare RawObject::StorePointer. + Label done; + if (can_be_smi == kValueCanBeSmi) { + testq(value, Immediate(kSmiTagMask)); + j(ZERO, &done, kNearJump); + } + movb(TMP, FieldAddress(object, Object::tags_offset())); + shrl(TMP, Immediate(RawObject::kBarrierOverlapShift)); + andl(TMP, Address(THR, Thread::write_barrier_mask_offset())); + testb(FieldAddress(value, Object::tags_offset()), TMP); + j(ZERO, &done, kNearJump); + movq(TMP, value); + call(Address(THR, Thread::write_barrier_wrappers_offset(object))); + Bind(&done); +#else movq(dest, value); Label done; StoreIntoObjectFilter(object, value, &done, can_be_smi, kJumpToNoUpdate); // A store buffer update is required. - call(Address(THR, Thread::update_store_buffer_wrappers_offset(object))); + call(Address(THR, Thread::write_barrier_wrappers_offset(object))); Bind(&done); +#endif } void Assembler::StoreIntoObjectNoBarrier(Register object, diff --git a/runtime/vm/compiler/assembler/assembler_x64.h b/runtime/vm/compiler/assembler/assembler_x64.h index bec61eda54b..cd5f054e7fd 100644 --- a/runtime/vm/compiler/assembler/assembler_x64.h +++ b/runtime/vm/compiler/assembler/assembler_x64.h @@ -719,7 +719,11 @@ class Assembler : public ValueObject { kValueCanBeSmi, }; - // Destroys value. + // Store into a heap object and apply the generational and incremental write + // barriers. All stores into heap objects must pass through this function or, + // if the value can be proven either Smi or old-and-premarked, its NoBarrier + // variants. + // Preserves object and value registers. void StoreIntoObject(Register object, // Object we are storing into. const Address& dest, // Where we are storing into. Register value, // Value we are storing. diff --git a/runtime/vm/compiler/backend/il.cc b/runtime/vm/compiler/backend/il.cc index 490f94b4d67..fb9dcdbd27a 100644 --- a/runtime/vm/compiler/backend/il.cc +++ b/runtime/vm/compiler/backend/il.cc @@ -1191,12 +1191,20 @@ void FlowGraphVisitor::VisitBlocks() { } } -bool Value::NeedsStoreBuffer() { +bool Value::NeedsWriteBarrier() { if (Type()->IsNull() || (Type()->ToNullableCid() == kSmiCid) || (Type()->ToNullableCid() == kBoolCid)) { return false; } + // Strictly speaking, the incremental barrier can only be skipped for + // immediate objects (Smis) or permanent objects (vm-isolate heap or + // image pages). Here we choose to skip the barrier for any constant on + // the assumption it will remain reachable through the object pool. + // TODO(concurrent-marking): Consider ensuring marking is not in progress + // when code is disabled or only omitting the barrier if code collection + // is disabled. + return !BindsToConstant(); } diff --git a/runtime/vm/compiler/backend/il.h b/runtime/vm/compiler/backend/il.h index 3e61638d116..ebac7b6496a 100644 --- a/runtime/vm/compiler/backend/il.h +++ b/runtime/vm/compiler/backend/il.h @@ -296,9 +296,9 @@ class Value : public ZoneAllocated { // Assert if BindsToConstant() is false, otherwise returns the constant value. const Object& BoundConstant() const; - // Compile time constants, Bool, Smi and Nulls do not need to update - // the store buffer. - bool NeedsStoreBuffer(); + // Return true if storing the value into a heap object requires applying the + // write barrier. + bool NeedsWriteBarrier(); bool Equals(Value* other) const; @@ -4222,7 +4222,7 @@ class StoreInstanceFieldInstr : public TemplateDefinition<2, NoThrow> { return false; } - return value()->NeedsStoreBuffer() && + return value()->NeedsWriteBarrier() && (emit_store_barrier_ == kEmitStoreBarrier); } @@ -4679,7 +4679,7 @@ class StoreIndexedInstr : public TemplateDefinition<3, NoThrow> { return false; } - return value()->NeedsStoreBuffer() && + return value()->NeedsWriteBarrier() && (emit_store_barrier_ == kEmitStoreBarrier); } diff --git a/runtime/vm/compiler/backend/il_arm.cc b/runtime/vm/compiler/backend/il_arm.cc index f22527839bb..6474df0ae9f 100644 --- a/runtime/vm/compiler/backend/il_arm.cc +++ b/runtime/vm/compiler/backend/il_arm.cc @@ -2218,9 +2218,15 @@ LocationSummary* StoreInstanceFieldInstr::MakeLocationSummary(Zone* zone, summary->set_temp(2, opt ? Location::RequiresFpuRegister() : Location::FpuRegisterLocation(Q1)); } else { +#if defined(CONCURRENT_MARKING) + summary->set_in(1, ShouldEmitStoreBarrier() + ? Location::RequiresRegister() + : Location::RegisterOrConstant(value())); +#else summary->set_in(1, ShouldEmitStoreBarrier() ? Location::WritableRegister() : Location::RegisterOrConstant(value())); +#endif } if (ShouldEmitStoreBarrier()) { summary->set_temp(kNumTemps - 1, Location::RegisterLocation(LR)); @@ -2428,13 +2434,17 @@ LocationSummary* StoreStaticFieldInstr::MakeLocationSummary(Zone* zone, bool opt) const { const intptr_t kNumInputs = 1; const intptr_t kNumTemps = - value()->NeedsStoreBuffer() ? 2 : 1; // Block LR for the store barrier. + value()->NeedsWriteBarrier() ? 2 : 1; // Block LR for the store barrier. LocationSummary* locs = new (zone) LocationSummary(zone, kNumInputs, kNumTemps, LocationSummary::kNoCall); - locs->set_in(0, value()->NeedsStoreBuffer() ? Location::WritableRegister() - : Location::RequiresRegister()); +#if defined(CONCURRENT_MARKING) + locs->set_in(0, Location::RequiresRegister()); +#else + locs->set_in(0, value()->NeedsWriteBarrier() ? Location::WritableRegister() + : Location::RequiresRegister()); +#endif locs->set_temp(0, Location::RequiresRegister()); - if (value()->NeedsStoreBuffer()) { + if (value()->NeedsWriteBarrier()) { locs->set_temp(kNumTemps - 1, Location::RegisterLocation(LR)); } return locs; @@ -2445,7 +2455,7 @@ void StoreStaticFieldInstr::EmitNativeCode(FlowGraphCompiler* compiler) { const Register temp = locs()->temp(0).reg(); __ LoadObject(temp, Field::ZoneHandle(Z, field().Original())); - if (this->value()->NeedsStoreBuffer()) { + if (this->value()->NeedsWriteBarrier()) { __ StoreIntoObject(temp, FieldAddress(temp, Field::static_value_offset()), value, CanValueBeSmi(), /*lr_reserved=*/!compiler->intrinsic_mode()); diff --git a/runtime/vm/compiler/backend/il_arm64.cc b/runtime/vm/compiler/backend/il_arm64.cc index e39d1650a06..24cd2b6ba9e 100644 --- a/runtime/vm/compiler/backend/il_arm64.cc +++ b/runtime/vm/compiler/backend/il_arm64.cc @@ -1929,9 +1929,15 @@ LocationSummary* StoreInstanceFieldInstr::MakeLocationSummary(Zone* zone, summary->set_temp(0, Location::RequiresRegister()); summary->set_temp(1, Location::RequiresRegister()); } else { +#if defined(CONCURRENT_MARKING) + summary->set_in(1, ShouldEmitStoreBarrier() + ? Location::RequiresRegister() + : Location::RegisterOrConstant(value())); +#else summary->set_in(1, ShouldEmitStoreBarrier() ? Location::WritableRegister() : Location::RegisterOrConstant(value())); +#endif } return summary; } @@ -2120,8 +2126,12 @@ LocationSummary* StoreStaticFieldInstr::MakeLocationSummary(Zone* zone, bool opt) const { LocationSummary* locs = new (zone) LocationSummary(zone, 1, 1, LocationSummary::kNoCall); - locs->set_in(0, value()->NeedsStoreBuffer() ? Location::WritableRegister() - : Location::RequiresRegister()); +#if defined(CONCURRENT_MARKING) + locs->set_in(0, Location::RequiresRegister()); +#else + locs->set_in(0, value()->NeedsWriteBarrier() ? Location::WritableRegister() + : Location::RequiresRegister()); +#endif locs->set_temp(0, Location::RequiresRegister()); return locs; } @@ -2131,7 +2141,7 @@ void StoreStaticFieldInstr::EmitNativeCode(FlowGraphCompiler* compiler) { const Register temp = locs()->temp(0).reg(); __ LoadObject(temp, Field::ZoneHandle(Z, field().Original())); - if (this->value()->NeedsStoreBuffer()) { + if (this->value()->NeedsWriteBarrier()) { __ StoreIntoObjectOffset(temp, Field::static_value_offset(), value, CanValueBeSmi()); } else { diff --git a/runtime/vm/compiler/backend/il_ia32.cc b/runtime/vm/compiler/backend/il_ia32.cc index 699fc6507f9..a0b36d6fa5d 100644 --- a/runtime/vm/compiler/backend/il_ia32.cc +++ b/runtime/vm/compiler/backend/il_ia32.cc @@ -2001,8 +2001,8 @@ LocationSummary* StoreStaticFieldInstr::MakeLocationSummary(Zone* zone, bool opt) const { LocationSummary* locs = new (zone) LocationSummary(zone, 1, 1, LocationSummary::kNoCall); - locs->set_in(0, value()->NeedsStoreBuffer() ? Location::WritableRegister() - : Location::RequiresRegister()); + locs->set_in(0, value()->NeedsWriteBarrier() ? Location::WritableRegister() + : Location::RequiresRegister()); locs->set_temp(0, Location::RequiresRegister()); return locs; } @@ -2012,7 +2012,7 @@ void StoreStaticFieldInstr::EmitNativeCode(FlowGraphCompiler* compiler) { Register temp = locs()->temp(0).reg(); __ LoadObject(temp, Field::ZoneHandle(Z, field().Original())); - if (this->value()->NeedsStoreBuffer()) { + if (this->value()->NeedsWriteBarrier()) { __ StoreIntoObject(temp, FieldAddress(temp, Field::static_value_offset()), value, CanValueBeSmi()); } else { diff --git a/runtime/vm/compiler/backend/il_x64.cc b/runtime/vm/compiler/backend/il_x64.cc index 32237fd7d2b..a5e521b0d33 100644 --- a/runtime/vm/compiler/backend/il_x64.cc +++ b/runtime/vm/compiler/backend/il_x64.cc @@ -1919,9 +1919,15 @@ LocationSummary* StoreInstanceFieldInstr::MakeLocationSummary(Zone* zone, summary->set_temp(2, opt ? Location::RequiresFpuRegister() : Location::FpuRegisterLocation(XMM1)); } else { +#if defined(CONCURRENT_MARKING) + summary->set_in(1, ShouldEmitStoreBarrier() + ? Location::RequiresRegister() + : Location::RegisterOrConstant(value())); +#else summary->set_in(1, ShouldEmitStoreBarrier() ? Location::WritableRegister() : Location::RegisterOrConstant(value())); +#endif } return summary; } @@ -2123,8 +2129,12 @@ LocationSummary* StoreStaticFieldInstr::MakeLocationSummary(Zone* zone, bool opt) const { LocationSummary* locs = new (zone) LocationSummary(zone, 1, 1, LocationSummary::kNoCall); - locs->set_in(0, value()->NeedsStoreBuffer() ? Location::WritableRegister() - : Location::RequiresRegister()); +#if defined(CONCURRENT_MARKING) + locs->set_in(0, Location::RequiresRegister()); +#else + locs->set_in(0, value()->NeedsWriteBarrier() ? Location::WritableRegister() + : Location::RequiresRegister()); +#endif locs->set_temp(0, Location::RequiresRegister()); return locs; } @@ -2134,7 +2144,7 @@ void StoreStaticFieldInstr::EmitNativeCode(FlowGraphCompiler* compiler) { Register temp = locs()->temp(0).reg(); __ LoadObject(temp, Field::ZoneHandle(Z, field().Original())); - if (this->value()->NeedsStoreBuffer()) { + if (this->value()->NeedsWriteBarrier()) { __ StoreIntoObject(temp, FieldAddress(temp, Field::static_value_offset()), value, CanValueBeSmi()); } else { diff --git a/runtime/vm/constants_arm64.h b/runtime/vm/constants_arm64.h index 64b09b68849..7ae393bcd3d 100644 --- a/runtime/vm/constants_arm64.h +++ b/runtime/vm/constants_arm64.h @@ -38,7 +38,7 @@ enum Register { R25 = 25, R26 = 26, // THR R27 = 27, // PP - R28 = 28, + R28 = 28, // BARRIER_MASK R29 = 29, // FP R30 = 30, // LR R31 = 31, // ZR, CSP @@ -116,6 +116,7 @@ const Register ARGS_DESC_REG = R4; // Arguments descriptor register. const Register THR = R26; // Caches current thread in generated code. const Register CALLEE_SAVED_TEMP = R19; const Register CALLEE_SAVED_TEMP2 = R20; +const Register BARRIER_MASK = R28; // Exception object is passed in this register to the catch handlers when an // exception is thrown. @@ -149,11 +150,11 @@ const VRegister kAbiFirstPreservedFpuReg = V8; const VRegister kAbiLastPreservedFpuReg = V15; const int kAbiPreservedFpuRegCount = 8; -const intptr_t kReservedCpuRegisters = (1 << SPREG) | // Dart SP - (1 << FPREG) | (1 << TMP) | (1 << TMP2) | - (1 << PP) | (1 << THR) | (1 << LR) | - (1 << R31) | // C++ SP - (1 << R18); // iOS platform register. +const intptr_t kReservedCpuRegisters = + (1 << SPREG) | // Dart SP + (1 << FPREG) | (1 << TMP) | (1 << TMP2) | (1 << PP) | (1 << THR) | + (1 << LR) | (1 << BARRIER_MASK) | (1 << R31) | // C++ SP + (1 << R18); // iOS platform register. // TODO(rmacnak): Only reserve on Mac & iOS. // CPU registers available to Dart allocator. const RegList kDartAvailableCpuRegs = diff --git a/runtime/vm/dart.cc b/runtime/vm/dart.cc index b366d4b9311..d2dcdb2fd43 100644 --- a/runtime/vm/dart.cc +++ b/runtime/vm/dart.cc @@ -92,7 +92,7 @@ static void CheckOffsets() { // These offsets are embedded in precompiled instructions. We need simarm // (compiler) and arm (runtime) to agree. CHECK_OFFSET(Thread::stack_limit_offset(), 4); - CHECK_OFFSET(Thread::object_null_offset(), 56); + CHECK_OFFSET(Thread::object_null_offset(), 64); CHECK_OFFSET(SingleTargetCache::upper_limit_offset(), 14); CHECK_OFFSET(Isolate::object_store_offset(), 28); NOT_IN_PRODUCT(CHECK_OFFSET(sizeof(ClassHeapStats), 168)); @@ -101,7 +101,7 @@ static void CheckOffsets() { // These offsets are embedded in precompiled instructions. We need simarm64 // (compiler) and arm64 (runtime) to agree. CHECK_OFFSET(Thread::stack_limit_offset(), 8); - CHECK_OFFSET(Thread::object_null_offset(), 104); + CHECK_OFFSET(Thread::object_null_offset(), 112); CHECK_OFFSET(SingleTargetCache::upper_limit_offset(), 26); CHECK_OFFSET(Isolate::object_store_offset(), 56); NOT_IN_PRODUCT(CHECK_OFFSET(sizeof(ClassHeapStats), 288)); diff --git a/runtime/vm/globals.h b/runtime/vm/globals.h index a2612113038..82f860cac67 100644 --- a/runtime/vm/globals.h +++ b/runtime/vm/globals.h @@ -85,6 +85,10 @@ const intptr_t kDefaultMaxOldGenHeapSize = (kWordSize <= 4) ? 1536 : 0; #define TAG_IC_DATA #endif +#if !defined(TARGET_OS_MACOS_IOS) && !defined(TARGET_OS_ANDROID) +#define CONCURRENT_MARKING 1 +#endif + // The expression OFFSET_OF(type, field) computes the byte-offset of // the specified field relative to the containing type. // diff --git a/runtime/vm/interpreter.cc b/runtime/vm/interpreter.cc index 88e76f27d53..81b85abb6b6 100644 --- a/runtime/vm/interpreter.cc +++ b/runtime/vm/interpreter.cc @@ -193,7 +193,8 @@ class InterpreterHelpers { RawSmi* index = static_cast(args[1]); RawArray* array = static_cast(args[0]); if (CheckIndex(index, array->ptr()->length_)) { - array->StorePointer(array->ptr()->data() + Smi::Value(index), args[2]); + array->StorePointer(array->ptr()->data() + Smi::Value(index), args[2], + thread); return true; } return false; @@ -228,7 +229,8 @@ class InterpreterHelpers { static_cast(args[0]); if (CheckIndex(index, array->ptr()->length_)) { RawArray* data = array->ptr()->data_; - data->StorePointer(data->ptr()->data() + Smi::Value(index), args[2]); + data->StorePointer(data->ptr()->data() + Smi::Value(index), args[2], + thread); return true; } return false; @@ -1042,7 +1044,7 @@ DART_NOINLINE bool Interpreter::ProcessInvocation(bool* invoked, } instance->StorePointer( reinterpret_cast(instance->ptr()) + offset_in_words, - value); + value, thread); *SP = call_base; **SP = null_value; *invoked = true; @@ -3473,7 +3475,7 @@ RawObject* Interpreter::Call(RawFunction* function, BYTECODE(StoreStaticTOS, A_D); RawField* field = reinterpret_cast(LOAD_CONSTANT(rD)); RawInstance* value = static_cast(*SP--); - field->StorePointer(&field->ptr()->value_.static_value_, value); + field->StorePointer(&field->ptr()->value_.static_value_, value, thread); DISPATCH(); } @@ -3497,8 +3499,8 @@ RawObject* Interpreter::Call(RawFunction* function, ASSERT(!thread->isolate()->use_field_guards()); instance->StorePointer( - reinterpret_cast(instance->ptr()) + offset_in_words, - value); + reinterpret_cast(instance->ptr()) + offset_in_words, value, + thread); DISPATCH(); } @@ -3512,8 +3514,8 @@ RawObject* Interpreter::Call(RawFunction* function, UNREACHABLE(); // TODO(regis): unused, remove. instance->StorePointer( - reinterpret_cast(instance->ptr()) + offset_in_words, - value); + reinterpret_cast(instance->ptr()) + offset_in_words, value, + thread); DISPATCH(); } @@ -3529,8 +3531,8 @@ RawObject* Interpreter::Call(RawFunction* function, ASSERT(!thread->isolate()->use_field_guards()); instance->StorePointer( - reinterpret_cast(instance->ptr()) + offset_in_words, - value); + reinterpret_cast(instance->ptr()) + offset_in_words, value, + thread); DISPATCH(); } @@ -3545,7 +3547,7 @@ RawObject* Interpreter::Call(RawFunction* function, instance->StorePointer( reinterpret_cast(instance->ptr()) + offset_in_words, - value); + value, thread); DISPATCH(); } @@ -3559,8 +3561,8 @@ RawObject* Interpreter::Call(RawFunction* function, SP -= 2; // Drop instance and value. instance->StorePointer( - reinterpret_cast(instance->ptr()) + offset_in_words, - value); + reinterpret_cast(instance->ptr()) + offset_in_words, value, + thread); DISPATCH(); } @@ -4490,7 +4492,8 @@ RawObject* Interpreter::Call(RawFunction* function, RawSmi* index = RAW_CAST(Smi, SP[2]); RawObject* value = SP[3]; ASSERT(InterpreterHelpers::CheckIndex(index, array->ptr()->length_)); - array->StorePointer(array->ptr()->data() + Smi::Value(index), value); + array->StorePointer(array->ptr()->data() + Smi::Value(index), value, + thread); DISPATCH(); } @@ -4500,7 +4503,8 @@ RawObject* Interpreter::Call(RawFunction* function, RawSmi* index = RAW_CAST(Smi, FP[rB]); RawObject* value = FP[rC]; ASSERT(InterpreterHelpers::CheckIndex(index, array->ptr()->length_)); - array->StorePointer(array->ptr()->data() + Smi::Value(index), value); + array->StorePointer(array->ptr()->data() + Smi::Value(index), value, + thread); DISPATCH(); } diff --git a/runtime/vm/raw_object.h b/runtime/vm/raw_object.h index 2529e8cec55..deda12f9a7a 100644 --- a/runtime/vm/raw_object.h +++ b/runtime/vm/raw_object.h @@ -309,6 +309,8 @@ class RawObject { #endif }; + static const intptr_t kGenerationalBarrierMask = 1 << kNewBit; + static const intptr_t kIncrementalBarrierMask = 1 << kOldAndNotMarkedBit; static const intptr_t kBarrierOverlapShift = 2; COMPILE_ASSERT(kOldAndNotMarkedBit + kBarrierOverlapShift == kOldBit); COMPILE_ASSERT(kNewBit + kBarrierOverlapShift == kOldAndNotRememberedBit); @@ -474,19 +476,10 @@ class RawObject { ASSERT(!IsRemembered()); UpdateTagBit(false); } - void SetRememberedBitUnsynchronized() { - ASSERT(!IsRemembered()); - uint32_t tags = ptr()->tags_; - ptr()->tags_ = OldAndNotRememberedBit::update(false, tags); - } void ClearRememberedBit() { ASSERT(IsOldObject()); UpdateTagBit(true); } - void ClearRememberedBitUnsynchronized() { - uint32_t tags = ptr()->tags_; - ptr()->tags_ = OldAndNotRememberedBit::update(true, tags); - } #define DEFINE_IS_CID(clazz) \ bool Is##clazz() const { return ((GetClassId() == k##clazz##Cid)); } @@ -698,11 +691,51 @@ class RawObject { template void StorePointer(type const* addr, type value) { *const_cast(addr) = value; - // Filter stores based on source and target. + if (!value->IsHeapObject()) return; - if (value->IsNewObject() && this->IsOldObject() && !this->IsRemembered()) { - this->SetRememberedBit(); - Thread::Current()->StoreBufferAddObject(this); + + uint32_t source_tags = this->ptr()->tags_; + uint32_t target_tags = value->ptr()->tags_; + Thread* thread = Thread::Current(); + if (((source_tags >> kBarrierOverlapShift) & target_tags & + thread->write_barrier_mask()) != 0) { + if (value->IsNewObject()) { + // Generational barrier: record when a store creates an + // old-and-not-remembered -> new reference. + ASSERT(!this->IsRemembered()); + this->SetRememberedBit(); + thread->StoreBufferAddObject(this); + } else { + // Incremental barrier: record when a store creates an + // old -> old-and-not-marked reference. + ASSERT(value->IsOldObject()); + UNREACHABLE(); + } + } + } + + template + void StorePointer(type const* addr, type value, Thread* thread) { + *const_cast(addr) = value; + + if (!value->IsHeapObject()) return; + + uint32_t source_tags = this->ptr()->tags_; + uint32_t target_tags = value->ptr()->tags_; + if (((source_tags >> kBarrierOverlapShift) & target_tags & + thread->write_barrier_mask()) != 0) { + if (value->IsNewObject()) { + // Generational barrier: record when a store creates an + // old-and-not-remembered -> new reference. + ASSERT(!this->IsRemembered()); + this->SetRememberedBit(); + thread->StoreBufferAddObject(this); + } else { + // Incremental barrier: record when a store creates an + // old -> old-and-not-marked reference. + ASSERT(value->IsOldObject()); + UNREACHABLE(); + } } } diff --git a/runtime/vm/simulator_dbc.cc b/runtime/vm/simulator_dbc.cc index 024d8b107ac..c9bb7444b0b 100644 --- a/runtime/vm/simulator_dbc.cc +++ b/runtime/vm/simulator_dbc.cc @@ -202,7 +202,8 @@ class SimulatorHelpers { RawSmi* index = static_cast(args[1]); RawArray* array = static_cast(args[0]); if (CheckIndex(index, array->ptr()->length_)) { - array->StorePointer(array->ptr()->data() + Smi::Value(index), args[2]); + array->StorePointer(array->ptr()->data() + Smi::Value(index), args[2], + thread); return true; } return false; @@ -237,7 +238,8 @@ class SimulatorHelpers { static_cast(args[0]); if (CheckIndex(index, array->ptr()->length_)) { RawArray* data = array->ptr()->data_; - data->StorePointer(data->ptr()->data() + Smi::Value(index), args[2]); + data->StorePointer(data->ptr()->data() + Smi::Value(index), args[2], + thread); return true; } return false; @@ -2627,7 +2629,7 @@ RawObject* Simulator::Call(const Code& code, BYTECODE(StoreStaticTOS, A_D); RawField* field = reinterpret_cast(LOAD_CONSTANT(rD)); RawInstance* value = static_cast(*SP--); - field->StorePointer(&field->ptr()->value_.static_value_, value); + field->StorePointer(&field->ptr()->value_.static_value_, value, thread); DISPATCH(); } @@ -2648,8 +2650,8 @@ RawObject* Simulator::Call(const Code& code, RawObject* value = FP[value_reg]; instance->StorePointer( - reinterpret_cast(instance->ptr()) + offset_in_words, - value); + reinterpret_cast(instance->ptr()) + offset_in_words, value, + thread); DISPATCH(); } @@ -2661,8 +2663,8 @@ RawObject* Simulator::Call(const Code& code, RawObject* value = FP[rD]; instance->StorePointer( - reinterpret_cast(instance->ptr()) + offset_in_words, - value); + reinterpret_cast(instance->ptr()) + offset_in_words, value, + thread); DISPATCH(); } @@ -2673,8 +2675,8 @@ RawObject* Simulator::Call(const Code& code, RawObject* value = reinterpret_cast(SP[0]); SP -= 2; // Drop instance and value. instance->StorePointer( - reinterpret_cast(instance->ptr()) + offset_in_words, - value); + reinterpret_cast(instance->ptr()) + offset_in_words, value, + thread); DISPATCH(); } @@ -3632,7 +3634,8 @@ RawObject* Simulator::Call(const Code& code, RawSmi* index = RAW_CAST(Smi, SP[2]); RawObject* value = SP[3]; ASSERT(SimulatorHelpers::CheckIndex(index, array->ptr()->length_)); - array->StorePointer(array->ptr()->data() + Smi::Value(index), value); + array->StorePointer(array->ptr()->data() + Smi::Value(index), value, + thread); DISPATCH(); } @@ -3642,7 +3645,8 @@ RawObject* Simulator::Call(const Code& code, RawSmi* index = RAW_CAST(Smi, FP[rB]); RawObject* value = FP[rC]; ASSERT(SimulatorHelpers::CheckIndex(index, array->ptr()->length_)); - array->StorePointer(array->ptr()->data() + Smi::Value(index), value); + array->StorePointer(array->ptr()->data() + Smi::Value(index), value, + thread); DISPATCH(); } diff --git a/runtime/vm/stub_code.h b/runtime/vm/stub_code.h index f796fbd9d74..73ff3dd15e4 100644 --- a/runtime/vm/stub_code.h +++ b/runtime/vm/stub_code.h @@ -26,8 +26,8 @@ class SnapshotWriter; V(JumpToFrame) \ V(RunExceptionHandler) \ V(DeoptForRewind) \ - V(UpdateStoreBuffer) \ - V(UpdateStoreBufferWrappers) \ + V(WriteBarrier) \ + V(WriteBarrierWrappers) \ V(PrintStopMessage) \ V(AllocateArray) \ V(AllocateContext) \ diff --git a/runtime/vm/stub_code_arm.cc b/runtime/vm/stub_code_arm.cc index f0c4243b0cf..3ba95316462 100644 --- a/runtime/vm/stub_code_arm.cc +++ b/runtime/vm/stub_code_arm.cc @@ -468,9 +468,9 @@ static void PushArrayOfArguments(Assembler* assembler) { __ b(&enter); Label loop; __ Bind(&loop); - __ ldr(IP, Address(R1, kWordSize, Address::PreIndex)); + __ ldr(R8, Address(R1, kWordSize, Address::PreIndex)); // Generational barrier is needed, array is not necessarily in new space. - __ StoreIntoObject(R0, Address(R3, R2, LSL, 1), IP); + __ StoreIntoObject(R0, Address(R3, R2, LSL, 1), R8); __ Bind(&enter); __ subs(R2, R2, Operand(Smi::RawValue(1))); // R2 is Smi. __ b(&loop, PL); @@ -1112,7 +1112,7 @@ void StubCode::GenerateAllocateContextStub(Assembler* assembler) { __ Ret(); } -void StubCode::GenerateUpdateStoreBufferWrappersStub(Assembler* assembler) { +void StubCode::GenerateWriteBarrierWrappersStub(Assembler* assembler) { RegList saved = (1 << LR) | (1 << R0); for (intptr_t i = 0; i < kNumberOfCpuRegisters; ++i) { if ((kDartAvailableCpuRegs & (1 << i)) == 0) continue; @@ -1121,7 +1121,7 @@ void StubCode::GenerateUpdateStoreBufferWrappersStub(Assembler* assembler) { intptr_t start = __ CodeSize(); __ PushList(saved); __ mov(R0, Operand(reg)); - __ ldr(LR, Address(THR, Thread::update_store_buffer_entry_point_offset())); + __ ldr(LR, Address(THR, Thread::write_barrier_entry_point_offset())); __ blx(LR); __ PopList(saved); __ bx(LR); @@ -1133,8 +1133,16 @@ void StubCode::GenerateUpdateStoreBufferWrappersStub(Assembler* assembler) { // Helper stub to implement Assembler::StoreIntoObject. // Input parameters: -// R0: address (i.e. object) being stored into. -void StubCode::GenerateUpdateStoreBufferStub(Assembler* assembler) { +// R0: Source object (old) +// TMP: Target object (old or new) +// If TMP is new, add R0 to the store buffer. Otherwise TMP is old, mark TMP +// and add it to the mark list. +void StubCode::GenerateWriteBarrierStub(Assembler* assembler) { +#if defined(CONCURRENT_MARKING) + Label add_to_mark_stack; + __ tst(TMP, Operand(1 << kNewObjectBitPosition)); + __ b(&add_to_mark_stack, ZERO); +#else Label add_to_buffer; // Check whether this object has already been remembered. Skip adding to the // store buffer if the object is in the store buffer already. @@ -1146,16 +1154,17 @@ void StubCode::GenerateUpdateStoreBufferStub(Assembler* assembler) { __ Ret(); __ Bind(&add_to_buffer); +#endif // Save values being destroyed. __ PushList((1 << R1) | (1 << R2) | (1 << R3)); - // R2: Header word. if (TargetCPUFeatures::arm_version() == ARMv5TE) { // TODO(21263): Implement 'swp' and use it below. #if !defined(USING_SIMULATOR) ASSERT(OS::NumberOfAvailableProcessors() <= 1); #endif + __ ldr(R2, FieldAddress(R0, Object::tags_offset())); __ bic(R2, R2, Operand(1 << RawObject::kOldAndNotRememberedBit)); __ str(R2, FieldAddress(R0, Object::tags_offset())); } else { @@ -1176,7 +1185,7 @@ void StubCode::GenerateUpdateStoreBufferStub(Assembler* assembler) { // StoreBufferBlock and add the address to the pointers_. __ ldr(R1, Address(THR, Thread::store_buffer_block_offset())); __ ldr(R2, Address(R1, StoreBufferBlock::top_offset())); - __ add(R3, R1, Operand(R2, LSL, 2)); + __ add(R3, R1, Operand(R2, LSL, kWordSizeLog2)); __ str(R0, Address(R3, StoreBufferBlock::pointers_offset())); // Increment top_ and check for overflow. @@ -1196,7 +1205,7 @@ void StubCode::GenerateUpdateStoreBufferStub(Assembler* assembler) { // Setup frame, push callee-saved registers. __ Push(CODE_REG); - __ ldr(CODE_REG, Address(THR, Thread::update_store_buffer_code_offset())); + __ ldr(CODE_REG, Address(THR, Thread::write_barrier_code_offset())); __ EnterCallRuntimeFrame(0 * kWordSize); __ mov(R0, Operand(THR)); __ CallRuntime(kStoreBufferBlockProcessRuntimeEntry, 1); @@ -1204,6 +1213,11 @@ void StubCode::GenerateUpdateStoreBufferStub(Assembler* assembler) { __ LeaveCallRuntimeFrame(); __ Pop(CODE_REG); __ Ret(); + +#if defined(CONCURRENT_MARKING) + __ Bind(&add_to_mark_stack); + __ Stop("Incremental barrier"); +#endif } // Called for inline allocation of objects. diff --git a/runtime/vm/stub_code_arm64.cc b/runtime/vm/stub_code_arm64.cc index 6a8cc121540..c63f0e85ce7 100644 --- a/runtime/vm/stub_code_arm64.cc +++ b/runtime/vm/stub_code_arm64.cc @@ -113,6 +113,9 @@ void StubCode::GenerateCallToRuntimeStub(Assembler* assembler) { __ mov(SP, CSP); __ mov(CSP, R25); + // Refresh write barrier mask. + __ ldr(BARRIER_MASK, Address(THR, Thread::write_barrier_mask_offset())); + // Retval is next to 1st argument. // Mark that the thread is executing Dart code. __ LoadImmediate(R2, VMTag::kDartTagId); @@ -302,6 +305,9 @@ static void GenerateCallNativeWithWrapperStub(Assembler* assembler, __ mov(SP, CSP); __ mov(CSP, R25); + // Refresh write barrier mask. + __ ldr(BARRIER_MASK, Address(THR, Thread::write_barrier_mask_offset())); + // Mark that the thread is executing Dart code. __ LoadImmediate(R2, VMTag::kDartTagId); __ StoreToOffset(R2, THR, Thread::vm_tag_offset()); @@ -404,6 +410,9 @@ void StubCode::GenerateCallBootstrapNativeStub(Assembler* assembler) { __ mov(SP, CSP); __ mov(CSP, R25); + // Refresh write barrier mask. + __ ldr(BARRIER_MASK, Address(THR, Thread::write_barrier_mask_offset())); + // Mark that the thread is executing Dart code. __ LoadImmediate(R2, VMTag::kDartTagId); __ StoreToOffset(R2, THR, Thread::vm_tag_offset()); @@ -947,6 +956,8 @@ void StubCode::GenerateInvokeDartCodeStub(Assembler* assembler) { if (THR != R3) { __ mov(THR, R3); } + // Refresh write barrier mask. + __ ldr(BARRIER_MASK, Address(THR, Thread::write_barrier_mask_offset())); // Save the current VMTag on the stack. __ LoadFromOffset(R4, THR, Thread::vm_tag_offset()); @@ -1088,6 +1099,8 @@ void StubCode::GenerateInvokeDartCodeFromBytecodeStub(Assembler* assembler) { if (THR != R3) { __ mov(THR, R3); } + // Refresh write barrier mask. + __ ldr(BARRIER_MASK, Address(THR, Thread::write_barrier_mask_offset())); // Save the current VMTag on the stack. __ LoadFromOffset(R4, THR, Thread::vm_tag_offset()); @@ -1296,7 +1309,7 @@ void StubCode::GenerateAllocateContextStub(Assembler* assembler) { __ ret(); } -void StubCode::GenerateUpdateStoreBufferWrappersStub(Assembler* assembler) { +void StubCode::GenerateWriteBarrierWrappersStub(Assembler* assembler) { for (intptr_t i = 0; i < kNumberOfCpuRegisters; ++i) { if ((kDartAvailableCpuRegs & (1 << i)) == 0) continue; @@ -1305,7 +1318,7 @@ void StubCode::GenerateUpdateStoreBufferWrappersStub(Assembler* assembler) { __ Push(LR); __ Push(R0); __ mov(R0, reg); - __ ldr(LR, Address(THR, Thread::update_store_buffer_entry_point_offset())); + __ ldr(LR, Address(THR, Thread::write_barrier_entry_point_offset())); __ blr(LR); __ Pop(R0); __ Pop(LR); @@ -1318,8 +1331,15 @@ void StubCode::GenerateUpdateStoreBufferWrappersStub(Assembler* assembler) { // Helper stub to implement Assembler::StoreIntoObject. // Input parameters: -// R0: Address being stored -void StubCode::GenerateUpdateStoreBufferStub(Assembler* assembler) { +// R0: Source object (old) +// TMP2: Target object (old or new) +// If TMP2 is new, add R0 to the store buffer. Otherwise TMP2 is old, mark TMP2 +// and add it to the mark list. +void StubCode::GenerateWriteBarrierStub(Assembler* assembler) { +#if defined(CONCURRENT_MARKING) + Label add_to_mark_stack; + __ tbz(&add_to_mark_stack, TMP2, kNewObjectBitPosition); +#else Label add_to_buffer; // Check whether this object has already been remembered. Skip adding to the // store buffer if the object is in the store buffer already. @@ -1328,6 +1348,8 @@ void StubCode::GenerateUpdateStoreBufferStub(Assembler* assembler) { __ ret(); __ Bind(&add_to_buffer); +#endif + // Save values being destroyed. __ Push(R1); __ Push(R2); @@ -1350,7 +1372,7 @@ void StubCode::GenerateUpdateStoreBufferStub(Assembler* assembler) { // StoreBufferBlock and add the address to the pointers_. __ LoadFromOffset(R1, THR, Thread::store_buffer_block_offset()); __ LoadFromOffset(R2, R1, StoreBufferBlock::top_offset(), kUnsignedWord); - __ add(R3, R1, Operand(R2, LSL, 3)); + __ add(R3, R1, Operand(R2, LSL, kWordSizeLog2)); __ StoreToOffset(R0, R3, StoreBufferBlock::pointers_offset()); // Increment top_ and check for overflow. @@ -1372,7 +1394,7 @@ void StubCode::GenerateUpdateStoreBufferStub(Assembler* assembler) { // Setup frame, push callee-saved registers. __ Push(CODE_REG); - __ ldr(CODE_REG, Address(THR, Thread::update_store_buffer_code_offset())); + __ ldr(CODE_REG, Address(THR, Thread::write_barrier_code_offset())); __ EnterCallRuntimeFrame(0 * kWordSize); __ mov(R0, THR); __ CallRuntime(kStoreBufferBlockProcessRuntimeEntry, 1); @@ -1380,6 +1402,11 @@ void StubCode::GenerateUpdateStoreBufferStub(Assembler* assembler) { __ LeaveCallRuntimeFrame(); __ Pop(CODE_REG); __ ret(); + +#if defined(CONCURRENT_MARKING) + __ Bind(&add_to_mark_stack); + __ Stop("Incremental barrier"); +#endif } // Called for inline allocation of objects. @@ -2030,6 +2057,9 @@ void StubCode::GenerateInterpretCallStub(Assembler* assembler) { __ mov(SP, CSP); __ mov(CSP, R25); + // Refresh write barrier mask. + __ ldr(BARRIER_MASK, Address(THR, Thread::write_barrier_mask_offset())); + // Mark that the thread is executing Dart code. __ LoadImmediate(R2, VMTag::kDartTagId); __ StoreToOffset(R2, THR, Thread::vm_tag_offset()); @@ -2517,6 +2547,7 @@ void StubCode::GenerateJumpToFrameStub(Assembler* assembler) { __ mov(SP, R1); // Stack pointer. __ mov(FP, R2); // Frame_pointer. __ mov(THR, R3); + __ ldr(BARRIER_MASK, Address(THR, Thread::write_barrier_mask_offset())); // Set the tag. __ LoadImmediate(R2, VMTag::kDartTagId); __ StoreToOffset(R2, THR, Thread::vm_tag_offset()); diff --git a/runtime/vm/stub_code_ia32.cc b/runtime/vm/stub_code_ia32.cc index fb9c43e4e08..369dad7ed91 100644 --- a/runtime/vm/stub_code_ia32.cc +++ b/runtime/vm/stub_code_ia32.cc @@ -972,7 +972,7 @@ void StubCode::GenerateAllocateContextStub(Assembler* assembler) { __ ret(); } -void StubCode::GenerateUpdateStoreBufferWrappersStub(Assembler* assembler) { +void StubCode::GenerateWriteBarrierWrappersStub(Assembler* assembler) { // Not used on IA32. __ Breakpoint(); } @@ -980,7 +980,7 @@ void StubCode::GenerateUpdateStoreBufferWrappersStub(Assembler* assembler) { // Helper stub to implement Assembler::StoreIntoObject. // Input parameters: // EDX: Address being stored -void StubCode::GenerateUpdateStoreBufferStub(Assembler* assembler) { +void StubCode::GenerateWriteBarrierStub(Assembler* assembler) { // Save values being destroyed. __ pushl(EAX); __ pushl(ECX); diff --git a/runtime/vm/stub_code_x64.cc b/runtime/vm/stub_code_x64.cc index 59fd88dfc02..f6a279076ec 100644 --- a/runtime/vm/stub_code_x64.cc +++ b/runtime/vm/stub_code_x64.cc @@ -1242,7 +1242,7 @@ void StubCode::GenerateAllocateContextStub(Assembler* assembler) { __ ret(); } -void StubCode::GenerateUpdateStoreBufferWrappersStub(Assembler* assembler) { +void StubCode::GenerateWriteBarrierWrappersStub(Assembler* assembler) { for (intptr_t i = 0; i < kNumberOfCpuRegisters; ++i) { if ((kDartAvailableCpuRegs & (1 << i)) == 0) continue; @@ -1250,7 +1250,7 @@ void StubCode::GenerateUpdateStoreBufferWrappersStub(Assembler* assembler) { intptr_t start = __ CodeSize(); __ pushq(RDX); __ movq(RDX, reg); - __ call(Address(THR, Thread::update_store_buffer_entry_point_offset())); + __ call(Address(THR, Thread::write_barrier_entry_point_offset())); __ popq(RDX); __ ret(); intptr_t end = __ CodeSize(); @@ -1261,8 +1261,16 @@ void StubCode::GenerateUpdateStoreBufferWrappersStub(Assembler* assembler) { // Helper stub to implement Assembler::StoreIntoObject. // Input parameters: -// RDX: Address being stored -void StubCode::GenerateUpdateStoreBufferStub(Assembler* assembler) { +// RDX: Source object (old) +// TMP: Target object (old or new) +// If TMP is new, add RDX to the store buffer. Otherwise TMP is old, mark TMP +// and add it to the mark list. +void StubCode::GenerateWriteBarrierStub(Assembler* assembler) { +#if defined(CONCURRENT_MARKING) + Label add_to_mark_stack; + __ testq(TMP, Immediate(1 << kNewObjectBitPosition)); + __ j(ZERO, &add_to_mark_stack); +#else Label add_to_buffer; // Check whether this object has already been remembered. Skip adding to the // store buffer if the object is in the store buffer already. @@ -1272,12 +1280,14 @@ void StubCode::GenerateUpdateStoreBufferStub(Assembler* assembler) { __ j(NOT_EQUAL, &add_to_buffer, Assembler::kNearJump); __ ret(); + __ Bind(&add_to_buffer); +#endif + // Update the tags that this object has been remembered. // Note that we use 32 bit operations here to match the size of the // background sweeper which is also manipulating this 32 bit word. // RDX: Address being stored // RAX: Current tag value - __ Bind(&add_to_buffer); // lock+andl is an atomic read-modify-write. __ lock(); __ andl(FieldAddress(RDX, Object::tags_offset()), @@ -1311,13 +1321,18 @@ void StubCode::GenerateUpdateStoreBufferStub(Assembler* assembler) { __ Bind(&overflow); // Setup frame, push callee-saved registers. __ pushq(CODE_REG); - __ movq(CODE_REG, Address(THR, Thread::update_store_buffer_code_offset())); + __ movq(CODE_REG, Address(THR, Thread::write_barrier_code_offset())); __ EnterCallRuntimeFrame(0); __ movq(CallingConventions::kArg1Reg, THR); __ CallRuntime(kStoreBufferBlockProcessRuntimeEntry, 1); __ LeaveCallRuntimeFrame(); __ popq(CODE_REG); __ ret(); + +#if defined(CONCURRENT_MARKING) + __ Bind(&add_to_mark_stack); + __ Stop("Incremental barrier"); +#endif } // Called for inline allocation of objects. diff --git a/runtime/vm/thread.cc b/runtime/vm/thread.cc index a188d2b1c8e..ba4ecc768ee 100644 --- a/runtime/vm/thread.cc +++ b/runtime/vm/thread.cc @@ -60,6 +60,7 @@ Thread::Thread(Isolate* isolate) : BaseThread(false), stack_limit_(0), stack_overflow_flags_(0), + write_barrier_mask_(RawObject::kGenerationalBarrierMask), isolate_(NULL), heap_(NULL), top_(0), @@ -119,7 +120,7 @@ Thread::Thread(Isolate* isolate) #if defined(TARGET_ARCH_ARM) || defined(TARGET_ARCH_ARM64) || \ defined(TARGET_ARCH_X64) for (intptr_t i = 0; i < kNumberOfDartAvailableCpuRegs; ++i) { - update_store_buffer_wrappers_entry_points_[i] = 0; + write_barrier_wrappers_entry_points_[i] = 0; } #endif @@ -213,8 +214,8 @@ void Thread::InitVMConstants() { #if defined(TARGET_ARCH_ARM) || defined(TARGET_ARCH_ARM64) || \ defined(TARGET_ARCH_X64) for (intptr_t i = 0; i < kNumberOfDartAvailableCpuRegs; ++i) { - update_store_buffer_wrappers_entry_points_[i] = - StubCode::UpdateStoreBufferWrappers_entry()->EntryPoint() + + write_barrier_wrappers_entry_points_[i] = + StubCode::WriteBarrierWrappers_entry()->EntryPoint() + i * kStoreBufferWrapperSize; } #endif diff --git a/runtime/vm/thread.h b/runtime/vm/thread.h index 65402deb06c..cb5d6c15058 100644 --- a/runtime/vm/thread.h +++ b/runtime/vm/thread.h @@ -83,8 +83,8 @@ class Zone; #define CACHED_VM_STUBS_LIST(V) #else #define CACHED_VM_STUBS_LIST(V) \ - V(RawCode*, update_store_buffer_code_, \ - StubCode::UpdateStoreBuffer_entry()->code(), NULL) \ + V(RawCode*, write_barrier_code_, StubCode::WriteBarrier_entry()->code(), \ + NULL) \ V(RawCode*, fix_callers_target_code_, \ StubCode::FixCallersTarget_entry()->code(), NULL) \ V(RawCode*, fix_allocation_stub_code_, \ @@ -139,8 +139,8 @@ class Zone; #define CACHED_VM_STUBS_ADDRESSES_LIST(V) #else #define CACHED_VM_STUBS_ADDRESSES_LIST(V) \ - V(uword, update_store_buffer_entry_point_, \ - StubCode::UpdateStoreBuffer_entry()->EntryPoint(), 0) \ + V(uword, write_barrier_entry_point_, \ + StubCode::WriteBarrier_entry()->EntryPoint(), 0) \ V(uword, call_to_runtime_entry_point_, \ StubCode::CallToRuntime_entry()->EntryPoint(), 0) \ V(uword, null_error_shared_without_fpu_regs_entry_point_, \ @@ -274,8 +274,10 @@ class Thread : public BaseThread { kOsrRequest = 0x1, // Current stack overflow caused by OSR request. }; - uword stack_overflow_flags_address() const { - return reinterpret_cast(&stack_overflow_flags_); + uword write_barrier_mask() const { return write_barrier_mask_; } + + static intptr_t write_barrier_mask_offset() { + return OFFSET_OF(Thread, write_barrier_mask_); } static intptr_t stack_overflow_flags_offset() { return OFFSET_OF(Thread, stack_overflow_flags_); @@ -522,7 +524,7 @@ class Thread : public BaseThread { #if defined(TARGET_ARCH_ARM) || defined(TARGET_ARCH_ARM64) || \ defined(TARGET_ARCH_X64) - static intptr_t update_store_buffer_wrappers_offset(Register reg) { + static intptr_t write_barrier_wrappers_offset(Register reg) { ASSERT((kDartAvailableCpuRegs & (1 << reg)) != 0); intptr_t index = 0; for (intptr_t i = 0; i < kNumberOfCpuRegisters; ++i) { @@ -530,7 +532,7 @@ class Thread : public BaseThread { if (i == reg) break; ++index; } - return OFFSET_OF(Thread, update_store_buffer_wrappers_entry_points_) + + return OFFSET_OF(Thread, write_barrier_wrappers_entry_points_) + index * sizeof(uword); } #endif @@ -799,6 +801,7 @@ class Thread : public BaseThread { // different architectures. See also CheckOffsets in dart.cc. uword stack_limit_; uword stack_overflow_flags_; + uword write_barrier_mask_; Isolate* isolate_; Heap* heap_; uword top_; @@ -812,7 +815,7 @@ class Thread : public BaseThread { // generated code to runtime. // TODO(dartbug.com/33549): Clean this up when unboxed values // could be passed as arguments. - int64_t unboxed_int64_runtime_arg_; + ALIGN8 int64_t unboxed_int64_runtime_arg_; // State that is cached in the TLS for fast access in generated code. #define DECLARE_MEMBERS(type_name, member_name, expr, default_init_value) \ @@ -830,8 +833,7 @@ class Thread : public BaseThread { #if defined(TARGET_ARCH_ARM) || defined(TARGET_ARCH_ARM64) || \ defined(TARGET_ARCH_X64) - uword - update_store_buffer_wrappers_entry_points_[kNumberOfDartAvailableCpuRegs]; + uword write_barrier_wrappers_entry_points_[kNumberOfDartAvailableCpuRegs]; #endif TimelineStream* dart_stream_;