diff --git a/runtime/vm/heap.cc b/runtime/vm/heap.cc index ad1f24deadd..1aa3a8e5c3a 100644 --- a/runtime/vm/heap.cc +++ b/runtime/vm/heap.cc @@ -57,23 +57,72 @@ Heap::~Heap() { } } +void Heap::FillRemainingTLAB(Thread* thread) { + uword start = thread->top(); + uword end = thread->end(); + ASSERT(end >= start); + intptr_t size = end - start; + ASSERT(Utils::IsAligned(size, kObjectAlignment)); + if (size >= kObjectAlignment) { + FreeListElement::AsElement(start, size); + ASSERT(RawObject::FromAddr(start)->Size() == size); + ASSERT((start + size) == new_space_.top()); + } +} + +void Heap::AbandonRemainingTLAB(Thread* thread) { + FillRemainingTLAB(thread); + thread->set_top(0); + thread->set_end(0); +} + +intptr_t Heap::CalculateTLABSize() { + intptr_t size = new_space_.end() - new_space_.top(); + return Utils::RoundDown(size, kObjectAlignment); +} + uword Heap::AllocateNew(intptr_t size) { ASSERT(Thread::Current()->no_safepoint_scope_depth() == 0); // Currently, only the Dart thread may allocate in new space. isolate()->AssertCurrentThreadIsMutator(); Thread* thread = Thread::Current(); uword addr = new_space_.TryAllocateInTLAB(thread, size); - if (addr == 0) { - // This call to CollectGarbage might end up "reusing" a collection spawned - // from a different thread and will be racing to allocate the requested - // memory with other threads being released after the collection. - CollectGarbage(kNew); - addr = new_space_.TryAllocateInTLAB(thread, size); - if (addr == 0) { - return AllocateOld(size, HeapPage::kData); + if (addr != 0) { + return addr; + } + + intptr_t tlab_size = CalculateTLABSize(); + if ((tlab_size > 0) && (size > tlab_size)) { + return AllocateOld(size, HeapPage::kData); + } + + AbandonRemainingTLAB(thread); + if (tlab_size > 0) { + uword tlab_top = new_space_.TryAllocateNewTLAB(thread, tlab_size); + if (tlab_top != 0) { + addr = new_space_.TryAllocateInTLAB(thread, size); + ASSERT(addr != 0); + return addr; } } - return addr; + + ASSERT(!thread->HasActiveTLAB()); + + // This call to CollectGarbage might end up "reusing" a collection spawned + // from a different thread and will be racing to allocate the requested + // memory with other threads being released after the collection. + CollectGarbage(kNew); + tlab_size = CalculateTLABSize(); + uword tlab_top = new_space_.TryAllocateNewTLAB(thread, tlab_size); + if (tlab_top != 0) { + addr = new_space_.TryAllocateInTLAB(thread, size); + // It is possible a GC doesn't clear enough space. + // In that case, we must fall through and allocate into old space. + if (addr != 0) { + return addr; + } + } + return AllocateOld(size, HeapPage::kData); } uword Heap::AllocateOld(intptr_t size, HeapPage::PageType type) { @@ -555,13 +604,14 @@ bool Heap::VerifyGC(MarkExpectation mark_expectation) const { StackZone stack_zone(Thread::Current()); // Change the new space's top_ with the more up-to-date thread's view of top_ - new_space_.FlushTLS(); + uword saved_top = new_space_.FlushTLS(); ObjectSet* allocated_set = CreateAllocatedObjectSet(stack_zone.GetZone(), mark_expectation); VerifyPointersVisitor visitor(isolate(), allocated_set); VisitObjectPointers(&visitor); + new_space_.UnflushTLS(saved_top); // Only returning a value so that Heap::Validate can be called from an ASSERT. return true; } diff --git a/runtime/vm/heap.h b/runtime/vm/heap.h index 2acc2b188b6..97240bf5010 100644 --- a/runtime/vm/heap.h +++ b/runtime/vm/heap.h @@ -256,6 +256,10 @@ class Heap { old_space_.SetupImagePage(pointer, size, is_executable); } + intptr_t CalculateTLABSize(); + void FillRemainingTLAB(Thread* thread); + void AbandonRemainingTLAB(Thread* thread); + private: class GCStats : public ValueObject { public: diff --git a/runtime/vm/isolate.cc b/runtime/vm/isolate.cc index 8589373d6fd..5971446ef6e 100644 --- a/runtime/vm/isolate.cc +++ b/runtime/vm/isolate.cc @@ -2574,9 +2574,10 @@ Thread* Isolate::ScheduleThread(bool is_mutator, bool bypass_safepoint) { os_thread->set_thread(thread); if (is_mutator) { mutator_thread_ = thread; - if (this != Dart::vm_isolate()) { - mutator_thread_->set_top(heap()->new_space()->top()); - mutator_thread_->set_end(heap()->new_space()->end()); + if ((Dart::vm_isolate() != NULL) && + (heap() != Dart::vm_isolate()->heap())) { + mutator_thread_->set_top(0); + mutator_thread_->set_end(0); } } Thread::SetCurrent(thread); @@ -2617,11 +2618,11 @@ void Isolate::UnscheduleThread(Thread* thread, OSThread::SetCurrent(os_thread); if (is_mutator) { if (this != Dart::vm_isolate()) { - heap()->new_space()->set_top(mutator_thread_->top_); - heap()->new_space()->set_end(mutator_thread_->end_); + if (mutator_thread_->HasActiveTLAB()) { + heap()->AbandonRemainingTLAB(mutator_thread_); + } } - mutator_thread_->top_ = 0; - mutator_thread_->end_ = 0; + ASSERT(!mutator_thread_->HasActiveTLAB()); mutator_thread_ = NULL; } thread->isolate_ = NULL; diff --git a/runtime/vm/scavenger.cc b/runtime/vm/scavenger.cc index 32d62e00090..da1792cde1b 100644 --- a/runtime/vm/scavenger.cc +++ b/runtime/vm/scavenger.cc @@ -394,13 +394,6 @@ SemiSpace* Scavenger::Prologue(Isolate* isolate, bool invoke_api_callbacks) { resolved_top_ = top_; end_ = to_->end(); - // Throw out the old information about the from space - if (isolate->IsMutatorThreadScheduled()) { - Thread* mutator_thread = isolate->mutator_thread(); - mutator_thread->set_top(top_); - mutator_thread->set_end(end_); - } - return from; } @@ -410,12 +403,11 @@ void Scavenger::Epilogue(Isolate* isolate, // All objects in the to space have been copied from the from space at this // moment. - // Ensure the mutator thread now has the up-to-date top_ and end_ of the - // semispace + // Ensure the mutator thread will fail the next allocation. This will force + // mutator to allocate a new TLAB if (isolate->IsMutatorThreadScheduled()) { - Thread* thread = isolate->mutator_thread(); - thread->set_top(top_); - thread->set_end(end_); + Thread* mutator_thread = isolate->mutator_thread(); + ASSERT(!mutator_thread->HasActiveTLAB()); } double avg_frac = stats_history_.Get(0).PromoCandidatesSuccessFraction(); @@ -722,31 +714,49 @@ void Scavenger::ProcessWeakReferences() { } } -void Scavenger::FlushTLS() const { +uword Scavenger::FlushTLS() const { ASSERT(heap_ != NULL); - if (heap_->isolate()->IsMutatorThreadScheduled()) { + uword saved_top = top_; + if (heap_->isolate()->IsMutatorThreadScheduled() && !scavenging_) { Thread* mutator_thread = heap_->isolate()->mutator_thread(); - mutator_thread->heap()->new_space()->set_top(mutator_thread->top()); + saved_top = mutator_thread->heap()->new_space()->top(); + if (mutator_thread->HasActiveTLAB()) { + ASSERT(mutator_thread->top() <= + mutator_thread->heap()->new_space()->top()); + heap_->FillRemainingTLAB(mutator_thread); + } + } + return saved_top; +} + +void Scavenger::UnflushTLS(uword value) const { + ASSERT(heap_ != NULL); + if (heap_->isolate()->IsMutatorThreadScheduled() && !scavenging_) { + Thread* mutator_thread = heap_->isolate()->mutator_thread(); + mutator_thread->heap()->new_space()->set_top(value); + ASSERT(mutator_thread->top() <= mutator_thread->heap()->new_space()->top()); } } void Scavenger::VisitObjectPointers(ObjectPointerVisitor* visitor) const { - FlushTLS(); + uword saved_top = FlushTLS(); uword cur = FirstObjectStart(); while (cur < top_) { RawObject* raw_obj = RawObject::FromAddr(cur); cur += raw_obj->VisitPointers(visitor); } + UnflushTLS(saved_top); } void Scavenger::VisitObjects(ObjectVisitor* visitor) const { - FlushTLS(); + uword saved_top = FlushTLS(); uword cur = FirstObjectStart(); while (cur < top_) { RawObject* raw_obj = RawObject::FromAddr(cur); visitor->VisitObject(raw_obj); cur += raw_obj->Size(); } + UnflushTLS(saved_top); } void Scavenger::AddRegionsToObjectSet(ObjectSet* set) const { @@ -755,19 +765,21 @@ void Scavenger::AddRegionsToObjectSet(ObjectSet* set) const { RawObject* Scavenger::FindObject(FindObjectVisitor* visitor) const { ASSERT(!scavenging_); - FlushTLS(); + uword saved_top = FlushTLS(); uword cur = FirstObjectStart(); if (visitor->VisitRange(cur, top_)) { while (cur < top_) { RawObject* raw_obj = RawObject::FromAddr(cur); uword next = cur + raw_obj->Size(); if (visitor->VisitRange(cur, next) && raw_obj->FindObject(visitor)) { + UnflushTLS(saved_top); return raw_obj; // Found object, return it. } cur = next; } ASSERT(cur == top_); } + UnflushTLS(saved_top); return Object::null(); } @@ -802,6 +814,13 @@ void Scavenger::Scavenge(bool invoke_api_callbacks) { int64_t post_safe_point = OS::GetCurrentMonotonicMicros(); heap_->RecordTime(kSafePoint, post_safe_point - pre_safe_point); + if (isolate->IsMutatorThreadScheduled()) { + Thread* mutator_thread = isolate->mutator_thread(); + if (mutator_thread->HasActiveTLAB()) { + heap_->AbandonRemainingTLAB(mutator_thread); + } + } + // TODO(koda): Make verification more compatible with concurrent sweep. if (FLAG_verify_before_gc && !FLAG_concurrent_sweep) { OS::PrintErr("Verifying before Scavenge..."); @@ -915,7 +934,9 @@ void Scavenger::Evacuate() { if (heap_->isolate()->IsMutatorThreadScheduled()) { Thread* mutator_thread = heap_->isolate()->mutator_thread(); - survivor_end_ = mutator_thread->top(); + if (mutator_thread->HasActiveTLAB()) { + survivor_end_ = mutator_thread->top(); + } } Scavenge(); @@ -925,4 +946,11 @@ void Scavenger::Evacuate() { ASSERT((UsedInWords() == 0) || failed_to_promote_); } +int64_t Scavenger::UsedInWords() const { + uword saved_top = FlushTLS(); + int64_t used_in_words = (top_ - FirstObjectStart()) >> kWordSizeLog2; + UnflushTLS(saved_top); + return used_in_words; +} + } // namespace dart diff --git a/runtime/vm/scavenger.h b/runtime/vm/scavenger.h index 1fd7c9fe2bd..0721fe2f916 100644 --- a/runtime/vm/scavenger.h +++ b/runtime/vm/scavenger.h @@ -121,6 +121,25 @@ class Scavenger { RawObject* FindObject(FindObjectVisitor* visitor) const; + uword TryAllocateNewTLAB(Thread* thread, intptr_t size) { + ASSERT(Utils::IsAligned(size, kObjectAlignment)); + ASSERT(heap_ != Dart::vm_isolate()->heap()); + ASSERT(!scavenging_); + uword result = top_; + intptr_t remaining = end_ - top_; + if (remaining < size) { + return 0; + } + ASSERT(to_->Contains(result)); + ASSERT((result & kObjectAlignmentMask) == object_alignment_); + top_ += size; + ASSERT(to_->Contains(top_) || (top_ == to_->end())); + ASSERT(result < top_); + thread->set_top(result); + thread->set_end(top_); + return result; + } + uword AllocateGC(intptr_t size) { ASSERT(Utils::IsAligned(size, kObjectAlignment)); ASSERT(heap_ != Dart::vm_isolate()->heap()); @@ -134,7 +153,7 @@ class Scavenger { ASSERT(to_->Contains(result)); ASSERT((result & kObjectAlignmentMask) == object_alignment_); top_ += size; - ASSERT(to_->Contains(top_) || (top_ == to_->end())); + ASSERT((to_->Contains(top_)) || (top_ == to_->end())); return result; } @@ -143,6 +162,8 @@ class Scavenger { ASSERT(heap_ != Dart::vm_isolate()->heap()); ASSERT(thread->IsMutatorThread()); ASSERT(thread->isolate()->IsMutatorThreadScheduled()); + ASSERT(thread->top() <= top_); + ASSERT((thread->end() == 0) || (thread->end() == top_)); #if defined(DEBUG) if (FLAG_gc_at_alloc) { ASSERT(!scavenging_); @@ -159,7 +180,7 @@ class Scavenger { ASSERT(to_->Contains(result)); ASSERT((result & kObjectAlignmentMask) == object_alignment_); top += size; - ASSERT(to_->Contains(top) || (top == to_->end())); + ASSERT((to_->Contains(top)) || (top == to_->end())); thread->set_top(top); return result; } @@ -180,9 +201,7 @@ class Scavenger { end_ = value; } - int64_t UsedInWords() const { - return (top_ - FirstObjectStart()) >> kWordSizeLog2; - } + int64_t UsedInWords() const; int64_t CapacityInWords() const { return to_->size_in_words(); } int64_t ExternalInWords() const { return external_size_ >> kWordSizeLog2; } SpaceUsage GetCurrentUsage() const { @@ -215,7 +234,9 @@ class Scavenger { void AllocateExternal(intptr_t size); void FreeExternal(intptr_t size); - void FlushTLS() const; + uword FlushTLS() const; + void UnflushTLS(uword value) const; + uword FirstObjectStart() const { return to_->start() | object_alignment_; } private: // Ids for time and data records in Heap::GCStats. @@ -234,7 +255,6 @@ class Scavenger { kToKBAfterStoreBuffer = 3 }; - uword FirstObjectStart() const { return to_->start() | object_alignment_; } SemiSpace* Prologue(Isolate* isolate, bool invoke_api_callbacks); void IterateStoreBuffers(Isolate* isolate, ScavengerVisitor* visitor); void IterateObjectIdTable(Isolate* isolate, ScavengerVisitor* visitor); diff --git a/runtime/vm/thread.h b/runtime/vm/thread.h index e3761cd753a..6f3100279f9 100644 --- a/runtime/vm/thread.h +++ b/runtime/vm/thread.h @@ -377,6 +377,8 @@ class Thread : public BaseThread { uword top() { return top_; } uword end() { return end_; } + bool HasActiveTLAB() { return end_ > 0; } + static intptr_t top_offset() { return OFFSET_OF(Thread, top_); } static intptr_t end_offset() { return OFFSET_OF(Thread, end_); }