From 9209f522d6a36bb4efb93491d699ebdfb29b5bb1 Mon Sep 17 00:00:00 2001 From: Ryan Macnak Date: Mon, 6 Apr 2020 20:14:54 +0000 Subject: [PATCH] [vm, gc] Parallel scavenge. Run N tasks that compete to copy or promote survivors. Installation of the forwarding pointer uses a CAS. Tasks do not share the list of copied objects to be processed. Tasks do share the list of promoted objects to be processed. Divide the freelist in N copies. Each scavenger task uses one without locking. The sweeper inserts into the freelists round-robin. Change-Id: I54c7839ac5f15829462b3be7cf8c9635c01b785f Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/135962 Commit-Queue: Ryan Macnak Reviewed-by: Martin Kustermann --- runtime/docs/gc.md | 6 + runtime/platform/globals.h | 8 - runtime/tests/vm/dart/splay_test.dart | 4 + runtime/vm/clustered_snapshot.cc | 9 +- runtime/vm/flag_list.h | 3 + runtime/vm/heap/freelist.cc | 3 +- runtime/vm/heap/freelist.h | 51 +- runtime/vm/heap/marker.cc | 61 +- runtime/vm/heap/pages.cc | 152 ++-- runtime/vm/heap/pages.h | 55 +- runtime/vm/heap/pointer_block.cc | 2 +- runtime/vm/heap/pointer_block.h | 91 ++- runtime/vm/heap/scavenger.cc | 839 +++++++++++++++------ runtime/vm/heap/scavenger.h | 98 +-- runtime/vm/heap/sweeper.cc | 16 +- runtime/vm/isolate.cc | 4 +- runtime/vm/object.cc | 15 +- runtime/vm/object.h | 16 +- runtime/vm/raw_object.cc | 10 +- runtime/vm/raw_object.h | 27 +- runtime/vm/snapshot.cc | 12 - runtime/vm/thread.h | 1 + tests/standalone_2/fragmentation_test.dart | 4 + 23 files changed, 978 insertions(+), 509 deletions(-) diff --git a/runtime/docs/gc.md b/runtime/docs/gc.md index 70c58ea9dd6..8e6d4636f88 100644 --- a/runtime/docs/gc.md +++ b/runtime/docs/gc.md @@ -27,6 +27,10 @@ On 64-bit architectures, the header of heap objects also contains a 32-bit ident See [Cheney's algorithm](https://en.wikipedia.org/wiki/Cheney's_algorithm). +## Parallel Scavenge + +FLAG_scavenger_tasks (default 2) workers are started on separate threads. Each worker competes to process parts of the root set (including the remembered set). When a worker copies an object to to-space, it allocates from a worker-local bump allocation region. The same worker will process the copied object. When a worker promotes an object to old-space, it allocates from a worker-local freelist, which uses bump allocation for large free blocks. The promoted object is added to a work list that implements work stealing, so some other worker may process the promoted object. After the object is evacuated, the worker using a compare-and-swap to install the forwarding pointer into the from-space object's header. If it loses the race, it un-allocates the to-space or old-space object it just allocated, and uses the winner's object to update the pointer it was processing. Workers run until all of the work set have been processed, and every worker have processed its to-space objects and local part of the promoted work list. + ## Mark-Sweep All objects have a bit in their header called the mark bit. At the start of a collection cycle, all objects have this bit clear. @@ -51,6 +55,8 @@ To perform these operations, all mutators need to temporarily stop accessing the Note that a mutator can be at a safepoint without being suspended. It might be performing a long task that doesn't access the heap. It will, however, need to wait for any safepoint operation to complete in order to leave its safepoint and resume accessing the heap. +Because a safepoint operation excludes excution of Dart code, it is sometimes used for non-GC tasks that requires only this property. For example, when a background compilation has completed and wants to install its result, it uses a safepoint operation to ensure no Dart execution sees the intermediate states during installation. + ## Concurrent Marking To reduce the time the mutator is paused for old-space GCs, we allow the mutator to continue running during most of the marking work. diff --git a/runtime/platform/globals.h b/runtime/platform/globals.h index 22e85f48175..e2a35ca79ab 100644 --- a/runtime/platform/globals.h +++ b/runtime/platform/globals.h @@ -251,14 +251,6 @@ typedef simd128_value_t fpu_register_t; #error Automatic compiler detection failed. #endif -#ifdef _MSC_VER -#define DART_FLATTEN -#elif __GNUC__ -#define DART_FLATTEN __attribute__((flatten)) -#else -#error Automatic compiler detection failed. -#endif - #ifdef _MSC_VER #elif __GNUC__ #define DART_HAS_COMPUTED_GOTO 1 diff --git a/runtime/tests/vm/dart/splay_test.dart b/runtime/tests/vm/dart/splay_test.dart index a91c853fb86..eae1cbb3971 100644 --- a/runtime/tests/vm/dart/splay_test.dart +++ b/runtime/tests/vm/dart/splay_test.dart @@ -19,6 +19,10 @@ // VMOptions=--concurrent_mark --concurrent_sweep // VMOptions=--concurrent_mark --use_compactor // VMOptions=--concurrent_mark --use_compactor --force_evacuation +// VMOptions=--scavenger_tasks=0 +// VMOptions=--scavenger_tasks=1 +// VMOptions=--scavenger_tasks=2 +// VMOptions=--scavenger_tasks=3 // VMOptions=--verify_before_gc // VMOptions=--verify_after_gc // VMOptions=--verify_before_gc --verify_after_gc diff --git a/runtime/vm/clustered_snapshot.cc b/runtime/vm/clustered_snapshot.cc index 8322a104f36..6bef8643c17 100644 --- a/runtime/vm/clustered_snapshot.cc +++ b/runtime/vm/clustered_snapshot.cc @@ -6058,13 +6058,16 @@ void Deserializer::Deserialize() { class HeapLocker : public StackResource { public: HeapLocker(Thread* thread, PageSpace* page_space) - : StackResource(thread), page_space_(page_space) { - page_space_->AcquireDataLock(); + : StackResource(thread), + page_space_(page_space), + freelist_(page_space->DataFreeList()) { + page_space_->AcquireLock(freelist_); } - ~HeapLocker() { page_space_->ReleaseDataLock(); } + ~HeapLocker() { page_space_->ReleaseLock(freelist_); } private: PageSpace* page_space_; + FreeList* freelist_; }; void Deserializer::AddVMIsolateBaseObjects() { diff --git a/runtime/vm/flag_list.h b/runtime/vm/flag_list.h index 310aa8a165f..f01d69058f6 100644 --- a/runtime/vm/flag_list.h +++ b/runtime/vm/flag_list.h @@ -143,6 +143,9 @@ constexpr bool kDartUseBackgroundCompilation = true; P(link_natives_lazily, bool, false, "Link native calls lazily") \ R(log_marker_tasks, false, bool, false, \ "Log debugging information for old gen GC marking tasks.") \ + P(scavenger_tasks, int, 2, \ + "The number of tasks to spawn during scavenging (0 means " \ + "perform all marking on main thread).") \ P(marker_tasks, int, 2, \ "The number of tasks to spawn during old gen GC marking (0 means " \ "perform all marking on main thread).") \ diff --git a/runtime/vm/heap/freelist.cc b/runtime/vm/heap/freelist.cc index d08ec12a7b1..20b02ccc507 100644 --- a/runtime/vm/heap/freelist.cc +++ b/runtime/vm/heap/freelist.cc @@ -54,8 +54,7 @@ intptr_t FreeListElement::HeaderSizeFor(intptr_t size) { return ((size > RawObject::SizeTag::kMaxSizeTag) ? 3 : 2) * kWordSize; } -FreeList::FreeList() - : mutex_(), freelist_search_budget_(kInitialFreeListSearchBudget) { +FreeList::FreeList() : mutex_() { Reset(); } diff --git a/runtime/vm/heap/freelist.h b/runtime/vm/heap/freelist.h index 7c885cf1147..5b2c14a4331 100644 --- a/runtime/vm/heap/freelist.h +++ b/runtime/vm/heap/freelist.h @@ -117,6 +117,46 @@ class FreeList { return 0; } + uword TryAllocateBumpLocked(intptr_t size) { + ASSERT(mutex_.IsOwnedByCurrentThread()); + uword result = top_; + uword new_top = result + size; + if (new_top <= end_) { + top_ = new_top; + unaccounted_size_ += size; + return result; + } + return 0; + } + intptr_t TakeUnaccountedSizeLocked() { + ASSERT(mutex_.IsOwnedByCurrentThread()); + intptr_t result = unaccounted_size_; + unaccounted_size_ = 0; + return result; + } + + // Ensures HeapPage::VisitObjects can successful walk over a partially + // allocated bump region. + void MakeIterable() { + if (top_ < end_) { + FreeListElement::AsElement(top_, end_ - top_); + } + } + // Returns the bump region to the free list. + void AbandonBumpAllocation() { + if (top_ < end_) { + Free(top_, end_ - top_); + top_ = 0; + end_ = 0; + } + } + + uword top() const { return top_; } + uword end() const { return end_; } + void set_top(uword value) { top_ = value; } + void set_end(uword value) { end_ = value; } + void AddUnaccountedSize(intptr_t size) { unaccounted_size_ += size; } + void MergeOtherFreelist(FreeList* freelist, bool is_protected); private: @@ -161,6 +201,15 @@ class FreeList { void PrintSmall() const; void PrintLarge() const; + // Bump pointer region. + uword top_ = 0; + uword end_ = 0; + + // Allocated from the bump pointer region, but not yet added to + // PageSpace::usage_. Used to avoid expensive atomic adds during parallel + // scavenge. + intptr_t unaccounted_size_ = 0; + // Lock protecting the free list data structures. mutable Mutex mutex_; @@ -168,7 +217,7 @@ class FreeList { FreeListElement* free_lists_[kNumLists + 1]; - intptr_t freelist_search_budget_; + intptr_t freelist_search_budget_ = kInitialFreeListSearchBudget; // The largest available small size in bytes, or negative if there is none. intptr_t last_free_small_size_; diff --git a/runtime/vm/heap/marker.cc b/runtime/vm/heap/marker.cc index 994a2681421..69dc98b8c51 100644 --- a/runtime/vm/heap/marker.cc +++ b/runtime/vm/heap/marker.cc @@ -22,65 +22,6 @@ namespace dart { -class MarkerWorkList : public ValueObject { - public: - explicit MarkerWorkList(MarkingStack* marking_stack) - : marking_stack_(marking_stack) { - work_ = marking_stack_->PopEmptyBlock(); - } - - ~MarkerWorkList() { - ASSERT(work_ == NULL); - ASSERT(marking_stack_ == NULL); - } - - // Returns NULL if no more work was found. - RawObject* Pop() { - ASSERT(work_ != NULL); - if (work_->IsEmpty()) { - // TODO(koda): Track over/underflow events and use in heuristics to - // distribute work and prevent degenerate flip-flopping. - MarkingStack::Block* new_work = marking_stack_->PopNonEmptyBlock(); - if (new_work == NULL) { - return NULL; - } - marking_stack_->PushBlock(work_); - work_ = new_work; - // Generated code appends to marking stacks; tell MemorySanitizer. - MSAN_UNPOISON(work_, sizeof(*work_)); - } - return work_->Pop(); - } - - void Push(RawObject* raw_obj) { - if (work_->IsFull()) { - // TODO(koda): Track over/underflow events and use in heuristics to - // distribute work and prevent degenerate flip-flopping. - marking_stack_->PushBlock(work_); - work_ = marking_stack_->PopEmptyBlock(); - } - work_->Push(raw_obj); - } - - void Finalize() { - ASSERT(work_->IsEmpty()); - marking_stack_->PushBlock(work_); - work_ = NULL; - // Fail fast on attempts to mark after finalizing. - marking_stack_ = NULL; - } - - void AbandonWork() { - marking_stack_->PushBlock(work_); - work_ = NULL; - marking_stack_ = NULL; - } - - private: - MarkingStack::Block* work_; - MarkingStack* marking_stack_; -}; - template class MarkingVisitorBase : public ObjectPointerVisitor { public: @@ -496,7 +437,7 @@ void GCMarker::ProcessRememberedSet(Thread* thread) { TIMELINE_FUNCTION_GC_DURATION(thread, "ProcessRememberedSet"); // Filter collected objects from the remembered set. StoreBuffer* store_buffer = isolate_group_->store_buffer(); - StoreBufferBlock* reading = store_buffer->Blocks(); + StoreBufferBlock* reading = store_buffer->TakeBlocks(); StoreBufferBlock* writing = store_buffer->PopNonFullBlock(); while (reading != NULL) { StoreBufferBlock* next = reading->next(); diff --git a/runtime/vm/heap/pages.cc b/runtime/vm/heap/pages.cc index 0461261b9ed..f1f59a010ec 100644 --- a/runtime/vm/heap/pages.cc +++ b/runtime/vm/heap/pages.cc @@ -118,7 +118,8 @@ void HeapPage::VisitObjectPointers(ObjectPointerVisitor* visitor) const { } void HeapPage::VisitRememberedCards(ObjectPointerVisitor* visitor) { - ASSERT(Thread::Current()->IsAtSafepoint()); + ASSERT(Thread::Current()->IsAtSafepoint() || + (Thread::Current()->task_kind() == Thread::kScavengerTask)); NoSafepointScope no_safepoint; if (card_table_ == NULL) { @@ -218,11 +219,10 @@ void HeapPage::WriteProtect(bool read_only) { static const intptr_t kConservativeInitialMarkSpeed = 20; PageSpace::PageSpace(Heap* heap, intptr_t max_capacity_in_words) - : freelist_(), - heap_(heap), + : heap_(heap), + num_freelists_(Utils::Maximum(FLAG_scavenger_tasks, 1) + 1), + freelists_(new FreeList[num_freelists_]), pages_lock_(), - bump_top_(0), - bump_end_(0), max_capacity_in_words_(max_capacity_in_words), usage_(), allocated_black_in_words_(0), @@ -245,6 +245,10 @@ PageSpace::PageSpace(Heap* heap, intptr_t max_capacity_in_words) // We aren't holding the lock but no one can reference us yet. UpdateMaxCapacityLocked(); UpdateMaxUsed(); + + for (intptr_t i = 0; i < num_freelists_; i++) { + freelists_[i].Reset(); + } } PageSpace::~PageSpace() { @@ -259,6 +263,7 @@ PageSpace::~PageSpace() { FreePages(large_pages_); FreePages(image_pages_); ASSERT(marker_ == NULL); + delete[] freelists_; } intptr_t PageSpace::LargePageSizeInWordsFor(intptr_t size) { @@ -458,6 +463,7 @@ void PageSpace::EvaluateConcurrentMarking(GrowthPolicy growth_policy) { } uword PageSpace::TryAllocateInFreshPage(intptr_t size, + FreeList* freelist, HeapPage::PageType type, GrowthPolicy growth_policy, bool is_locked) { @@ -485,9 +491,9 @@ uword PageSpace::TryAllocateInFreshPage(intptr_t size, intptr_t free_size = page->object_end() - free_start; if (free_size > 0) { if (is_locked) { - freelist_[type].FreeLocked(free_start, free_size); + freelist->FreeLocked(free_start, free_size); } else { - freelist_[type].Free(free_start, free_size); + freelist->Free(free_start, free_size); } } } @@ -524,6 +530,7 @@ uword PageSpace::TryAllocateInFreshLargePage(intptr_t size, } uword PageSpace::TryAllocateInternal(intptr_t size, + FreeList* freelist, HeapPage::PageType type, GrowthPolicy growth_policy, bool is_protected, @@ -533,12 +540,13 @@ uword PageSpace::TryAllocateInternal(intptr_t size, uword result = 0; if (Heap::IsAllocatableViaFreeLists(size)) { if (is_locked) { - result = freelist_[type].TryAllocateLocked(size, is_protected); + result = freelist->TryAllocateLocked(size, is_protected); } else { - result = freelist_[type].TryAllocate(size, is_protected); + result = freelist->TryAllocate(size, is_protected); } if (result == 0) { - result = TryAllocateInFreshPage(size, type, growth_policy, is_locked); + result = TryAllocateInFreshPage(size, freelist, type, growth_policy, + is_locked); // usage_ is updated by the call above. } else { usage_.used_in_words += (size >> kWordSizeLog2); @@ -551,20 +559,16 @@ uword PageSpace::TryAllocateInternal(intptr_t size, return result; } -void PageSpace::AcquireDataLock() { - freelist_[HeapPage::kData].mutex()->Lock(); +void PageSpace::AcquireLock(FreeList* freelist) { + freelist->mutex()->Lock(); } -void PageSpace::ReleaseDataLock() { - freelist_[HeapPage::kData].mutex()->Unlock(); +void PageSpace::ReleaseLock(FreeList* freelist) { + intptr_t size = freelist->TakeUnaccountedSizeLocked(); + usage_.used_in_words += (size >> kWordSizeLog2); + freelist->mutex()->Unlock(); } -#if defined(DEBUG) -bool PageSpace::CurrentThreadOwnsDataLock() { - return freelist_[HeapPage::kData].mutex()->IsOwnedByCurrentThread(); -} -#endif - void PageSpace::AllocateExternal(intptr_t cid, intptr_t size) { intptr_t size_in_words = size >> kWordSizeLog2; usage_.external_in_words += size_in_words; @@ -681,16 +685,14 @@ void PageSpace::MakeIterable() const { // Assert not called from concurrent sweeper task. // TODO(koda): Use thread/task identity when implemented. ASSERT(IsolateGroup::Current()->heap() != NULL); - if (bump_top_ < bump_end_) { - FreeListElement::AsElement(bump_top_, bump_end_ - bump_top_); + for (intptr_t i = 0; i < num_freelists_; i++) { + freelists_[i].MakeIterable(); } } void PageSpace::AbandonBumpAllocation() { - if (bump_top_ < bump_end_) { - freelist_[HeapPage::kData].Free(bump_top_, bump_end_ - bump_top_); - bump_top_ = 0; - bump_end_ = 0; + for (intptr_t i = 0; i < num_freelists_; i++) { + freelists_[i].AbandonBumpAllocation(); } } @@ -808,7 +810,8 @@ void PageSpace::VisitObjectPointers(ObjectPointerVisitor* visitor) const { } void PageSpace::VisitRememberedCards(ObjectPointerVisitor* visitor) const { - ASSERT(Thread::Current()->IsAtSafepoint()); + ASSERT(Thread::Current()->IsAtSafepoint() || + (Thread::Current()->task_kind() == Thread::kScavengerTask)); // Wait for the sweeper to finish mutating the large page list. MonitorLocker ml(tasks_lock()); @@ -1105,10 +1108,10 @@ void PageSpace::CollectGarbageAtSafepoint(bool compact, NoSafepointScope no_safepoints; if (FLAG_print_free_list_before_gc) { - OS::PrintErr("Data Freelist (before GC):\n"); - freelist_[HeapPage::kData].Print(); - OS::PrintErr("Executable Freelist (before GC):\n"); - freelist_[HeapPage::kExecutable].Print(); + for (intptr_t i = 0; i < num_freelists_; i++) { + OS::PrintErr("Before GC: Freelist %" Pd "\n", i); + freelists_[i].Print(); + } } if (FLAG_verify_before_gc) { @@ -1149,8 +1152,9 @@ void PageSpace::CollectGarbageAtSafepoint(bool compact, // Abandon the remainder of the bump allocation block. AbandonBumpAllocation(); // Reset the freelists and setup sweeping. - freelist_[HeapPage::kData].Reset(); - freelist_[HeapPage::kExecutable].Reset(); + for (intptr_t i = 0; i < num_freelists_; i++) { + freelists_[i].Reset(); + } int64_t mid2 = OS::GetCurrentMonotonicMicros(); int64_t mid3 = 0; @@ -1169,7 +1173,7 @@ void PageSpace::CollectGarbageAtSafepoint(bool compact, GCSweeper sweeper; HeapPage* prev_page = NULL; HeapPage* page = exec_pages_; - FreeList* freelist = &freelist_[HeapPage::kExecutable]; + FreeList* freelist = &freelists_[HeapPage::kExecutable]; MutexLocker ml(freelist->mutex()); while (page != NULL) { HeapPage* next_page = page->next(); @@ -1215,10 +1219,10 @@ void PageSpace::CollectGarbageAtSafepoint(bool compact, heap_->RecordTime(kSweepLargePages, end - mid3); if (FLAG_print_free_list_after_gc) { - OS::PrintErr("Data Freelist (after GC):\n"); - freelist_[HeapPage::kData].Print(); - OS::PrintErr("Executable Freelist (after GC):\n"); - freelist_[HeapPage::kExecutable].Print(); + for (intptr_t i = 0; i < num_freelists_; i++) { + OS::PrintErr("After GC: Freelist %" Pd "\n", i); + freelists_[i].Print(); + } } UpdateMaxUsed(); @@ -1251,14 +1255,21 @@ void PageSpace::Sweep() { TIMELINE_FUNCTION_GC_DURATION(Thread::Current(), "Sweep"); GCSweeper sweeper; + + intptr_t shard = 0; + const intptr_t num_shards = Utils::Maximum(FLAG_scavenger_tasks, 1); + for (intptr_t i = 0; i < num_shards; i++) { + DataFreeList(i)->mutex()->Lock(); + } + HeapPage* prev_page = nullptr; HeapPage* page = pages_; - FreeList* freelist = &freelist_[HeapPage::kData]; - MutexLocker ml(freelist_->mutex()); while (page != nullptr) { HeapPage* next_page = page->next(); ASSERT(page->type() == HeapPage::kData); - bool page_in_use = sweeper.SweepPage(page, freelist, true /*is_locked*/); + shard = (shard + 1) % num_shards; + bool page_in_use = + sweeper.SweepPage(page, DataFreeList(shard), true /*is_locked*/); if (page_in_use) { prev_page = page; } else { @@ -1268,6 +1279,10 @@ void PageSpace::Sweep() { page = next_page; } + for (intptr_t i = 0; i < num_shards; i++) { + DataFreeList(i)->mutex()->Unlock(); + } + if (FLAG_verify_after_gc) { OS::PrintErr("Verifying after sweeping..."); heap_->VerifyGC(kForbidMarked); @@ -1278,13 +1293,13 @@ void PageSpace::Sweep() { void PageSpace::ConcurrentSweep(IsolateGroup* isolate_group) { // Start the concurrent sweeper task now. GCSweeper::SweepConcurrent(isolate_group, pages_, pages_tail_, large_pages_, - large_pages_tail_, &freelist_[HeapPage::kData]); + large_pages_tail_, &freelists_[HeapPage::kData]); } void PageSpace::Compact(Thread* thread) { thread->isolate_group()->set_compaction_in_progress(true); GCCompactor compactor(thread, heap_); - compactor.Compact(pages_, &freelist_[HeapPage::kData], &pages_lock_); + compactor.Compact(pages_, &freelists_[HeapPage::kData], &pages_lock_); thread->isolate_group()->set_compaction_in_progress(false); if (FLAG_verify_after_gc) { @@ -1294,63 +1309,57 @@ void PageSpace::Compact(Thread* thread) { } } -uword PageSpace::TryAllocateDataBumpLocked(intptr_t size) { +uword PageSpace::TryAllocateDataBumpLocked(FreeList* freelist, intptr_t size) { ASSERT(size >= kObjectAlignment); ASSERT(Utils::IsAligned(size, kObjectAlignment)); - intptr_t remaining = bump_end_ - bump_top_; + + intptr_t remaining = freelist->end() - freelist->top(); if (UNLIKELY(remaining < size)) { // Checking this first would be logical, but needlessly slow. if (!Heap::IsAllocatableViaFreeLists(size)) { - return TryAllocateDataLocked(size, kForceGrowth); + return TryAllocateDataLocked(freelist, size, kForceGrowth); } - FreeListElement* block = - freelist_[HeapPage::kData].TryAllocateLargeLocked(size); + FreeListElement* block = freelist->TryAllocateLargeLocked(size); if (block == NULL) { // Allocating from a new page (if growth policy allows) will have the // side-effect of populating the freelist with a large block. The next // bump allocation request will have a chance to consume that block. // TODO(koda): Could take freelist lock just once instead of twice. - return TryAllocateInFreshPage(size, HeapPage::kData, kForceGrowth, - true /* is_locked*/); + return TryAllocateInFreshPage(size, freelist, HeapPage::kData, + kForceGrowth, true /* is_locked*/); } intptr_t block_size = block->HeapSize(); if (remaining > 0) { - freelist_[HeapPage::kData].FreeLocked(bump_top_, remaining); + freelist->FreeLocked(freelist->top(), remaining); } - bump_top_ = reinterpret_cast(block); - bump_end_ = bump_top_ + block_size; + freelist->set_top(reinterpret_cast(block)); + freelist->set_end(freelist->top() + block_size); remaining = block_size; } ASSERT(remaining >= size); - uword result = bump_top_; - bump_top_ += size; + uword result = freelist->top(); + freelist->set_top(result + size); - // No need for atomic operation: This is either running during a scavenge or - // isolate snapshot loading. Note that operator+= is atomic. - usage_.used_in_words = usage_.used_in_words + (size >> kWordSizeLog2); + freelist->AddUnaccountedSize(size); // Note: Remaining block is unwalkable until MakeIterable is called. #ifdef DEBUG - if (bump_top_ < bump_end_) { + if (freelist->top() < freelist->end()) { // Fail fast if we try to walk the remaining block. COMPILE_ASSERT(kIllegalCid == 0); - *reinterpret_cast(bump_top_) = 0; + *reinterpret_cast(freelist->top()) = 0; } #endif // DEBUG return result; } -DART_FLATTEN -uword PageSpace::TryAllocatePromoLocked(intptr_t size) { - FreeList* freelist = &freelist_[HeapPage::kData]; +uword PageSpace::TryAllocatePromoLockedSlow(FreeList* freelist, intptr_t size) { uword result = freelist->TryAllocateSmallLocked(size); if (result != 0) { - // No need for atomic operation: we're at a safepoint. Note that - // operator+= is atomic. - usage_.used_in_words = usage_.used_in_words + (size >> kWordSizeLog2); + freelist->AddUnaccountedSize(size); return result; } - return TryAllocateDataBumpLocked(size); + return TryAllocateDataBumpLocked(freelist, size); } void PageSpace::SetupImagePage(void* pointer, uword size, bool is_executable) { @@ -1438,18 +1447,19 @@ static void EnsureEqualImagePages(HeapPage* pages, HeapPage* other_pages) { void PageSpace::MergeOtherPageSpace(PageSpace* other) { other->AbandonBumpAllocation(); - ASSERT(other->bump_top_ == 0 && other->bump_end_ == 0); ASSERT(other->tasks_ == 0); ASSERT(other->concurrent_marker_tasks_ == 0); ASSERT(other->phase_ == kDone); DEBUG_ASSERT(other->iterating_thread_ == nullptr); ASSERT(other->marker_ == nullptr); - for (intptr_t i = 0; i < HeapPage::kNumPageTypes; ++i) { + for (intptr_t i = 0; i < num_freelists_; ++i) { + ASSERT(other->freelists_[i].top() == 0); + ASSERT(other->freelists_[i].end() == 0); const bool is_protected = FLAG_write_protect_code && i == HeapPage::kExecutable; - freelist_[i].MergeOtherFreelist(&other->freelist_[i], is_protected); - other->freelist_[i].Reset(); + freelists_[i].MergeOtherFreelist(&other->freelists_[i], is_protected); + other->freelists_[i].Reset(); } // The freelist locks will be taken in MergeOtherFreelist above, and the diff --git a/runtime/vm/heap/pages.h b/runtime/vm/heap/pages.h index d51e6975dc3..7843b35f4e8 100644 --- a/runtime/vm/heap/pages.h +++ b/runtime/vm/heap/pages.h @@ -39,7 +39,7 @@ static const intptr_t kBlocksPerPage = kPageSize / kBlockSize; // A page containing old generation objects. class HeapPage { public: - enum PageType { kData = 0, kExecutable, kNumPageTypes }; + enum PageType { kExecutable = 0, kData }; HeapPage* next() const { return next_; } void set_next(HeapPage* next) { next_ = next; } @@ -303,8 +303,8 @@ class PageSpace { bool is_protected = (type == HeapPage::kExecutable) && FLAG_write_protect_code; bool is_locked = false; - return TryAllocateInternal(size, type, growth_policy, is_protected, - is_locked); + return TryAllocateInternal(size, &freelists_[type], type, growth_policy, + is_protected, is_locked); } bool NeedsGarbageCollection() const { @@ -415,16 +415,18 @@ class PageSpace { void FreeExternal(intptr_t size); // Bulk data allocation. - void AcquireDataLock(); - void ReleaseDataLock(); -#if defined(DEBUG) - bool CurrentThreadOwnsDataLock(); -#endif + FreeList* DataFreeList(intptr_t i = 0) { + return &freelists_[HeapPage::kData + i]; + } + void AcquireLock(FreeList* freelist); + void ReleaseLock(FreeList* freelist); - uword TryAllocateDataLocked(intptr_t size, GrowthPolicy growth_policy) { + uword TryAllocateDataLocked(FreeList* freelist, + intptr_t size, + GrowthPolicy growth_policy) { bool is_protected = false; bool is_locked = true; - return TryAllocateInternal(size, HeapPage::kData, growth_policy, + return TryAllocateInternal(size, freelist, HeapPage::kData, growth_policy, is_protected, is_locked); } @@ -443,9 +445,19 @@ class PageSpace { void set_phase(Phase val) { phase_ = val; } // Attempt to allocate from bump block rather than normal freelist. - uword TryAllocateDataBumpLocked(intptr_t size); - // Prefer small freelist blocks, then chip away at the bump block. - uword TryAllocatePromoLocked(intptr_t size); + uword TryAllocateDataBumpLocked(intptr_t size) { + return TryAllocateDataBumpLocked(&freelists_[HeapPage::kData], size); + } + uword TryAllocateDataBumpLocked(FreeList* freelist, intptr_t size); + DART_FORCE_INLINE + uword TryAllocatePromoLocked(FreeList* freelist, intptr_t size) { + uword result = freelist->TryAllocateBumpLocked(size); + if (result != 0) { + return result; + } + return TryAllocatePromoLockedSlow(freelist, size); + } + uword TryAllocatePromoLockedSlow(FreeList* freelist, intptr_t size); void SetupImagePage(void* pointer, uword size, bool is_executable); @@ -481,11 +493,13 @@ class PageSpace { }; uword TryAllocateInternal(intptr_t size, + FreeList* freelist, HeapPage::PageType type, GrowthPolicy growth_policy, bool is_protected, bool is_locked); uword TryAllocateInFreshPage(intptr_t size, + FreeList* freelist, HeapPage::PageType type, GrowthPolicy growth_policy, bool is_locked); @@ -535,9 +549,15 @@ class PageSpace { (increase_in_words <= free_capacity_in_words)); } - FreeList freelist_[HeapPage::kNumPageTypes]; + Heap* const heap_; - Heap* heap_; + // One list for executable pages at freelists_[HeapPage::kExecutable]. + // FLAG_scavenger_tasks count of lists for data pages starting at + // freelists_[HeapPage::kData]. The sweeper inserts into the data page + // lists round-robin. The scavenger workers each use one of them without + // locking. + const intptr_t num_freelists_; + FreeList* freelists_; // Use ExclusivePageIterator for safe access to these. mutable Mutex pages_lock_; @@ -549,11 +569,6 @@ class PageSpace { HeapPage* large_pages_tail_ = nullptr; HeapPage* image_pages_ = nullptr; - // A block of memory in a data page, managed by bump allocation. The remainder - // is kept formatted as a FreeListElement, but is not in any freelist. - uword bump_top_; - uword bump_end_; - // Various sizes being tracked for this generation. intptr_t max_capacity_in_words_; diff --git a/runtime/vm/heap/pointer_block.cc b/runtime/vm/heap/pointer_block.cc index 69bc1f515cf..2814465d321 100644 --- a/runtime/vm/heap/pointer_block.cc +++ b/runtime/vm/heap/pointer_block.cc @@ -69,7 +69,7 @@ void BlockStack::Reset() { } template -typename BlockStack::Block* BlockStack::Blocks() { +typename BlockStack::Block* BlockStack::TakeBlocks() { MutexLocker ml(&mutex_); while (!partial_.IsEmpty()) { full_.Push(partial_.Pop()); diff --git a/runtime/vm/heap/pointer_block.h b/runtime/vm/heap/pointer_block.h index b5fa276586c..a5505e2dec9 100644 --- a/runtime/vm/heap/pointer_block.h +++ b/runtime/vm/heap/pointer_block.h @@ -24,7 +24,7 @@ class PointerBlock { void Reset() { top_ = 0; - next_ = NULL; + next_ = nullptr; } PointerBlock* next() const { return next_; } @@ -64,7 +64,7 @@ class PointerBlock { void VisitObjectPointers(ObjectPointerVisitor* visitor); private: - PointerBlock() : next_(NULL), top_(0) {} + PointerBlock() : next_(nullptr), top_(0) {} ~PointerBlock() { ASSERT(IsEmpty()); // Guard against unintentionally discarding pointers. } @@ -100,7 +100,7 @@ class BlockStack { Block* PopNonEmptyBlock(); // Pops and returns all non-empty blocks as a linked list (owned by caller). - Block* Blocks(); + Block* TakeBlocks(); // Discards the contents of all non-empty blocks. void Reset(); @@ -110,12 +110,12 @@ class BlockStack { protected: class List { public: - List() : head_(NULL), length_(0) {} + List() : head_(nullptr), length_(0) {} ~List(); void Push(Block* block); Block* Pop(); intptr_t length() const { return length_; } - bool IsEmpty() const { return head_ == NULL; } + bool IsEmpty() const { return head_ == nullptr; } Block* PopAll(); Block* Peek() { return head_; } @@ -144,6 +144,74 @@ class BlockStack { DISALLOW_COPY_AND_ASSIGN(BlockStack); }; +template +class BlockWorkList : public ValueObject { + public: + typedef typename Stack::Block Block; + + explicit BlockWorkList(Stack* stack) : stack_(stack) { + work_ = stack_->PopEmptyBlock(); + } + + ~BlockWorkList() { + ASSERT(work_ == nullptr); + ASSERT(stack_ == nullptr); + } + + // Returns nullptr if no more work was found. + RawObject* Pop() { + ASSERT(work_ != nullptr); + if (work_->IsEmpty()) { + // TODO(koda): Track over/underflow events and use in heuristics to + // distribute work and prevent degenerate flip-flopping. + Block* new_work = stack_->PopNonEmptyBlock(); + if (new_work == nullptr) { + return nullptr; + } + stack_->PushBlock(work_); + work_ = new_work; + // Generated code appends to marking stacks; tell MemorySanitizer. + MSAN_UNPOISON(work_, sizeof(*work_)); + } + return work_->Pop(); + } + + void Push(RawObject* raw_obj) { + if (work_->IsFull()) { + // TODO(koda): Track over/underflow events and use in heuristics to + // distribute work and prevent degenerate flip-flopping. + stack_->PushBlock(work_); + work_ = stack_->PopEmptyBlock(); + } + work_->Push(raw_obj); + } + + void Finalize() { + ASSERT(work_->IsEmpty()); + stack_->PushBlock(work_); + work_ = nullptr; + // Fail fast on attempts to mark after finalizing. + stack_ = nullptr; + } + + void AbandonWork() { + stack_->PushBlock(work_); + work_ = nullptr; + stack_ = nullptr; + } + + bool IsEmpty() { + if (!work_->IsEmpty()) { + return false; + } + return stack_->IsEmpty(); + } + + private: + Block* work_; + Stack* stack_; +}; + static const int kStoreBufferBlockSize = 1024; class StoreBuffer : public BlockStack { public: @@ -176,6 +244,19 @@ class MarkingStack : public BlockStack { }; typedef MarkingStack::Block MarkingStackBlock; +typedef BlockWorkList MarkerWorkList; + +static const int kPromotionStackBlockSize = 64; +class PromotionStack : public BlockStack { + public: + // Adds and transfers ownership of the block to the buffer. + void PushBlock(Block* block) { + BlockStack::PushBlockImpl(block); + } +}; + +typedef PromotionStack::Block PromotionStackBlock; +typedef BlockWorkList PromotionWorkList; } // namespace dart diff --git a/runtime/vm/heap/scavenger.cc b/runtime/vm/heap/scavenger.cc index 3de0aa3b5a6..19283e2b2a6 100644 --- a/runtime/vm/heap/scavenger.cc +++ b/runtime/vm/heap/scavenger.cc @@ -18,6 +18,7 @@ #include "vm/object_id_ring.h" #include "vm/object_set.h" #include "vm/stack_frame.h" +#include "vm/thread_barrier.h" #include "vm/thread_registry.h" #include "vm/timeline.h" #include "vm/visitor.h" @@ -55,12 +56,19 @@ static inline uword ForwardedAddr(uword header) { return header & ~kForwardingMask; } -static inline void ForwardTo(uword original, uword target) { +static inline uword ForwardingHeader(uword target) { // Make sure forwarding can be encoded. ASSERT((target & kForwardingMask) == 0); - *reinterpret_cast(original) = target | kForwarded; + return target | kForwarded; } +// Races: The first word in the copied region is a header word that may be +// updated by the scavenger worker in another thread, so we might copy either +// the original object header or an installed forwarding pointer. This race is +// harmless because if copy the installed forwarding pointer, the scavenge +// worker in the current thread will abandon this copy. We not mark the loads +// here as relaxed so the C++ compiler still has the freedom to reorder them. +NO_SANITIZE_THREAD static inline void objcpy(void* dst, const void* src, size_t size) { // A memcopy specialized for objects. We can assume: // - dst and src do not overlap @@ -86,19 +94,28 @@ static inline void objcpy(void* dst, const void* src, size_t size) { } while (size > 0); } -class ScavengerVisitor : public ObjectPointerVisitor { +template +class ScavengerVisitorBase : public ObjectPointerVisitor { public: - explicit ScavengerVisitor(IsolateGroup* isolate_group, - Scavenger* scavenger, - SemiSpace* from) + explicit ScavengerVisitorBase(IsolateGroup* isolate_group, + Scavenger* scavenger, + SemiSpace* from, + FreeList* freelist, + PromotionStack* promotion_stack) : ObjectPointerVisitor(isolate_group), - thread_(Thread::Current()), + thread_(nullptr), scavenger_(scavenger), from_(from), - heap_(scavenger->heap_), page_space_(scavenger->heap_->old_space()), + freelist_(freelist), bytes_promoted_(0), - visiting_old_object_(NULL) {} + visiting_old_object_(NULL), + promoted_list_(promotion_stack), + labs_(8) { + ASSERT(labs_.length() == 0); + labs_.Add({0, 0, 0}); + ASSERT(labs_.length() == 1); + } virtual void VisitTypedDataViewPointers(RawTypedDataView* view, RawObject** first, @@ -157,6 +174,65 @@ class ScavengerVisitor : public ObjectPointerVisitor { intptr_t bytes_promoted() const { return bytes_promoted_; } + void AddNewTLAB(uword top, uword end) { + producer_index_++; + ScavengerLAB lab; + lab.top = top; + lab.end = end; + lab.resolved_top = top; + labs_.Add(lab); + } + + void ProcessRoots() { + thread_ = Thread::Current(); + page_space_->AcquireLock(freelist_); + scavenger_->IterateRoots(this); + } + + void ProcessSurvivors() { + // Iterate until all work has been drained. + do { + ProcessToSpace(); + ProcessPromotedList(); + } while (HasWork()); + } + + void ProcessAll() { + do { + ProcessSurvivors(); + ProcessWeakProperties(); + } while (HasWork()); + } + + inline void ProcessWeakProperties(); + + bool HasWork() { + return (labs_[producer_index_].top != + labs_[producer_index_].resolved_top) || + !promoted_list_.IsEmpty(); + } + + void Finalize() { + ASSERT(!HasWork()); + + for (intptr_t i = 0; i <= producer_index_; i++) { + ASSERT(labs_[i].top <= labs_[i].end); + ASSERT(labs_[i].resolved_top == labs_[i].top); + } + + MakeProducerTLABIterable(); + + promoted_list_.Finalize(); + + MournWeakProperties(); + + page_space_->ReleaseLock(freelist_); + thread_ = nullptr; + } + + uword last_top() { return labs_[producer_index_].top; } + uword last_end() { return labs_[producer_index_].end; } + private: void UpdateStoreBuffer(RawObject** p, RawObject* obj) { ASSERT(obj->IsHeapObject()); @@ -182,39 +258,40 @@ class ScavengerVisitor : public ObjectPointerVisitor { ASSERT(from_->Contains(raw_addr)); // Read the header word of the object and determine if the object has // already been copied. - uword header = *reinterpret_cast(raw_addr); + uword header = reinterpret_cast*>(raw_addr)->load( + std::memory_order_relaxed); uword new_addr = 0; if (IsForwarding(header)) { // Get the new location of the object. new_addr = ForwardedAddr(header); } else { - intptr_t size = raw_obj->HeapSize(); + intptr_t size = raw_obj->HeapSize(header); // Check whether object should be promoted. - if (scavenger_->survivor_end_ <= raw_addr) { + if (raw_addr >= scavenger_->survivor_end_) { // Not a survivor of a previous scavenge. Just copy the object into the // to space. - new_addr = scavenger_->AllocateGC(size); - } else { - // TODO(iposva): Experiment with less aggressive promotion. For example - // a coin toss determines if an object is promoted or whether it should - // survive in this generation. - // + new_addr = TryAllocateCopy(size); + } + if (new_addr == 0) { // This object is a survivor of a previous scavenge. Attempt to promote - // the object. - new_addr = page_space_->TryAllocatePromoLocked(size); - if (new_addr != 0) { + // the object. (Or, unlikely, to-space was exhausted by fragmentation.) + new_addr = page_space_->TryAllocatePromoLocked(freelist_, size); + if (LIKELY(new_addr != 0)) { // If promotion succeeded then we need to remember it so that it can // be traversed later. - scavenger_->PushToPromotedStack(new_addr); + promoted_list_.Push(RawObject::FromAddr(new_addr)); bytes_promoted_ += size; } else { // Promotion did not succeed. Copy into the to space instead. scavenger_->failed_to_promote_ = true; - new_addr = scavenger_->AllocateGC(size); + new_addr = TryAllocateCopy(size); + // To-space was exhausted by fragmentation and old-space could not + // grow. + if (UNLIKELY(new_addr == 0)) { + FATAL("Failed to allocate during scavenge"); + } } } - // During a scavenge we always succeed to at least copy all of the - // current objects to the to space. ASSERT(new_addr != 0); // Copy the object to the new location. objcpy(reinterpret_cast(new_addr), @@ -223,7 +300,7 @@ class ScavengerVisitor : public ObjectPointerVisitor { RawObject* new_obj = RawObject::FromAddr(new_addr); if (new_obj->IsOldObject()) { // Promoted: update age/barrier tags. - uint32_t tags = new_obj->ptr()->tags_; + uint32_t tags = static_cast(header); tags = RawObject::OldBit::update(true, tags); tags = RawObject::OldAndNotRememberedBit::update(true, tags); tags = RawObject::NewBit::update(false, tags); @@ -235,18 +312,39 @@ class ScavengerVisitor : public ObjectPointerVisitor { tags = RawObject::OldAndNotMarkedBit::update(!thread_->is_marking(), tags); new_obj->ptr()->tags_ = tags; + } else { + ASSERT(scavenger_->to_->Contains(new_addr)); } - if (RawObject::IsTypedDataClassId(new_obj->GetClassId())) { + intptr_t cid = RawObject::ClassIdTag::decode(header); + if (RawObject::IsTypedDataClassId(cid)) { reinterpret_cast(new_obj)->RecomputeDataField(); } - // Remember forwarding address. - ForwardTo(raw_addr, new_addr); + // Try to install forwarding address. + uword forwarding_header = ForwardingHeader(new_addr); + if (!InstallForwardingPointer(raw_addr, &header, forwarding_header)) { + ASSERT(IsForwarding(header)); + if (new_obj->IsOldObject()) { + // Abandon as a free list element. + FreeListElement::AsElement(new_addr, size); + bytes_promoted_ -= size; + } else { + // Undo to-space allocation. + ASSERT(labs_[producer_index_].top == (new_addr + size)); + labs_[producer_index_].top = new_addr; + } + // Use the winner's forwarding target. + new_addr = ForwardedAddr(header); + if (RawObject::FromAddr(new_addr)->IsNewObject()) { + ASSERT(scavenger_->to_->Contains(new_addr)); + } + } } + // Update the reference. RawObject* new_obj = RawObject::FromAddr(new_addr); - if (new_obj->IsOldObject()) { + if (!new_obj->IsNewObject()) { // Setting the mark bit above must not be ordered after a publishing store // of this object. Note this could be a publishing store even if the // object was promoted by an early invocation of ScavengePointer. Compare @@ -254,6 +352,7 @@ class ScavengerVisitor : public ObjectPointerVisitor { reinterpret_cast*>(p)->store( new_obj, std::memory_order_release); } else { + ASSERT(scavenger_->to_->Contains(RawObject::ToAddr(new_obj))); *p = new_obj; } // Update the store buffer as needed. @@ -262,20 +361,82 @@ class ScavengerVisitor : public ObjectPointerVisitor { } } + DART_FORCE_INLINE + bool InstallForwardingPointer(uword addr, + uword* old_header, + uword new_header) { + if (parallel) { + return reinterpret_cast*>(addr) + ->compare_exchange_strong(*old_header, new_header, + std::memory_order_relaxed); + } else { + *reinterpret_cast(addr) = new_header; + return true; + } + } + + DART_FORCE_INLINE + uword TryAllocateCopy(intptr_t size) { + ASSERT(Utils::IsAligned(size, kObjectAlignment)); + ScavengerLAB& lab = labs_[producer_index_]; + uword result = lab.top; + uword new_top = result + size; + if (LIKELY(new_top <= lab.end)) { + ASSERT(scavenger_->to_->Contains(result)); + ASSERT((result & kObjectAlignmentMask) == kNewObjectAlignmentOffset); + lab.top = new_top; + ASSERT((scavenger_->to_->Contains(new_top)) || + (new_top == scavenger_->to_->end())); + return result; + } + return TryAllocateCopySlow(size); + } + + DART_NOINLINE inline uword TryAllocateCopySlow(intptr_t size); + + void MakeProducerTLABIterable() { + uword top = labs_[producer_index_].top; + uword end = labs_[producer_index_].end; + intptr_t size = end - top; + if (size != 0) { + ASSERT(Utils::IsAligned(size, kObjectAlignment)); + ForwardingCorpse::AsForwarder(top, size); + ASSERT(RawObject::FromAddr(top)->HeapSize() == size); + } + } + + inline void ProcessToSpace(); + DART_FORCE_INLINE intptr_t ProcessCopied(RawObject* raw_obj); + inline void ProcessPromotedList(); + inline void EnqueueWeakProperty(RawWeakProperty* raw_weak); + inline void MournWeakProperties(); + Thread* thread_; Scavenger* scavenger_; SemiSpace* from_; - Heap* heap_; PageSpace* page_space_; - RawWeakProperty* delayed_weak_properties_; + FreeList* freelist_; intptr_t bytes_promoted_; RawObject* visiting_old_object_; - friend class Scavenger; + PromotionWorkList promoted_list_; + RawWeakProperty* delayed_weak_properties_ = nullptr; - DISALLOW_COPY_AND_ASSIGN(ScavengerVisitor); + struct ScavengerLAB { + uword top; + uword end; + uword resolved_top; + }; + MallocGrowableArray labs_; + intptr_t consumer_index_ = 1; + intptr_t producer_index_ = 0; + + DISALLOW_COPY_AND_ASSIGN(ScavengerVisitorBase); }; +typedef ScavengerVisitorBase SerialScavengerVisitor; +typedef ScavengerVisitorBase ParallelScavengerVisitor; + class ScavengerWeakVisitor : public HandleVisitor { public: ScavengerWeakVisitor(Thread* thread, Scavenger* scavenger) @@ -303,6 +464,103 @@ class ScavengerWeakVisitor : public HandleVisitor { DISALLOW_COPY_AND_ASSIGN(ScavengerWeakVisitor); }; +class ParallelScavengerTask : public ThreadPool::Task { + public: + ParallelScavengerTask(IsolateGroup* isolate_group, + ThreadBarrier* barrier, + ParallelScavengerVisitor* visitor, + RelaxedAtomic* num_busy) + : isolate_group_(isolate_group), + barrier_(barrier), + visitor_(visitor), + num_busy_(num_busy) {} + + virtual void Run() { + bool result = Thread::EnterIsolateGroupAsHelper( + isolate_group_, Thread::kScavengerTask, /*bypass_safepoint=*/true); + ASSERT(result); + + RunEnteredIsolateGroup(); + + Thread::ExitIsolateGroupAsHelper(/*bypass_safepoint=*/true); + + // This task is done. Notify the original thread. + barrier_->Exit(); + } + + void RunEnteredIsolateGroup() { + TIMELINE_FUNCTION_GC_DURATION(Thread::Current(), "ParallelScavenge"); + + visitor_->ProcessRoots(); + + // Phase 1: Copying. + bool more_to_scavenge = false; + do { + do { + visitor_->ProcessSurvivors(); + + // I can't find more work right now. If no other task is busy, + // then there will never be more work (NB: 1 is *before* decrement). + if (num_busy_->fetch_sub(1u) == 1) break; + + // Wait for some work to appear. + // TODO(iposva): Replace busy-waiting with a solution using Monitor, + // and redraw the boundaries between stack/visitor/task as needed. + while (!visitor_->HasWork() && num_busy_->load() > 0) { + } + + // If no tasks are busy, there will never be more work. + if (num_busy_->load() == 0) break; + + // I saw some work; get busy and compete for it. + num_busy_->fetch_add(1u); + } while (true); + // Wait for all scavengers to stop. + barrier_->Sync(); +#if defined(DEBUG) + ASSERT(num_busy_->load() == 0); + // Caveat: must not allow any marker to continue past the barrier + // before we checked num_busy, otherwise one of them might rush + // ahead and increment it. + barrier_->Sync(); +#endif + // Check if we have any pending properties with marked keys. + // Those might have been marked by another marker. + visitor_->ProcessWeakProperties(); + more_to_scavenge = visitor_->HasWork(); + if (more_to_scavenge) { + // We have more work to do. Notify others. + num_busy_->fetch_add(1u); + } + + // Wait for all other scavengers to finish processing their pending + // weak properties and decide if they need to continue marking. + // Caveat: we need two barriers here to make this decision in lock step + // between all scavengers and the main thread. + barrier_->Sync(); + if (!more_to_scavenge && (num_busy_->load() > 0)) { + // All scavengers continue to mark as long as any single marker has + // some work to do. + num_busy_->fetch_add(1u); + more_to_scavenge = true; + } + barrier_->Sync(); + } while (more_to_scavenge); + + // Phase 2: Weak processing, statistics. + visitor_->Finalize(); + barrier_->Sync(); + } + + private: + IsolateGroup* isolate_group_; + ThreadBarrier* barrier_; + ParallelScavengerVisitor* visitor_; + RelaxedAtomic* num_busy_; + + DISALLOW_COPY_AND_ASSIGN(ParallelScavengerTask); +}; + SemiSpace::SemiSpace(VirtualMemory* reserved) : reserved_(reserved), region_(NULL, 0) { if (reserved != NULL) { @@ -384,6 +642,9 @@ void SemiSpace::Delete() { old_cache = cache_; cache_ = this; } + // TODO(rmacnak): This can take an order of magnitude longer the rest of + // a scavenge. Consider moving it to another thread, perhaps the idle + // notifier. delete old_cache; } @@ -404,7 +665,6 @@ Scavenger::Scavenger(Heap* heap, intptr_t max_semi_capacity_in_words) : heap_(heap), max_semi_capacity_in_words_(max_semi_capacity_in_words), scavenging_(false), - delayed_weak_properties_(NULL), gc_time_micros_(0), collections_(0), scavenge_words_per_micro_(kConservativeInitialScavengeSpeed), @@ -543,8 +803,10 @@ void Scavenger::VerifyStoreBuffers() { } } -SemiSpace* Scavenger::Prologue(IsolateGroup* isolate_group) { - isolate_group->ReleaseStoreBuffers(); +SemiSpace* Scavenger::Prologue() { + TIMELINE_FUNCTION_GC_DURATION(Thread::Current(), "Prologue"); + + heap_->isolate_group()->ReleaseStoreBuffers(); if (FLAG_verify_store_buffer) { OS::PrintErr("Verifying remembered set before Scavenge..."); @@ -553,6 +815,10 @@ SemiSpace* Scavenger::Prologue(IsolateGroup* isolate_group) { OS::PrintErr(" done.\n"); } + // Need to stash the old remembered set before any worker begins adding to the + // new remembered set. + blocks_ = heap_->isolate_group()->store_buffer()->TakeBlocks(); + // Flip the two semi-spaces so that to_ is always the space for allocating // objects. SemiSpace* from = to_; @@ -572,13 +838,15 @@ SemiSpace* Scavenger::Prologue(IsolateGroup* isolate_group) { return from; } -void Scavenger::Epilogue(IsolateGroup* isolate_group, SemiSpace* from) { +void Scavenger::Epilogue(SemiSpace* from) { + TIMELINE_FUNCTION_GC_DURATION(Thread::Current(), "Epilogue"); + // All objects in the to space have been copied from the from space at this // moment. // Ensure the mutator thread will fail the next allocation. This will force // mutator to allocate a new TLAB - isolate_group->ForEachIsolate( + heap_->isolate_group()->ForEachIsolate( [&](Isolate* isolate) { Thread* mutator_thread = isolate->mutator_thread(); ASSERT((mutator_thread == NULL) || (!mutator_thread->HasActiveTLAB())); @@ -646,7 +914,7 @@ void Scavenger::Epilogue(IsolateGroup* isolate_group, SemiSpace* from) { // a program to hit a store buffer overflow a bit sooner than it might // otherwise, since overflow is measured in blocks. Store buffer overflows // are very rare. - isolate_group->ReleaseStoreBuffers(); + heap_->isolate_group()->ReleaseStoreBuffers(); OS::PrintErr("Verifying remembered set after Scavenge..."); heap_->WaitForSweeperTasksAtSafepoint(Thread::Current()); @@ -677,11 +945,21 @@ bool Scavenger::ShouldPerformIdleScavenge(int64_t deadline) { return estimated_scavenge_completion <= deadline; } -void Scavenger::IterateStoreBuffers(IsolateGroup* isolate_group, - ScavengerVisitor* visitor) { +void Scavenger::IterateIsolateRoots(ObjectPointerVisitor* visitor) { + TIMELINE_FUNCTION_GC_DURATION(Thread::Current(), "IterateIsolateRoots"); + heap_->isolate_group()->VisitObjectPointers( + visitor, ValidationPolicy::kDontValidateFrames); +} + +template +void Scavenger::IterateStoreBuffers(ScavengerVisitorBase* visitor) { + TIMELINE_FUNCTION_GC_DURATION(Thread::Current(), "IterateStoreBuffers"); + // Iterating through the store buffers. // Grab the deduplication sets out of the isolate's consolidated store buffer. - StoreBufferBlock* pending = isolate_group->store_buffer()->Blocks(); + StoreBuffer* store_buffer = heap_->isolate_group()->store_buffer(); + StoreBufferBlock* pending = blocks_; + blocks_ = nullptr; intptr_t total_count = 0; while (pending != NULL) { StoreBufferBlock* next = pending->next(); @@ -695,29 +973,35 @@ void Scavenger::IterateStoreBuffers(IsolateGroup* isolate_group, ASSERT(raw_object->IsRemembered()); raw_object->ClearRememberedBit(); visitor->VisitingOldObject(raw_object); + // Note that this treats old-space WeakProperties as strong. A dead key + // won't be reclaimed until after the key is promoted. raw_object->VisitPointersNonvirtual(visitor); } pending->Reset(); // Return the emptied block for recycling (no need to check threshold). - isolate_group->store_buffer()->PushBlock(pending, - StoreBuffer::kIgnoreThreshold); + store_buffer->PushBlock(pending, StoreBuffer::kIgnoreThreshold); pending = next; } - + // Done iterating through old objects remembered in the store buffers. visitor->VisitingOldObject(NULL); - heap_->old_space()->VisitRememberedCards(visitor); heap_->RecordData(kStoreBufferEntries, total_count); heap_->RecordData(kDataUnused1, 0); heap_->RecordData(kDataUnused2, 0); - // Done iterating through old objects remembered in the store buffers. +} + +template +void Scavenger::IterateRememberedCards( + ScavengerVisitorBase* visitor) { + TIMELINE_FUNCTION_GC_DURATION(Thread::Current(), "IterateRememberedCards"); + heap_->old_space()->VisitRememberedCards(visitor); visitor->VisitingOldObject(NULL); } -void Scavenger::IterateObjectIdTable(IsolateGroup* isolate_group, - ScavengerVisitor* visitor) { +void Scavenger::IterateObjectIdTable(ObjectPointerVisitor* visitor) { #ifndef PRODUCT - isolate_group->ForEachIsolate( + TIMELINE_FUNCTION_GC_DURATION(Thread::Current(), "IterateObjectIdTable"); + heap_->isolate_group()->ForEachIsolate( [&](Isolate* isolate) { isolate->object_id_ring()->VisitPointers(visitor); }, @@ -725,28 +1009,39 @@ void Scavenger::IterateObjectIdTable(IsolateGroup* isolate_group, #endif // !PRODUCT } -void Scavenger::IterateRoots(IsolateGroup* isolate_group, - ScavengerVisitor* visitor) { -#ifdef SUPPORT_TIMELINE - Thread* thread = Thread::Current(); -#endif - int64_t start = OS::GetCurrentMonotonicMicros(); - { - TIMELINE_FUNCTION_GC_DURATION(thread, "ProcessRoots"); - isolate_group->VisitObjectPointers(visitor, - ValidationPolicy::kDontValidateFrames); +enum RootSlices { + kIsolate = 0, + kObjectIdRing, + kCards, + kStoreBuffer, + kNumRootSlices, +}; + +template +void Scavenger::IterateRoots(ScavengerVisitorBase* visitor) { + for (;;) { + intptr_t slice = root_slices_started_.fetch_add(1); + if (slice >= kNumRootSlices) { + return; // No more slices. + } + + switch (slice) { + case kIsolate: + IterateIsolateRoots(visitor); + break; + case kObjectIdRing: + IterateObjectIdTable(visitor); + break; + case kCards: + IterateRememberedCards(visitor); + break; + case kStoreBuffer: + IterateStoreBuffers(visitor); + break; + default: + UNREACHABLE(); + } } - int64_t middle = OS::GetCurrentMonotonicMicros(); - { - TIMELINE_FUNCTION_GC_DURATION(thread, "ProcessRememberedSet"); - IterateStoreBuffers(isolate_group, visitor); - } - IterateObjectIdTable(isolate_group, visitor); - int64_t end = OS::GetCurrentMonotonicMicros(); - heap_->RecordData(kToKBAfterStoreBuffer, RoundWordsToKB(UsedInWords())); - heap_->RecordTime(kVisitIsolateRoots, middle - start); - heap_->RecordTime(kIterateStoreBuffers, end - middle); - heap_->RecordTime(kDummyScavengeTime, 0); } bool Scavenger::IsUnreachable(RawObject** p) { @@ -770,81 +1065,85 @@ bool Scavenger::IsUnreachable(RawObject** p) { return true; } -void Scavenger::IterateWeakRoots(IsolateGroup* isolate_group, - HandleVisitor* visitor) { - isolate_group->VisitWeakPersistentHandles(visitor); +void Scavenger::MournWeakHandles() { + Thread* thread = Thread::Current(); + TIMELINE_FUNCTION_GC_DURATION(thread, "MournWeakHandles"); + ScavengerWeakVisitor weak_visitor(thread, this); + heap_->isolate_group()->VisitWeakPersistentHandles(&weak_visitor); } -void Scavenger::ProcessToSpace(ScavengerVisitor* visitor) { - Thread* thread = Thread::Current(); +template +void ScavengerVisitorBase::ProcessToSpace() { + intptr_t i = consumer_index_; + while (i <= producer_index_) { + uword resolved_top = labs_[i].resolved_top; + while (resolved_top < labs_[i].top) { + RawObject* raw_obj = RawObject::FromAddr(resolved_top); + resolved_top += ProcessCopied(raw_obj); + } + labs_[i].resolved_top = resolved_top; - // Iterate until all work has been drained. - while ((resolved_top_ < top_) || PromotedStackHasMore()) { - while (resolved_top_ < top_) { - RawObject* raw_obj = RawObject::FromAddr(resolved_top_); - intptr_t class_id = raw_obj->GetClassId(); - intptr_t size; - if (class_id != kWeakPropertyCid) { - size = raw_obj->VisitPointersNonvirtual(visitor); - } else { - RawWeakProperty* raw_weak = reinterpret_cast(raw_obj); - size = ProcessWeakProperty(raw_weak, visitor); - } - resolved_top_ += size; + if (i == producer_index_) { + return; // More objects may yet be copied to this TLAB. } - { - // Visit all the promoted objects and update/scavenge their internal - // pointers. Potentially this adds more objects to the to space. - while (PromotedStackHasMore()) { - RawObject* raw_object = RawObject::FromAddr(PopFromPromotedStack()); - // Resolve or copy all objects referred to by the current object. This - // can potentially push more objects on this stack as well as add more - // objects to be resolved in the to space. - ASSERT(!raw_object->IsRemembered()); - visitor->VisitingOldObject(raw_object); - raw_object->VisitPointersNonvirtual(visitor); - if (raw_object->IsMarked()) { - // Complete our promise from ScavengePointer. Note that marker cannot - // visit this object until it pops a block from the mark stack, which - // involves a memory fence from the mutex, so even on architectures - // with a relaxed memory model, the marker will see the fully - // forwarded contents of this object. - thread->MarkingStackAddObject(raw_object); - } - } - visitor->VisitingOldObject(NULL); + + i++; + consumer_index_ = i; + ASSERT(consumer_index_ < labs_.length()); + } +} + +template +void ScavengerVisitorBase::ProcessPromotedList() { + while (RawObject* raw_object = promoted_list_.Pop()) { + // Resolve or copy all objects referred to by the current object. This + // can potentially push more objects on this stack as well as add more + // objects to be resolved in the to space. + ASSERT(!raw_object->IsRemembered()); + VisitingOldObject(raw_object); + raw_object->VisitPointersNonvirtual(this); + if (raw_object->IsMarked()) { + // Complete our promise from ScavengePointer. Note that marker cannot + // visit this object until it pops a block from the mark stack, which + // involves a memory fence from the mutex, so even on architectures + // with a relaxed memory model, the marker will see the fully + // forwarded contents of this object. + thread_->MarkingStackAddObject(raw_object); } - { - // Finished this round of scavenging. Process the pending weak properties - // for which the keys have become reachable. Potentially this adds more - // objects to the to space. - RawWeakProperty* cur_weak = delayed_weak_properties_; - delayed_weak_properties_ = NULL; - while (cur_weak != NULL) { - uword next_weak = cur_weak->ptr()->next_; - // Promoted weak properties are not enqueued. So we can guarantee that - // we do not need to think about store barriers here. - ASSERT(cur_weak->IsNewObject()); - RawObject* raw_key = cur_weak->ptr()->key_; - ASSERT(raw_key->IsHeapObject()); - // Key still points into from space even if the object has been - // promoted to old space by now. The key will be updated accordingly - // below when VisitPointers is run. - ASSERT(raw_key->IsNewObject()); - uword raw_addr = RawObject::ToAddr(raw_key); - ASSERT(visitor->from_->Contains(raw_addr)); - uword header = *reinterpret_cast(raw_addr); - // Reset the next pointer in the weak property. - cur_weak->ptr()->next_ = 0; - if (IsForwarding(header)) { - cur_weak->VisitPointersNonvirtual(visitor); - } else { - EnqueueWeakProperty(cur_weak); - } - // Advance to next weak property in the queue. - cur_weak = reinterpret_cast(next_weak); - } + } + VisitingOldObject(NULL); +} + +template +void ScavengerVisitorBase::ProcessWeakProperties() { + // Finished this round of scavenging. Process the pending weak properties + // for which the keys have become reachable. Potentially this adds more + // objects to the to space. + RawWeakProperty* cur_weak = delayed_weak_properties_; + delayed_weak_properties_ = NULL; + while (cur_weak != NULL) { + uword next_weak = cur_weak->ptr()->next_; + // Promoted weak properties are not enqueued. So we can guarantee that + // we do not need to think about store barriers here. + ASSERT(cur_weak->IsNewObject()); + RawObject* raw_key = cur_weak->ptr()->key_; + ASSERT(raw_key->IsHeapObject()); + // Key still points into from space even if the object has been + // promoted to old space by now. The key will be updated accordingly + // below when VisitPointers is run. + ASSERT(raw_key->IsNewObject()); + uword raw_addr = RawObject::ToAddr(raw_key); + ASSERT(from_->Contains(raw_addr)); + uword header = *reinterpret_cast(raw_addr); + // Reset the next pointer in the weak property. + cur_weak->ptr()->next_ = 0; + if (IsForwarding(header)) { + cur_weak->VisitPointersNonvirtual(this); + } else { + EnqueueWeakProperty(cur_weak); } + // Advance to next weak property in the queue. + cur_weak = reinterpret_cast(next_weak); } } @@ -877,7 +1176,9 @@ void Scavenger::UpdateMaxHeapUsage() { #endif // !defined(PRODUCT) } -void Scavenger::EnqueueWeakProperty(RawWeakProperty* raw_weak) { +template +void ScavengerVisitorBase::EnqueueWeakProperty( + RawWeakProperty* raw_weak) { ASSERT(raw_weak->IsHeapObject()); ASSERT(raw_weak->IsNewObject()); ASSERT(raw_weak->IsWeakProperty()); @@ -891,24 +1192,30 @@ void Scavenger::EnqueueWeakProperty(RawWeakProperty* raw_weak) { delayed_weak_properties_ = raw_weak; } -uword Scavenger::ProcessWeakProperty(RawWeakProperty* raw_weak, - ScavengerVisitor* visitor) { - // The fate of the weak property is determined by its key. - RawObject* raw_key = raw_weak->ptr()->key_; - if (raw_key->IsHeapObject() && raw_key->IsNewObject()) { - uword raw_addr = RawObject::ToAddr(raw_key); - uword header = *reinterpret_cast(raw_addr); - if (!IsForwarding(header)) { - // Key is white. Enqueue the weak property. - EnqueueWeakProperty(raw_weak); - return raw_weak->HeapSize(); +template +intptr_t ScavengerVisitorBase::ProcessCopied(RawObject* raw_obj) { + intptr_t class_id = raw_obj->GetClassId(); + if (UNLIKELY(class_id == kWeakPropertyCid)) { + RawWeakProperty* raw_weak = reinterpret_cast(raw_obj); + // The fate of the weak property is determined by its key. + RawObject* raw_key = raw_weak->ptr()->key_; + if (raw_key->IsHeapObject() && raw_key->IsNewObject()) { + uword raw_addr = RawObject::ToAddr(raw_key); + uword header = *reinterpret_cast(raw_addr); + if (!IsForwarding(header)) { + // Key is white. Enqueue the weak property. + EnqueueWeakProperty(raw_weak); + return raw_weak->HeapSize(); + } } + // Key is gray or black. Make the weak property black. } - // Key is gray or black. Make the weak property black. - return raw_weak->VisitPointersNonvirtual(visitor); + return raw_obj->VisitPointersNonvirtual(this); } -void Scavenger::ProcessWeakReferences() { +void Scavenger::MournWeakTables() { + TIMELINE_FUNCTION_GC_DURATION(Thread::Current(), "MournWeakTables"); + auto rehash_weak_table = [](WeakTable* table, WeakTable* replacement_new, WeakTable* replacement_old) { intptr_t size = table->size(); @@ -958,31 +1265,32 @@ void Scavenger::ProcessWeakReferences() { } }, /*at_safepoint=*/true); +} +template +void ScavengerVisitorBase::MournWeakProperties() { // The queued weak properties at this point do not refer to reachable keys, // so we clear their key and value fields. - { - RawWeakProperty* cur_weak = delayed_weak_properties_; - delayed_weak_properties_ = NULL; - while (cur_weak != NULL) { - uword next_weak = cur_weak->ptr()->next_; - // Reset the next pointer in the weak property. - cur_weak->ptr()->next_ = 0; + RawWeakProperty* cur_weak = delayed_weak_properties_; + delayed_weak_properties_ = NULL; + while (cur_weak != NULL) { + uword next_weak = cur_weak->ptr()->next_; + // Reset the next pointer in the weak property. + cur_weak->ptr()->next_ = 0; #if defined(DEBUG) - RawObject* raw_key = cur_weak->ptr()->key_; - uword raw_addr = RawObject::ToAddr(raw_key); - uword header = *reinterpret_cast(raw_addr); - ASSERT(!IsForwarding(header)); - ASSERT(raw_key->IsHeapObject()); - ASSERT(raw_key->IsNewObject()); // Key still points into from space. -#endif // defined(DEBUG) + RawObject* raw_key = cur_weak->ptr()->key_; + uword raw_addr = RawObject::ToAddr(raw_key); + uword header = *reinterpret_cast(raw_addr); + ASSERT(!IsForwarding(header)); + ASSERT(raw_key->IsHeapObject()); + ASSERT(raw_key->IsNewObject()); // Key still points into from space. +#endif // defined(DEBUG) - WeakProperty::Clear(cur_weak); + WeakProperty::Clear(cur_weak); - // Advance to next weak property in the queue. - cur_weak = reinterpret_cast(next_weak); - } + // Advance to next weak property in the queue. + cur_weak = reinterpret_cast(next_weak); } } @@ -1009,8 +1317,9 @@ void Scavenger::MakeNewSpaceIterable() const { /*at_safepoint=*/true); } -void Scavenger::AbandonTLABsLocked(IsolateGroup* isolate_group) { +void Scavenger::AbandonTLABsLocked() { ASSERT(Thread::Current()->IsAtSafepoint()); + IsolateGroup* isolate_group = heap_->isolate_group(); MonitorLocker ml(isolate_group->threads_lock(), false); Thread* current = isolate_group->thread_registry()->active_list(); while (current != NULL) { @@ -1125,31 +1434,67 @@ void Scavenger::AbandonRemainingTLABLocked(Thread* thread) { thread->set_end(0); } +template +uword ScavengerVisitorBase::TryAllocateCopySlow(intptr_t size) { + MakeProducerTLABIterable(); + + if (!scavenger_->TryAllocateNewTLAB(this)) { + return 0; + } + + const uword result = labs_[producer_index_].top; + const intptr_t remaining = + labs_[producer_index_].end - labs_[producer_index_].top; + ASSERT(size <= remaining); + ASSERT(scavenger_->to_->Contains(result)); + ASSERT((result & kObjectAlignmentMask) == kNewObjectAlignmentOffset); + labs_[producer_index_].top = result + size; + return result; +} + +template +bool Scavenger::TryAllocateNewTLAB(ScavengerVisitorBase* visitor) { + intptr_t size = kTLABSize; + ASSERT(Utils::IsAligned(size, kObjectAlignment)); + ASSERT(heap_ != Dart::vm_isolate()->heap()); + ASSERT(scavenging_); + MutexLocker ml(&space_lock_); + const uword result = top_; + const intptr_t remaining = end_ - top_; + if (remaining < size) { + // Grab whatever is remaining + size = Utils::RoundDown(remaining, kObjectAlignment); + } + if (size == 0) { + return false; + } + ASSERT(to_->Contains(result)); + ASSERT((result & kObjectAlignmentMask) == kNewObjectAlignmentOffset); + top_ += size; + ASSERT(to_->Contains(top_) || (top_ == to_->end())); + ASSERT(result < top_); + visitor->AddNewTLAB(result, top_); + return true; +} + void Scavenger::Scavenge() { - auto isolate_group = heap_->isolate_group(); + int64_t start = OS::GetCurrentMonotonicMicros(); + // Ensure that all threads for this isolate are at a safepoint (either stopped // or in native code). If two threads are racing at this point, the loser // will continue with its scavenge after waiting for the winner to complete. // TODO(koda): Consider moving SafepointThreads into allocation failure/retry // logic to avoid needless collections. - - int64_t start = OS::GetCurrentMonotonicMicros(); - Thread* thread = Thread::Current(); SafepointOperationScope safepoint_scope(thread); + int64_t safe_point = OS::GetCurrentMonotonicMicros(); + heap_->RecordTime(kSafePoint, safe_point - start); + // Scavenging is not reentrant. Make sure that is the case. ASSERT(!scavenging_); scavenging_ = true; - failed_to_promote_ = false; - - PageSpace* page_space = heap_->old_space(); - NoSafepointScope no_safepoints; - - int64_t safe_point = OS::GetCurrentMonotonicMicros(); - heap_->RecordTime(kSafePoint, safe_point - start); - if (FLAG_verify_before_gc) { OS::PrintErr("Verifying before Scavenge..."); heap_->WaitForSweeperTasksAtSafepoint(thread); @@ -1158,48 +1503,33 @@ void Scavenger::Scavenge() { } // Prepare for a scavenge. - AbandonTLABsLocked(isolate_group); + AbandonTLABsLocked(); + failed_to_promote_ = false; + root_slices_started_ = 0; intptr_t abandoned_bytes = GetAndResetAbandonedInBytes(); - SpaceUsage usage_before = GetCurrentUsage(); intptr_t promo_candidate_words = (survivor_end_ - FirstObjectStart()) / kWordSize; - SemiSpace* from = Prologue(isolate_group); - // The API prologue/epilogue may create/destroy zones, so we must not - // depend on zone allocations surviving beyond the epilogue callback. - { - StackZone zone(thread); - // Setup the visitor and run the scavenge. - ScavengerVisitor visitor(isolate_group, this, from); - page_space->AcquireDataLock(); - IterateRoots(isolate_group, &visitor); - int64_t iterate_roots = OS::GetCurrentMonotonicMicros(); - { - TIMELINE_FUNCTION_GC_DURATION(thread, "ProcessToSpace"); - ProcessToSpace(&visitor); - } - int64_t process_to_space = OS::GetCurrentMonotonicMicros(); - { - TIMELINE_FUNCTION_GC_DURATION(thread, "ProcessWeakHandles"); - ScavengerWeakVisitor weak_visitor(thread, this); - IterateWeakRoots(isolate_group, &weak_visitor); - } - ProcessWeakReferences(); - page_space->ReleaseDataLock(); + SemiSpace* from = Prologue(); - // Restore write-barrier assumptions. - isolate_group->RememberLiveTemporaries(); - - // Scavenge finished. Run accounting. - int64_t end = OS::GetCurrentMonotonicMicros(); - heap_->RecordTime(kProcessToSpace, process_to_space - iterate_roots); - heap_->RecordTime(kIterateWeaks, end - process_to_space); - stats_history_.Add(ScavengeStats(start, end, usage_before, - GetCurrentUsage(), promo_candidate_words, - visitor.bytes_promoted() >> kWordSizeLog2, - abandoned_bytes >> kWordSizeLog2)); + intptr_t bytes_promoted; + if (FLAG_scavenger_tasks == 0) { + bytes_promoted = SerialScavenge(from); + } else { + bytes_promoted = ParallelScavenge(from); } - Epilogue(isolate_group, from); + MournWeakHandles(); + MournWeakTables(); + + // Restore write-barrier assumptions. + heap_->isolate_group()->RememberLiveTemporaries(); + + // Scavenge finished. Run accounting. + int64_t end = OS::GetCurrentMonotonicMicros(); + stats_history_.Add(ScavengeStats( + start, end, usage_before, GetCurrentUsage(), promo_candidate_words, + bytes_promoted >> kWordSizeLog2, abandoned_bytes >> kWordSizeLog2)); + Epilogue(from); if (FLAG_verify_after_gc) { OS::PrintErr("Verifying after Scavenge..."); @@ -1213,6 +1543,69 @@ void Scavenger::Scavenge() { scavenging_ = false; } +intptr_t Scavenger::SerialScavenge(SemiSpace* from) { + FreeList* freelist = heap_->old_space()->DataFreeList(0); + SerialScavengerVisitor visitor(heap_->isolate_group(), this, from, freelist, + &promotion_stack_); + visitor.ProcessRoots(); + { + TIMELINE_FUNCTION_GC_DURATION(Thread::Current(), "ProcessToSpace"); + visitor.ProcessAll(); + } + visitor.Finalize(); + + // Donate last bit of TLAB. + uword top = visitor.last_top(); + uword end = visitor.last_end(); + if (end == top_) { + top_ = top; + } + return visitor.bytes_promoted(); +} + +intptr_t Scavenger::ParallelScavenge(SemiSpace* from) { + intptr_t bytes_promoted = 0; + const intptr_t num_tasks = FLAG_scavenger_tasks; + ASSERT(num_tasks > 0); + + ThreadBarrier barrier(num_tasks, heap_->barrier(), heap_->barrier_done()); + RelaxedAtomic num_busy = num_tasks; + + ParallelScavengerVisitor** visitors = + new ParallelScavengerVisitor*[num_tasks]; + for (intptr_t i = 0; i < num_tasks; i++) { + FreeList* freelist = heap_->old_space()->DataFreeList(i); + visitors[i] = new ParallelScavengerVisitor( + heap_->isolate_group(), this, from, freelist, &promotion_stack_); + if (i < (num_tasks - 1)) { + // Begin scavenging on a helper thread. + bool result = Dart::thread_pool()->Run( + heap_->isolate_group(), &barrier, visitors[i], &num_busy); + ASSERT(result); + } else { + // Last workers is the main thread. + ParallelScavengerTask task(heap_->isolate_group(), &barrier, visitors[i], + &num_busy); + task.RunEnteredIsolateGroup(); + barrier.Exit(); + } + } + + for (intptr_t i = 0; i < num_tasks; i++) { + bytes_promoted += visitors[i]->bytes_promoted(); + // Donate last bit of TLAB. + uword top = visitors[i]->last_top(); + uword end = visitors[i]->last_end(); + if (end == top_) { + top_ = top; + } + delete visitors[i]; + } + + delete[] visitors; + return bytes_promoted; +} + void Scavenger::WriteProtect(bool read_only) { ASSERT(!scavenging_); to_->WriteProtect(read_only); diff --git a/runtime/vm/heap/scavenger.h b/runtime/vm/heap/scavenger.h index 9dacdd363db..d02d6231cbd 100644 --- a/runtime/vm/heap/scavenger.h +++ b/runtime/vm/heap/scavenger.h @@ -24,7 +24,8 @@ class Heap; class Isolate; class JSONObject; class ObjectSet; -class ScavengerVisitor; +template +class ScavengerVisitorBase; // Wrapper around VirtualMemory that adds caching and handles the empty case. class SemiSpace { @@ -139,23 +140,8 @@ class Scavenger { } void MakeTLABIterable(Thread* thread); void AbandonRemainingTLAB(Thread* thread); - - uword AllocateGC(intptr_t size) { - ASSERT(Utils::IsAligned(size, kObjectAlignment)); - ASSERT(heap_ != Dart::vm_isolate()->heap()); - ASSERT(scavenging_); - uword result = top_; - intptr_t remaining = end_ - top_; - - // This allocation happens only in GC and only when copying objects to - // the new to_ space. It must succeed. - ASSERT(size <= remaining); - ASSERT(to_->Contains(result)); - ASSERT((result & kObjectAlignmentMask) == kNewObjectAlignmentOffset); - top_ += size; - ASSERT((to_->Contains(top_)) || (top_ == to_->end())); - return result; - } + template + bool TryAllocateNewTLAB(ScavengerVisitorBase* visitor); // Collect the garbage in this scavenger. void Scavenge(); @@ -163,15 +149,6 @@ class Scavenger { // Promote all live objects. void Evacuate(); - uword top() { return top_; } - uword end() { return end_; } - - void set_top(uword value) { top_ = value; } - void set_end(uword value) { - ASSERT(to_->end() == value); - end_ = value; - } - // Report (TLAB) abandoned bytes that should be taken account when // deciding whether to grow new space or not. void AddAbandonedInBytes(intptr_t value) { @@ -225,6 +202,8 @@ class Scavenger { void MakeNewSpaceIterable() const; int64_t FreeSpaceInWords(Isolate* isolate) const; + bool scavenging() const { return scavenging_; } + private: static const intptr_t kTLABSize = 512 * KB; @@ -264,58 +243,33 @@ class Scavenger { void TryAllocateNewTLAB(Thread* thread); void AddAbandonedInBytesLocked(intptr_t value) { abandoned_ += value; } void AbandonRemainingTLABLocked(Thread* thread); - void AbandonTLABsLocked(IsolateGroup* isolate_group); + void AbandonTLABsLocked(); uword FirstObjectStart() const { return to_->start() + kNewObjectAlignmentOffset; } - SemiSpace* Prologue(IsolateGroup* isolate_group); - void IterateStoreBuffers(IsolateGroup* isolate_group, - ScavengerVisitor* visitor); - void IterateObjectIdTable(IsolateGroup* isolate_group, - ScavengerVisitor* visitor); - void IterateRoots(IsolateGroup* isolate_group, ScavengerVisitor* visitor); - void IterateWeakProperties(IsolateGroup* isolate_group, - ScavengerVisitor* visitor); - void IterateWeakReferences(IsolateGroup* isolate_group, - ScavengerVisitor* visitor); - void IterateWeakRoots(IsolateGroup* isolate_group, HandleVisitor* visitor); - void ProcessToSpace(ScavengerVisitor* visitor); - void EnqueueWeakProperty(RawWeakProperty* raw_weak); - uword ProcessWeakProperty(RawWeakProperty* raw_weak, - ScavengerVisitor* visitor); - void Epilogue(IsolateGroup* isolate_group, SemiSpace* from); + SemiSpace* Prologue(); + intptr_t ParallelScavenge(SemiSpace* from); + intptr_t SerialScavenge(SemiSpace* from); + void IterateIsolateRoots(ObjectPointerVisitor* visitor); + template + void IterateStoreBuffers(ScavengerVisitorBase* visitor); + template + void IterateRememberedCards(ScavengerVisitorBase* visitor); + void IterateObjectIdTable(ObjectPointerVisitor* visitor); + template + void IterateRoots(ScavengerVisitorBase* visitor); + void MournWeakHandles(); + void Epilogue(SemiSpace* from); bool IsUnreachable(RawObject** p); void VerifyStoreBuffers(); - // During a scavenge we need to remember the promoted objects. - // This is implemented as a stack of objects at the end of the to space. As - // object sizes are always greater than sizeof(uword) and promoted objects do - // not consume space in the to space they leave enough room for this stack. - void PushToPromotedStack(uword addr) { - ASSERT(scavenging_); - end_ -= sizeof(addr); - ASSERT(end_ > top_); - *reinterpret_cast(end_) = addr; - } - uword PopFromPromotedStack() { - ASSERT(scavenging_); - uword result = *reinterpret_cast(end_); - end_ += sizeof(result); - ASSERT(end_ <= to_->end()); - return result; - } - bool PromotedStackHasMore() const { - ASSERT(scavenging_); - return end_ < to_->end(); - } - void UpdateMaxHeapCapacity(); void UpdateMaxHeapUsage(); - void ProcessWeakReferences(); + void MournWeakTables(); intptr_t NewSizeInWords(intptr_t old_size_in_words) const; @@ -337,13 +291,14 @@ class Scavenger { // whether to grow newspace or not. intptr_t abandoned_ = 0; + PromotionStack promotion_stack_; + intptr_t max_semi_capacity_in_words_; // Keep track whether a scavenge is currently running. bool scavenging_; - - // Keep track of pending weak properties discovered while scagenging. - RawWeakProperty* delayed_weak_properties_; + RelaxedAtomic root_slices_started_; + StoreBufferBlock* blocks_; int64_t gc_time_micros_; intptr_t collections_; @@ -361,7 +316,8 @@ class Scavenger { // Protects new space during the allocation of new TLABs mutable Mutex space_lock_; - friend class ScavengerVisitor; + template + friend class ScavengerVisitorBase; friend class ScavengerWeakVisitor; DISALLOW_COPY_AND_ASSIGN(Scavenger); diff --git a/runtime/vm/heap/sweeper.cc b/runtime/vm/heap/sweeper.cc index 405357d521b..2a19fba538a 100644 --- a/runtime/vm/heap/sweeper.cc +++ b/runtime/vm/heap/sweeper.cc @@ -111,20 +111,17 @@ class ConcurrentSweeperTask : public ThreadPool::Task { HeapPage* first, HeapPage* last, HeapPage* large_first, - HeapPage* large_last, - FreeList* freelist) + HeapPage* large_last) : task_isolate_group_(isolate_group), old_space_(old_space), first_(first), last_(last), large_first_(large_first), - large_last_(large_last), - freelist_(freelist) { + large_last_(large_last) { ASSERT(task_isolate_group_ != NULL); ASSERT(first_ != NULL); ASSERT(old_space_ != NULL); ASSERT(last_ != NULL); - ASSERT(freelist_ != NULL); MonitorLocker ml(old_space_->tasks_lock()); old_space_->set_tasks(old_space_->tasks() + 1); old_space_->set_phase(PageSpace::kSweepingLarge); @@ -169,6 +166,8 @@ class ConcurrentSweeperTask : public ThreadPool::Task { ml.NotifyAll(); } + intptr_t shard = 0; + const intptr_t num_shards = Utils::Maximum(FLAG_scavenger_tasks, 1); page = first_; prev_page = NULL; while (page != NULL) { @@ -181,7 +180,9 @@ class ConcurrentSweeperTask : public ThreadPool::Task { next_page = page->next(); } ASSERT(page->type() == HeapPage::kData); - bool page_in_use = sweeper.SweepPage(page, freelist_, false); + shard = (shard + 1) % num_shards; + bool page_in_use = + sweeper.SweepPage(page, old_space_->DataFreeList(shard), false); if (page_in_use) { prev_page = page; } else { @@ -215,7 +216,6 @@ class ConcurrentSweeperTask : public ThreadPool::Task { HeapPage* last_; HeapPage* large_first_; HeapPage* large_last_; - FreeList* freelist_; }; void GCSweeper::SweepConcurrent(IsolateGroup* isolate_group, @@ -226,7 +226,7 @@ void GCSweeper::SweepConcurrent(IsolateGroup* isolate_group, FreeList* freelist) { bool result = Dart::thread_pool()->Run( isolate_group, isolate_group->heap()->old_space(), first, last, - large_first, large_last, freelist); + large_first, large_last); ASSERT(result); } diff --git a/runtime/vm/isolate.cc b/runtime/vm/isolate.cc index 8cbd8a00522..eff7c914a4b 100644 --- a/runtime/vm/isolate.cc +++ b/runtime/vm/isolate.cc @@ -79,6 +79,7 @@ static void DeterministicModeHandler(bool value) { FLAG_background_compilation = false; // Timing dependent. FLAG_concurrent_mark = false; // Timing dependent. FLAG_concurrent_sweep = false; // Timing dependent. + FLAG_scavenger_tasks = 0; // Timing dependent. FLAG_random_seed = 0x44617274; // "Dart" } } @@ -2494,7 +2495,8 @@ void IsolateGroup::ForEachIsolate( ASSERT(Thread::Current()->IsAtSafepoint() || (Thread::Current()->task_kind() == Thread::kMutatorTask) || (Thread::Current()->task_kind() == Thread::kMarkerTask) || - (Thread::Current()->task_kind() == Thread::kCompactorTask)); + (Thread::Current()->task_kind() == Thread::kCompactorTask) || + (Thread::Current()->task_kind() == Thread::kScavengerTask)); for (Isolate* isolate : isolates_) { function(isolate); } diff --git a/runtime/vm/object.cc b/runtime/vm/object.cc index bc1883df307..44a69d3a946 100644 --- a/runtime/vm/object.cc +++ b/runtime/vm/object.cc @@ -2588,12 +2588,15 @@ void Object::CheckHandle() const { ASSERT(vtable() == builtin_vtables_[cid]); if (FLAG_verify_handles && raw_->IsHeapObject()) { Heap* isolate_heap = IsolateGroup::Current()->heap(); - Heap* vm_isolate_heap = Dart::vm_isolate()->heap(); - uword addr = RawObject::ToAddr(raw_); - if (!isolate_heap->Contains(addr) && !vm_isolate_heap->Contains(addr)) { - ASSERT(FLAG_write_protect_code); - addr = RawObject::ToAddr(HeapPage::ToWritable(raw_)); - ASSERT(isolate_heap->Contains(addr) || vm_isolate_heap->Contains(addr)); + if (!isolate_heap->new_space()->scavenging()) { + Heap* vm_isolate_heap = Dart::vm_isolate()->heap(); + uword addr = RawObject::ToAddr(raw_); + if (!isolate_heap->Contains(addr) && !vm_isolate_heap->Contains(addr)) { + ASSERT(FLAG_write_protect_code); + addr = RawObject::ToAddr(HeapPage::ToWritable(raw_)); + ASSERT(isolate_heap->Contains(addr) || + vm_isolate_heap->Contains(addr)); + } } } } diff --git a/runtime/vm/object.h b/runtime/vm/object.h index 1bbc3a3ee35..9c7af6b130c 100644 --- a/runtime/vm/object.h +++ b/runtime/vm/object.h @@ -10853,12 +10853,16 @@ void Object::SetRaw(RawObject* value) { if (FLAG_verify_handles && raw_->IsHeapObject()) { Isolate* isolate = Isolate::Current(); Heap* isolate_heap = isolate->heap(); - Heap* vm_isolate_heap = Dart::vm_isolate()->heap(); - uword addr = RawObject::ToAddr(raw_); - if (!isolate_heap->Contains(addr) && !vm_isolate_heap->Contains(addr)) { - ASSERT(FLAG_write_protect_code); - addr = RawObject::ToAddr(HeapPage::ToWritable(raw_)); - ASSERT(isolate_heap->Contains(addr) || vm_isolate_heap->Contains(addr)); + // TODO(rmacnak): Remove after rewriting StackFrame::VisitObjectPointers + // to not use handles. + if (!isolate_heap->new_space()->scavenging()) { + Heap* vm_isolate_heap = Dart::vm_isolate()->heap(); + uword addr = RawObject::ToAddr(raw_); + if (!isolate_heap->Contains(addr) && !vm_isolate_heap->Contains(addr)) { + ASSERT(FLAG_write_protect_code); + addr = RawObject::ToAddr(HeapPage::ToWritable(raw_)); + ASSERT(isolate_heap->Contains(addr) || vm_isolate_heap->Contains(addr)); + } } } #endif diff --git a/runtime/vm/raw_object.cc b/runtime/vm/raw_object.cc index e155a11de9f..809e67276df 100644 --- a/runtime/vm/raw_object.cc +++ b/runtime/vm/raw_object.cc @@ -73,7 +73,7 @@ void RawObject::Validate(IsolateGroup* isolate_group) const { return; } intptr_t size_from_tags = SizeTag::decode(tags); - intptr_t size_from_class = HeapSizeFromClass(); + intptr_t size_from_class = HeapSizeFromClass(tags); if ((size_from_tags != 0) && (size_from_tags != size_from_class)) { FATAL3( "Inconsistent size encountered " @@ -85,11 +85,12 @@ void RawObject::Validate(IsolateGroup* isolate_group) const { // Can't look at the class object because it can be called during // compaction when the class objects are moving. Can use the class // id in the header and the sizes in the Class Table. -intptr_t RawObject::HeapSizeFromClass() const { +// Cannot deference ptr()->tags_. May dereference other parts of the object. +intptr_t RawObject::HeapSizeFromClass(uint32_t tags) const { // Only reasonable to be called on heap objects. ASSERT(IsHeapObject()); - intptr_t class_id = GetClassId(); + intptr_t class_id = ClassIdTag::decode(tags); intptr_t instance_size = 0; switch (class_id) { case kCodeCid: { @@ -245,7 +246,7 @@ intptr_t RawObject::HeapSizeFromClass() const { if (!class_table->IsValidIndex(class_id) || (!class_table->HasValidClassAt(class_id) && !use_saved_class_table)) { FATAL3("Invalid cid: %" Pd ", obj: %p, tags: %x. Corrupt heap?", - class_id, this, static_cast(ptr()->tags_)); + class_id, this, static_cast(tags)); } #endif // DEBUG instance_size = isolate_group->GetClassSizeForHeapWalkAt(class_id); @@ -253,7 +254,6 @@ intptr_t RawObject::HeapSizeFromClass() const { } ASSERT(instance_size != 0); #if defined(DEBUG) - uint32_t tags = ptr()->tags_; intptr_t tags_size = SizeTag::decode(tags); if ((class_id == kArrayCid) && (instance_size > tags_size && tags_size > 0)) { // TODO(22501): Array::MakeFixedLength could be in the process of shrinking diff --git a/runtime/vm/raw_object.h b/runtime/vm/raw_object.h index 625fab1d54d..1a4d9ada437 100644 --- a/runtime/vm/raw_object.h +++ b/runtime/vm/raw_object.h @@ -437,7 +437,7 @@ class RawObject { // leading to inconsistency between HeapSizeFromClass() and // SizeTag::decode(tags). We are working around it by reloading tags_ and // recomputing size from tags. - const intptr_t size_from_class = HeapSizeFromClass(); + const intptr_t size_from_class = HeapSizeFromClass(tags); if ((result > size_from_class) && (GetClassId() == kArrayCid) && (ptr()->tags_) != tags) { result = SizeTag::decode(ptr()->tags_); @@ -446,7 +446,19 @@ class RawObject { #endif return result; } - result = HeapSizeFromClass(); + result = HeapSizeFromClass(tags); + ASSERT(result > SizeTag::kMaxSizeTag); + return result; + } + + // This variant must not deference ptr()->tags_. + intptr_t HeapSize(uint32_t tags) const { + ASSERT(IsHeapObject()); + intptr_t result = SizeTag::decode(tags); + if (result != 0) { + return result; + } + result = HeapSizeFromClass(tags); ASSERT(result > SizeTag::kMaxSizeTag); return result; } @@ -605,7 +617,7 @@ class RawObject { intptr_t VisitPointersPredefined(ObjectPointerVisitor* visitor, intptr_t class_id); - intptr_t HeapSizeFromClass() const; + intptr_t HeapSizeFromClass(uint32_t tags) const; void SetClassId(intptr_t new_cid) { ptr()->tags_.UpdateUnsynchronized(new_cid); @@ -761,7 +773,8 @@ class RawObject { friend class OneByteString; // StoreSmi friend class RawInstance; friend class Scavenger; - friend class ScavengerVisitor; + template + friend class ScavengerVisitorBase; friend class ImageReader; // tags_ check friend class ImageWriter; friend class AssemblyImageWriter; @@ -2602,7 +2615,8 @@ class RawTypedDataView : public RawTypedDataBase { friend class ObjectPoolSerializationCluster; friend class RawObjectPool; friend class GCCompactor; - friend class ScavengerVisitor; + template + friend class ScavengerVisitorBase; friend class SnapshotReader; }; @@ -2896,7 +2910,8 @@ class RawWeakProperty : public RawInstance { template friend class MarkingVisitorBase; friend class Scavenger; - friend class ScavengerVisitor; + template + friend class ScavengerVisitorBase; }; // MirrorReferences are used by mirrors to hold reflectees that are VM diff --git a/runtime/vm/snapshot.cc b/runtime/vm/snapshot.cc index 8968b17568a..ad552386349 100644 --- a/runtime/vm/snapshot.cc +++ b/runtime/vm/snapshot.cc @@ -685,18 +685,6 @@ Object* SnapshotReader::GetBackRef(intptr_t id) { return NULL; } -class HeapLocker : public StackResource { - public: - HeapLocker(Thread* thread, PageSpace* page_space) - : StackResource(thread), page_space_(page_space) { - page_space_->AcquireDataLock(); - } - ~HeapLocker() { page_space_->ReleaseDataLock(); } - - private: - PageSpace* page_space_; -}; - RawApiError* SnapshotReader::VerifyVersionAndFeatures(Isolate* isolate) { // If the version string doesn't match, return an error. // Note: New things are allocated only if we're going to return an error. diff --git a/runtime/vm/thread.h b/runtime/vm/thread.h index 911e5e8b21f..b2284a7df0d 100644 --- a/runtime/vm/thread.h +++ b/runtime/vm/thread.h @@ -239,6 +239,7 @@ class Thread : public ThreadState { kMarkerTask = 0x4, kSweeperTask = 0x8, kCompactorTask = 0x10, + kScavengerTask = 0x20, }; // Converts a TaskKind to its corresponding C-String name. static const char* TaskKindToCString(TaskKind kind); diff --git a/tests/standalone_2/fragmentation_test.dart b/tests/standalone_2/fragmentation_test.dart index b083ec16d53..f24ea098327 100644 --- a/tests/standalone_2/fragmentation_test.dart +++ b/tests/standalone_2/fragmentation_test.dart @@ -18,6 +18,10 @@ // VMOptions=--concurrent_mark --concurrent_sweep // VMOptions=--concurrent_mark --use_compactor // VMOptions=--concurrent_mark --use_compactor --force_evacuation +// VMOptions=--scavenger_tasks=0 +// VMOptions=--scavenger_tasks=1 +// VMOptions=--scavenger_tasks=2 +// VMOptions=--scavenger_tasks=3 main() { final List arrays = [];