From 0844cbcfbedecbeb2b5aace53d0e6e10455bd0cd Mon Sep 17 00:00:00 2001 From: "koda@google.com" Date: Fri, 29 Aug 2014 03:54:12 +0000 Subject: [PATCH] Proof-of-concept pretenuring of some strings: - Add bump-pointer allocated block in page space. - Pretenure num.toString whenever >98% of strings are being promoted. - Fix deadlock in freelist printing. Review URL: https://codereview.chromium.org//511963007 git-svn-id: https://dart.googlecode.com/svn/branches/bleeding_edge/dart@39688 260f80e4-7a28-3924-810f-c04153c831b5 --- runtime/lib/num.cc | 4 +- runtime/vm/freelist.cc | 28 +++++++++- runtime/vm/freelist.h | 5 +- runtime/vm/heap.cc | 43 ++++++++++++++- runtime/vm/heap.h | 9 +++ runtime/vm/pages.cc | 122 +++++++++++++++++++++++++++++++---------- runtime/vm/pages.h | 12 ++++ 7 files changed, 189 insertions(+), 34 deletions(-) diff --git a/runtime/lib/num.cc b/runtime/lib/num.cc index 85d9426cbfa..8004c8c9d9e 100644 --- a/runtime/lib/num.cc +++ b/runtime/lib/num.cc @@ -11,7 +11,9 @@ namespace dart { DEFINE_NATIVE_ENTRY(Num_toString, 1) { const Number& number = Number::CheckedHandle(arguments->NativeArgAt(0)); - return number.ToString(Heap::kNew); + Heap::Space space = isolate->heap()->ShouldPretenure(kOneByteStringCid) ? + Heap::kPretenured : Heap::kNew; + return number.ToString(space); } } // namespace dart diff --git a/runtime/vm/freelist.cc b/runtime/vm/freelist.cc index eb1ce0a2a8e..a78359b9739 100644 --- a/runtime/vm/freelist.cc +++ b/runtime/vm/freelist.cc @@ -235,8 +235,8 @@ FreeListElement* FreeList::DequeueElement(intptr_t index) { } -intptr_t FreeList::Length(int index) const { - MutexLocker ml(mutex_); +intptr_t FreeList::LengthLocked(int index) const { + DEBUG_ASSERT(mutex_->Owner() == Isolate::Current()); ASSERT(index >= 0); ASSERT(index < kNumLists); intptr_t result = 0; @@ -258,7 +258,7 @@ void FreeList::PrintSmall() const { continue; } small_sizes += 1; - intptr_t list_length = Length(i); + intptr_t list_length = LengthLocked(i); small_objects += list_length; intptr_t list_bytes = list_length * i * kObjectAlignment; small_bytes += list_bytes; @@ -341,4 +341,26 @@ void FreeList::SplitElementAfterAndEnqueue(FreeListElement* element, } } + +FreeListElement* FreeList::TryAllocateLarge(intptr_t minimum_size) { + MutexLocker ml(mutex_); + FreeListElement* previous = NULL; + FreeListElement* current = free_lists_[kNumLists]; + // TODO(koda): Find largest. + while (current != NULL) { + FreeListElement* next = current->next(); + if (current->Size() >= minimum_size) { + if (previous == NULL) { + free_lists_[kNumLists] = next; + } else { + previous->set_next(next); + } + return current; + } + previous = current; + current = next; + } + return NULL; +} + } // namespace dart diff --git a/runtime/vm/freelist.h b/runtime/vm/freelist.h index 0216e07b647..38451e56d74 100644 --- a/runtime/vm/freelist.h +++ b/runtime/vm/freelist.h @@ -92,12 +92,15 @@ class FreeList { uword TryAllocateLocked(intptr_t size, bool is_protected); void FreeLocked(uword addr, intptr_t size); + // Returns a large element, at least 'minimum_size', or NULL if none exists. + FreeListElement* TryAllocateLarge(intptr_t minimum_size); + private: static const int kNumLists = 128; static intptr_t IndexForSize(intptr_t size); - intptr_t Length(int index) const; + intptr_t LengthLocked(int index) const; void EnqueueElement(FreeListElement* element, intptr_t index); FreeListElement* DequeueElement(intptr_t index); diff --git a/runtime/vm/heap.cc b/runtime/vm/heap.cc index 08ea60afcf6..a60d35ff3dc 100644 --- a/runtime/vm/heap.cc +++ b/runtime/vm/heap.cc @@ -33,11 +33,18 @@ DEFINE_FLAG(bool, verify_after_gc, false, DEFINE_FLAG(bool, gc_at_alloc, false, "GC at every allocation."); DEFINE_FLAG(int, new_gen_ext_limit, 64, "maximum total external size (MB) in new gen before triggering GC"); +DEFINE_FLAG(int, pretenure_threshold, 98, + "Trigger pretenuring when this many percent are promoted."); +DEFINE_FLAG(int, pretenure_interval, 10, + "Back off pretenuring after this many cycles."); Heap::Heap(Isolate* isolate, intptr_t max_new_gen_semi_words, intptr_t max_old_gen_words) - : isolate_(isolate), read_only_(false), gc_in_progress_(false) { + : isolate_(isolate), + read_only_(false), + gc_in_progress_(false), + pretenure_policy_(0) { for (int sel = 0; sel < kNumWeakSelectors; sel++) { @@ -138,6 +145,15 @@ uword Heap::AllocateOld(intptr_t size, HeapPage::PageType type) { return 0; } + +uword Heap::AllocatePretenured(intptr_t size) { + ASSERT(isolate()->no_gc_scope_depth() == 0); + uword addr = old_space_->TryAllocateDataBump(size, PageSpace::kControlGrowth); + if (addr != 0) return addr; + return AllocateOld(size, HeapPage::kData); +} + + void Heap::AllocateExternal(intptr_t size, Space space) { ASSERT(isolate()->no_gc_scope_depth() == 0); if (space == kNew) { @@ -269,6 +285,7 @@ void Heap::CollectGarbage(Space space, UpdateClassHeapStatsBeforeGC(kNew); new_space_->Scavenge(invoke_api_callbacks); isolate()->class_table()->UpdatePromoted(); + UpdatePretenurePolicy(); RecordAfterGC(); PrintStats(); if (old_space_->NeedsGarbageCollection()) { @@ -321,6 +338,7 @@ void Heap::CollectAllGarbage() { UpdateClassHeapStatsBeforeGC(kNew); new_space_->Scavenge(kInvokeApiCallbacks); isolate()->class_table()->UpdatePromoted(); + UpdatePretenurePolicy(); RecordAfterGC(); PrintStats(); } @@ -335,6 +353,29 @@ void Heap::CollectAllGarbage() { } +bool Heap::ShouldPretenure(intptr_t class_id) const { + if (class_id == kOneByteStringCid) { + return pretenure_policy_ > 0; + } else { + return false; + } +} + + +void Heap::UpdatePretenurePolicy() { + ClassHeapStats* stats = + isolate_->class_table()->StatsWithUpdatedSize(kOneByteStringCid); + int allocated = stats->pre_gc.new_count; + int promo_percent = (allocated == 0) ? 0 : + (100 * stats->promoted_count) / allocated; + if (promo_percent >= FLAG_pretenure_threshold) { + pretenure_policy_ += FLAG_pretenure_interval; + } else { + pretenure_policy_ = Utils::Maximum(0, pretenure_policy_ - 1); + } +} + + void Heap::SetGrowthControlState(bool state) { old_space_->SetGrowthControlState(state); } diff --git a/runtime/vm/heap.h b/runtime/vm/heap.h index 1d508c4a151..133de794ae2 100644 --- a/runtime/vm/heap.h +++ b/runtime/vm/heap.h @@ -33,6 +33,7 @@ class Heap { kNew, kOld, kCode, + kPretenured, }; enum WeakSelector { @@ -78,6 +79,8 @@ class Heap { return AllocateOld(size, HeapPage::kData); case kCode: return AllocateOld(size, HeapPage::kExecutable); + case kPretenured: + return AllocatePretenured(size); default: UNREACHABLE(); } @@ -236,6 +239,8 @@ class Heap { Isolate* isolate() const { return isolate_; } + bool ShouldPretenure(intptr_t class_id) const; + private: class GCStats : public ValueObject { public: @@ -275,12 +280,14 @@ class Heap { uword AllocateNew(intptr_t size); uword AllocateOld(intptr_t size, HeapPage::PageType type); + uword AllocatePretenured(intptr_t size); // GC stats collection. void RecordBeforeGC(Space space, GCReason reason); void RecordAfterGC(); void PrintStats(); void UpdateClassHeapStatsBeforeGC(Heap::Space space); + void UpdatePretenurePolicy(); // If this heap is non-empty, updates start and end to the smallest range that // contains both the original [start, end) and the [lowest, highest) addresses @@ -305,6 +312,8 @@ class Heap { // GC on the heap is in progress. bool gc_in_progress_; + int pretenure_policy_; + friend class GCEvent; friend class GCTestHelper; DISALLOW_COPY_AND_ASSIGN(Heap); diff --git a/runtime/vm/pages.cc b/runtime/vm/pages.cc index cf6870ecd50..41bf2d422a4 100644 --- a/runtime/vm/pages.cc +++ b/runtime/vm/pages.cc @@ -19,7 +19,7 @@ DEFINE_FLAG(int, heap_growth_space_ratio, 20, "The desired maximum percentage of free space after GC"); DEFINE_FLAG(int, heap_growth_time_ratio, 3, "The desired maximum percentage of time spent in GC"); -DEFINE_FLAG(int, heap_growth_rate, 256, +DEFINE_FLAG(int, heap_growth_rate, 280, "The max number of pages the heap can grow at a time"); DEFINE_FLAG(bool, print_free_list_before_gc, false, "Print free list statistics before a GC"); @@ -35,6 +35,7 @@ DEFINE_FLAG(bool, always_drop_code, false, "Always try to drop code if the function's usage counter is >= 0"); DEFINE_FLAG(bool, concurrent_sweep, false, "Concurrent sweep for old generation."); +DEFINE_FLAG(bool, log_growth, false, "Log PageSpace growth policy decisions."); HeapPage* HeapPage::Initialize(VirtualMemory* memory, PageType type) { ASSERT(memory->size() > VirtualMemory::PageSize()); @@ -129,6 +130,8 @@ PageSpace::PageSpace(Heap* heap, intptr_t max_capacity_in_words) exec_pages_(NULL), exec_pages_tail_(NULL), large_pages_(NULL), + bump_top_(0), + bump_end_(0), max_capacity_in_words_(max_capacity_in_words), tasks_lock_(new Monitor()), tasks_(0), @@ -281,6 +284,39 @@ void PageSpace::FreePages(HeapPage* pages) { } +uword PageSpace::TryAllocateInFreshPage(intptr_t size, + HeapPage::PageType type, + GrowthPolicy growth_policy, + bool is_locked) { + ASSERT(size < kAllocatablePageSize); + uword result = 0; + SpaceUsage after_allocation = usage_; + after_allocation.used_in_words += size >> kWordSizeLog2; + // Can we grow by one page? + after_allocation.capacity_in_words += kPageSizeInWords; + if ((growth_policy == kForceGrowth || + !page_space_controller_.NeedsGarbageCollection(after_allocation)) && + CanIncreaseCapacityInWords(kPageSizeInWords)) { + HeapPage* page = AllocatePage(type); + ASSERT(page != NULL); + // Start of the newly allocated page is the allocated object. + result = page->object_start(); + usage_ = after_allocation; + // Enqueue the remainder in the free list. + uword free_start = result + size; + intptr_t free_size = page->object_end() - free_start; + if (free_size > 0) { + if (is_locked) { + freelist_[type].FreeLocked(free_start, free_size); + } else { + freelist_[type].Free(free_start, free_size); + } + } + } + return result; +} + + uword PageSpace::TryAllocateInternal(intptr_t size, HeapPage::PageType type, GrowthPolicy growth_policy, @@ -289,8 +325,6 @@ uword PageSpace::TryAllocateInternal(intptr_t size, ASSERT(size >= kObjectAlignment); ASSERT(Utils::IsAligned(size, kObjectAlignment)); uword result = 0; - SpaceUsage after_allocation = usage_; - after_allocation.used_in_words += size >> kWordSizeLog2; if (size < kAllocatablePageSize) { if (is_locked) { result = freelist_[type].TryAllocateLocked(size, is_protected); @@ -298,26 +332,7 @@ uword PageSpace::TryAllocateInternal(intptr_t size, result = freelist_[type].TryAllocate(size, is_protected); } if (result == 0) { - // Can we grow by one page? - after_allocation.capacity_in_words += kPageSizeInWords; - if ((!page_space_controller_.NeedsGarbageCollection(after_allocation) || - growth_policy == kForceGrowth) && - CanIncreaseCapacityInWords(kPageSizeInWords)) { - HeapPage* page = AllocatePage(type); - ASSERT(page != NULL); - // Start of the newly allocated page is the allocated object. - result = page->object_start(); - // Enqueue the remainder in the free list. - uword free_start = result + size; - intptr_t free_size = page->object_end() - free_start; - if (free_size > 0) { - if (is_locked) { - freelist_[type].FreeLocked(free_start, free_size); - } else { - freelist_[type].Free(free_start, free_size); - } - } - } + result = TryAllocateInFreshPage(size, type, growth_policy, is_locked); } } else { // Large page allocation. @@ -326,18 +341,20 @@ uword PageSpace::TryAllocateInternal(intptr_t size, // On overflow we fail to allocate. return 0; } + SpaceUsage after_allocation = usage_; + after_allocation.used_in_words += size >> kWordSizeLog2; after_allocation.capacity_in_words += page_size_in_words; - if ((!page_space_controller_.NeedsGarbageCollection(after_allocation) || - growth_policy == kForceGrowth) && + if ((growth_policy == kForceGrowth || + !page_space_controller_.NeedsGarbageCollection(after_allocation)) && CanIncreaseCapacityInWords(page_size_in_words)) { HeapPage* page = AllocateLargePage(size, type); if (page != NULL) { result = page->object_start(); + usage_ = after_allocation; } } } if (result != 0) { - usage_ = after_allocation; if (FLAG_compiler_stats && (type == HeapPage::kExecutable)) { CompilerStats::code_allocated += size; } @@ -628,7 +645,9 @@ void PageSpace::MarkSweep(bool invoke_api_callbacks) { int64_t mid1 = OS::GetCurrentTimeMicros(); - // Reset the bump allocation page to unused. + // Abandon the remainder of the bump allocation block. + bump_top_ = 0; + bump_end_ = 0; // Reset the freelists and setup sweeping. freelist_[HeapPage::kData].Reset(); freelist_[HeapPage::kExecutable].Reset(); @@ -735,6 +754,44 @@ void PageSpace::MarkSweep(bool invoke_api_callbacks) { } +uword PageSpace::TryAllocateDataBump(intptr_t size, + GrowthPolicy growth_policy) { + ASSERT(size >= kObjectAlignment); + ASSERT(Utils::IsAligned(size, kObjectAlignment)); + intptr_t remaining = bump_end_ - bump_top_; + if (remaining < size) { + // Checking this first would be logical, but needlessly slow. + if (size >= kAllocatablePageSize) { + return TryAllocate(size, HeapPage::kData, growth_policy); + } + FreeListElement* block = freelist_[HeapPage::kData].TryAllocateLarge(size); + if (block == NULL) { + // Allocating from a new page (if growth policy allows) will have the + // side-effect of populating the freelist with a large block. The next + // bump allocation request will have a chance to consume that block. + // TODO(koda): Could take freelist lock just once instead of twice. + return TryAllocateInFreshPage(size, + HeapPage::kData, + growth_policy, + /* is_locked = */ false); + } + intptr_t block_size = block->Size(); + bump_top_ = reinterpret_cast(block); + bump_end_ = bump_top_ + block_size; + remaining = block_size; + } + ASSERT(remaining >= size); + uword result = bump_top_; + bump_top_ += size; + usage_.used_in_words += size >> kWordSizeLog2; + remaining -= size; + if (remaining > 0) { + FreeListElement::AsElement(bump_top_, remaining); + } + return result; +} + + PageSpaceController::PageSpaceController(Heap* heap, int heap_growth_ratio, int heap_growth_max, @@ -780,7 +837,16 @@ bool PageSpaceController::NeedsGarbageCollection(SpaceUsage after) const { multiplier *= seconds_since_init / kInitialTimeoutSeconds; } } - return capacity_increase_in_pages * multiplier > grow_heap_; + bool needs_gc = capacity_increase_in_pages * multiplier > grow_heap_; + if (FLAG_log_growth) { + OS::PrintErr("%s: %" Pd " * %f %s %" Pd "\n", + needs_gc ? "NEEDS GC" : "grow", + capacity_increase_in_pages, + multiplier, + needs_gc ? ">" : "<=", + grow_heap_); + } + return needs_gc; } diff --git a/runtime/vm/pages.h b/runtime/vm/pages.h index 225e0a17030..0b1b0d23089 100644 --- a/runtime/vm/pages.h +++ b/runtime/vm/pages.h @@ -303,6 +303,9 @@ class PageSpace { tasks_ = val; } + // Attempt to allocate from bump block rather than normal freelist. + uword TryAllocateDataBump(intptr_t size, GrowthPolicy growth_policy); + private: // Ids for time and data records in Heap::GCStats. enum { @@ -325,6 +328,10 @@ class PageSpace { GrowthPolicy growth_policy, bool is_protected, bool is_locked); + uword TryAllocateInFreshPage(intptr_t size, + HeapPage::PageType type, + GrowthPolicy growth_policy, + bool is_locked); HeapPage* AllocatePage(HeapPage::PageType type); void FreePage(HeapPage* page, HeapPage* previous_page); HeapPage* AllocateLargePage(intptr_t size, HeapPage::PageType type); @@ -358,6 +365,11 @@ class PageSpace { HeapPage* exec_pages_tail_; HeapPage* large_pages_; + // A block of memory in a data page, managed by bump allocation. The remainder + // is kept formatted as a FreeListElement, but is not in any freelist. + uword bump_top_; + uword bump_end_; + // Various sizes being tracked for this generation. intptr_t max_capacity_in_words_; SpaceUsage usage_;