Proof-of-concept pretenuring of some strings:

- Add bump-pointer allocated block in page space.
- Pretenure num.toString whenever >98% of strings are being promoted.
- Fix deadlock in freelist printing.

Review URL: https://codereview.chromium.org//511963007

git-svn-id: https://dart.googlecode.com/svn/branches/bleeding_edge/dart@39688 260f80e4-7a28-3924-810f-c04153c831b5

runtime/lib/num.cc

@@ -11,7 +11,9 @@ namespace dart {
 
 DEFINE_NATIVE_ENTRY(Num_toString, 1) {
   const Number& number = Number::CheckedHandle(arguments->NativeArgAt(0));
-  return number.ToString(Heap::kNew);
+  Heap::Space space = isolate->heap()->ShouldPretenure(kOneByteStringCid) ?
+      Heap::kPretenured : Heap::kNew;
+  return number.ToString(space);
 }
 
 }  // namespace dart

runtime/vm/freelist.cc

@@ -235,8 +235,8 @@ FreeListElement* FreeList::DequeueElement(intptr_t index) {
 }
 
 
-intptr_t FreeList::Length(int index) const {
-  MutexLocker ml(mutex_);
+intptr_t FreeList::LengthLocked(int index) const {
+  DEBUG_ASSERT(mutex_->Owner() == Isolate::Current());
   ASSERT(index >= 0);
   ASSERT(index < kNumLists);
   intptr_t result = 0;
@@ -258,7 +258,7 @@ void FreeList::PrintSmall() const {
       continue;
     }
     small_sizes += 1;
-    intptr_t list_length = Length(i);
+    intptr_t list_length = LengthLocked(i);
     small_objects += list_length;
     intptr_t list_bytes = list_length * i * kObjectAlignment;
     small_bytes += list_bytes;
@@ -341,4 +341,26 @@ void FreeList::SplitElementAfterAndEnqueue(FreeListElement* element,
   }
 }
 
+
+FreeListElement* FreeList::TryAllocateLarge(intptr_t minimum_size) {
+  MutexLocker ml(mutex_);
+  FreeListElement* previous = NULL;
+  FreeListElement* current = free_lists_[kNumLists];
+  // TODO(koda): Find largest.
+  while (current != NULL) {
+    FreeListElement* next = current->next();
+    if (current->Size() >= minimum_size) {
+      if (previous == NULL) {
+        free_lists_[kNumLists] = next;
+      } else {
+        previous->set_next(next);
+      }
+      return current;
+    }
+    previous = current;
+    current = next;
+  }
+  return NULL;
+}
+
 }  // namespace dart

runtime/vm/freelist.h

@@ -92,12 +92,15 @@ class FreeList {
   uword TryAllocateLocked(intptr_t size, bool is_protected);
   void FreeLocked(uword addr, intptr_t size);
 
+  // Returns a large element, at least 'minimum_size', or NULL if none exists.
+  FreeListElement* TryAllocateLarge(intptr_t minimum_size);
+
  private:
   static const int kNumLists = 128;
 
   static intptr_t IndexForSize(intptr_t size);
 
-  intptr_t Length(int index) const;
+  intptr_t LengthLocked(int index) const;
 
   void EnqueueElement(FreeListElement* element, intptr_t index);
   FreeListElement* DequeueElement(intptr_t index);

runtime/vm/heap.cc

@@ -33,11 +33,18 @@ DEFINE_FLAG(bool, verify_after_gc, false,
 DEFINE_FLAG(bool, gc_at_alloc, false, "GC at every allocation.");
 DEFINE_FLAG(int, new_gen_ext_limit, 64,
             "maximum total external size (MB) in new gen before triggering GC");
+DEFINE_FLAG(int, pretenure_threshold, 98,
+            "Trigger pretenuring when this many percent are promoted.");
+DEFINE_FLAG(int, pretenure_interval, 10,
+            "Back off pretenuring after this many cycles.");
 
 Heap::Heap(Isolate* isolate,
            intptr_t max_new_gen_semi_words,
            intptr_t max_old_gen_words)
-    : isolate_(isolate), read_only_(false), gc_in_progress_(false) {
+    : isolate_(isolate),
+      read_only_(false),
+      gc_in_progress_(false),
+      pretenure_policy_(0) {
   for (int sel = 0;
        sel < kNumWeakSelectors;
        sel++) {
@@ -138,6 +145,15 @@ uword Heap::AllocateOld(intptr_t size, HeapPage::PageType type) {
   return 0;
 }
 
+
+uword Heap::AllocatePretenured(intptr_t size) {
+  ASSERT(isolate()->no_gc_scope_depth() == 0);
+  uword addr = old_space_->TryAllocateDataBump(size, PageSpace::kControlGrowth);
+  if (addr != 0) return addr;
+  return AllocateOld(size, HeapPage::kData);
+}
+
+
 void Heap::AllocateExternal(intptr_t size, Space space) {
   ASSERT(isolate()->no_gc_scope_depth() == 0);
   if (space == kNew) {
@@ -269,6 +285,7 @@ void Heap::CollectGarbage(Space space,
       UpdateClassHeapStatsBeforeGC(kNew);
       new_space_->Scavenge(invoke_api_callbacks);
       isolate()->class_table()->UpdatePromoted();
+      UpdatePretenurePolicy();
       RecordAfterGC();
       PrintStats();
       if (old_space_->NeedsGarbageCollection()) {
@@ -321,6 +338,7 @@ void Heap::CollectAllGarbage() {
     UpdateClassHeapStatsBeforeGC(kNew);
     new_space_->Scavenge(kInvokeApiCallbacks);
     isolate()->class_table()->UpdatePromoted();
+    UpdatePretenurePolicy();
     RecordAfterGC();
     PrintStats();
   }
@@ -335,6 +353,29 @@ void Heap::CollectAllGarbage() {
 }
 
 
+bool Heap::ShouldPretenure(intptr_t class_id) const {
+  if (class_id == kOneByteStringCid) {
+    return pretenure_policy_ > 0;
+  } else {
+    return false;
+  }
+}
+
+
+void Heap::UpdatePretenurePolicy() {
+  ClassHeapStats* stats =
+      isolate_->class_table()->StatsWithUpdatedSize(kOneByteStringCid);
+  int allocated = stats->pre_gc.new_count;
+  int promo_percent = (allocated == 0) ? 0 :
+      (100 * stats->promoted_count) / allocated;
+  if (promo_percent >= FLAG_pretenure_threshold) {
+    pretenure_policy_ += FLAG_pretenure_interval;
+  } else {
+    pretenure_policy_ = Utils::Maximum(0, pretenure_policy_ - 1);
+  }
+}
+
+
 void Heap::SetGrowthControlState(bool state) {
   old_space_->SetGrowthControlState(state);
 }

runtime/vm/heap.h

@@ -33,6 +33,7 @@ class Heap {
     kNew,
     kOld,
     kCode,
+    kPretenured,
   };
 
   enum WeakSelector {
@@ -78,6 +79,8 @@ class Heap {
         return AllocateOld(size, HeapPage::kData);
       case kCode:
         return AllocateOld(size, HeapPage::kExecutable);
+      case kPretenured:
+        return AllocatePretenured(size);
       default:
         UNREACHABLE();
     }
@@ -236,6 +239,8 @@ class Heap {
 
   Isolate* isolate() const { return isolate_; }
 
+  bool ShouldPretenure(intptr_t class_id) const;
+
  private:
   class GCStats : public ValueObject {
    public:
@@ -275,12 +280,14 @@ class Heap {
 
   uword AllocateNew(intptr_t size);
   uword AllocateOld(intptr_t size, HeapPage::PageType type);
+  uword AllocatePretenured(intptr_t size);
 
   // GC stats collection.
   void RecordBeforeGC(Space space, GCReason reason);
   void RecordAfterGC();
   void PrintStats();
   void UpdateClassHeapStatsBeforeGC(Heap::Space space);
+  void UpdatePretenurePolicy();
 
   // If this heap is non-empty, updates start and end to the smallest range that
   // contains both the original [start, end) and the [lowest, highest) addresses
@@ -305,6 +312,8 @@ class Heap {
   // GC on the heap is in progress.
   bool gc_in_progress_;
 
+  int pretenure_policy_;
+
   friend class GCEvent;
   friend class GCTestHelper;
   DISALLOW_COPY_AND_ASSIGN(Heap);

runtime/vm/pages.cc

@@ -19,7 +19,7 @@ DEFINE_FLAG(int, heap_growth_space_ratio, 20,
             "The desired maximum percentage of free space after GC");
 DEFINE_FLAG(int, heap_growth_time_ratio, 3,
             "The desired maximum percentage of time spent in GC");
-DEFINE_FLAG(int, heap_growth_rate, 256,
+DEFINE_FLAG(int, heap_growth_rate, 280,
             "The max number of pages the heap can grow at a time");
 DEFINE_FLAG(bool, print_free_list_before_gc, false,
             "Print free list statistics before a GC");
@@ -35,6 +35,7 @@ DEFINE_FLAG(bool, always_drop_code, false,
             "Always try to drop code if the function's usage counter is >= 0");
 DEFINE_FLAG(bool, concurrent_sweep, false,
             "Concurrent sweep for old generation.");
+DEFINE_FLAG(bool, log_growth, false, "Log PageSpace growth policy decisions.");
 
 HeapPage* HeapPage::Initialize(VirtualMemory* memory, PageType type) {
   ASSERT(memory->size() > VirtualMemory::PageSize());
@@ -129,6 +130,8 @@ PageSpace::PageSpace(Heap* heap, intptr_t max_capacity_in_words)
       exec_pages_(NULL),
       exec_pages_tail_(NULL),
       large_pages_(NULL),
+      bump_top_(0),
+      bump_end_(0),
       max_capacity_in_words_(max_capacity_in_words),
       tasks_lock_(new Monitor()),
       tasks_(0),
@@ -281,6 +284,39 @@ void PageSpace::FreePages(HeapPage* pages) {
 }
 
 
+uword PageSpace::TryAllocateInFreshPage(intptr_t size,
+                                        HeapPage::PageType type,
+                                        GrowthPolicy growth_policy,
+                                        bool is_locked) {
+  ASSERT(size < kAllocatablePageSize);
+  uword result = 0;
+  SpaceUsage after_allocation = usage_;
+  after_allocation.used_in_words += size >> kWordSizeLog2;
+  // Can we grow by one page?
+  after_allocation.capacity_in_words += kPageSizeInWords;
+  if ((growth_policy == kForceGrowth ||
+       !page_space_controller_.NeedsGarbageCollection(after_allocation)) &&
+      CanIncreaseCapacityInWords(kPageSizeInWords)) {
+    HeapPage* page = AllocatePage(type);
+    ASSERT(page != NULL);
+    // Start of the newly allocated page is the allocated object.
+    result = page->object_start();
+    usage_ = after_allocation;
+    // Enqueue the remainder in the free list.
+    uword free_start = result + size;
+    intptr_t free_size = page->object_end() - free_start;
+    if (free_size > 0) {
+      if (is_locked) {
+        freelist_[type].FreeLocked(free_start, free_size);
+      } else {
+        freelist_[type].Free(free_start, free_size);
+      }
+    }
+  }
+  return result;
+}
+
+
 uword PageSpace::TryAllocateInternal(intptr_t size,
                             HeapPage::PageType type,
                             GrowthPolicy growth_policy,
@@ -289,8 +325,6 @@ uword PageSpace::TryAllocateInternal(intptr_t size,
   ASSERT(size >= kObjectAlignment);
   ASSERT(Utils::IsAligned(size, kObjectAlignment));
   uword result = 0;
-  SpaceUsage after_allocation = usage_;
-  after_allocation.used_in_words += size >> kWordSizeLog2;
   if (size < kAllocatablePageSize) {
     if (is_locked) {
       result = freelist_[type].TryAllocateLocked(size, is_protected);
@@ -298,26 +332,7 @@ uword PageSpace::TryAllocateInternal(intptr_t size,
       result = freelist_[type].TryAllocate(size, is_protected);
     }
     if (result == 0) {
-      // Can we grow by one page?
-      after_allocation.capacity_in_words += kPageSizeInWords;
-      if ((!page_space_controller_.NeedsGarbageCollection(after_allocation) ||
-           growth_policy == kForceGrowth) &&
-          CanIncreaseCapacityInWords(kPageSizeInWords)) {
-        HeapPage* page = AllocatePage(type);
-        ASSERT(page != NULL);
-        // Start of the newly allocated page is the allocated object.
-        result = page->object_start();
-        // Enqueue the remainder in the free list.
-        uword free_start = result + size;
-        intptr_t free_size = page->object_end() - free_start;
-        if (free_size > 0) {
-          if (is_locked) {
-            freelist_[type].FreeLocked(free_start, free_size);
-          } else {
-            freelist_[type].Free(free_start, free_size);
-          }
-        }
-      }
+      result = TryAllocateInFreshPage(size, type, growth_policy, is_locked);
     }
   } else {
     // Large page allocation.
@@ -326,18 +341,20 @@ uword PageSpace::TryAllocateInternal(intptr_t size,
       // On overflow we fail to allocate.
       return 0;
     }
+    SpaceUsage after_allocation = usage_;
+    after_allocation.used_in_words += size >> kWordSizeLog2;
     after_allocation.capacity_in_words += page_size_in_words;
-    if ((!page_space_controller_.NeedsGarbageCollection(after_allocation) ||
-         growth_policy == kForceGrowth) &&
+    if ((growth_policy == kForceGrowth ||
+         !page_space_controller_.NeedsGarbageCollection(after_allocation)) &&
         CanIncreaseCapacityInWords(page_size_in_words)) {
       HeapPage* page = AllocateLargePage(size, type);
       if (page != NULL) {
         result = page->object_start();
+        usage_ = after_allocation;
       }
     }
   }
   if (result != 0) {
-    usage_ = after_allocation;
     if (FLAG_compiler_stats && (type == HeapPage::kExecutable)) {
       CompilerStats::code_allocated += size;
     }
@@ -628,7 +645,9 @@ void PageSpace::MarkSweep(bool invoke_api_callbacks) {
 
   int64_t mid1 = OS::GetCurrentTimeMicros();
 
-  // Reset the bump allocation page to unused.
+  // Abandon the remainder of the bump allocation block.
+  bump_top_ = 0;
+  bump_end_ = 0;
   // Reset the freelists and setup sweeping.
   freelist_[HeapPage::kData].Reset();
   freelist_[HeapPage::kExecutable].Reset();
@@ -735,6 +754,44 @@ void PageSpace::MarkSweep(bool invoke_api_callbacks) {
 }
 
 
+uword PageSpace::TryAllocateDataBump(intptr_t size,
+                                     GrowthPolicy growth_policy) {
+  ASSERT(size >= kObjectAlignment);
+  ASSERT(Utils::IsAligned(size, kObjectAlignment));
+  intptr_t remaining = bump_end_ - bump_top_;
+  if (remaining < size) {
+    // Checking this first would be logical, but needlessly slow.
+    if (size >= kAllocatablePageSize) {
+      return TryAllocate(size, HeapPage::kData, growth_policy);
+    }
+    FreeListElement* block = freelist_[HeapPage::kData].TryAllocateLarge(size);
+    if (block == NULL) {
+      // Allocating from a new page (if growth policy allows) will have the
+      // side-effect of populating the freelist with a large block. The next
+      // bump allocation request will have a chance to consume that block.
+      // TODO(koda): Could take freelist lock just once instead of twice.
+      return TryAllocateInFreshPage(size,
+                                    HeapPage::kData,
+                                    growth_policy,
+                                    /* is_locked = */ false);
+    }
+    intptr_t block_size = block->Size();
+    bump_top_ = reinterpret_cast<uword>(block);
+    bump_end_ = bump_top_ + block_size;
+    remaining = block_size;
+  }
+  ASSERT(remaining >= size);
+  uword result = bump_top_;
+  bump_top_ += size;
+  usage_.used_in_words += size >> kWordSizeLog2;
+  remaining -= size;
+  if (remaining > 0) {
+    FreeListElement::AsElement(bump_top_, remaining);
+  }
+  return result;
+}
+
+
 PageSpaceController::PageSpaceController(Heap* heap,
                                          int heap_growth_ratio,
                                          int heap_growth_max,
@@ -780,7 +837,16 @@ bool PageSpaceController::NeedsGarbageCollection(SpaceUsage after) const {
       multiplier *= seconds_since_init / kInitialTimeoutSeconds;
     }
   }
-  return capacity_increase_in_pages * multiplier > grow_heap_;
+  bool needs_gc = capacity_increase_in_pages * multiplier > grow_heap_;
+  if (FLAG_log_growth) {
+    OS::PrintErr("%s: %" Pd " * %f %s %" Pd "\n",
+                 needs_gc ? "NEEDS GC" : "grow",
+                 capacity_increase_in_pages,
+                 multiplier,
+                 needs_gc ? ">" : "<=",
+                 grow_heap_);
+  }
+  return needs_gc;
 }
 
 

runtime/vm/pages.h

@@ -303,6 +303,9 @@ class PageSpace {
     tasks_ = val;
   }
 
+  // Attempt to allocate from bump block rather than normal freelist.
+  uword TryAllocateDataBump(intptr_t size, GrowthPolicy growth_policy);
+
  private:
   // Ids for time and data records in Heap::GCStats.
   enum {
@@ -325,6 +328,10 @@ class PageSpace {
                             GrowthPolicy growth_policy,
                             bool is_protected,
                             bool is_locked);
+  uword TryAllocateInFreshPage(intptr_t size,
+                               HeapPage::PageType type,
+                               GrowthPolicy growth_policy,
+                               bool is_locked);
   HeapPage* AllocatePage(HeapPage::PageType type);
   void FreePage(HeapPage* page, HeapPage* previous_page);
   HeapPage* AllocateLargePage(intptr_t size, HeapPage::PageType type);
@@ -358,6 +365,11 @@ class PageSpace {
   HeapPage* exec_pages_tail_;
   HeapPage* large_pages_;
 
+  // A block of memory in a data page, managed by bump allocation. The remainder
+  // is kept formatted as a FreeListElement, but is not in any freelist.
+  uword bump_top_;
+  uword bump_end_;
+
   // Various sizes being tracked for this generation.
   intptr_t max_capacity_in_words_;
   SpaceUsage usage_;