[vm, gc] Reserve some memory to use during handling of OutOfMemoryErrors.

This reserved space can only be allocated from after an allocation has failed from OutOfMemory, and once some portion of this space is used, refilling it is the first allocation performed after GC.

Also avoid greatly slowing down from ineffective scavenges as the memory limit is reached.

Bug: https://github.com/dart-lang/sdk/issues/43543
Bug: https://github.com/dart-lang/sdk/issues/43642
Bug: b/169880355
Change-Id: Ic7132cb34d7a7d13c67661f057f00dd74306251c
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/165862
Commit-Queue: Ryan Macnak <rmacnak@google.com>
Reviewed-by: Martin Kustermann <kustermann@google.com>

runtime/vm/heap/compactor.cc

@@ -296,6 +296,8 @@ void GCCompactor::Compact(OldPage* pages,
     ForwardStackPointers();
   }
 
+  heap_->old_space()->VisitRoots(this);
+
   {
     MutexLocker ml(pages_lock);
 

runtime/vm/heap/heap.cc

@@ -58,9 +58,11 @@ class NoActiveIsolateScope {
 };
 
 Heap::Heap(IsolateGroup* isolate_group,
+           bool is_vm_isolate,
            intptr_t max_new_gen_semi_words,
            intptr_t max_old_gen_words)
     : isolate_group_(isolate_group),
+      is_vm_isolate_(is_vm_isolate),
       new_space_(this, max_new_gen_semi_words),
       old_space_(this, max_old_gen_words),
       barrier_(),
@@ -150,6 +152,9 @@ uword Heap::AllocateOld(intptr_t size, OldPage::PageType type) {
   if (addr != 0) {
     return addr;
   }
+
+  old_space_.TryReleaseReservation();
+
   // Give up allocating this object.
   OS::PrintErr("Exhausted heap space, trying to allocate %" Pd " bytes.\n",
                size);
@@ -678,11 +683,12 @@ void Heap::WriteProtect(bool read_only) {
 }
 
 void Heap::Init(IsolateGroup* isolate_group,
+                bool is_vm_isolate,
                 intptr_t max_new_gen_words,
                 intptr_t max_old_gen_words) {
   ASSERT(isolate_group->heap() == nullptr);
-  std::unique_ptr<Heap> heap(
-      new Heap(isolate_group, max_new_gen_words, max_old_gen_words));
+  std::unique_ptr<Heap> heap(new Heap(isolate_group, is_vm_isolate,
+                                      max_new_gen_words, max_old_gen_words));
   isolate_group->set_heap(std::move(heap));
 }
 

runtime/vm/heap/heap.h

@@ -167,6 +167,7 @@ class Heap {
 
   // Initialize the heap and register it with the isolate.
   static void Init(IsolateGroup* isolate_group,
+                   bool is_vm_isolate,
                    intptr_t max_new_gen_words,
                    intptr_t max_old_gen_words);
 
@@ -305,6 +306,7 @@ class Heap {
 #endif  // PRODUCT
 
   IsolateGroup* isolate_group() const { return isolate_group_; }
+  bool is_vm_isolate() const { return is_vm_isolate_; }
 
   Monitor* barrier() const { return &barrier_; }
   Monitor* barrier_done() const { return &barrier_done_; }
@@ -354,6 +356,7 @@ class Heap {
   };
 
   Heap(IsolateGroup* isolate_group,
+       bool is_vm_isolate,
        intptr_t max_new_gen_semi_words,  // Max capacity of new semi-space.
        intptr_t max_old_gen_words);
 
@@ -391,6 +394,7 @@ class Heap {
   void CollectForDebugging();
 
   IsolateGroup* isolate_group_;
+  bool is_vm_isolate_;
 
   // The different spaces used for allocation.
   Scavenger new_space_;

runtime/vm/heap/pages.cc

@@ -249,6 +249,8 @@ PageSpace::PageSpace(Heap* heap, intptr_t max_capacity_in_words)
   for (intptr_t i = 0; i < num_freelists_; i++) {
     freelists_[i].Reset();
   }
+
+  TryReserveForOOM();
 }
 
 PageSpace::~PageSpace() {
@@ -366,7 +368,7 @@ OldPage* PageSpace::AllocatePage(OldPage::PageType type, bool link) {
 
   page->set_object_end(page->memory_->end());
   if ((type != OldPage::kExecutable) && (heap_ != nullptr) &&
-      (heap_->isolate_group() != Dart::vm_isolate()->group())) {
+      (!heap_->is_vm_isolate())) {
     page->AllocateForwardingPage();
   }
   return page;
@@ -1020,6 +1022,47 @@ bool PageSpace::ShouldPerformIdleMarkCompact(int64_t deadline) {
   return estimated_mark_compact_completion <= deadline;
 }
 
+void PageSpace::TryReleaseReservation() {
+  if (oom_reservation_ == nullptr) return;
+  uword addr = reinterpret_cast<uword>(oom_reservation_);
+  intptr_t size = oom_reservation_->HeapSize();
+  oom_reservation_ = nullptr;
+  freelists_[OldPage::kData].Free(addr, size);
+}
+
+bool PageSpace::MarkReservation() {
+  if (oom_reservation_ == nullptr) {
+    return false;
+  }
+  ObjectLayout* ptr = reinterpret_cast<ObjectLayout*>(oom_reservation_);
+  if (!ptr->IsMarked()) {
+    ptr->SetMarkBit();
+  }
+  return true;
+}
+
+void PageSpace::TryReserveForOOM() {
+  if (oom_reservation_ == nullptr) {
+    uword addr = TryAllocate(kOOMReservationSize, OldPage::kData,
+                             kForceGrowth /* Don't re-enter GC */);
+    if (addr != 0) {
+      oom_reservation_ = FreeListElement::AsElement(addr, kOOMReservationSize);
+    }
+  }
+}
+
+void PageSpace::VisitRoots(ObjectPointerVisitor* visitor) {
+  if (oom_reservation_ != nullptr) {
+    // FreeListElements are generally held untagged, but ObjectPointerVisitors
+    // expect tagged pointers.
+    ObjectPtr ptr =
+        ObjectLayout::FromAddr(reinterpret_cast<uword>(oom_reservation_));
+    visitor->VisitPointer(&ptr);
+    oom_reservation_ =
+        reinterpret_cast<FreeListElement*>(ObjectLayout::ToAddr(ptr));
+  }
+}
+
 void PageSpace::CollectGarbage(bool compact, bool finalize) {
   ASSERT(GrowthControlState());
 
@@ -1186,11 +1229,13 @@ void PageSpace::CollectGarbageHelper(bool compact,
     mid3 = OS::GetCurrentMonotonicMicros();
   }
 
+  bool has_reservation = MarkReservation();
+
   if (compact) {
     SweepLarge();
     Compact(thread);
     set_phase(kDone);
-  } else if (FLAG_concurrent_sweep) {
+  } else if (FLAG_concurrent_sweep && has_reservation) {
     ConcurrentSweep(isolate_group);
   } else {
     SweepLarge();
@@ -1198,6 +1243,8 @@ void PageSpace::CollectGarbageHelper(bool compact,
     set_phase(kDone);
   }
 
+  TryReserveForOOM();
+
   // Make code pages read-only.
   if (finalize) WriteProtectCode(true);
 
@@ -1551,9 +1598,10 @@ void PageSpaceController::EvaluateGarbageCollection(SpaceUsage before,
       before.CombinedUsedInWords() - last_usage_.CombinedUsedInWords();
   intptr_t grow_heap;
   if (allocated_since_previous_gc > 0) {
-    const intptr_t garbage =
+    intptr_t garbage =
         before.CombinedUsedInWords() - after.CombinedUsedInWords();
-    ASSERT(garbage >= 0);
+    // Garbage may be negative if when the OOM reservation is refilled.
+    garbage = Utils::Maximum(static_cast<intptr_t>(0), garbage);
     // It makes no sense to expect that each kb allocated will cause more than
     // one kb of garbage, so we clamp k at 1.0.
     const double k = Utils::Minimum(

runtime/vm/heap/pages.h

@@ -307,6 +307,11 @@ class PageSpace {
                                is_protected, is_locked);
   }
 
+  void TryReleaseReservation();
+  bool MarkReservation();
+  void TryReserveForOOM();
+  void VisitRoots(ObjectPointerVisitor* visitor);
+
   bool ReachedHardThreshold() const {
     return page_space_controller_.ReachedHardThreshold(usage_);
   }
@@ -569,6 +574,8 @@ class PageSpace {
   // page freelists without locking.
   const intptr_t num_freelists_;
   FreeList* freelists_;
+  static constexpr intptr_t kOOMReservationSize = 32 * KB;
+  FreeListElement* oom_reservation_ = nullptr;
 
   // Use ExclusivePageIterator for safe access to these.
   mutable Mutex pages_lock_;

runtime/vm/heap/scavenger.cc

@@ -1512,6 +1512,10 @@ void Scavenger::Scavenge() {
   if (abort_) {
     ReverseScavenge(&from);
     bytes_promoted = 0;
+  } else if ((CapacityInWords() - UsedInWords()) < KBInWords) {
+    // Don't scavenge again until the next old-space GC has occurred. Prevents
+    // performing one scavenge per allocation as the heap limit is approached.
+    heap_->assume_scavenge_will_fail_ = true;
   }
   ASSERT(promotion_stack_.IsEmpty());
   MournWeakHandles();

runtime/vm/isolate.cc

@@ -448,7 +448,7 @@ bool IsolateGroup::UnregisterIsolateDecrementCount(Isolate* isolate) {
 
 void IsolateGroup::CreateHeap(bool is_vm_isolate,
                               bool is_service_or_kernel_isolate) {
-  Heap::Init(this,
+  Heap::Init(this, is_vm_isolate,
              is_vm_isolate
                  ? 0  // New gen size 0; VM isolate should only allocate in old.
                  : FLAG_new_gen_semi_max_size * MBInWords,

tests/standalone/out_of_memory_recovery_synchronous_test.dart

@@ -0,0 +1,49 @@
+// Copyright (c) 2020, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+// VMOptions=--old_gen_heap_size=20
+
+import "package:expect/expect.dart";
+
+main() {
+  var leak;
+  var exceptionThrown = false;
+  try {
+    leak = [];
+    while (true) {
+      leak = [leak];
+    }
+  } on OutOfMemoryError catch (exception) {
+    leak = null;
+    exceptionThrown = true;
+    print("Okay");
+  }
+  Expect.isTrue(exceptionThrown);
+
+  exceptionThrown = false;
+  try {
+    leak = [];
+    while (true) {
+      leak = [leak];
+    }
+  } on OutOfMemoryError catch (exception) {
+    leak = null;
+    exceptionThrown = true;
+    print("Okay");
+  }
+  Expect.isTrue(exceptionThrown);
+
+  exceptionThrown = false;
+  try {
+    leak = [];
+    while (true) {
+      leak = [leak];
+    }
+  } on OutOfMemoryError catch (exception) {
+    leak = null;
+    exceptionThrown = true;
+    print("Okay");
+  }
+  Expect.isTrue(exceptionThrown);
+}

tests/standalone/out_of_memory_recovery_test.dart

@@ -0,0 +1,55 @@
+// Copyright (c) 2020, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+// VMOptions=--old_gen_heap_size=20
+
+import "dart:io";
+import "dart:isolate";
+
+import "package:expect/expect.dart";
+
+handleRequest(request) {
+  if (request % 2 == 0) {
+    var leak = [];
+    while (true) {
+      leak = [leak];
+    }
+  }
+  return "Okay";
+}
+
+handleMessage(message) {
+  print(">> $message");
+  var responsePort = message[0];
+  var request = message[1];
+  try {
+    responsePort.send(<dynamic>[request, handleRequest(request)]);
+  } catch (e, st) {
+    responsePort.send(<dynamic>[request, "Failed: $e\n$st"]);
+  }
+}
+
+main(args) async {
+  var child = new RawReceivePort(handleMessage);
+
+  var parent;
+  parent = new RawReceivePort((message) {
+    print("<< $message");
+    var request = message[0];
+    var response = message[1];
+    if (request % 2 == 0) {
+      Expect.isTrue(response.contains("Out of Memory"));
+    } else {
+      Expect.equals("Okay", response);
+    }
+    if (request == 5) {
+      child.close();
+      parent.close();
+    } else {
+      child.sendPort.send(<dynamic>[parent.sendPort, request + 1]);
+    }
+  });
+
+  child.sendPort.send(<dynamic>[parent.sendPort, 1]);
+}

tests/standalone/standalone_kernel.status

@@ -6,7 +6,6 @@
 
 fragmentation_typed_data_test: Pass, Slow # GC heavy
 io/process_sync_test: Pass, Slow # Spawns synchronously subprocesses in sequence.
-out_of_memory_unhandled_exception_test: Pass, Slow
 
 [ $system == android ]
 entrypoints_verification_test: Skip # Requires shared objects which the test script doesn't "adb push".

tests/standalone_2/out_of_memory_recovery_synchronous_test.dart

@@ -0,0 +1,49 @@
+// Copyright (c) 2020, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+// VMOptions=--old_gen_heap_size=20
+
+import "package:expect/expect.dart";
+
+main() {
+  var leak;
+  var exceptionThrown = false;
+  try {
+    leak = [];
+    while (true) {
+      leak = [leak];
+    }
+  } on OutOfMemoryError catch (exception) {
+    leak = null;
+    exceptionThrown = true;
+    print("Okay");
+  }
+  Expect.isTrue(exceptionThrown);
+
+  exceptionThrown = false;
+  try {
+    leak = [];
+    while (true) {
+      leak = [leak];
+    }
+  } on OutOfMemoryError catch (exception) {
+    leak = null;
+    exceptionThrown = true;
+    print("Okay");
+  }
+  Expect.isTrue(exceptionThrown);
+
+  exceptionThrown = false;
+  try {
+    leak = [];
+    while (true) {
+      leak = [leak];
+    }
+  } on OutOfMemoryError catch (exception) {
+    leak = null;
+    exceptionThrown = true;
+    print("Okay");
+  }
+  Expect.isTrue(exceptionThrown);
+}

tests/standalone_2/out_of_memory_recovery_test.dart

@@ -0,0 +1,55 @@
+// Copyright (c) 2020, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+// VMOptions=--old_gen_heap_size=20
+
+import "dart:io";
+import "dart:isolate";
+
+import "package:expect/expect.dart";
+
+handleRequest(request) {
+  if (request % 2 == 0) {
+    var leak = [];
+    while (true) {
+      leak = [leak];
+    }
+  }
+  return "Okay";
+}
+
+handleMessage(message) {
+  print(">> $message");
+  var responsePort = message[0];
+  var request = message[1];
+  try {
+    responsePort.send(<dynamic>[request, handleRequest(request)]);
+  } catch (e, st) {
+    responsePort.send(<dynamic>[request, "Failed: $e\n$st"]);
+  }
+}
+
+main(args) async {
+  var child = new RawReceivePort(handleMessage);
+
+  var parent;
+  parent = new RawReceivePort((message) {
+    print("<< $message");
+    var request = message[0];
+    var response = message[1];
+    if (request % 2 == 0) {
+      Expect.isTrue(response.contains("Out of Memory"));
+    } else {
+      Expect.equals("Okay", response);
+    }
+    if (request == 5) {
+      child.close();
+      parent.close();
+    } else {
+      child.sendPort.send(<dynamic>[parent.sendPort, request + 1]);
+    }
+  });
+
+  child.sendPort.send(<dynamic>[parent.sendPort, 1]);
+}

tests/standalone_2/standalone_2_kernel.status

@@ -7,7 +7,6 @@
 fragmentation_test: Pass, Slow # GC heavy
 fragmentation_typed_data_test: Pass, Slow # GC heavy
 io/process_sync_test: Pass, Slow # Spawns synchronously subprocesses in sequence.
-out_of_memory_unhandled_exception_test: Pass, Slow
 
 [ $system == android ]
 entrypoints_verification_test: Skip # Requires shared objects which the test script doesn't "adb push".