[vm, gc] Also apply back pressure during concurrent sweep.

When running without concurrent marking, the growth policy sets an allocation limit that triggers a blocking GC. When running with concurrent marking, the growth policy sets an allocation limit that triggers the start of concurrent marking, but the mutator will continue to allocate until marking completes. By itself, this would allow the heap to grow far beyond its working set size, so back pressure is applied to the mutator by making it do some incremental marking work in proportion to the amount it allocates. A similar race between mutator and GC exists for concurrent sweep, so make the mutator do some incremental sweeping too.

Cf. 305c3e30c6.

TEST=golem
Bug: https://github.com/dart-lang/sdk/issues/62762
Change-Id: I6f882e6893e1cd3b3591db8cd03b61eb06bfa02f
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/486828
Reviewed-by: Slava Egorov <vegorov@google.com>
Commit-Queue: Ryan Macnak <rmacnak@google.com>

runtime/vm/heap/heap.cc

@@ -616,12 +616,17 @@ void Heap::CheckConcurrentMarking(Thread* thread,
   switch (phase) {
     case PageSpace::kMarking:
       if (mode_ != Dart_PerformanceMode_Latency) {
-        old_space_.IncrementalMarkWithSizeBudget(size);
+        // Back pressure: do slightly more marking work than allocation work.
+        old_space_.IncrementalMarkWithSizeBudget(size + (size >> 2));
       }
       return;
     case PageSpace::kSweepingLarge:
     case PageSpace::kSweepingRegular:
-      return;  // Busy.
+      if (mode_ != Dart_PerformanceMode_Latency) {
+        // Back pressure: do some sweeping work.
+        old_space_.IncrementalSweepWithSizeBudget(size);
+      }
+      return;
     case PageSpace::kAwaitingFinalization:
       CollectOldSpaceGarbage(thread, GCType::kMarkSweep, GCReason::kFinalize);
       return;

runtime/vm/heap/pages.cc

@@ -899,6 +899,13 @@ void PageSpace::IncrementalMarkWithTimeBudget(int64_t deadline) {
   }
 }
 
+void PageSpace::IncrementalSweepWithSizeBudget(intptr_t size) {
+  if (size >= kAllocatablePageSize) {
+    // Sweeping work is less divisible than marking work.
+    Sweep(/*exclusive=*/false, /*one_page=*/true);
+  }
+}
+
 void PageSpace::AssistTasks(MonitorLocker* ml) {
   if (phase() == PageSpace::kMarking) {
     ml->Exit();
@@ -1390,7 +1397,7 @@ void PageSpace::SweepLarge() {
   }
 }
 
-void PageSpace::Sweep(bool exclusive) {
+void PageSpace::Sweep(bool exclusive, bool one_page) {
   TIMELINE_FUNCTION_GC_DURATION(Thread::Current(), "Sweep");
 
   GCSweeper sweeper;
@@ -1436,6 +1443,8 @@ void PageSpace::Sweep(bool exclusive) {
     } else {
       IncreaseCapacityInWordsLocked(-(size >> kWordSizeLog2));
     }
+
+    if (one_page) break;
   }
 
   if (exclusive) {

runtime/vm/heap/pages.h

@@ -259,6 +259,7 @@ class PageSpace {
   bool ShouldPerformIdleMarkCompact(int64_t deadline);
   void IncrementalMarkWithSizeBudget(intptr_t size);
   void IncrementalMarkWithTimeBudget(int64_t deadline);
+  void IncrementalSweepWithSizeBudget(intptr_t size);
   void AssistTasks(MonitorLocker* ml);
 
   void AddGCTime(int64_t micros) { gc_time_micros_ += micros; }
@@ -426,7 +427,7 @@ class PageSpace {
   void SweepExecutable();
   void SweepNew();
   void SweepLarge();
-  void Sweep(bool exclusive);
+  void Sweep(bool exclusive, bool one_page = false);
   void ConcurrentSweep(IsolateGroup* isolate_group);
   void Compact(Thread* thread);