Reapply "[vm, gc] Sweep non-executable large pages concurrently."

Wait for the concurrent sweeper to visit processing large pages before visiting the card tables during a scavenge. Bug: https://github.com/flutter/flutter/issues/48360 Bug: b/147582727 Change-Id: Iaec22f05e22d9ded75017aa1d8463c1be64858aa Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/131703 Commit-Queue: Ryan Macnak <rmacnak@google.com> Reviewed-by: Siva Annamalai <asiva@google.com>
2020-01-27 19:43:09 +00:00
parent c352c6c0b8
commit 155c0d1b92
6 changed files with 286 additions and 168 deletions
@@ -500,7 +500,7 @@ class _SnapshotClass implements SnapshotClass {
  Iterable<SnapshotObject> get instances sync* {
    final N = _graph._N;
    for (var id = 1; id <= N; id++) {
-      if (_graph._cids[id] == _cid) {
+      if (_graph._cids[id] == _cid && _graph._retainedSizes[id] > 0) {
        yield _SnapshotObject._new(id, _graph, "");
      }
    }
@@ -569,7 +569,9 @@ class _SnapshotGraph implements SnapshotGraph {
  Iterable<SnapshotObject> get objects sync* {
    final N = _N;
    for (var id = 1; id <= N; id++) {
-      yield _SnapshotObject._new(id, this, "");
+      if (_retainedSizes[id] > 0) {
+        yield _SnapshotObject._new(id, this, "");
+      }
    }
  }

@@ -1302,10 +1304,14 @@ class _SnapshotGraph implements SnapshotGraph {
      cls.liveInstanceCount++;
    }

-    // Start with retained size as shallow size + external size.
-    final retainedSizes = _newUint32Array(N + 1);
-    for (var i = 0; i < N + 1; i++) {
-      retainedSizes[i] = internalSizes[i] + externalSizes[i];
+    // Start with retained size as shallow size + external size. For reachable
+    // objects only; leave unreachable objects with a retained size of 0 so
+    // they can be filtered during graph iterations.
+    var retainedSizes = new Uint32List(N + 1);
+    assert(Nconnected <= N);
+    for (var i = 0; i <= Nconnected; i++) {
+      var v = vertex[i];
+      retainedSizes[v] = internalSizes[v] + externalSizes[v];
    }

    // In post order (bottom up), add retained size to dominator's retained
@@ -78,6 +78,69 @@ var tests = <IsolateTest>[
    // Check that the short list retains more than the long list inside.
    // and specifically, that it retains exactly itself + the long one.
    expect(first.retainedSize, equals(first.shallowSize + second.shallowSize));
+
+    // Verify sizes of classes are the appropriates sums of their instances.
+    // This also verifies that the class instance iterators are visiting the
+    // correct set of objects (e.g., not including dead objects).
+    for (SnapshotClass klass in graph.classes) {
+      int shallowSum = 0;
+      int internalSum = 0;
+      int externalSum = 0;
+      for (SnapshotObject instance in klass.instances) {
+        if (instance == graph.root) {
+          // The root may have 0 self size.
+          expect(instance.internalSize, greaterThanOrEqualTo(0));
+          expect(instance.externalSize, greaterThanOrEqualTo(0));
+          expect(instance.shallowSize, greaterThanOrEqualTo(0));
+        } else {
+          // All other objects are heap objects with positive size.
+          expect(instance.internalSize, greaterThan(0));
+          expect(instance.externalSize, greaterThanOrEqualTo(0));
+          expect(instance.shallowSize, greaterThan(0));
+        }
+        expect(instance.retainedSize, greaterThan(0));
+        expect(instance.shallowSize,
+            equals(instance.internalSize + instance.externalSize));
+        shallowSum += instance.shallowSize;
+        internalSum += instance.internalSize;
+        externalSum += instance.externalSize;
+      }
+      expect(shallowSum, equals(klass.shallowSize));
+      expect(internalSum, equals(klass.internalSize));
+      expect(externalSum, equals(klass.externalSize));
+      expect(
+          klass.shallowSize, equals(klass.internalSize + klass.externalSize));
+    }
+
+    // Verify sizes of the overall graph are the appropriates sums of all
+    // instances. This also verifies that the all instances iterator is visiting
+    // the correct set of objects (e.g., not including dead objects).
+    int shallowSum = 0;
+    int internalSum = 0;
+    int externalSum = 0;
+    for (SnapshotObject instance in graph.objects) {
+      if (instance == graph.root) {
+        // The root may have 0 self size.
+        expect(instance.internalSize, greaterThanOrEqualTo(0));
+        expect(instance.externalSize, greaterThanOrEqualTo(0));
+        expect(instance.shallowSize, greaterThanOrEqualTo(0));
+      } else {
+        // All other objects are heap objects with positive size.
+        expect(instance.internalSize, greaterThan(0));
+        expect(instance.externalSize, greaterThanOrEqualTo(0));
+        expect(instance.shallowSize, greaterThan(0));
+      }
+      expect(instance.retainedSize, greaterThan(0));
+      expect(instance.shallowSize,
+          equals(instance.internalSize + instance.externalSize));
+      shallowSum += instance.shallowSize;
+      internalSum += instance.internalSize;
+      externalSum += instance.externalSize;
+    }
+    expect(shallowSum, equals(graph.size));
+    expect(internalSum, equals(graph.internalSize));
+    expect(externalSum, equals(graph.externalSize));
+    expect(graph.size, equals(graph.internalSize + graph.externalSize));
  },
 ];

@@ -221,12 +221,6 @@ PageSpace::PageSpace(Heap* heap, intptr_t max_capacity_in_words)
    : freelist_(),
      heap_(heap),
      pages_lock_(),
-      pages_(NULL),
-      pages_tail_(NULL),
-      exec_pages_(NULL),
-      exec_pages_tail_(NULL),
-      large_pages_(NULL),
-      image_pages_(NULL),
      bump_top_(0),
      bump_end_(0),
      max_capacity_in_words_(max_capacity_in_words),
@@ -273,49 +267,95 @@ intptr_t PageSpace::LargePageSizeInWordsFor(intptr_t size) {
  return page_size >> kWordSizeLog2;
 }

+void PageSpace::AddPageLocked(HeapPage* page) {
+  if (pages_ == nullptr) {
+    pages_ = page;
+  } else {
+    pages_tail_->set_next(page);
+  }
+  pages_tail_ = page;
+}
+
+void PageSpace::AddLargePageLocked(HeapPage* page) {
+  if (large_pages_ == nullptr) {
+    large_pages_ = page;
+  } else {
+    large_pages_tail_->set_next(page);
+  }
+  large_pages_tail_ = page;
+}
+
+void PageSpace::AddExecPageLocked(HeapPage* page) {
+  if (exec_pages_ == nullptr) {
+    exec_pages_ = page;
+  } else {
+    if (FLAG_write_protect_code) {
+      exec_pages_tail_->WriteProtect(false);
+    }
+    exec_pages_tail_->set_next(page);
+    if (FLAG_write_protect_code) {
+      exec_pages_tail_->WriteProtect(true);
+    }
+  }
+  exec_pages_tail_ = page;
+}
+
+void PageSpace::RemovePageLocked(HeapPage* page, HeapPage* previous_page) {
+  if (previous_page != NULL) {
+    previous_page->set_next(page->next());
+  } else {
+    pages_ = page->next();
+  }
+  if (page == pages_tail_) {
+    pages_tail_ = previous_page;
+  }
+}
+
+void PageSpace::RemoveLargePageLocked(HeapPage* page, HeapPage* previous_page) {
+  if (previous_page != NULL) {
+    previous_page->set_next(page->next());
+  } else {
+    large_pages_ = page->next();
+  }
+  if (page == large_pages_tail_) {
+    large_pages_tail_ = previous_page;
+  }
+}
+
+void PageSpace::RemoveExecPageLocked(HeapPage* page, HeapPage* previous_page) {
+  if (previous_page != NULL) {
+    previous_page->set_next(page->next());
+  } else {
+    exec_pages_ = page->next();
+  }
+  if (page == exec_pages_tail_) {
+    exec_pages_tail_ = previous_page;
+  }
+}
+
 HeapPage* PageSpace::AllocatePage(HeapPage::PageType type, bool link) {
  {
    MutexLocker ml(&pages_lock_);
    if (!CanIncreaseCapacityInWordsLocked(kPageSizeInWords)) {
-      return NULL;
+      return nullptr;
    }
    IncreaseCapacityInWordsLocked(kPageSizeInWords);
  }
  const bool is_exec = (type == HeapPage::kExecutable);
  const char* name = Heap::RegionName(is_exec ? Heap::kCode : Heap::kOld);
  HeapPage* page = HeapPage::Allocate(kPageSizeInWords, type, name);
-  if (page == NULL) {
+  if (page == nullptr) {
    RELEASE_ASSERT(!FLAG_abort_on_oom);
    IncreaseCapacityInWords(-kPageSizeInWords);
-    return NULL;
+    return nullptr;
  }

  MutexLocker ml(&pages_lock_);
  if (link) {
-    if (!is_exec) {
-      if (pages_ == NULL) {
-        pages_ = page;
-      } else {
-        pages_tail_->set_next(page);
-      }
-      pages_tail_ = page;
+    if (is_exec) {
+      AddExecPageLocked(page);
    } else {
-      // Should not allocate executable pages when running from a precompiled
-      // snapshot.
-      ASSERT(Dart::vm_snapshot_kind() != Snapshot::kFullAOT);
-
-      if (exec_pages_ == NULL) {
-        exec_pages_ = page;
-      } else {
-        if (FLAG_write_protect_code) {
-          exec_pages_tail_->WriteProtect(false);
-        }
-        exec_pages_tail_->set_next(page);
-        if (FLAG_write_protect_code) {
-          exec_pages_tail_->WriteProtect(true);
-        }
-      }
-      exec_pages_tail_ = page;
+      AddPageLocked(page);
    }
  }

@@ -332,26 +372,28 @@ HeapPage* PageSpace::AllocateLargePage(intptr_t size, HeapPage::PageType type) {
  {
    MutexLocker ml(&pages_lock_);
    if (!CanIncreaseCapacityInWordsLocked(page_size_in_words)) {
-      return NULL;
+      return nullptr;
    }
    IncreaseCapacityInWordsLocked(page_size_in_words);
  }
  const bool is_exec = (type == HeapPage::kExecutable);
  const char* name = Heap::RegionName(is_exec ? Heap::kCode : Heap::kOld);
  HeapPage* page = HeapPage::Allocate(page_size_in_words, type, name);
-  {
-    MutexLocker ml(&pages_lock_);
-    if (page == nullptr) {
-      IncreaseCapacityInWordsLocked(-page_size_in_words);
-      return nullptr;
-    }
-    page->set_next(large_pages_);
-    large_pages_ = page;

-    // Only one object in this page (at least until Array::MakeFixedLength
-    // is called).
-    page->set_object_end(page->object_start() + size);
+  MutexLocker ml(&pages_lock_);
+  if (page == nullptr) {
+    IncreaseCapacityInWordsLocked(-page_size_in_words);
+    return nullptr;
  }
+  if (is_exec) {
+    AddExecPageLocked(page);
+  } else {
+    AddLargePageLocked(page);
+  }
+
+  // Only one object in this page (at least until Array::MakeFixedLength
+  // is called).
+  page->set_object_end(page->object_start() + size);
  return page;
 }

@@ -376,26 +418,10 @@ void PageSpace::FreePage(HeapPage* page, HeapPage* previous_page) {
  {
    MutexLocker ml(&pages_lock_);
    IncreaseCapacityInWordsLocked(-(page->memory_->size() >> kWordSizeLog2));
-    if (!is_exec) {
-      // Remove the page from the list of data pages.
-      if (previous_page != NULL) {
-        previous_page->set_next(page->next());
-      } else {
-        pages_ = page->next();
-      }
-      if (page == pages_tail_) {
-        pages_tail_ = previous_page;
-      }
+    if (is_exec) {
+      RemoveExecPageLocked(page, previous_page);
    } else {
-      // Remove the page from the list of executable pages.
-      if (previous_page != NULL) {
-        previous_page->set_next(page->next());
-      } else {
-        exec_pages_ = page->next();
-      }
-      if (page == exec_pages_tail_) {
-        exec_pages_tail_ = previous_page;
-      }
+      RemovePageLocked(page, previous_page);
    }
  }
  // TODO(iposva): Consider adding to a pool of empty pages.
@@ -403,16 +429,10 @@ void PageSpace::FreePage(HeapPage* page, HeapPage* previous_page) {
 }

 void PageSpace::FreeLargePage(HeapPage* page, HeapPage* previous_page) {
-  // Thread should be at a safepoint when this code is called and hence
-  // it is not necessary to lock large_pages_.
-  ASSERT(Thread::Current()->IsAtSafepoint());
-  IncreaseCapacityInWords(-(page->memory_->size() >> kWordSizeLog2));
-  // Remove the page from the list.
-  if (previous_page != NULL) {
-    previous_page->set_next(page->next());
-  } else {
-    large_pages_ = page->next();
-  }
+  ASSERT(page->type() != HeapPage::kExecutable);
+  MutexLocker ml(&pages_lock_);
+  IncreaseCapacityInWordsLocked(-(page->memory_->size() >> kWordSizeLog2));
+  RemoveLargePageLocked(page, previous_page);
  page->Deallocate();
 }

@@ -634,28 +654,6 @@ class ExclusiveCodePageIterator : ValueObject {
  HeapPage* page_;
 };

-// Provides exclusive access to large pages, and ensures they are walkable.
-class ExclusiveLargePageIterator : ValueObject {
- public:
-  explicit ExclusiveLargePageIterator(const PageSpace* space)
-      : space_(space), ml_(&space->pages_lock_) {
-    space_->MakeIterable();
-    page_ = space_->large_pages_;
-  }
-  HeapPage* page() const { return page_; }
-  bool Done() const { return page_ == NULL; }
-  void Advance() {
-    ASSERT(!Done());
-    page_ = page_->next();
-  }
-
- private:
-  const PageSpace* space_;
-  MutexLocker ml_;
-  NoSafepointScope no_safepoint;
-  HeapPage* page_;
-};
-
 void PageSpace::MakeIterable() const {
  // Assert not called from concurrent sweeper task.
  // TODO(koda): Use thread/task identity when implemented.
@@ -722,12 +720,6 @@ bool PageSpace::Contains(uword addr, HeapPage::PageType type) const {
        return true;
      }
    }
-    // Large pages can be executable, walk them too.
-    for (ExclusiveLargePageIterator it(this); !it.Done(); it.Advance()) {
-      if ((it.page()->type() == type) && it.page()->Contains(addr)) {
-        return true;
-      }
-    }
    return false;
  }
  for (ExclusivePageIterator it(this); !it.Done(); it.Advance()) {
@@ -785,7 +777,14 @@ void PageSpace::VisitObjectPointers(ObjectPointerVisitor* visitor) const {

 void PageSpace::VisitRememberedCards(ObjectPointerVisitor* visitor) const {
  ASSERT(Thread::Current()->IsAtSafepoint());
-  for (HeapPage* page = large_pages_; page != NULL; page = page->next()) {
+
+  // Wait for the sweeper to finish mutating the large page list.
+  MonitorLocker ml(tasks_lock());
+  while (phase() == kSweepingLarge) {
+    ml.Wait();  // No safepoint check.
+  }
+
+  for (HeapPage* page = large_pages_; page != nullptr; page = page->next()) {
    page->VisitRememberedCards(visitor);
  }
 }
@@ -800,15 +799,6 @@ RawObject* PageSpace::FindObject(FindObjectVisitor* visitor,
        return obj;
      }
    }
-    // Large pages can be executable, walk them too.
-    for (ExclusiveLargePageIterator it(this); !it.Done(); it.Advance()) {
-      if (it.page()->type() == type) {
-        RawObject* obj = it.page()->FindObject(visitor);
-        if (obj != Object::null()) {
-          return obj;
-        }
-      }
-    }
    return Object::null();
  }

@@ -1132,36 +1122,18 @@ void PageSpace::CollectGarbageAtSafepoint(bool compact,
      OS::PrintErr(" done.\n");
    }

-    TIMELINE_FUNCTION_GC_DURATION(thread, "SweepLargeAndExecutablePages");
+    // Executable pages are always swept immediately to simplify
+    // code protection.
+
+    TIMELINE_FUNCTION_GC_DURATION(thread, "SweepExecutable");
    GCSweeper sweeper;
-
-    // During stop-the-world phases we should use bulk lock when adding
-    // elements to the free list.
-    MutexLocker mld(freelist_[HeapPage::kData].mutex());
-    MutexLocker mle(freelist_[HeapPage::kExecutable].mutex());
-
-    // Large and executable pages are always swept immediately.
    HeapPage* prev_page = NULL;
-    HeapPage* page = large_pages_;
-    while (page != NULL) {
-      HeapPage* next_page = page->next();
-      const intptr_t words_to_end = sweeper.SweepLargePage(page);
-      if (words_to_end == 0) {
-        FreeLargePage(page, prev_page);
-      } else {
-        TruncateLargePage(page, words_to_end << kWordSizeLog2);
-        prev_page = page;
-      }
-      // Advance to the next page.
-      page = next_page;
-    }
-
-    prev_page = NULL;
-    page = exec_pages_;
+    HeapPage* page = exec_pages_;
    FreeList* freelist = &freelist_[HeapPage::kExecutable];
+    MutexLocker ml(freelist->mutex());
    while (page != NULL) {
      HeapPage* next_page = page->next();
-      bool page_in_use = sweeper.SweepPage(page, freelist, true);
+      bool page_in_use = sweeper.SweepPage(page, freelist, true /*is_locked*/);
      if (page_in_use) {
        prev_page = page;
      } else {
@@ -1175,12 +1147,14 @@ void PageSpace::CollectGarbageAtSafepoint(bool compact,
  }

  if (compact) {
+    SweepLarge();
    Compact(thread);
    set_phase(kDone);
  } else if (FLAG_concurrent_sweep) {
    ConcurrentSweep(isolate);
  } else {
-    BlockingSweep();
+    SweepLarge();
+    Sweep();
    set_phase(kDone);
  }

@@ -1213,19 +1187,38 @@ void PageSpace::CollectGarbageAtSafepoint(bool compact,
  }
 }

-void PageSpace::BlockingSweep() {
+void PageSpace::SweepLarge() {
+  TIMELINE_FUNCTION_GC_DURATION(Thread::Current(), "SweepLarge");
+
+  GCSweeper sweeper;
+  HeapPage* prev_page = nullptr;
+  HeapPage* page = large_pages_;
+  while (page != nullptr) {
+    HeapPage* next_page = page->next();
+    const intptr_t words_to_end = sweeper.SweepLargePage(page);
+    if (words_to_end == 0) {
+      FreeLargePage(page, prev_page);
+    } else {
+      TruncateLargePage(page, words_to_end << kWordSizeLog2);
+      prev_page = page;
+    }
+    // Advance to the next page.
+    page = next_page;
+  }
+}
+
+void PageSpace::Sweep() {
  TIMELINE_FUNCTION_GC_DURATION(Thread::Current(), "Sweep");

-  MutexLocker mld(freelist_[HeapPage::kData].mutex());
-  MutexLocker mle(freelist_[HeapPage::kExecutable].mutex());
-
-  // Sweep all regular sized pages now.
  GCSweeper sweeper;
-  HeapPage* prev_page = NULL;
+  HeapPage* prev_page = nullptr;
  HeapPage* page = pages_;
-  while (page != NULL) {
+  FreeList* freelist = &freelist_[HeapPage::kData];
+  MutexLocker ml(freelist_->mutex());
+  while (page != nullptr) {
    HeapPage* next_page = page->next();
-    bool page_in_use = sweeper.SweepPage(page, &freelist_[page->type()], true);
+    ASSERT(page->type() == HeapPage::kData);
+    bool page_in_use = sweeper.SweepPage(page, freelist, true /*is_locked*/);
    if (page_in_use) {
      prev_page = page;
    } else {
@@ -1244,8 +1237,8 @@ void PageSpace::BlockingSweep() {

 void PageSpace::ConcurrentSweep(Isolate* isolate) {
  // Start the concurrent sweeper task now.
-  GCSweeper::SweepConcurrent(isolate, pages_, pages_tail_,
-                             &freelist_[HeapPage::kData]);
+  GCSweeper::SweepConcurrent(isolate, pages_, pages_tail_, large_pages_,
+                             large_pages_tail_, &freelist_[HeapPage::kData]);
 }

 void PageSpace::Compact(Thread* thread) {
@@ -275,7 +275,13 @@ class PageSpaceController {
 class PageSpace {
 public:
  enum GrowthPolicy { kControlGrowth, kForceGrowth };
-  enum Phase { kDone, kMarking, kAwaitingFinalization, kSweeping };
+  enum Phase {
+    kDone,
+    kMarking,
+    kAwaitingFinalization,
+    kSweepingLarge,
+    kSweepingRegular
+  };

  PageSpace(Heap* heap, intptr_t max_capacity_in_words);
  ~PageSpace();
@@ -479,10 +485,19 @@ class PageSpace {

  // Makes bump block walkable; do not call concurrently with mutator.
  void MakeIterable() const;
+
+  void AddPageLocked(HeapPage* page);
+  void AddLargePageLocked(HeapPage* page);
+  void AddExecPageLocked(HeapPage* page);
+  void RemovePageLocked(HeapPage* page, HeapPage* previous_page);
+  void RemoveLargePageLocked(HeapPage* page, HeapPage* previous_page);
+  void RemoveExecPageLocked(HeapPage* page, HeapPage* previous_page);
+
  HeapPage* AllocatePage(HeapPage::PageType type, bool link = true);
-  void FreePage(HeapPage* page, HeapPage* previous_page);
  HeapPage* AllocateLargePage(intptr_t size, HeapPage::PageType type);
+
  void TruncateLargePage(HeapPage* page, intptr_t new_object_size_in_bytes);
+  void FreePage(HeapPage* page, HeapPage* previous_page);
  void FreeLargePage(HeapPage* page, HeapPage* previous_page);
  void FreePages(HeapPage* pages);

@@ -490,7 +505,8 @@ class PageSpace {
                                 bool finalize,
                                 int64_t pre_wait_for_sweepers,
                                 int64_t pre_safe_point);
-  void BlockingSweep();
+  void SweepLarge();
+  void Sweep();
  void ConcurrentSweep(Isolate* isolate);
  void Compact(Thread* thread);

@@ -513,12 +529,13 @@ class PageSpace {

  // Use ExclusivePageIterator for safe access to these.
  mutable Mutex pages_lock_;
-  HeapPage* pages_;
-  HeapPage* pages_tail_;
-  HeapPage* exec_pages_;
-  HeapPage* exec_pages_tail_;
-  HeapPage* large_pages_;
-  HeapPage* image_pages_;
+  HeapPage* pages_ = nullptr;
+  HeapPage* pages_tail_ = nullptr;
+  HeapPage* exec_pages_ = nullptr;
+  HeapPage* exec_pages_tail_ = nullptr;
+  HeapPage* large_pages_ = nullptr;
+  HeapPage* large_pages_tail_ = nullptr;
+  HeapPage* image_pages_ = nullptr;

  // A block of memory in a data page, managed by bump allocation. The remainder
  // is kept formatted as a FreeListElement, but is not in any freelist.
@@ -110,11 +110,15 @@ class ConcurrentSweeperTask : public ThreadPool::Task {
                        PageSpace* old_space,
                        HeapPage* first,
                        HeapPage* last,
+                        HeapPage* large_first,
+                        HeapPage* large_last,
                        FreeList* freelist)
      : task_isolate_(isolate),
        old_space_(old_space),
        first_(first),
        last_(last),
+        large_first_(large_first),
+        large_last_(large_last),
        freelist_(freelist) {
    ASSERT(task_isolate_ != NULL);
    ASSERT(first_ != NULL);
@@ -123,7 +127,7 @@ class ConcurrentSweeperTask : public ThreadPool::Task {
    ASSERT(freelist_ != NULL);
    MonitorLocker ml(old_space_->tasks_lock());
    old_space_->set_tasks(old_space_->tasks() + 1);
-    old_space_->set_phase(PageSpace::kSweeping);
+    old_space_->set_phase(PageSpace::kSweepingLarge);
  }

  virtual void Run() {
@@ -132,14 +136,42 @@ class ConcurrentSweeperTask : public ThreadPool::Task {
    ASSERT(result);
    {
      Thread* thread = Thread::Current();
+      ASSERT(thread->BypassSafepoints());  // Or we should be checking in.
      TIMELINE_FUNCTION_GC_DURATION(thread, "ConcurrentSweep");
      GCSweeper sweeper;

-      HeapPage* page = first_;
+      HeapPage* page = large_first_;
      HeapPage* prev_page = NULL;
-
      while (page != NULL) {
-        ASSERT(thread->BypassSafepoints());  // Or we should be checking in.
+        HeapPage* next_page;
+        if (page == large_last_) {
+          // Don't access page->next(), which would be a race with mutator
+          // allocating new pages.
+          next_page = NULL;
+        } else {
+          next_page = page->next();
+        }
+        ASSERT(page->type() == HeapPage::kData);
+        const intptr_t words_to_end = sweeper.SweepLargePage(page);
+        if (words_to_end == 0) {
+          old_space_->FreeLargePage(page, prev_page);
+        } else {
+          old_space_->TruncateLargePage(page, words_to_end << kWordSizeLog2);
+          prev_page = page;
+        }
+        page = next_page;
+      }
+
+      {
+        MonitorLocker ml(old_space_->tasks_lock());
+        ASSERT(old_space_->phase() == PageSpace::kSweepingLarge);
+        old_space_->set_phase(PageSpace::kSweepingRegular);
+        ml.NotifyAll();
+      }
+
+      page = first_;
+      prev_page = NULL;
+      while (page != NULL) {
        HeapPage* next_page;
        if (page == last_) {
          // Don't access page->next(), which would be a race with mutator
@@ -170,7 +202,7 @@ class ConcurrentSweeperTask : public ThreadPool::Task {
    {
      MonitorLocker ml(old_space_->tasks_lock());
      old_space_->set_tasks(old_space_->tasks() - 1);
-      ASSERT(old_space_->phase() == PageSpace::kSweeping);
+      ASSERT(old_space_->phase() == PageSpace::kSweepingRegular);
      old_space_->set_phase(PageSpace::kDone);
      ml.NotifyAll();
    }
@@ -181,15 +213,20 @@ class ConcurrentSweeperTask : public ThreadPool::Task {
  PageSpace* old_space_;
  HeapPage* first_;
  HeapPage* last_;
+  HeapPage* large_first_;
+  HeapPage* large_last_;
  FreeList* freelist_;
 };

 void GCSweeper::SweepConcurrent(Isolate* isolate,
                                HeapPage* first,
                                HeapPage* last,
+                                HeapPage* large_first,
+                                HeapPage* large_last,
                                FreeList* freelist) {
  bool result = Dart::thread_pool()->Run<ConcurrentSweeperTask>(
-      isolate, isolate->heap()->old_space(), first, last, freelist);
+      isolate, isolate->heap()->old_space(), first, last, large_first,
+      large_last, freelist);
  ASSERT(result);
 }

@@ -38,6 +38,8 @@ class GCSweeper {
  static void SweepConcurrent(Isolate* isolate,
                              HeapPage* first,
                              HeapPage* last,
+                              HeapPage* large_first,
+                              HeapPage* large_last,
                              FreeList* freelist);
 };