dart-lang/sdk
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Reland "[vm, gc] Divide new-space into pages like old-space."

Browse Source 
Bug: b/155227688
Change-Id: I3f7bb5eaf090622749869f405efd19674c6d74d6
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/145640
Reviewed-by: Alexander Aprelev <aam@google.com>
Commit-Queue: Ryan Macnak <rmacnak@google.com>
This commit is contained in:
Ryan Macnak
2020-05-01 02:01:27 +00:00
committed by commit-bot@chromium.org
parent ff6724285c
commit 1483df9ff5
7 changed files with 423 additions and 511 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/vm/heap/heap.cc
-3
View File
@@ -776,9 +776,6 @@ bool Heap::VerifyGC(MarkExpectation mark_expectation) {
auto thread = Thread::Current();
StackZone stack_zone(thread);
// Change the new space's top_ with the more up-to-date thread's view of top_
new_space_.MakeNewSpaceIterable();
ObjectSet* allocated_set =
CreateAllocatedObjectSet(stack_zone.GetZone(), mark_expectation);
VerifyPointersVisitor visitor(isolate_group(), allocated_set);
runtime/vm/heap/scavenger.cc
+379 -358
View File
@@ -94,6 +94,142 @@ static inline void objcpy(void* dst, const void* src, size_t size) {
} while (size > 0);
}
static const intptr_t kNewPageSize = 512 * KB;
static const intptr_t kNewPageSizeInWords = kNewPageSize / kWordSize;
static const intptr_t kNewPageMask = ~(kNewPageSize - 1);
// A page containing new generation objects.
class NewPage {
public:
static NewPage* Allocate();
void Deallocate();
uword start() const { return memory_->start(); }
uword end() const { return memory_->end(); }
bool Contains(uword addr) const { return memory_->Contains(addr); }
void WriteProtect(bool read_only) {
memory_->Protect(read_only ? VirtualMemory::kReadOnly
: VirtualMemory::kReadWrite);
}
NewPage* next() const { return next_; }
void set_next(NewPage* next) { next_ = next; }
Thread* owner() const { return owner_; }
uword object_start() const { return start() + ObjectStartOffset(); }
uword object_end() const { return owner_ != nullptr ? owner_->top() : top_; }
void VisitObjects(ObjectVisitor* visitor) const {
uword addr = object_start();
uword end = object_end();
while (addr < end) {
ObjectPtr obj = ObjectLayout::FromAddr(addr);
visitor->VisitObject(obj);
addr += obj->ptr()->HeapSize();
}
}
void VisitObjectPointers(ObjectPointerVisitor* visitor) const {
uword addr = object_start();
uword end = object_end();
while (addr < end) {
ObjectPtr obj = ObjectLayout::FromAddr(addr);
intptr_t size = obj->ptr()->VisitPointers(visitor);
addr += size;
}
}
static intptr_t ObjectStartOffset() {
return Utils::RoundUp(sizeof(NewPage), kObjectAlignment) +
kNewObjectAlignmentOffset;
}
static NewPage* Of(ObjectPtr obj) {
ASSERT(obj->IsHeapObject());
ASSERT(obj->IsNewObject());
return Of(static_cast<uword>(obj));
}
static NewPage* Of(uword addr) {
return reinterpret_cast<NewPage*>(addr & kNewPageMask);
}
// Remember the limit to which objects have been copied.
void RecordSurvivors() { survivor_end_ = object_end(); }
// Move survivor end to the end of the to_ space, making all surviving
// objects candidates for promotion next time.
void EarlyTenure() { survivor_end_ = end_; }
uword promo_candidate_words() const {
return (survivor_end_ - object_start()) / kWordSize;
}
void Acquire(Thread* thread) {
ASSERT(owner_ == nullptr);
owner_ = thread;
thread->set_top(top_);
thread->set_end(end_);
}
void Release(Thread* thread) {
ASSERT(owner_ == thread);
owner_ = nullptr;
top_ = thread->top();
thread->set_top(0);
thread->set_end(0);
}
void Release() {
if (owner_ != nullptr) {
Release(owner_);
}
}
uword TryAllocateGC(intptr_t size) {
ASSERT(owner_ == nullptr);
uword result = top_;
uword new_top = result + size;
if (LIKELY(new_top < end_)) {
top_ = new_top;
return result;
}
return 0;
}
void Unallocate(uword addr, intptr_t size) {
ASSERT((addr + size) == top_);
top_ -= size;
}
bool IsSurvivor(uword raw_addr) const { return raw_addr < survivor_end_; }
bool IsResolved() const { return top_ == resolved_top_; }
private:
VirtualMemory* memory_;
NewPage* next_;
// The thread using this page for allocation, otherwise NULL.
Thread* owner_;
// The address of the next allocation. If owner is non-NULL, this value is
// stale and the current value is at owner->top_. Called "NEXT" in the
// original Cheney paper.
uword top_;
// The address after the last allocatable byte in this page.
uword end_;
// Objects below this address have survived a scavenge.
uword survivor_end_;
// A pointer to the first unprocessed object. Resolution completes when this
// value meets the allocation top. Called "SCAN" in the original Cheney paper.
uword resolved_top_;
template <bool>
friend class ScavengerVisitorBase;
DISALLOW_ALLOCATION();
DISALLOW_IMPLICIT_CONSTRUCTORS(NewPage);
};
template <bool parallel>
class ScavengerVisitorBase : public ObjectPointerVisitor {
public:
@@ -110,12 +246,7 @@ class ScavengerVisitorBase : public ObjectPointerVisitor {
freelist_(freelist),
bytes_promoted_(0),
visiting_old_object_(nullptr),
promoted_list_(promotion_stack),
labs_(8) {
ASSERT(labs_.length() == 0);
labs_.Add({0, 0, 0});
ASSERT(labs_.length() == 1);
}
promoted_list_(promotion_stack) {}
virtual void VisitTypedDataViewPointers(TypedDataViewPtr view,
ObjectPtr* first,
@@ -174,15 +305,6 @@ class ScavengerVisitorBase : public ObjectPointerVisitor {
intptr_t bytes_promoted() const { return bytes_promoted_; }
void AddNewTLAB(uword top, uword end) {
producer_index_++;
ScavengerLAB lab;
lab.top = top;
lab.end = end;
lab.resolved_top = top;
labs_.Add(lab);
}
void ProcessRoots() {
thread_ = Thread::Current();
page_space_->AcquireLock(freelist_);
@@ -207,28 +329,18 @@ class ScavengerVisitorBase : public ObjectPointerVisitor {
inline void ProcessWeakProperties();
bool HasWork() {
// N.B.: Normally if any TLABs have things left to resolve, then the
// TLAB we are allocating from (producer_index_) will too because we
// always immediately allocate when we switch to a new TLAB. However,
// this first allocation may be undone if we lose the race to install
// the forwarding pointer, so we must also check that there aren't
// any TLABs after the resolution cursor.
return (consumer_index_ < producer_index_) ||
(labs_[producer_index_].top !=
labs_[producer_index_].resolved_top) ||
return (scan_ != tail_) || (scan_ != nullptr && !scan_->IsResolved()) ||
!promoted_list_.IsEmpty();
}
void Finalize() {
ASSERT(!HasWork());
for (intptr_t i = 0; i <= producer_index_; i++) {
ASSERT(labs_[i].top <= labs_[i].end);
ASSERT(labs_[i].resolved_top == labs_[i].top);
for (NewPage* page = head_; page != nullptr; page = page->next()) {
ASSERT(page->IsResolved());
page->RecordSurvivors();
}
MakeProducerTLABIterable();
promoted_list_.Finalize();
MournWeakProperties();
@@ -237,15 +349,8 @@ class ScavengerVisitorBase : public ObjectPointerVisitor {
thread_ = nullptr;
}
void DonateTLABs() {
MutexLocker ml(&scavenger_->space_lock_);
// NOTE: We could make all [labs_] re-usable after a scavenge if we remember
// the promotion pointer of each TLAB.
const auto& lab = labs_[producer_index_];
if (lab.end == scavenger_->top_) {
scavenger_->top_ = lab.top;
}
}
NewPage* head() const { return head_; }
NewPage* tail() const { return tail_; }
private:
void UpdateStoreBuffer(ObjectPtr* p, ObjectPtr obj) {
@@ -281,7 +386,7 @@ class ScavengerVisitorBase : public ObjectPointerVisitor {
} else {
intptr_t size = raw_obj->ptr()->HeapSize(header);
// Check whether object should be promoted.
if (raw_addr >= scavenger_->survivor_end_) {
if (!NewPage::Of(raw_obj)->IsSurvivor(raw_addr)) {
// Not a survivor of a previous scavenge. Just copy the object into the
// to space.
new_addr = TryAllocateCopy(size);
@@ -326,8 +431,6 @@ class ScavengerVisitorBase : public ObjectPointerVisitor {
tags = ObjectLayout::OldAndNotMarkedBit::update(!thread_->is_marking(),
tags);
new_obj->ptr()->tags_ = tags;
} else {
ASSERT(scavenger_->to_->Contains(new_addr));
}
intptr_t cid = ObjectLayout::ClassIdTag::decode(header);
@@ -345,14 +448,10 @@ class ScavengerVisitorBase : public ObjectPointerVisitor {
bytes_promoted_ -= size;
} else {
// Undo to-space allocation.
ASSERT(labs_[producer_index_].top == (new_addr + size));
labs_[producer_index_].top = new_addr;
tail_->Unallocate(new_addr, size);
}
// Use the winner's forwarding target.
new_addr = ForwardedAddr(header);
if (ObjectLayout::FromAddr(new_addr)->IsNewObject()) {
ASSERT(scavenger_->to_->Contains(new_addr));
}
}
}
@@ -366,7 +465,6 @@ class ScavengerVisitorBase : public ObjectPointerVisitor {
reinterpret_cast<std::atomic<ObjectPtr>*>(p)->store(
new_obj, std::memory_order_release);
} else {
ASSERT(scavenger_->to_->Contains(ObjectLayout::ToAddr(new_obj)));
*p = new_obj;
}
// Update the store buffer as needed.
@@ -392,33 +490,21 @@ class ScavengerVisitorBase : public ObjectPointerVisitor {
DART_FORCE_INLINE
uword TryAllocateCopy(intptr_t size) {
ASSERT(Utils::IsAligned(size, kObjectAlignment));
ScavengerLAB& lab = labs_[producer_index_];
uword result = lab.top;
uword new_top = result + size;
if (LIKELY(new_top <= lab.end)) {
ASSERT(scavenger_->to_->Contains(result));
// TODO(rmacnak): Allocate one to start?
if (tail_ != nullptr) {
uword result = tail_->top_;
ASSERT((result & kObjectAlignmentMask) == kNewObjectAlignmentOffset);
lab.top = new_top;
ASSERT((scavenger_->to_->Contains(new_top)) ||
(new_top == scavenger_->to_->end()));
return result;
uword new_top = result + size;
if (LIKELY(new_top <= tail_->end_)) {
tail_->top_ = new_top;
return result;
}
}
return TryAllocateCopySlow(size);
}
DART_NOINLINE inline uword TryAllocateCopySlow(intptr_t size);
void MakeProducerTLABIterable() {
uword top = labs_[producer_index_].top;
uword end = labs_[producer_index_].end;
intptr_t size = end - top;
if (size != 0) {
ASSERT(Utils::IsAligned(size, kObjectAlignment));
ForwardingCorpse::AsForwarder(top, size);
ASSERT(ObjectLayout::FromAddr(top)->ptr()->HeapSize() == size);
}
}
inline void ProcessToSpace();
DART_FORCE_INLINE intptr_t ProcessCopied(ObjectPtr raw_obj);
inline void ProcessPromotedList();
@@ -436,14 +522,9 @@ class ScavengerVisitorBase : public ObjectPointerVisitor {
PromotionWorkList promoted_list_;
WeakPropertyPtr delayed_weak_properties_ = nullptr;
struct ScavengerLAB {
uword top;
uword end;
uword resolved_top;
};
MallocGrowableArray<ScavengerLAB> labs_;
intptr_t consumer_index_ = 1;
intptr_t producer_index_ = 0;
NewPage* head_ = nullptr;
NewPage* tail_ = nullptr; // Allocating from here.
NewPage* scan_ = nullptr; // Resolving from here.
DISALLOW_COPY_AND_ASSIGN(ScavengerVisitorBase);
};
@@ -575,100 +656,153 @@ class ParallelScavengerTask : public ThreadPool::Task {
DISALLOW_COPY_AND_ASSIGN(ParallelScavengerTask);
};
SemiSpace::SemiSpace(VirtualMemory* reserved)
: reserved_(reserved), region_(NULL, 0) {
if (reserved != NULL) {
region_ = MemoryRegion(reserved_->address(), reserved_->size());
}
}
SemiSpace::SemiSpace(intptr_t max_capacity_in_words)
: max_capacity_in_words_(max_capacity_in_words), head_(nullptr) {}
SemiSpace::~SemiSpace() {
delete reserved_;
NewPage* page = head_;
while (page != nullptr) {
NewPage* next = page->next();
page->Deallocate();
page = next;
}
}
Mutex* SemiSpace::mutex_ = NULL;
SemiSpace* SemiSpace::cache_ = NULL;
// TODO(rmacnak): Unify this with old-space pages, and possibly zone segments.
// This cache needs to be at least as big as FLAG_new_gen_semi_max_size or
// munmap will noticably impact performance.
static constexpr intptr_t kPageCacheCapacity = 8 * kWordSize;
static Mutex* page_cache_mutex = nullptr;
static VirtualMemory* page_cache[kPageCacheCapacity] = {nullptr};
static intptr_t page_cache_size = 0;
void SemiSpace::Init() {
if (mutex_ == NULL) {
mutex_ = new Mutex();
}
ASSERT(mutex_ != NULL);
ASSERT(page_cache_mutex == nullptr);
page_cache_mutex = new Mutex(NOT_IN_PRODUCT("page_cache_mutex"));
}
void SemiSpace::Cleanup() {
MutexLocker locker(mutex_);
delete cache_;
cache_ = NULL;
{
MutexLocker ml(page_cache_mutex);
ASSERT(page_cache_size >= 0);
ASSERT(page_cache_size <= kPageCacheCapacity);
while (page_cache_size > 0) {
delete page_cache[--page_cache_size];
}
}
delete page_cache_mutex;
page_cache_mutex = nullptr;
}
SemiSpace* SemiSpace::New(intptr_t size_in_words, const char* name) {
SemiSpace* result = nullptr;
NewPage* NewPage::Allocate() {
const intptr_t size = kNewPageSize;
VirtualMemory* memory = nullptr;
{
MutexLocker locker(mutex_);
// TODO(koda): Cache one entry per size.
if (cache_ != nullptr && cache_->size_in_words() == size_in_words) {
result = cache_;
cache_ = nullptr;
MutexLocker ml(page_cache_mutex);
ASSERT(page_cache_size >= 0);
ASSERT(page_cache_size <= kPageCacheCapacity);
if (page_cache_size > 0) {
memory = page_cache[--page_cache_size];
}
}
if (result != nullptr) {
#ifdef DEBUG
result->reserved_->Protect(VirtualMemory::kReadWrite);
#endif
// Initialized by generated code.
MSAN_UNPOISON(result->reserved_->address(), size_in_words << kWordSizeLog2);
return result;
if (memory == nullptr) {
const intptr_t alignment = kNewPageSize;
const bool is_executable = false;
const char* const name = Heap::RegionName(Heap::kNew);
memory =
VirtualMemory::AllocateAligned(size, alignment, is_executable, name);
}
if (memory == nullptr) {
// TODO(koda): We could try to recover (collect old space, wait for another
// isolate to finish scavenge, etc.).
OUT_OF_MEMORY();
}
if (size_in_words == 0) {
return new SemiSpace(nullptr);
} else {
intptr_t size_in_bytes = size_in_words << kWordSizeLog2;
const bool kExecutable = false;
VirtualMemory* memory =
VirtualMemory::Allocate(size_in_bytes, kExecutable, name);
if (memory == nullptr) {
// TODO(koda): If cache_ is not empty, we could try to delete it.
return nullptr;
}
#if defined(DEBUG)
memset(memory->address(), Heap::kZapByte, size_in_bytes);
#endif // defined(DEBUG)
// Initialized by generated code.
MSAN_UNPOISON(memory->address(), size_in_bytes);
return new SemiSpace(memory);
}
memset(memory->address(), Heap::kZapByte, size);
#endif
// Initialized by generated code.
MSAN_UNPOISON(memory->address(), size);
NewPage* result = reinterpret_cast<NewPage*>(memory->address());
result->memory_ = memory;
result->next_ = nullptr;
result->owner_ = nullptr;
uword top = result->object_start();
result->top_ = top;
result->end_ = memory->end() - kNewObjectAlignmentOffset;
result->survivor_end_ = top;
result->resolved_top_ = top;
LSAN_REGISTER_ROOT_REGION(result, sizeof(*result));
return result;
}
void SemiSpace::Delete() {
if (reserved_ != nullptr) {
const intptr_t size_in_bytes = size_in_words() << kWordSizeLog2;
#ifdef DEBUG
memset(reserved_->address(), Heap::kZapByte, size_in_bytes);
reserved_->Protect(VirtualMemory::kNoAccess);
#endif
MSAN_POISON(reserved_->address(), size_in_bytes);
}
SemiSpace* old_cache = nullptr;
void NewPage::Deallocate() {
LSAN_UNREGISTER_ROOT_REGION(this, sizeof(*this));
VirtualMemory* memory = memory_;
{
MutexLocker locker(mutex_);
old_cache = cache_;
cache_ = this;
MutexLocker ml(page_cache_mutex);
ASSERT(page_cache_size >= 0);
ASSERT(page_cache_size <= kPageCacheCapacity);
if (page_cache_size < kPageCacheCapacity) {
intptr_t size = memory->size();
#if defined(DEBUG)
memset(memory->address(), Heap::kZapByte, size);
#endif
MSAN_POISON(memory->address(), size);
page_cache[page_cache_size++] = memory;
memory = nullptr;
}
}
// TODO(rmacnak): This can take an order of magnitude longer the rest of
// a scavenge. Consider moving it to another thread, perhaps the idle
// notifier.
delete old_cache;
delete memory;
}
NewPage* SemiSpace::TryAllocatePageLocked(bool link) {
if (capacity_in_words_ >= max_capacity_in_words_) {
return nullptr; // Full.
}
NewPage* page = NewPage::Allocate();
capacity_in_words_ += kNewPageSizeInWords;
if (link) {
if (head_ == nullptr) {
head_ = tail_ = page;
} else {
tail_->set_next(page);
tail_ = page;
}
}
return page;
}
bool SemiSpace::Contains(uword addr) const {
for (NewPage* page = head_; page != nullptr; page = page->next()) {
if (page->Contains(addr)) return true;
}
return false;
}
void SemiSpace::WriteProtect(bool read_only) {
if (reserved_ != NULL) {
reserved_->Protect(read_only ? VirtualMemory::kReadOnly
: VirtualMemory::kReadWrite);
for (NewPage* page = head_; page != nullptr; page = page->next()) {
page->WriteProtect(read_only);
}
}
void SemiSpace::AddList(NewPage* head, NewPage* tail) {
if (head == nullptr) {
return;
}
if (head_ == nullptr) {
head_ = head;
tail_ = tail;
return;
}
tail_->set_next(head);
tail_ = tail;
}
// The initial estimate of how many words we can scavenge per microsecond (usage
// before / scavenge time). This is a conservative value observed running
// Flutter on a Nexus 4. After the first scavenge, we instead use a value based
@@ -693,17 +827,7 @@ Scavenger::Scavenger(Heap* heap, intptr_t max_semi_capacity_in_words)
const intptr_t initial_semi_capacity_in_words = Utils::Minimum(
max_semi_capacity_in_words, FLAG_new_gen_semi_initial_size * MBInWords);
const char* name = Heap::RegionName(Heap::kNew);
to_ = SemiSpace::New(initial_semi_capacity_in_words, name);
if (to_ == NULL) {
OUT_OF_MEMORY();
}
// Setup local fields.
top_ = FirstObjectStart();
resolved_top_ = top_;
end_ = to_->end();
survivor_end_ = FirstObjectStart();
to_ = new SemiSpace(initial_semi_capacity_in_words);
idle_scavenge_threshold_in_words_ = initial_semi_capacity_in_words;
UpdateMaxHeapCapacity();
@@ -712,7 +836,7 @@ Scavenger::Scavenger(Heap* heap, intptr_t max_semi_capacity_in_words)
Scavenger::~Scavenger() {
ASSERT(!scavenging_);
to_->Delete();
delete to_;
}
intptr_t Scavenger::NewSizeInWords(intptr_t old_size_in_words) const {
@@ -838,20 +962,8 @@ SemiSpace* Scavenger::Prologue() {
// objects.
SemiSpace* from = to_;
const char* name = Heap::RegionName(Heap::kNew);
to_ = SemiSpace::New(NewSizeInWords(from->size_in_words()), name);
if (to_ == NULL) {
// TODO(koda): We could try to recover (collect old space, wait for another
// isolate to finish scavenge, etc.).
OUT_OF_MEMORY();
}
to_ = new SemiSpace(NewSizeInWords(from->max_capacity_in_words()));
UpdateMaxHeapCapacity();
{
MutexLocker ml(&space_lock_);
top_ = FirstObjectStart();
resolved_top_ = top_;
end_ = to_->end();
}
return from;
}
@@ -861,20 +973,26 @@ void Scavenger::Epilogue(SemiSpace* from) {
// All objects in the to space have been copied from the from space at this
// moment.
// Ensure the mutator thread will fail the next allocation. This will force
// mutator to allocate a new TLAB
#if defined(DEBUG)
heap_->isolate_group()->ForEachIsolate(
[&](Isolate* isolate) {
Thread* mutator_thread = isolate->mutator_thread();
ASSERT(mutator_thread == nullptr || mutator_thread->top() == 0);
},
/*at_safepoint=*/true);
#endif // DEBUG
double avg_frac = stats_history_.Get(0).PromoCandidatesSuccessFraction();
if (stats_history_.Size() >= 2) {
// Previous scavenge is only given half as much weight.
avg_frac += 0.5 * stats_history_.Get(1).PromoCandidatesSuccessFraction();
avg_frac /= 1.0 + 0.5; // Normalize.
}
if (avg_frac < (FLAG_early_tenuring_threshold / 100.0)) {
// Remember the limit to which objects have been copied.
survivor_end_ = top_;
} else {
// Move survivor end to the end of the to_ space, making all surviving
// objects candidates for promotion next time.
survivor_end_ = end_;
}
early_tenure_ = avg_frac >= (FLAG_early_tenuring_threshold / 100.0);
// Update estimate of scavenger speed. This statistic assumes survivorship
// rates don't change much.
@@ -930,7 +1048,7 @@ void Scavenger::Epilogue(SemiSpace* from) {
OS::PrintErr(" done.\n");
}
from->Delete();
delete from;
UpdateMaxHeapUsage();
if (heap_ != NULL) {
heap_->UpdateGlobalMaxUsed();
@@ -1078,22 +1196,20 @@ void Scavenger::MournWeakHandles() {
template <bool parallel>
void ScavengerVisitorBase<parallel>::ProcessToSpace() {
intptr_t i = consumer_index_;
while (i <= producer_index_) {
uword resolved_top = labs_[i].resolved_top;
while (resolved_top < labs_[i].top) {
while (scan_ != nullptr) {
uword resolved_top = scan_->resolved_top_;
while (resolved_top < scan_->top_) {
ObjectPtr raw_obj = ObjectLayout::FromAddr(resolved_top);
resolved_top += ProcessCopied(raw_obj);
}
labs_[i].resolved_top = resolved_top;
scan_->resolved_top_ = resolved_top;
if (i == producer_index_) {
return; // More objects may yet be copied to this TLAB.
NewPage* next = scan_->next();
if (next == nullptr) {
// Don't update scan_. More objects may yet be copied to this TLAB.
return;
}
i++;
consumer_index_ = i;
ASSERT(consumer_index_ < labs_.length());
scan_ = next;
}
}
@@ -1162,8 +1278,8 @@ void Scavenger::UpdateMaxHeapCapacity() {
ASSERT(heap_ != NULL);
auto isolate_group = heap_->isolate_group();
ASSERT(isolate_group != NULL);
isolate_group->GetHeapNewCapacityMaxMetric()->SetValue(to_->size_in_words() *
kWordSize);
isolate_group->GetHeapNewCapacityMaxMetric()->SetValue(
to_->max_capacity_in_words() * kWordSize);
#endif // !defined(PRODUCT)
}
@@ -1299,81 +1415,35 @@ void ScavengerVisitorBase<parallel>::MournWeakProperties() {
}
}
void Scavenger::MakeNewSpaceIterable() {
void Scavenger::VisitObjectPointers(ObjectPointerVisitor* visitor) const {
ASSERT(Thread::Current()->IsAtSafepoint() ||
(Thread::Current()->task_kind() == Thread::kMarkerTask) ||
(Thread::Current()->task_kind() == Thread::kCompactorTask));
auto isolate_group = heap_->isolate_group();
MonitorLocker ml(isolate_group->threads_lock(), false);
// Make all scheduled thread's TLABs iterable.
Thread* current = heap_->isolate_group()->thread_registry()->active_list();
while (current != NULL) {
const TLAB tlab = current->tlab();
if (!tlab.IsAbandoned()) {
MakeTLABIterable(tlab);
}
current = current->next();
}
for (intptr_t i = 0; i < free_tlabs_.length(); ++i) {
MakeTLABIterable(free_tlabs_[i]);
for (NewPage* page = to_->head(); page != nullptr; page = page->next()) {
page->VisitObjectPointers(visitor);
}
}
void Scavenger::AbandonTLABsLocked() {
ASSERT(Thread::Current()->IsAtSafepoint());
IsolateGroup* isolate_group = heap_->isolate_group();
MonitorLocker ml(isolate_group->threads_lock(), false);
// Abandon TLABs of all scheduled threads.
Thread* current = isolate_group->thread_registry()->active_list();
while (current != NULL) {
const TLAB tlab = current->tlab();
AddAbandonedInBytesLocked(tlab.RemainingSize());
current->set_tlab(TLAB());
current = current->next();
}
while (free_tlabs_.length() > 0) {
const TLAB tlab = free_tlabs_.RemoveLast();
AddAbandonedInBytesLocked(tlab.RemainingSize());
}
}
void Scavenger::VisitObjectPointers(ObjectPointerVisitor* visitor) {
ASSERT(Thread::Current()->IsAtSafepoint() ||
(Thread::Current()->task_kind() == Thread::kMarkerTask) ||
(Thread::Current()->task_kind() == Thread::kCompactorTask));
MakeNewSpaceIterable();
uword cur = FirstObjectStart();
while (cur < top_) {
ObjectPtr raw_obj = ObjectLayout::FromAddr(cur);
cur += raw_obj->ptr()->VisitPointers(visitor);
}
}
void Scavenger::VisitObjects(ObjectVisitor* visitor) {
void Scavenger::VisitObjects(ObjectVisitor* visitor) const {
ASSERT(Thread::Current()->IsAtSafepoint() ||
(Thread::Current()->task_kind() == Thread::kMarkerTask));
MakeNewSpaceIterable();
uword cur = FirstObjectStart();
while (cur < top_) {
ObjectPtr raw_obj = ObjectLayout::FromAddr(cur);
visitor->VisitObject(raw_obj);
cur += raw_obj->ptr()->HeapSize();
for (NewPage* page = to_->head(); page != nullptr; page = page->next()) {
page->VisitObjects(visitor);
}
}
void Scavenger::AddRegionsToObjectSet(ObjectSet* set) const {
set->AddRegion(to_->start(), to_->end());
for (NewPage* page = to_->head(); page != nullptr; page = page->next()) {
set->AddRegion(page->start(), page->end());
}
}
ObjectPtr Scavenger::FindObject(FindObjectVisitor* visitor) {
ASSERT(!scavenging_);
MakeNewSpaceIterable();
uword cur = FirstObjectStart();
if (visitor->VisitRange(cur, top_)) {
while (cur < top_) {
for (NewPage* page = to_->head(); page != nullptr; page = page->next()) {
uword cur = page->object_start();
if (!visitor->VisitRange(cur, page->object_end())) continue;
while (cur < page->object_end()) {
ObjectPtr raw_obj = ObjectLayout::FromAddr(cur);
uword next = cur + raw_obj->ptr()->HeapSize();
if (visitor->VisitRange(cur, next) &&
@@ -1382,131 +1452,77 @@ ObjectPtr Scavenger::FindObject(FindObjectVisitor* visitor) {
}
cur = next;
}
ASSERT(cur == top_);
ASSERT(cur == page->object_end());
}
return Object::null();
}
void Scavenger::TryAllocateNewTLAB(Thread* thread) {
void Scavenger::TryAllocateNewTLAB(Thread* thread, intptr_t min_size) {
ASSERT(heap_ != Dart::vm_isolate()->heap());
ASSERT(!scavenging_);
AbandonRemainingTLAB(thread);
MutexLocker ml(&space_lock_);
// We might need a new TLAB not because the current TLAB is empty but because
// we failed to allocate alarge object in new space. So in case the remaining
// TLAB is still big enough to be useful we cache it.
CacheTLABLocked(thread->tlab());
thread->set_tlab(TLAB());
uword result = top_;
intptr_t remaining = end_ - top_;
intptr_t size = kTLABSize;
if (remaining < size) {
// Grab whatever is remaining
size = Utils::RoundDown(remaining, kObjectAlignment);
for (NewPage* page = to_->head(); page != nullptr; page = page->next()) {
if (page->owner() != nullptr) continue;
intptr_t available = page->end() - page->object_end();
if (available >= min_size) {
page->Acquire(thread);
return;
}
}
ASSERT(Utils::IsAligned(size, kObjectAlignment));
if (size == 0) {
thread->set_tlab(TryAcquireCachedTLABLocked());
NewPage* page = to_->TryAllocatePageLocked(true);
if (page == nullptr) {
return;
}
ASSERT(to_->Contains(result));
ASSERT((result & kObjectAlignmentMask) == kNewObjectAlignmentOffset);
top_ += size;
ASSERT(to_->Contains(top_) || (top_ == to_->end()));
ASSERT(result < top_);
thread->set_tlab(TLAB(result, top_));
}
void Scavenger::MakeTLABIterable(const TLAB& tlab) {
ASSERT(tlab.end >= tlab.top);
const intptr_t size = tlab.RemainingSize();
ASSERT(Utils::IsAligned(size, kObjectAlignment));
if (size >= kObjectAlignment) {
// ForwardingCorpse(forwarding to default null) will work as filler.
ForwardingCorpse::AsForwarder(tlab.top, size);
ASSERT(ObjectLayout::FromAddr(tlab.top)->ptr()->HeapSize() == size);
}
page->Acquire(thread);
}
void Scavenger::AbandonRemainingTLABForDebugging(Thread* thread) {
MutexLocker ml(&space_lock_);
const TLAB tlab = thread->tlab();
MakeTLABIterable(tlab);
AddAbandonedInBytesLocked(tlab.RemainingSize());
thread->set_tlab(TLAB());
// Allocate any remaining space so the TLAB won't be reused. Write a filler
// object so it remains iterable.
uword top = thread->top();
intptr_t size = thread->end() - thread->top();
if (size > 0) {
thread->set_top(top + size);
ForwardingCorpse::AsForwarder(top, size);
}
AbandonRemainingTLAB(thread);
}
void Scavenger::AbandonRemainingTLAB(Thread* thread) {
if (thread->top() == 0) return;
NewPage* page = NewPage::Of(thread->top() - 1);
{
MutexLocker ml(&space_lock_);
page->Release(thread);
}
ASSERT(thread->top() == 0);
}
template <bool parallel>
uword ScavengerVisitorBase<parallel>::TryAllocateCopySlow(intptr_t size) {
MakeProducerTLABIterable();
if (!scavenger_->TryAllocateNewTLAB(this)) {
NewPage* page;
{
MutexLocker ml(&scavenger_->space_lock_);
page = scavenger_->to_->TryAllocatePageLocked(false);
}
if (page == nullptr) {
return 0;
}
const uword result = labs_[producer_index_].top;
const intptr_t remaining =
labs_[producer_index_].end - labs_[producer_index_].top;
ASSERT(size <= remaining);
ASSERT(scavenger_->to_->Contains(result));
ASSERT((result & kObjectAlignmentMask) == kNewObjectAlignmentOffset);
labs_[producer_index_].top = result + size;
return result;
}
template <bool parallel>
bool Scavenger::TryAllocateNewTLAB(ScavengerVisitorBase<parallel>* visitor) {
intptr_t size = kTLABSize;
ASSERT(Utils::IsAligned(size, kObjectAlignment));
ASSERT(heap_ != Dart::vm_isolate()->heap());
ASSERT(scavenging_);
MutexLocker ml(&space_lock_);
const uword result = top_;
const intptr_t remaining = end_ - top_;
if (remaining < size) {
// Grab whatever is remaining
size = Utils::RoundDown(remaining, kObjectAlignment);
if (head_ == nullptr) {
head_ = scan_ = page;
} else {
ASSERT(scan_ != nullptr);
tail_->set_next(page);
}
if (size == 0) {
return false;
}
ASSERT(to_->Contains(result));
ASSERT((result & kObjectAlignmentMask) == kNewObjectAlignmentOffset);
top_ += size;
ASSERT(to_->Contains(top_) || (top_ == to_->end()));
ASSERT(result < top_);
visitor->AddNewTLAB(result, top_);
return true;
}
tail_ = page;
TLAB Scavenger::TryAcquireCachedTLABLocked() {
if (free_tlabs_.length() == 0) {
return TLAB();
}
return free_tlabs_.RemoveLast();
}
void Scavenger::CacheTLABLocked(TLAB tlab) {
// If the memory following this TLAB is the unused new space, we'll merge the
// bytes into there.
if (tlab.end == top_) {
top_ = tlab.top;
return;
}
MakeTLABIterable(tlab);
// If this TLAB is lare enough to be useful in the future, we'll make it
// reusable, otherwise we abandon it.
const uword size = tlab.RemainingSize();
if (size > (50 * KB)) {
free_tlabs_.Add(tlab);
return;
}
// Else we discard the memory.
AddAbandonedInBytesLocked(size);
return tail_->TryAllocateGC(size);
}
void Scavenger::Scavenge() {
@@ -1535,13 +1551,18 @@ void Scavenger::Scavenge() {
}
// Prepare for a scavenge.
AbandonTLABsLocked();
failed_to_promote_ = false;
root_slices_started_ = 0;
intptr_t abandoned_bytes = GetAndResetAbandonedInBytes();
intptr_t abandoned_bytes = 0; // TODO(rmacnak): Count fragmentation?
SpaceUsage usage_before = GetCurrentUsage();
intptr_t promo_candidate_words =
(survivor_end_ - FirstObjectStart()) / kWordSize;
intptr_t promo_candidate_words = 0;
for (NewPage* page = to_->head(); page != nullptr; page = page->next()) {
page->Release();
if (early_tenure_) {
page->EarlyTenure();
}
promo_candidate_words += page->promo_candidate_words();
}
SemiSpace* from = Prologue();
intptr_t bytes_promoted;
@@ -1585,8 +1606,8 @@ intptr_t Scavenger::SerialScavenge(SemiSpace* from) {
visitor.ProcessAll();
}
visitor.Finalize();
visitor.DonateTLABs();
to_->AddList(visitor.head(), visitor.tail());
return visitor.bytes_promoted();
}
@@ -1619,8 +1640,8 @@ intptr_t Scavenger::ParallelScavenge(SemiSpace* from) {
}
for (intptr_t i = 0; i < num_tasks; i++) {
to_->AddList(visitors[i]->head(), visitors[i]->tail());
bytes_promoted += visitors[i]->bytes_promoted();
visitors[i]->DonateTLABs();
delete visitors[i];
}
@@ -1681,7 +1702,7 @@ void Scavenger::Evacuate() {
SafepointOperationScope scope(Thread::Current());
// Forces the next scavenge to promote all the objects in the new space.
survivor_end_ = top_;
early_tenure_ = true;
Scavenge();
runtime/vm/heap/scavenger.h
+35 -97
View File
@@ -12,7 +12,6 @@
#include "vm/flags.h"
#include "vm/globals.h"
#include "vm/heap/spaces.h"
#include "vm/heap/tlab.h"
#include "vm/lockers.h"
#include "vm/raw_object.h"
#include "vm/ring_buffer.h"
@@ -25,47 +24,40 @@ namespace dart {
class Heap;
class Isolate;
class JSONObject;
class NewPage;
class ObjectSet;
template <bool parallel>
class ScavengerVisitorBase;
// Wrapper around VirtualMemory that adds caching and handles the empty case.
class SemiSpace {
public:
static void Init();
static void Cleanup();
// Get a space of the given size. Returns NULL on out of memory. If size is 0,
// returns an empty space: pointer(), start() and end() all return NULL.
// The name parameter may be NULL. If non-NULL it is ued to give the OS a name
// for the underlying virtual memory region.
static SemiSpace* New(intptr_t size_in_words, const char* name);
// Hand back an unused space.
void Delete();
void* pointer() const { return region_.pointer(); }
uword start() const { return region_.start(); }
uword end() const { return region_.end(); }
intptr_t size_in_words() const {
return static_cast<intptr_t>(region_.size()) >> kWordSizeLog2;
}
bool Contains(uword address) const { return region_.Contains(address); }
// Set write protection mode for this space. The space must not be protected
// when Delete is called.
// TODO(koda): Remember protection mode in VirtualMemory and assert this.
void WriteProtect(bool read_only);
private:
explicit SemiSpace(VirtualMemory* reserved);
explicit SemiSpace(intptr_t max_capacity_in_words);
~SemiSpace();
VirtualMemory* reserved_; // NULL for an empty space.
MemoryRegion region_;
NewPage* TryAllocatePageLocked(bool link);
static SemiSpace* cache_;
static Mutex* mutex_;
bool Contains(uword addr) const;
void WriteProtect(bool read_only);
intptr_t capacity_in_words() const { return capacity_in_words_; }
intptr_t max_capacity_in_words() const { return max_capacity_in_words_; }
NewPage* head() const { return head_; }
void AddList(NewPage* head, NewPage* tail);
private:
// Size of NewPages in this semi-space.
intptr_t capacity_in_words_ = 0;
// Size of NewPages before we trigger a scavenge.
intptr_t max_capacity_in_words_;
NewPage* head_ = nullptr;
NewPage* tail_ = nullptr;
};
// Statistics for a particular scavenge.
@@ -137,28 +129,11 @@ class Scavenger {
if (LIKELY(addr != 0)) {
return addr;
}
TryAllocateNewTLAB(thread);
TryAllocateNewTLAB(thread, size);
return TryAllocateFromTLAB(thread, size);
}
void MakeTLABIterable(const TLAB& tlab);
void AbandonRemainingTLAB(Thread* thread);
void AbandonRemainingTLABForDebugging(Thread* thread);
template <bool parallel>
bool TryAllocateNewTLAB(ScavengerVisitorBase<parallel>* visitor);
// When a thread gets scheduled it will try to acquire a TLAB.
void TryAcquireCachedTLAB(Thread* thread) {
MutexLocker ml(&space_lock_);
thread->set_tlab(TryAcquireCachedTLABLocked());
}
TLAB TryAcquireCachedTLABLocked();
// When a thread gets unscheduled it will release it's TLAB.
void ReleaseAndCacheTLAB(Thread* thread) {
MutexLocker ml(&space_lock_);
CacheTLABLocked(thread->tlab());
thread->set_tlab(TLAB());
}
void CacheTLABLocked(TLAB tlab);
// Collect the garbage in this scavenger.
void Scavenge();
@@ -166,23 +141,11 @@ class Scavenger {
// Promote all live objects.
void Evacuate();
// Report (TLAB) abandoned bytes that should be taken account when
// deciding whether to grow new space or not.
void AddAbandonedInBytes(intptr_t value) {
MutexLocker ml(&space_lock_);
AddAbandonedInBytesLocked(value);
}
int64_t GetAndResetAbandonedInBytes() {
int64_t result = abandoned_;
abandoned_ = 0;
return result;
}
int64_t UsedInWords() const {
MutexLocker ml(&space_lock_);
return (top_ - FirstObjectStart()) >> kWordSizeLog2;
return to_->capacity_in_words();
}
int64_t CapacityInWords() const { return to_->size_in_words(); }
int64_t CapacityInWords() const { return to_->max_capacity_in_words(); }
int64_t ExternalInWords() const { return external_size_ >> kWordSizeLog2; }
SpaceUsage GetCurrentUsage() const {
SpaceUsage usage;
@@ -192,8 +155,8 @@ class Scavenger {
return usage;
}
void VisitObjects(ObjectVisitor* visitor);
void VisitObjectPointers(ObjectPointerVisitor* visitor);
void VisitObjects(ObjectVisitor* visitor) const;
void VisitObjectPointers(ObjectPointerVisitor* visitor) const;
void AddRegionsToObjectSet(ObjectSet* set) const;
@@ -232,8 +195,6 @@ class Scavenger {
bool scavenging() const { return scavenging_; }
private:
static const intptr_t kTLABSize = 512 * KB;
// Ids for time and data records in Heap::GCStats.
enum {
// Time
@@ -253,27 +214,20 @@ class Scavenger {
uword TryAllocateFromTLAB(Thread* thread, intptr_t size) {
ASSERT(Utils::IsAligned(size, kObjectAlignment));
ASSERT(heap_ != Dart::vm_isolate()->heap());
TLAB tlab = thread->tlab();
const intptr_t remaining = tlab.RemainingSize();
const uword result = thread->top();
const intptr_t remaining = thread->end() - result;
if (UNLIKELY(remaining < size)) {
return 0;
}
const uword result = tlab.top;
ASSERT(to_->Contains(result));
ASSERT((result & kObjectAlignmentMask) == kNewObjectAlignmentOffset);
const uword new_top = tlab.top + size;
ASSERT(to_->Contains(new_top) || new_top == to_->end());
thread->set_tlab(tlab.BumpAllocate(size));
thread->set_top(result + size);
return result;
}
void TryAllocateNewTLAB(Thread* thread);
void AddAbandonedInBytesLocked(intptr_t value) { abandoned_ += value; }
void AbandonTLABsLocked();
void TryAllocateNewTLAB(Thread* thread, intptr_t size);
uword FirstObjectStart() const {
return to_->start() + kNewObjectAlignmentOffset;
}
SemiSpace* Prologue();
intptr_t ParallelScavenge(SemiSpace* from);
intptr_t SerialScavenge(SemiSpace* from);
@@ -299,26 +253,9 @@ class Scavenger {
intptr_t NewSizeInWords(intptr_t old_size_in_words) const;
uword top_;
uword end_;
MallocGrowableArray<TLAB> abandoned_tlabs_;
MallocGrowableArray<TLAB> free_tlabs_;
SemiSpace* to_;
Heap* heap_;
// A pointer to the first unscanned object. Scanning completes when
// this value meets the allocation top.
uword resolved_top_;
// Objects below this address have survived a scavenge.
uword survivor_end_;
// Abandoned (TLAB) bytes that need to be accounted for when deciding
// whether to grow newspace or not.
intptr_t abandoned_ = 0;
SemiSpace* to_;
PromotionStack promotion_stack_;
@@ -326,6 +263,7 @@ class Scavenger {
// Keep track whether a scavenge is currently running.
bool scavenging_;
bool early_tenure_ = false;
RelaxedAtomic<intptr_t> root_slices_started_;
StoreBufferBlock* blocks_;
runtime/vm/heap/tlab.h
-37
View File
@@ -1,37 +0,0 @@
// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#ifndef RUNTIME_VM_HEAP_TLAB_H_
#define RUNTIME_VM_HEAP_TLAB_H_
#include "platform/assert.h"
#include "platform/globals.h"
namespace dart {
struct TLAB {
TLAB() : top(0), end(0) {}
TLAB(uword top, uword end) : top(top), end(end) {}
TLAB(const TLAB& other) : top(other.top), end(other.end) {}
TLAB& operator=(const TLAB& other) {
top = other.top;
end = other.end;
return *this;
}
intptr_t RemainingSize() const { return end - top; }
bool IsAbandoned() const { return top == 0 && end == 0; }
TLAB BumpAllocate(intptr_t size) const {
ASSERT(RemainingSize() >= size);
return TLAB(top + size, end);
}
uword top;
uword end;
};
} // namespace dart
#endif // RUNTIME_VM_HEAP_TLAB_H_
runtime/vm/interpreter.cc
+4 -5
View File
@@ -272,18 +272,17 @@ DART_FORCE_INLINE static bool TryAllocate(Thread* thread,
ASSERT(instance_size > 0);
ASSERT(Utils::IsAligned(instance_size, kObjectAlignment));
const TLAB tlab = thread->tlab();
#ifndef PRODUCT
auto table = thread->isolate_group()->shared_class_table();
if (UNLIKELY(table->TraceAllocationFor(class_id))) {
return false;
}
#endif
const intptr_t remaining = tlab.RemainingSize();
const uword top = thread->top();
const intptr_t remaining = thread->end() - top;
if (LIKELY(remaining >= instance_size)) {
const uword old_top = tlab.top;
thread->set_tlab(tlab.BumpAllocate(instance_size));
*result = InitializeHeader(old_top, class_id, instance_size);
thread->set_top(top + instance_size);
*result = InitializeHeader(top, class_id, instance_size);
return true;
}
return false;
runtime/vm/isolate.cc
+1 -3
View File
@@ -422,8 +422,6 @@ Thread* IsolateGroup::ScheduleThreadLocked(MonitorLocker* ml,
os_thread->set_thread(thread);
Thread::SetCurrent(thread);
os_thread->EnableThreadInterrupts();
thread->heap()->new_space()->TryAcquireCachedTLAB(thread);
}
return thread;
}
@@ -432,7 +430,7 @@ void IsolateGroup::UnscheduleThreadLocked(MonitorLocker* ml,
Thread* thread,
bool is_mutator,
bool bypass_safepoint) {
thread->heap()->new_space()->ReleaseAndCacheTLAB(thread);
thread->heap()->new_space()->AbandonRemainingTLAB(thread);
// Clear since GC will not visit the thread once it is unscheduled. Do this
// under the thread lock to prevent races with the GC visiting thread roots.
runtime/vm/thread.h
+4 -8
View File
@@ -19,7 +19,6 @@
#include "vm/globals.h"
#include "vm/handles.h"
#include "vm/heap/pointer_block.h"
#include "vm/heap/tlab.h"
#include "vm/os_thread.h"
#include "vm/random.h"
#include "vm/runtime_entry_list.h"
@@ -495,13 +494,10 @@ class Thread : public ThreadState {
Heap* heap() const { return heap_; }
static intptr_t heap_offset() { return OFFSET_OF(Thread, heap_); }
void set_tlab(TLAB tlab) {
top_ = tlab.top;
end_ = tlab.end;
}
TLAB tlab() { return TLAB(top_, end_); }
uword top() const { return top_; }
uword end() const { return end_; }
void set_top(uword top) { top_ = top; }
void set_end(uword end) { end_ = end; }
static intptr_t top_offset() { return OFFSET_OF(Thread, top_); }
static intptr_t end_offset() { return OFFSET_OF(Thread, end_); }