From dae308461c546c9dd763dcd1dc5a16335ff48bc2 Mon Sep 17 00:00:00 2001 From: Martin Kustermann Date: Thu, 20 Feb 2020 21:08:35 +0000 Subject: [PATCH] [vm/concurrency] Share [Heap] and [SharedClassTable] between all isolates within one isolate group This CL: * Moves [Heap]/[SharedClassTable] from [Isolate] to [IsolateGroup], which will make all isolates in the group use the same heap. The GC will use the shared class table for object size information. * Adds support for entering/leaving an isolate group as a helper thread (e.g. via [Thread::EnterIsolateGroupAsHelper]). The current active isolate group can be accessed via TLS `IsolateGroup::Current()` or `Thread::isolate_group_`. When entering as a helper thread there will be no current isolate. * Changes the GC to use the above mechanism and ensures GC works without a currently active isolate. The GC will use information purely available via [IsolateGroup]. The GC will iterate all isolates within an isolate group e.g. for scanning roots. * Makes spawning of new isolates start in their own isolate group. Once the isolate is fully functional it's heap will be merged into the original isolate group * Moves ApiState, containing persistent and weak persistent handles, from [Isolate] to [IsolateGroup], plus adds appropriate locking. Issue https://github.com/dart-lang/sdk/issues/36097 Change-Id: Ia8e1d8aa78750e8400864200f4825395a182c004 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/126646 Commit-Queue: Martin Kustermann Reviewed-by: Ryan Macnak --- runtime/bin/main.cc | 8 - runtime/bin/process_android.cc | 3 - runtime/bin/process_linux.cc | 3 - runtime/bin/process_macos.cc | 3 - .../service/get_vm_timeline_rpc_test.dart | 5 +- runtime/vm/class_finalizer.cc | 51 +- runtime/vm/class_finalizer.h | 4 +- runtime/vm/class_table.cc | 37 +- runtime/vm/class_table.h | 10 +- runtime/vm/clustered_snapshot.cc | 18 +- runtime/vm/compiler/aot/precompiler.cc | 2 +- .../vm/compiler/runtime_offsets_extracted.h | 70 +-- runtime/vm/dart.cc | 15 +- runtime/vm/dart_api_impl.cc | 245 ++++++-- runtime/vm/dart_api_impl.h | 4 +- runtime/vm/dart_api_impl_test.cc | 6 +- runtime/vm/dart_api_state.h | 33 +- runtime/vm/gdb_helpers.cc | 3 +- runtime/vm/heap/become.cc | 29 +- runtime/vm/heap/compactor.cc | 33 +- runtime/vm/heap/compactor.h | 2 +- runtime/vm/heap/freelist.cc | 37 ++ runtime/vm/heap/freelist.h | 2 + runtime/vm/heap/heap.cc | 119 ++-- runtime/vm/heap/heap.h | 12 +- runtime/vm/heap/heap_test.cc | 14 +- runtime/vm/heap/marker.cc | 115 ++-- runtime/vm/heap/marker.h | 6 +- runtime/vm/heap/pages.cc | 140 ++++- runtime/vm/heap/pages.h | 9 +- runtime/vm/heap/pointer_block.cc | 5 +- runtime/vm/heap/safepoint.cc | 29 +- runtime/vm/heap/scavenger.cc | 170 +++--- runtime/vm/heap/scavenger.h | 22 +- runtime/vm/heap/sweeper.cc | 20 +- runtime/vm/heap/sweeper.h | 4 +- runtime/vm/heap/verifier.cc | 35 +- runtime/vm/heap/verifier.h | 13 +- runtime/vm/heap/weak_table.cc | 8 + runtime/vm/heap/weak_table.h | 2 + runtime/vm/interpreter.cc | 2 +- runtime/vm/isolate.cc | 528 +++++++++++------- runtime/vm/isolate.h | 233 +++++--- runtime/vm/isolate_reload.h | 3 +- runtime/vm/lockers.cc | 21 +- runtime/vm/lockers.h | 95 +++- runtime/vm/message_handler.cc | 14 +- runtime/vm/metrics.cc | 183 ++---- runtime/vm/metrics.h | 45 +- runtime/vm/native_entry.cc | 2 +- runtime/vm/object.cc | 27 +- runtime/vm/object.h | 4 +- runtime/vm/object_graph.cc | 55 +- runtime/vm/object_id_ring.cc | 12 +- runtime/vm/object_id_ring.h | 6 +- runtime/vm/object_reload.cc | 5 +- runtime/vm/profiler.cc | 2 +- runtime/vm/raw_object.cc | 26 +- runtime/vm/raw_object.h | 5 +- runtime/vm/raw_object_snapshot.cc | 2 +- runtime/vm/service.cc | 54 +- runtime/vm/snapshot.cc | 5 +- runtime/vm/snapshot.h | 4 +- runtime/vm/stack_frame.cc | 40 +- runtime/vm/stack_frame.h | 8 +- runtime/vm/tags.cc | 4 +- runtime/vm/thread.cc | 70 ++- runtime/vm/thread.h | 7 + runtime/vm/thread_registry.cc | 40 +- runtime/vm/thread_registry.h | 8 +- runtime/vm/thread_stack_resource.cc | 4 + runtime/vm/thread_stack_resource.h | 2 + runtime/vm/thread_test.cc | 10 +- runtime/vm/timeline.cc | 33 +- runtime/vm/timeline.h | 3 + runtime/vm/virtual_memory_fuchsia.cc | 3 +- runtime/vm/virtual_memory_posix.cc | 3 +- runtime/vm/virtual_memory_win.cc | 3 +- runtime/vm/visitor.cc | 6 +- runtime/vm/visitor.h | 7 +- .../standalone/io/socket_finalizer_test.dart | 10 + .../io/socket_finalizer_test.dart | 14 +- 82 files changed, 1815 insertions(+), 1139 deletions(-) diff --git a/runtime/bin/main.cc b/runtime/bin/main.cc index 98d59744966..6624a4e7b58 100644 --- a/runtime/bin/main.cc +++ b/runtime/bin/main.cc @@ -267,14 +267,6 @@ static bool OnIsolateInitialize(void** child_callback_data, char** error) { if (Dart_IsError(result)) goto failed; } - if (Options::gen_snapshot_kind() == kAppJIT) { - // If we sort, we must do it for all isolates, not just the main isolate, - // otherwise isolates related by spawnFunction will disagree on CIDs and - // cannot correctly send each other messages. - result = Dart_SortClasses(); - if (Dart_IsError(result)) goto failed; - } - // Make the isolate runnable so that it is ready to handle messages. Dart_ExitScope(); Dart_ExitIsolate(); diff --git a/runtime/bin/process_android.cc b/runtime/bin/process_android.cc index bd539e8733b..44df4d692d2 100644 --- a/runtime/bin/process_android.cc +++ b/runtime/bin/process_android.cc @@ -1027,9 +1027,6 @@ intptr_t Process::SetSignalHandler(intptr_t signal) { } void Process::ClearSignalHandler(intptr_t signal, Dart_Port port) { - // Either the port is illegal or there is no current isolate, but not both. - ASSERT((port != ILLEGAL_PORT) || (Dart_CurrentIsolate() == NULL)); - ASSERT((port == ILLEGAL_PORT) || (Dart_CurrentIsolate() != NULL)); ThreadSignalBlocker blocker(kSignalsCount, kSignals); MutexLocker lock(signal_mutex); SignalInfo* handler = signal_handlers; diff --git a/runtime/bin/process_linux.cc b/runtime/bin/process_linux.cc index f6ec06d24d4..5d82e1b8f8d 100644 --- a/runtime/bin/process_linux.cc +++ b/runtime/bin/process_linux.cc @@ -1023,9 +1023,6 @@ intptr_t Process::SetSignalHandler(intptr_t signal) { } void Process::ClearSignalHandler(intptr_t signal, Dart_Port port) { - // Either the port is illegal or there is no current isolate, but not both. - ASSERT((port != ILLEGAL_PORT) || (Dart_CurrentIsolate() == NULL)); - ASSERT((port == ILLEGAL_PORT) || (Dart_CurrentIsolate() != NULL)); ThreadSignalBlocker blocker(kSignalsCount, kSignals); MutexLocker lock(signal_mutex); SignalInfo* handler = signal_handlers; diff --git a/runtime/bin/process_macos.cc b/runtime/bin/process_macos.cc index 0f9287ba75f..536b1b33c01 100644 --- a/runtime/bin/process_macos.cc +++ b/runtime/bin/process_macos.cc @@ -1055,9 +1055,6 @@ intptr_t Process::SetSignalHandler(intptr_t signal) { } void Process::ClearSignalHandler(intptr_t signal, Dart_Port port) { - // Either the port is illegal or there is no current isolate, but not both. - ASSERT((port != ILLEGAL_PORT) || (Dart_CurrentIsolate() == NULL)); - ASSERT((port == ILLEGAL_PORT) || (Dart_CurrentIsolate() != NULL)); signal = SignalMap(signal); if (signal == -1) { return; diff --git a/runtime/observatory/tests/service/get_vm_timeline_rpc_test.dart b/runtime/observatory/tests/service/get_vm_timeline_rpc_test.dart index a5fec01d178..f68010b6c6e 100644 --- a/runtime/observatory/tests/service/get_vm_timeline_rpc_test.dart +++ b/runtime/observatory/tests/service/get_vm_timeline_rpc_test.dart @@ -110,7 +110,10 @@ void allEventsHaveIsolateNumber(List events) { } Map arguments = event['args']; expect(arguments, new isInstanceOf()); - expect(arguments['isolateId'], new isInstanceOf()); + expect(arguments['isolateGroupId'], new isInstanceOf()); + if (event['cat'] != 'GC') { + expect(arguments['isolateId'], new isInstanceOf()); + } } } diff --git a/runtime/vm/class_finalizer.cc b/runtime/vm/class_finalizer.cc index ae43c7bf3af..dab3350986c 100644 --- a/runtime/vm/class_finalizer.cc +++ b/runtime/vm/class_finalizer.cc @@ -2,6 +2,9 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +#include +#include + #include "vm/class_finalizer.h" #include "vm/compiler/jit/compiler.h" @@ -1389,7 +1392,9 @@ void ClassFinalizer::SortClasses() { ClassTable* table = I->class_table(); intptr_t num_cids = table->NumCids(); - intptr_t* old_to_new_cid = new intptr_t[num_cids]; + + std::unique_ptr old_to_new_cid(new intptr_t[num_cids]); + for (intptr_t cid = 0; cid < kNumPredefinedCids; cid++) { old_to_new_cid[cid] = cid; // The predefined classes cannot change cids. } @@ -1450,9 +1455,7 @@ void ClassFinalizer::SortClasses() { } } ASSERT(next_new_cid == num_cids); - - RemapClassIds(old_to_new_cid); - delete[] old_to_new_cid; + RemapClassIds(std::move(old_to_new_cid)); RehashTypes(); // Types use cid's as part of their hashes. I->RehashConstants(); // Const objects use cid's as part of their hashes. } @@ -1501,34 +1504,52 @@ class CidRewriteVisitor : public ObjectVisitor { intptr_t* old_to_new_cids_; }; -void ClassFinalizer::RemapClassIds(intptr_t* old_to_new_cid) { +void ClassFinalizer::RemapClassIds(std::unique_ptr old_to_new_cid) { Thread* T = Thread::Current(); - Isolate* I = T->isolate(); + IsolateGroup* IG = T->isolate_group(); // Code, ICData, allocation stubs have now-invalid cids. ClearAllCode(); { + // The [HeapIterationScope] also safepoints all threads. HeapIterationScope his(T); - I->set_remapping_cids(true); - // Update the class table. Do it before rewriting cids in headers, as the - // heap walkers load an object's size *after* calling the visitor. - I->class_table()->Remap(old_to_new_cid); + IG->class_table()->Remap(old_to_new_cid.get()); + IG->ForEachIsolate( + [&](Isolate* I) { + I->set_remapping_cids(true); + + // Update the class table. Do it before rewriting cids in headers, as + // the heap walkers load an object's size *after* calling the visitor. + I->class_table()->Remap(old_to_new_cid.get()); + }, + /*is_at_safepoint=*/true); // Rewrite cids in headers and cids in Classes, Fields, Types and // TypeParameters. { - CidRewriteVisitor visitor(old_to_new_cid); - I->heap()->VisitObjects(&visitor); + CidRewriteVisitor visitor(old_to_new_cid.get()); + IG->heap()->VisitObjects(&visitor); } - I->set_remapping_cids(false); + + IG->ForEachIsolate( + [&](Isolate* I) { + I->set_remapping_cids(false); +#if defined(DEBUG) + I->class_table()->Validate(); +#endif + }, + /*is_at_safepoint=*/true); } #if defined(DEBUG) - I->class_table()->Validate(); - I->heap()->Verify(); + IG->heap()->Verify(); #endif + + // Ensure any newly spawned isolate will apply this permutation map right + // after kernel loading. + IG->source()->cid_permutation_map = std::move(old_to_new_cid); } // Clears the cached canonicalized hash codes for all instances which directly diff --git a/runtime/vm/class_finalizer.h b/runtime/vm/class_finalizer.h index 4a82e1c458c..840bb7b77fc 100644 --- a/runtime/vm/class_finalizer.h +++ b/runtime/vm/class_finalizer.h @@ -5,6 +5,8 @@ #ifndef RUNTIME_VM_CLASS_FINALIZER_H_ #define RUNTIME_VM_CLASS_FINALIZER_H_ +#include + #include "vm/allocation.h" #include "vm/growable_array.h" #include "vm/object.h" @@ -41,7 +43,7 @@ class ClassFinalizer : public AllStatic { // Useful for sorting classes to make dispatch faster. static void SortClasses(); - static void RemapClassIds(intptr_t* old_to_new_cid); + static void RemapClassIds(std::unique_ptr old_to_new_cid); static void RehashTypes(); static void ClearAllCode(bool including_nonchanging_cids = false); diff --git a/runtime/vm/class_table.cc b/runtime/vm/class_table.cc index 929a4b8e6bf..df84820a5a6 100644 --- a/runtime/vm/class_table.cc +++ b/runtime/vm/class_table.cc @@ -33,7 +33,7 @@ SharedClassTable::SharedClassTable() table_ = static_cast(calloc(capacity_, sizeof(intptr_t))); } else { // Duplicate the class table from the VM isolate. - auto vm_shared_class_table = Dart::vm_isolate()->shared_class_table(); + auto vm_shared_class_table = Dart::vm_isolate()->group()->class_table(); capacity_ = vm_shared_class_table->capacity_; // Note that [calloc] will zero-initialize the memory. table_ = static_cast(calloc(capacity_, sizeof(RawClass*))); @@ -114,14 +114,6 @@ ClassTable::ClassTable(SharedClassTable* shared_class_table) } } -ClassTable::ClassTable(ClassTable* original, - SharedClassTable* shared_class_table) - : top_(original->top_), - capacity_(original->top_), - table_(original->table_), - old_class_tables_(nullptr), - shared_class_table_(shared_class_table) {} - ClassTable::~ClassTable() { if (old_class_tables_ != nullptr) { FreeOldTables(); @@ -308,26 +300,23 @@ void SharedClassTable::Unregister(intptr_t index) { void ClassTable::Remap(intptr_t* old_to_new_cid) { ASSERT(Thread::Current()->IsAtSafepoint()); - shared_class_table_->Remap(old_to_new_cid); - const intptr_t num_cids = NumCids(); - auto cls_by_old_cid = new RawClass*[num_cids]; - memmove(cls_by_old_cid, table_, sizeof(RawClass*) * num_cids); + std::unique_ptr cls_by_old_cid(new RawClass*[num_cids]); + memmove(cls_by_old_cid.get(), table_, sizeof(RawClass*) * num_cids); for (intptr_t i = 0; i < num_cids; i++) { table_[old_to_new_cid[i]] = cls_by_old_cid[i]; } - delete[] cls_by_old_cid; } void SharedClassTable::Remap(intptr_t* old_to_new_cid) { ASSERT(Thread::Current()->IsAtSafepoint()); const intptr_t num_cids = NumCids(); - std::unique_ptr cls_by_old_cid(new intptr_t[num_cids]); + std::unique_ptr size_by_old_cid(new intptr_t[num_cids]); for (intptr_t i = 0; i < num_cids; i++) { - cls_by_old_cid[i] = table_[i]; + size_by_old_cid[i] = table_[i]; } for (intptr_t i = 0; i < num_cids; i++) { - table_[old_to_new_cid[i]] = cls_by_old_cid[i]; + table_[old_to_new_cid[i]] = size_by_old_cid[i]; } #if defined(SUPPORT_UNBOXED_INSTANCE_FIELDS) @@ -428,18 +417,20 @@ intptr_t SharedClassTable::ClassOffsetFor(intptr_t cid) { void ClassTable::AllocationProfilePrintJSON(JSONStream* stream, bool internal) { Isolate* isolate = Isolate::Current(); ASSERT(isolate != NULL); - Heap* heap = isolate->heap(); + auto isolate_group = isolate->group(); + Heap* heap = isolate_group->heap(); ASSERT(heap != NULL); JSONObject obj(stream); obj.AddProperty("type", "AllocationProfile"); - if (isolate->last_allocationprofile_accumulator_reset_timestamp() != 0) { + if (isolate_group->last_allocationprofile_accumulator_reset_timestamp() != + 0) { obj.AddPropertyF( "dateLastAccumulatorReset", "%" Pd64 "", - isolate->last_allocationprofile_accumulator_reset_timestamp()); + isolate_group->last_allocationprofile_accumulator_reset_timestamp()); } - if (isolate->last_allocationprofile_gc_timestamp() != 0) { + if (isolate_group->last_allocationprofile_gc_timestamp() != 0) { obj.AddPropertyF("dateLastServiceGC", "%" Pd64 "", - isolate->last_allocationprofile_gc_timestamp()); + isolate_group->last_allocationprofile_gc_timestamp()); } if (internal) { @@ -458,7 +449,7 @@ void ClassTable::AllocationProfilePrintJSON(JSONStream* stream, bool internal) { { HeapIterationScope iter(thread); iter.IterateObjects(&visitor); - isolate->VisitWeakPersistentHandles(&visitor); + isolate->group()->VisitWeakPersistentHandles(&visitor); } { diff --git a/runtime/vm/class_table.h b/runtime/vm/class_table.h index 62383d3794e..990560bea09 100644 --- a/runtime/vm/class_table.h +++ b/runtime/vm/class_table.h @@ -5,6 +5,8 @@ #ifndef RUNTIME_VM_CLASS_TABLE_H_ #define RUNTIME_VM_CLASS_TABLE_H_ +#include + #include "platform/assert.h" #include "platform/atomic.h" #include "platform/utils.h" @@ -80,6 +82,7 @@ class SharedClassTable { void SetSizeAt(intptr_t index, intptr_t size) { ASSERT(IsValidIndex(index)); + // Ensure we never change size for a given cid from one non-zero size to // another non-zero size. RELEASE_ASSERT(table_[index] == 0 || table_[index] == size); @@ -230,10 +233,6 @@ class SharedClassTable { class ClassTable { public: explicit ClassTable(SharedClassTable* shared_class_table_); - - // Creates a shallow copy of the original class table for some read-only - // access, without support for stats data. - ClassTable(ClassTable* original, SharedClassTable* shared_class_table); ~ClassTable(); SharedClassTable* shared_class_table() const { return shared_class_table_; } @@ -357,6 +356,9 @@ class ClassTable { friend class MarkingWeakVisitor; friend class Scavenger; friend class ScavengerWeakVisitor; + friend Isolate* CreateWithinExistingIsolateGroup(IsolateGroup* group, + const char* name, + char** error); static const int kInitialCapacity = SharedClassTable::kInitialCapacity; static const int kCapacityIncrement = SharedClassTable::kCapacityIncrement; diff --git a/runtime/vm/clustered_snapshot.cc b/runtime/vm/clustered_snapshot.cc index a5a7bba1897..a791ce11ff7 100644 --- a/runtime/vm/clustered_snapshot.cc +++ b/runtime/vm/clustered_snapshot.cc @@ -206,7 +206,7 @@ class ClassSerializationCluster : public SerializationCluster { UnboxedFieldBitmap CalculateTargetUnboxedFieldsBitmap(Serializer* s, intptr_t class_id) { const auto unboxed_fields_bitmap_host = - s->isolate()->shared_class_table()->GetUnboxedFieldsMapAt(class_id); + s->isolate()->group()->class_table()->GetUnboxedFieldsMapAt(class_id); UnboxedFieldBitmap unboxed_fields_bitmap; if (unboxed_fields_bitmap_host.IsEmpty() || @@ -310,6 +310,7 @@ class ClassDeserializationCluster : public DeserializationCluster { } } + auto shared_class_table = d->isolate()->group()->class_table(); for (intptr_t id = start_index_; id < stop_index_; id++) { RawClass* cls = reinterpret_cast(d->Ref(id)); Deserializer::InitializeHeader(cls, kClassCid, Class::InstanceSize()); @@ -347,8 +348,7 @@ class ClassDeserializationCluster : public DeserializationCluster { if (FLAG_precompiled_mode) { const UnboxedFieldBitmap unboxed_fields_map(d->ReadUnsigned64()); - d->isolate()->shared_class_table()->SetUnboxedFieldsMapAt( - class_id, unboxed_fields_map); + shared_class_table->SetUnboxedFieldsMapAt(class_id, unboxed_fields_map); } } } @@ -2802,7 +2802,7 @@ class InstanceSerializationCluster : public SerializationCluster { const intptr_t next_field_offset = host_next_field_offset_in_words_ << kWordSizeLog2; const auto unboxed_fields_bitmap = - s->isolate()->shared_class_table()->GetUnboxedFieldsMapAt(cid_); + s->isolate()->group()->class_table()->GetUnboxedFieldsMapAt(cid_); intptr_t offset = Instance::NextFieldOffset(); while (offset < next_field_offset) { // Skips unboxed fields @@ -2837,13 +2837,12 @@ class InstanceSerializationCluster : public SerializationCluster { intptr_t next_field_offset = host_next_field_offset_in_words_ << kWordSizeLog2; const intptr_t count = objects_.length(); - const auto shared_class_table = s->isolate()->shared_class_table(); + const auto unboxed_fields_bitmap = + s->isolate()->group()->class_table()->GetUnboxedFieldsMapAt(cid_); for (intptr_t i = 0; i < count; i++) { RawInstance* instance = objects_[i]; AutoTraceObject(instance); s->Write(instance->IsCanonical()); - const auto unboxed_fields_bitmap = - shared_class_table->GetUnboxedFieldsMapAt(cid_); intptr_t offset = Instance::NextFieldOffset(); while (offset < next_field_offset) { if (unboxed_fields_bitmap.Get(offset / kWordSize)) { @@ -2896,14 +2895,13 @@ class InstanceDeserializationCluster : public DeserializationCluster { intptr_t instance_size = Object::RoundedAllocationSize(instance_size_in_words_ * kWordSize); - const auto shared_class_table = d->isolate()->shared_class_table(); + const auto unboxed_fields_bitmap = + d->isolate()->group()->class_table()->GetUnboxedFieldsMapAt(cid_); for (intptr_t id = start_index_; id < stop_index_; id++) { RawInstance* instance = reinterpret_cast(d->Ref(id)); bool is_canonical = d->Read(); Deserializer::InitializeHeader(instance, cid_, instance_size, is_canonical); - const auto unboxed_fields_bitmap = - shared_class_table->GetUnboxedFieldsMapAt(cid_); intptr_t offset = Instance::NextFieldOffset(); while (offset < next_field_offset) { if (unboxed_fields_bitmap.Get(offset / kWordSize)) { diff --git a/runtime/vm/compiler/aot/precompiler.cc b/runtime/vm/compiler/aot/precompiler.cc index 8ca56074109..f7663d5e0a0 100644 --- a/runtime/vm/compiler/aot/precompiler.cc +++ b/runtime/vm/compiler/aot/precompiler.cc @@ -946,7 +946,7 @@ void Precompiler::AddConstObject(const class Instance& instance) { class ConstObjectVisitor : public ObjectPointerVisitor { public: ConstObjectVisitor(Precompiler* precompiler, Isolate* isolate) - : ObjectPointerVisitor(isolate), + : ObjectPointerVisitor(isolate->group()), precompiler_(precompiler), subinstance_(Object::Handle()) {} diff --git a/runtime/vm/compiler/runtime_offsets_extracted.h b/runtime/vm/compiler/runtime_offsets_extracted.h index b6f385e750b..128aa1405f0 100644 --- a/runtime/vm/compiler/runtime_offsets_extracted.h +++ b/runtime/vm/compiler/runtime_offsets_extracted.h @@ -135,12 +135,12 @@ static constexpr dart::compiler::target::word ICData_owner_offset = 20; static constexpr dart::compiler::target::word ICData_state_bits_offset = 28; static constexpr dart::compiler::target::word ICData_receivers_static_type_offset = 16; -static constexpr dart::compiler::target::word Isolate_class_table_offset = 40; +static constexpr dart::compiler::target::word Isolate_class_table_offset = 36; static constexpr dart::compiler::target::word Isolate_current_tag_offset = 20; static constexpr dart::compiler::target::word Isolate_default_tag_offset = 24; static constexpr dart::compiler::target::word Isolate_ic_miss_code_offset = 28; -static constexpr dart::compiler::target::word Isolate_object_store_offset = 36; -static constexpr dart::compiler::target::word Isolate_single_step_offset = 64; +static constexpr dart::compiler::target::word Isolate_object_store_offset = 32; +static constexpr dart::compiler::target::word Isolate_single_step_offset = 60; static constexpr dart::compiler::target::word Isolate_user_tag_offset = 16; static constexpr dart::compiler::target::word LinkedHashMap_data_offset = 16; static constexpr dart::compiler::target::word @@ -600,12 +600,12 @@ static constexpr dart::compiler::target::word ICData_owner_offset = 40; static constexpr dart::compiler::target::word ICData_state_bits_offset = 52; static constexpr dart::compiler::target::word ICData_receivers_static_type_offset = 32; -static constexpr dart::compiler::target::word Isolate_class_table_offset = 80; +static constexpr dart::compiler::target::word Isolate_class_table_offset = 72; static constexpr dart::compiler::target::word Isolate_current_tag_offset = 40; static constexpr dart::compiler::target::word Isolate_default_tag_offset = 48; static constexpr dart::compiler::target::word Isolate_ic_miss_code_offset = 56; -static constexpr dart::compiler::target::word Isolate_object_store_offset = 72; -static constexpr dart::compiler::target::word Isolate_single_step_offset = 128; +static constexpr dart::compiler::target::word Isolate_object_store_offset = 64; +static constexpr dart::compiler::target::word Isolate_single_step_offset = 120; static constexpr dart::compiler::target::word Isolate_user_tag_offset = 32; static constexpr dart::compiler::target::word LinkedHashMap_data_offset = 32; static constexpr dart::compiler::target::word @@ -1067,12 +1067,12 @@ static constexpr dart::compiler::target::word ICData_owner_offset = 20; static constexpr dart::compiler::target::word ICData_state_bits_offset = 28; static constexpr dart::compiler::target::word ICData_receivers_static_type_offset = 16; -static constexpr dart::compiler::target::word Isolate_class_table_offset = 40; +static constexpr dart::compiler::target::word Isolate_class_table_offset = 36; static constexpr dart::compiler::target::word Isolate_current_tag_offset = 20; static constexpr dart::compiler::target::word Isolate_default_tag_offset = 24; static constexpr dart::compiler::target::word Isolate_ic_miss_code_offset = 28; -static constexpr dart::compiler::target::word Isolate_object_store_offset = 36; -static constexpr dart::compiler::target::word Isolate_single_step_offset = 64; +static constexpr dart::compiler::target::word Isolate_object_store_offset = 32; +static constexpr dart::compiler::target::word Isolate_single_step_offset = 60; static constexpr dart::compiler::target::word Isolate_user_tag_offset = 16; static constexpr dart::compiler::target::word LinkedHashMap_data_offset = 16; static constexpr dart::compiler::target::word @@ -1529,12 +1529,12 @@ static constexpr dart::compiler::target::word ICData_owner_offset = 40; static constexpr dart::compiler::target::word ICData_state_bits_offset = 52; static constexpr dart::compiler::target::word ICData_receivers_static_type_offset = 32; -static constexpr dart::compiler::target::word Isolate_class_table_offset = 80; +static constexpr dart::compiler::target::word Isolate_class_table_offset = 72; static constexpr dart::compiler::target::word Isolate_current_tag_offset = 40; static constexpr dart::compiler::target::word Isolate_default_tag_offset = 48; static constexpr dart::compiler::target::word Isolate_ic_miss_code_offset = 56; -static constexpr dart::compiler::target::word Isolate_object_store_offset = 72; -static constexpr dart::compiler::target::word Isolate_single_step_offset = 128; +static constexpr dart::compiler::target::word Isolate_object_store_offset = 64; +static constexpr dart::compiler::target::word Isolate_single_step_offset = 120; static constexpr dart::compiler::target::word Isolate_user_tag_offset = 32; static constexpr dart::compiler::target::word LinkedHashMap_data_offset = 32; static constexpr dart::compiler::target::word @@ -1997,11 +1997,11 @@ static constexpr dart::compiler::target::word ICData_owner_offset = 20; static constexpr dart::compiler::target::word ICData_state_bits_offset = 28; static constexpr dart::compiler::target::word ICData_receivers_static_type_offset = 16; -static constexpr dart::compiler::target::word Isolate_class_table_offset = 40; +static constexpr dart::compiler::target::word Isolate_class_table_offset = 36; static constexpr dart::compiler::target::word Isolate_current_tag_offset = 20; static constexpr dart::compiler::target::word Isolate_default_tag_offset = 24; static constexpr dart::compiler::target::word Isolate_ic_miss_code_offset = 28; -static constexpr dart::compiler::target::word Isolate_object_store_offset = 36; +static constexpr dart::compiler::target::word Isolate_object_store_offset = 32; static constexpr dart::compiler::target::word Isolate_user_tag_offset = 16; static constexpr dart::compiler::target::word LinkedHashMap_data_offset = 16; static constexpr dart::compiler::target::word @@ -2456,11 +2456,11 @@ static constexpr dart::compiler::target::word ICData_owner_offset = 40; static constexpr dart::compiler::target::word ICData_state_bits_offset = 52; static constexpr dart::compiler::target::word ICData_receivers_static_type_offset = 32; -static constexpr dart::compiler::target::word Isolate_class_table_offset = 80; +static constexpr dart::compiler::target::word Isolate_class_table_offset = 72; static constexpr dart::compiler::target::word Isolate_current_tag_offset = 40; static constexpr dart::compiler::target::word Isolate_default_tag_offset = 48; static constexpr dart::compiler::target::word Isolate_ic_miss_code_offset = 56; -static constexpr dart::compiler::target::word Isolate_object_store_offset = 72; +static constexpr dart::compiler::target::word Isolate_object_store_offset = 64; static constexpr dart::compiler::target::word Isolate_user_tag_offset = 32; static constexpr dart::compiler::target::word LinkedHashMap_data_offset = 32; static constexpr dart::compiler::target::word @@ -2917,11 +2917,11 @@ static constexpr dart::compiler::target::word ICData_owner_offset = 20; static constexpr dart::compiler::target::word ICData_state_bits_offset = 28; static constexpr dart::compiler::target::word ICData_receivers_static_type_offset = 16; -static constexpr dart::compiler::target::word Isolate_class_table_offset = 40; +static constexpr dart::compiler::target::word Isolate_class_table_offset = 36; static constexpr dart::compiler::target::word Isolate_current_tag_offset = 20; static constexpr dart::compiler::target::word Isolate_default_tag_offset = 24; static constexpr dart::compiler::target::word Isolate_ic_miss_code_offset = 28; -static constexpr dart::compiler::target::word Isolate_object_store_offset = 36; +static constexpr dart::compiler::target::word Isolate_object_store_offset = 32; static constexpr dart::compiler::target::word Isolate_user_tag_offset = 16; static constexpr dart::compiler::target::word LinkedHashMap_data_offset = 16; static constexpr dart::compiler::target::word @@ -3373,11 +3373,11 @@ static constexpr dart::compiler::target::word ICData_owner_offset = 40; static constexpr dart::compiler::target::word ICData_state_bits_offset = 52; static constexpr dart::compiler::target::word ICData_receivers_static_type_offset = 32; -static constexpr dart::compiler::target::word Isolate_class_table_offset = 80; +static constexpr dart::compiler::target::word Isolate_class_table_offset = 72; static constexpr dart::compiler::target::word Isolate_current_tag_offset = 40; static constexpr dart::compiler::target::word Isolate_default_tag_offset = 48; static constexpr dart::compiler::target::word Isolate_ic_miss_code_offset = 56; -static constexpr dart::compiler::target::word Isolate_object_store_offset = 72; +static constexpr dart::compiler::target::word Isolate_object_store_offset = 64; static constexpr dart::compiler::target::word Isolate_user_tag_offset = 32; static constexpr dart::compiler::target::word LinkedHashMap_data_offset = 32; static constexpr dart::compiler::target::word @@ -3836,7 +3836,7 @@ static constexpr dart::compiler::target::word AOT_ICData_arguments_descriptor_offset = 12; static constexpr dart::compiler::target::word AOT_ICData_entries_offset = 4; static constexpr dart::compiler::target::word AOT_Isolate_class_table_offset = - 40; + 36; static constexpr dart::compiler::target::word AOT_Isolate_current_tag_offset = 20; static constexpr dart::compiler::target::word AOT_Isolate_default_tag_offset = @@ -3844,9 +3844,9 @@ static constexpr dart::compiler::target::word AOT_Isolate_default_tag_offset = static constexpr dart::compiler::target::word AOT_Isolate_ic_miss_code_offset = 28; static constexpr dart::compiler::target::word AOT_Isolate_object_store_offset = - 36; + 32; static constexpr dart::compiler::target::word AOT_Isolate_single_step_offset = - 64; + 60; static constexpr dart::compiler::target::word AOT_Isolate_user_tag_offset = 16; static constexpr dart::compiler::target::word AOT_LinkedHashMap_data_offset = 16; @@ -4344,7 +4344,7 @@ static constexpr dart::compiler::target::word AOT_ICData_arguments_descriptor_offset = 24; static constexpr dart::compiler::target::word AOT_ICData_entries_offset = 8; static constexpr dart::compiler::target::word AOT_Isolate_class_table_offset = - 80; + 72; static constexpr dart::compiler::target::word AOT_Isolate_current_tag_offset = 40; static constexpr dart::compiler::target::word AOT_Isolate_default_tag_offset = @@ -4352,9 +4352,9 @@ static constexpr dart::compiler::target::word AOT_Isolate_default_tag_offset = static constexpr dart::compiler::target::word AOT_Isolate_ic_miss_code_offset = 56; static constexpr dart::compiler::target::word AOT_Isolate_object_store_offset = - 72; + 64; static constexpr dart::compiler::target::word AOT_Isolate_single_step_offset = - 128; + 120; static constexpr dart::compiler::target::word AOT_Isolate_user_tag_offset = 32; static constexpr dart::compiler::target::word AOT_LinkedHashMap_data_offset = 32; @@ -4858,7 +4858,7 @@ static constexpr dart::compiler::target::word AOT_ICData_arguments_descriptor_offset = 24; static constexpr dart::compiler::target::word AOT_ICData_entries_offset = 8; static constexpr dart::compiler::target::word AOT_Isolate_class_table_offset = - 80; + 72; static constexpr dart::compiler::target::word AOT_Isolate_current_tag_offset = 40; static constexpr dart::compiler::target::word AOT_Isolate_default_tag_offset = @@ -4866,9 +4866,9 @@ static constexpr dart::compiler::target::word AOT_Isolate_default_tag_offset = static constexpr dart::compiler::target::word AOT_Isolate_ic_miss_code_offset = 56; static constexpr dart::compiler::target::word AOT_Isolate_object_store_offset = - 72; + 64; static constexpr dart::compiler::target::word AOT_Isolate_single_step_offset = - 128; + 120; static constexpr dart::compiler::target::word AOT_Isolate_user_tag_offset = 32; static constexpr dart::compiler::target::word AOT_LinkedHashMap_data_offset = 32; @@ -5370,7 +5370,7 @@ static constexpr dart::compiler::target::word AOT_ICData_arguments_descriptor_offset = 12; static constexpr dart::compiler::target::word AOT_ICData_entries_offset = 4; static constexpr dart::compiler::target::word AOT_Isolate_class_table_offset = - 40; + 36; static constexpr dart::compiler::target::word AOT_Isolate_current_tag_offset = 20; static constexpr dart::compiler::target::word AOT_Isolate_default_tag_offset = @@ -5378,7 +5378,7 @@ static constexpr dart::compiler::target::word AOT_Isolate_default_tag_offset = static constexpr dart::compiler::target::word AOT_Isolate_ic_miss_code_offset = 28; static constexpr dart::compiler::target::word AOT_Isolate_object_store_offset = - 36; + 32; static constexpr dart::compiler::target::word AOT_Isolate_user_tag_offset = 16; static constexpr dart::compiler::target::word AOT_LinkedHashMap_data_offset = 16; @@ -5871,7 +5871,7 @@ static constexpr dart::compiler::target::word AOT_ICData_arguments_descriptor_offset = 24; static constexpr dart::compiler::target::word AOT_ICData_entries_offset = 8; static constexpr dart::compiler::target::word AOT_Isolate_class_table_offset = - 80; + 72; static constexpr dart::compiler::target::word AOT_Isolate_current_tag_offset = 40; static constexpr dart::compiler::target::word AOT_Isolate_default_tag_offset = @@ -5879,7 +5879,7 @@ static constexpr dart::compiler::target::word AOT_Isolate_default_tag_offset = static constexpr dart::compiler::target::word AOT_Isolate_ic_miss_code_offset = 56; static constexpr dart::compiler::target::word AOT_Isolate_object_store_offset = - 72; + 64; static constexpr dart::compiler::target::word AOT_Isolate_user_tag_offset = 32; static constexpr dart::compiler::target::word AOT_LinkedHashMap_data_offset = 32; @@ -6378,7 +6378,7 @@ static constexpr dart::compiler::target::word AOT_ICData_arguments_descriptor_offset = 24; static constexpr dart::compiler::target::word AOT_ICData_entries_offset = 8; static constexpr dart::compiler::target::word AOT_Isolate_class_table_offset = - 80; + 72; static constexpr dart::compiler::target::word AOT_Isolate_current_tag_offset = 40; static constexpr dart::compiler::target::word AOT_Isolate_default_tag_offset = @@ -6386,7 +6386,7 @@ static constexpr dart::compiler::target::word AOT_Isolate_default_tag_offset = static constexpr dart::compiler::target::word AOT_Isolate_ic_miss_code_offset = 56; static constexpr dart::compiler::target::word AOT_Isolate_object_store_offset = - 72; + 64; static constexpr dart::compiler::target::word AOT_Isolate_user_tag_offset = 32; static constexpr dart::compiler::target::word AOT_LinkedHashMap_data_offset = 32; diff --git a/runtime/vm/dart.cc b/runtime/vm/dart.cc index 457c2194f88..badb1ebbef1 100644 --- a/runtime/vm/dart.cc +++ b/runtime/vm/dart.cc @@ -249,6 +249,8 @@ char* Dart::Init(const uint8_t* vm_isolate_snapshot, new IsolateGroupSource(nullptr, kVmIsolateName, vm_isolate_snapshot, instructions_snapshot, nullptr, -1, api_flags)); auto group = new IsolateGroup(std::move(source), /*embedder_data=*/nullptr); + group->CreateHeap(/*is_vm_isolate=*/true, + /*is_service_or_kernel_isolate=*/false); IsolateGroup::RegisterIsolateGroup(group); vm_isolate_ = Isolate::InitIsolate(kVmIsolateName, group, api_flags, is_vm_isolate); @@ -947,18 +949,7 @@ void Dart::ShutdownIsolate(Isolate* isolate) { } void Dart::ShutdownIsolate() { - Isolate* isolate = Isolate::Current(); - const bool is_application_isolate = !Isolate::IsVMInternalIsolate(isolate); - isolate->Shutdown(); - if (KernelIsolate::IsKernelIsolate(isolate)) { - KernelIsolate::SetKernelIsolate(NULL); - } - delete isolate; - - // Only now do we know for sure that the isolate and all it's resources have - // been deleted. So we can let any potential Dart::Cleanup() know it's safe to - // proceed shutdown of the VM. - Isolate::MarkIsolateDead(is_application_isolate); + Isolate::Current()->Shutdown(); } bool Dart::VmIsolateNameEquals(const char* name) { diff --git a/runtime/vm/dart_api_impl.cc b/runtime/vm/dart_api_impl.cc index 5ebade9ae5b..c6de3da8afc 100644 --- a/runtime/vm/dart_api_impl.cc +++ b/runtime/vm/dart_api_impl.cc @@ -374,7 +374,7 @@ RawObject* Api::UnwrapHandle(Dart_Handle object) { ASSERT(thread->IsMutatorThread()); ASSERT(thread->isolate() != NULL); ASSERT(!FLAG_verify_handles || thread->IsValidLocalHandle(object) || - thread->isolate()->api_state()->IsActivePersistentHandle( + thread->isolate()->group()->api_state()->IsActivePersistentHandle( reinterpret_cast(object)) || Dart::IsReadOnlyApiHandle(object)); ASSERT(FinalizablePersistentHandle::raw_offset() == 0 && @@ -477,9 +477,8 @@ Dart_Handle Api::NewArgumentError(const char* format, ...) { return Api::NewHandle(T, error.raw()); } -Dart_Handle Api::AcquiredError(Isolate* isolate) { - ASSERT(isolate != NULL); - ApiState* state = isolate->api_state(); +Dart_Handle Api::AcquiredError(IsolateGroup* isolate_group) { + ApiState* state = isolate_group->api_state(); ASSERT(state != NULL); PersistentHandle* acquired_error_handle = state->AcquiredError(); return reinterpret_cast(acquired_error_handle); @@ -494,9 +493,9 @@ bool Api::IsValid(Dart_Handle handle) { // Check against all of the handles in the current isolate as well as the // read-only handles. return thread->IsValidHandle(handle) || - isolate->api_state()->IsActivePersistentHandle( + isolate->group()->api_state()->IsActivePersistentHandle( reinterpret_cast(handle)) || - isolate->api_state()->IsActiveWeakPersistentHandle( + isolate->group()->api_state()->IsActiveWeakPersistentHandle( reinterpret_cast(handle)) || Dart::IsReadOnlyApiHandle(handle) || Dart::IsReadOnlyHandle(reinterpret_cast(handle)); @@ -527,7 +526,7 @@ void Api::InitHandles() { Isolate* isolate = Isolate::Current(); ASSERT(isolate != NULL); ASSERT(isolate == Dart::vm_isolate()); - ApiState* state = isolate->api_state(); + ApiState* state = isolate->group()->api_state(); ASSERT(state != NULL); ASSERT(true_handle_ == NULL); @@ -688,21 +687,21 @@ void Api::SetWeakHandleReturnValue(NativeArguments* args, } PersistentHandle* PersistentHandle::Cast(Dart_PersistentHandle handle) { - ASSERT(Isolate::Current()->api_state()->IsValidPersistentHandle(handle)); + ASSERT(IsolateGroup::Current()->api_state()->IsValidPersistentHandle(handle)); return reinterpret_cast(handle); } FinalizablePersistentHandle* FinalizablePersistentHandle::Cast( Dart_WeakPersistentHandle handle) { #if defined(DEBUG) - ApiState* state = Isolate::Current()->api_state(); + ApiState* state = IsolateGroup::Current()->api_state(); ASSERT(state->IsValidWeakPersistentHandle(handle)); #endif return reinterpret_cast(handle); } void FinalizablePersistentHandle::Finalize( - Isolate* isolate, + IsolateGroup* isolate_group, FinalizablePersistentHandle* handle) { if (!handle->raw()->IsHeapObject()) { return; // Free handle. @@ -711,8 +710,8 @@ void FinalizablePersistentHandle::Finalize( ASSERT(callback != NULL); void* peer = handle->peer(); Dart_WeakPersistentHandle object = handle->apiHandle(); - (*callback)(isolate->init_callback_data(), object, peer); - ApiState* state = isolate->api_state(); + (*callback)(isolate_group->embedder_data(), object, peer); + ApiState* state = isolate_group->api_state(); ASSERT(state != NULL); state->FreeWeakPersistentHandle(handle); } @@ -908,7 +907,7 @@ Dart_HandleFromPersistent(Dart_PersistentHandle object) { Thread* thread = Thread::Current(); Isolate* isolate = thread->isolate(); CHECK_ISOLATE(isolate); - ApiState* state = isolate->api_state(); + ApiState* state = isolate->group()->api_state(); ASSERT(state != NULL); TransitionNativeToVM transition(thread); NoSafepointScope no_safepoint_scope; @@ -921,7 +920,7 @@ Dart_HandleFromWeakPersistent(Dart_WeakPersistentHandle object) { Thread* thread = Thread::Current(); Isolate* isolate = thread->isolate(); CHECK_ISOLATE(isolate); - ApiState* state = isolate->api_state(); + ApiState* state = isolate->group()->api_state(); ASSERT(state != NULL); TransitionNativeToVM transition(thread); NoSafepointScope no_safepoint_scope; @@ -933,7 +932,7 @@ Dart_HandleFromWeakPersistent(Dart_WeakPersistentHandle object) { DART_EXPORT Dart_PersistentHandle Dart_NewPersistentHandle(Dart_Handle object) { DARTSCOPE(Thread::Current()); Isolate* I = T->isolate(); - ApiState* state = I->api_state(); + ApiState* state = I->group()->api_state(); ASSERT(state != NULL); const Object& old_ref = Object::Handle(Z, Api::UnwrapHandle(object)); PersistentHandle* new_ref = state->AllocatePersistentHandle(); @@ -945,7 +944,7 @@ DART_EXPORT void Dart_SetPersistentHandle(Dart_PersistentHandle obj1, Dart_Handle obj2) { DARTSCOPE(Thread::Current()); Isolate* I = T->isolate(); - ApiState* state = I->api_state(); + ApiState* state = I->group()->api_state(); ASSERT(state != NULL); ASSERT(state->IsValidPersistentHandle(obj1)); const Object& obj2_ref = Object::Handle(Z, Api::UnwrapHandle(obj2)); @@ -1000,7 +999,7 @@ DART_EXPORT void Dart_DeletePersistentHandle(Dart_PersistentHandle object) { Isolate* isolate = Isolate::Current(); CHECK_ISOLATE(isolate); NoSafepointScope no_safepoint_scope; - ApiState* state = isolate->api_state(); + ApiState* state = isolate->group()->api_state(); ASSERT(state != NULL); PersistentHandle* ref = PersistentHandle::Cast(object); ASSERT(!state->IsProtectedHandle(ref)); @@ -1016,10 +1015,10 @@ DART_EXPORT void Dart_DeleteWeakPersistentHandle( CHECK_ISOLATE(isolate); NoSafepointScope no_safepoint_scope; ASSERT(isolate == Isolate::Current()); - ApiState* state = isolate->api_state(); + ApiState* state = isolate->group()->api_state(); ASSERT(state != NULL); auto weak_ref = FinalizablePersistentHandle::Cast(object); - weak_ref->EnsureFreeExternal(isolate); + weak_ref->EnsureFreeExternal(isolate->group()); state->FreeWeakPersistentHandle(weak_ref); } @@ -1073,6 +1072,17 @@ DART_EXPORT bool Dart_IsVMFlagSet(const char* flag_name) { VM_METRIC_LIST(VM_METRIC_API); #undef VM_METRIC_API +#define ISOLATE_GROUP_METRIC_API(type, variable, name, unit) \ + DART_EXPORT int64_t Dart_Isolate##variable##Metric(Dart_Isolate isolate) { \ + if (isolate == nullptr) { \ + FATAL1("%s expects argument 'isolate' to be non-null.", CURRENT_FUNC); \ + } \ + Isolate* iso = reinterpret_cast(isolate); \ + return iso->group()->Get##variable##Metric()->Value(); \ + } +ISOLATE_GROUP_METRIC_LIST(ISOLATE_GROUP_METRIC_API) +#undef ISOLATE_GROUP_METRIC_API + #define ISOLATE_METRIC_API(type, variable, name, unit) \ DART_EXPORT int64_t Dart_Isolate##variable##Metric(Dart_Isolate isolate) { \ if (isolate == NULL) { \ @@ -1081,24 +1091,28 @@ VM_METRIC_LIST(VM_METRIC_API); Isolate* iso = reinterpret_cast(isolate); \ return iso->Get##variable##Metric()->Value(); \ } -ISOLATE_METRIC_LIST(ISOLATE_METRIC_API); +ISOLATE_METRIC_LIST(ISOLATE_METRIC_API) #undef ISOLATE_METRIC_API + #else // !defined(PRODUCT) + #define VM_METRIC_API(type, variable, name, unit) \ DART_EXPORT int64_t Dart_VM##variable##Metric() { return -1; } -VM_METRIC_LIST(VM_METRIC_API); +VM_METRIC_LIST(VM_METRIC_API) #undef VM_METRIC_API #define ISOLATE_METRIC_API(type, variable, name, unit) \ DART_EXPORT int64_t Dart_Isolate##variable##Metric(Dart_Isolate isolate) { \ return -1; \ } -ISOLATE_METRIC_LIST(ISOLATE_METRIC_API); +ISOLATE_METRIC_LIST(ISOLATE_METRIC_API) +ISOLATE_GROUP_METRIC_LIST(ISOLATE_METRIC_API) #endif // !defined(PRODUCT) // --- Isolates --- static Dart_Isolate CreateIsolate(IsolateGroup* group, + bool is_new_group, const char* name, void* isolate_data, char** error) { @@ -1142,6 +1156,9 @@ static Dart_Isolate CreateIsolate(IsolateGroup* group, } if (success) { + if (is_new_group) { + I->heap()->InitGrowthControl(); + } // A Thread structure has been associated to the thread, we do the // safepoint transition explicitly here instead of using the // TransitionXXX scope objects as the reverse transition happens @@ -1158,17 +1175,42 @@ static Dart_Isolate CreateIsolate(IsolateGroup* group, return reinterpret_cast(NULL); } +static bool IsServiceOrKernelIsolateName(const char* name) { + if (ServiceIsolate::NameEquals(name)) { + ASSERT(!ServiceIsolate::Exists()); + return true; + } +#if !defined(DART_PRECOMPILED_RUNTIME) + if (KernelIsolate::NameEquals(name)) { + ASSERT(!KernelIsolate::Exists()); + return true; + } +#endif // !defined(DART_PRECOMPILED_RUNTIME) + return false; +} + Isolate* CreateWithinExistingIsolateGroup(IsolateGroup* group, const char* name, char** error) { API_TIMELINE_DURATION(Thread::Current()); CHECK_NO_ISOLATE(Isolate::Current()); + // During isolate start we'll make a temporary anonymous group from the same + // [source]. Once the isolate has been fully loaded we will merge it's heap + // into the shared heap. + auto spawning_group = new IsolateGroup(group->shareable_source(), + /*isolate_group_data=*/nullptr); + IsolateGroup::RegisterIsolateGroup(spawning_group); + spawning_group->CreateHeap( + /*is_vm_isolate=*/false, + IsServiceOrKernelIsolateName(group->source()->name)); + Isolate* isolate = reinterpret_cast( - CreateIsolate(group, name, /*isolate_data=*/nullptr, error)); + CreateIsolate(spawning_group, /*is_new_group=*/false, name, + /*isolate_data=*/nullptr, error)); if (isolate == nullptr) return nullptr; - auto source = group->source(); + auto source = spawning_group->source(); ASSERT(isolate->source() == source); if (source->script_kernel_buffer != nullptr) { @@ -1207,6 +1249,100 @@ Isolate* CreateWithinExistingIsolateGroup(IsolateGroup* group, #endif // defined(DART_PRECOMPILED_RUNTIME) } + // If we are running in AppJIT training mode we'll have to remap class ids. + if (auto permutation_map = group->source()->cid_permutation_map.get()) { + Dart_EnterScope(); + { + auto T = Thread::Current(); + TransitionNativeToVM transition(T); + HANDLESCOPE(T); + + const intptr_t num_cids = group->class_table()->NumCids(); + std::unique_ptr permutation_map_copy(new intptr_t[num_cids]); + for (intptr_t i = 0; i < num_cids; ++i) { + permutation_map_copy[i] = permutation_map[i]; + } + + // Remap all class ids loaded atm (e.g. from snapshot) and do appropriate + // re-hashing of constants and types. + ClassFinalizer::RemapClassIds(std::move(permutation_map_copy)); + // Types use cid's as part of their hashes. + ClassFinalizer::RehashTypes(); + // Const objects use cid's as part of their hashes. + isolate->RehashConstants(); + } + Dart_ExitScope(); + } + + auto thread = Thread::Current(); + { + TransitionNativeToVM native_to_vm(thread); + + // Ensure new space is empty and there are no threads running. + BackgroundCompiler::Stop(isolate); + isolate->heap()->new_space()->Evacuate(); + isolate->heap()->WaitForMarkerTasks(thread); + isolate->heap()->WaitForSweeperTasks(thread); + RELEASE_ASSERT(isolate->heap()->new_space()->UsedInWords() == 0); + RELEASE_ASSERT(isolate->heap()->old_space()->tasks() == 0); + } + + Dart_ExitIsolate(); + { + const bool kBypassSafepoint = false; + Thread::EnterIsolateGroupAsHelper(group, Thread::kUnknownTask, + kBypassSafepoint); + ASSERT(group == IsolateGroup::Current()); + + { + auto thread = Thread::Current(); + + // Prevent additions of new isolates to [group] until we're done. + group->RunWithLockedGroup([&]() { + // Ensure no other old space GC tasks are running and "occupy" the old + // space. + { + auto old_space = group->heap()->old_space(); + MonitorLocker ml(old_space->tasks_lock()); + while (old_space->tasks() > 0) { + ml.WaitWithSafepointCheck(thread); + } + old_space->set_tasks(1); + } + + // Merge the heap from [spawning_group] to [group]. + { + SafepointOperationScope safepoint_scope(thread); + group->heap()->MergeOtherHeap(isolate->group()->heap()); + } + + spawning_group->UnregisterIsolate(isolate); + const bool shutdown_group = + spawning_group->UnregisterIsolateDecrementCount(isolate); + ASSERT(shutdown_group); + + isolate->isolate_group_ = group; + group->RegisterIsolateLocked(isolate); + isolate->class_table()->shared_class_table_ = group->class_table(); + + // Allow other old space GC tasks to run again. + { + auto old_space = group->heap()->old_space(); + MonitorLocker ml(old_space->tasks_lock()); + ASSERT(old_space->tasks() == 1); + old_space->set_tasks(0); + ml.NotifyAll(); + } + + spawning_group->Shutdown(); + }); + } + + Thread::ExitIsolateGroupAsHelper(kBypassSafepoint); + } + Dart_EnterIsolate(Api::CastIsolate(isolate)); + ASSERT(Thread::Current()->isolate_group() == isolate->group()); + return isolate; } @@ -1236,9 +1372,11 @@ Dart_CreateIsolateGroup(const char* script_uri, new IsolateGroupSource(script_uri, non_null_name, snapshot_data, snapshot_instructions, nullptr, -1, *flags)); auto group = new IsolateGroup(std::move(source), isolate_group_data); + group->CreateHeap( + /*is_vm_isolate=*/false, IsServiceOrKernelIsolateName(non_null_name)); IsolateGroup::RegisterIsolateGroup(group); - Dart_Isolate isolate = - CreateIsolate(group, non_null_name, isolate_data, error); + Dart_Isolate isolate = CreateIsolate(group, /*is_new_group=*/true, + non_null_name, isolate_data, error); if (isolate != nullptr) { group->set_initial_spawn_successful(); } @@ -1263,13 +1401,15 @@ Dart_CreateIsolateGroupFromKernel(const char* script_uri, } const char* non_null_name = name == nullptr ? "isolate" : name; - std::unique_ptr source( + std::shared_ptr source( new IsolateGroupSource(script_uri, non_null_name, nullptr, nullptr, kernel_buffer, kernel_buffer_size, *flags)); - auto group = new IsolateGroup(std::move(source), isolate_group_data); + auto group = new IsolateGroup(source, isolate_group_data); IsolateGroup::RegisterIsolateGroup(group); - Dart_Isolate isolate = - CreateIsolate(group, non_null_name, isolate_data, error); + group->CreateHeap( + /*is_vm_isolate=*/false, IsServiceOrKernelIsolateName(non_null_name)); + Dart_Isolate isolate = CreateIsolate(group, /*is_new_group=*/true, + non_null_name, isolate_data, error); if (isolate != nullptr) { group->set_initial_spawn_successful(); } @@ -1607,7 +1747,7 @@ DART_EXPORT void Dart_NotifyIdle(int64_t deadline) { CHECK_ISOLATE(T->isolate()); API_TIMELINE_BEGIN_END(T); TransitionNativeToVM transition(T); - T->isolate()->idle_time_handler()->NotifyIdle(deadline); + T->isolate()->group()->idle_time_handler()->NotifyIdle(deadline); } DART_EXPORT void Dart_NotifyLowMemory() { @@ -3952,7 +4092,7 @@ DART_EXPORT Dart_Handle Dart_TypedDataAcquireData(Dart_Handle object, ASSERT(I->heap()->Contains(reinterpret_cast(data_tmp))); } const Object& obj = Object::Handle(Z, Api::UnwrapHandle(object)); - WeakTable* table = I->api_state()->acquired_table(); + WeakTable* table = I->group()->api_state()->acquired_table(); intptr_t current = table->GetValue(obj.raw()); if (current != 0) { return Api::NewError("Data was already acquired for this object."); @@ -3982,7 +4122,7 @@ DART_EXPORT Dart_Handle Dart_TypedDataReleaseData(Dart_Handle object) { END_NO_CALLBACK_SCOPE(T); if (FLAG_verify_acquired_data) { const Object& obj = Object::Handle(Z, Api::UnwrapHandle(object)); - WeakTable* table = I->api_state()->acquired_table(); + WeakTable* table = I->group()->api_state()->acquired_table(); intptr_t current = table->GetValue(obj.raw()); if (current == 0) { return Api::NewError("Data was not acquired for this object."); @@ -5062,8 +5202,8 @@ DART_EXPORT void Dart_SetWeakHandleReturnValue(Dart_NativeArguments args, #if defined(DEBUG) Isolate* isolate = arguments->thread()->isolate(); ASSERT(isolate == Isolate::Current()); - ASSERT(isolate->api_state() != NULL && - (isolate->api_state()->IsValidWeakPersistentHandle(rval))); + ASSERT(isolate->group()->api_state() != NULL && + (isolate->group()->api_state()->IsValidWeakPersistentHandle(rval))); #endif Api::SetWeakHandleReturnValue(arguments, rval); } @@ -5647,7 +5787,13 @@ DART_EXPORT Dart_Handle Dart_FinalizeLoading(bool complete_futures) { I->debugger()->NotifyDoneLoading(); #endif - I->heap()->old_space()->EvaluateAfterLoading(); + // After having loaded all the code, we can let the GC set reaonsable limits + // for the heap growth. + // If this is an auxiliary isolate inside a larger isolate group, we will not + // re-initialize the growth policy. + if (I->group()->ContainsOnlyOneIsolate()) { + I->heap()->old_space()->EvaluateAfterLoading(); + } #if !defined(DART_PRECOMPILED_RUNTIME) if (FLAG_enable_mirrors) { @@ -6486,6 +6632,28 @@ DART_EXPORT Dart_Handle Dart_CreateCoreJITSnapshotAsBlobs( #endif } +#if !defined(TARGET_ARCH_IA32) && !defined(DART_PRECOMPILED_RUNTIME) +static void KillNonMainIsolatesSlow(Thread* thread, Isolate* main_isolate) { + auto group = main_isolate->group(); + while (true) { + bool non_main_isolates_alive = false; + { + SafepointOperationScope safepoint(thread); + group->ForEachIsolate([&](Isolate* isolate) { + if (isolate != main_isolate) { + Isolate::KillIfExists(isolate, Isolate::kKillMsg); + non_main_isolates_alive = true; + } + }); + if (!non_main_isolates_alive) { + break; + } + } + OS::SleepMicros(10 * 1000); + } +} +#endif // !defined(TARGET_ARCH_IA32) && !defined(DART_PRECOMPILED_RUNTIME) + DART_EXPORT Dart_Handle Dart_CreateAppJITSnapshotAsBlobs(uint8_t** isolate_snapshot_data_buffer, intptr_t* isolate_snapshot_data_size, @@ -6503,11 +6671,16 @@ Dart_CreateAppJITSnapshotAsBlobs(uint8_t** isolate_snapshot_data_buffer, CHECK_NULL(isolate_snapshot_data_size); CHECK_NULL(isolate_snapshot_instructions_buffer); CHECK_NULL(isolate_snapshot_instructions_size); + // Finalize all classes if needed. Dart_Handle state = Api::CheckAndFinalizePendingClasses(T); if (Api::IsError(state)) { return state; } + + // Kill off any auxiliary isolates before starting with deduping. + KillNonMainIsolatesSlow(T, I); + BackgroundCompiler::Stop(I); DropRegExpMatchCode(Z); diff --git a/runtime/vm/dart_api_impl.h b/runtime/vm/dart_api_impl.h index 93a24431da5..68bb50b2278 100644 --- a/runtime/vm/dart_api_impl.h +++ b/runtime/vm/dart_api_impl.h @@ -177,7 +177,7 @@ class Api : AllStatic { static Dart_Handle Success() { return Api::True(); } // Gets the handle which holds the pre-created acquired error object. - static Dart_Handle AcquiredError(Isolate* isolate); + static Dart_Handle AcquiredError(IsolateGroup* isolate_group); // Returns true if the handle holds a Smi. static bool IsSmi(Dart_Handle handle) { @@ -325,7 +325,7 @@ class Api : AllStatic { #define CHECK_CALLBACK_STATE(thread) \ if (thread->no_callback_scope_depth() != 0) { \ return reinterpret_cast( \ - Api::AcquiredError(thread->isolate())); \ + Api::AcquiredError(thread->isolate_group())); \ } #define CHECK_COMPILATION_ALLOWED(isolate) \ diff --git a/runtime/vm/dart_api_impl_test.cc b/runtime/vm/dart_api_impl_test.cc index 6fe05e331e8..512027867ef 100644 --- a/runtime/vm/dart_api_impl_test.cc +++ b/runtime/vm/dart_api_impl_test.cc @@ -2933,7 +2933,7 @@ VM_UNIT_TEST_CASE(DartAPI_PersistentHandles) { Thread* thread = Thread::Current(); Isolate* isolate = thread->isolate(); EXPECT(isolate != NULL); - ApiState* state = isolate->api_state(); + ApiState* state = isolate->group()->api_state(); EXPECT(state != NULL); ApiLocalScope* scope = thread->api_top_scope(); @@ -3001,7 +3001,7 @@ VM_UNIT_TEST_CASE(DartAPI_NewPersistentHandle_FromPersistentHandle) { Isolate* isolate = Isolate::Current(); EXPECT(isolate != NULL); - ApiState* state = isolate->api_state(); + ApiState* state = isolate->group()->api_state(); EXPECT(state != NULL); Thread* thread = Thread::Current(); CHECK_API_SCOPE(thread); @@ -3034,7 +3034,7 @@ VM_UNIT_TEST_CASE(DartAPI_AssignToPersistentHandle) { CHECK_API_SCOPE(T); Isolate* isolate = T->isolate(); EXPECT(isolate != NULL); - ApiState* state = isolate->api_state(); + ApiState* state = isolate->group()->api_state(); EXPECT(state != NULL); // Start with a known persistent handle. diff --git a/runtime/vm/dart_api_state.h b/runtime/vm/dart_api_state.h index 77e0f3cff66..3e48e722a38 100644 --- a/runtime/vm/dart_api_state.h +++ b/runtime/vm/dart_api_state.h @@ -211,35 +211,35 @@ class FinalizablePersistentHandle { return ExternalSizeInWordsBits::decode(external_data_) * kWordSize; } - void SetExternalSize(intptr_t size, Isolate* isolate) { + void SetExternalSize(intptr_t size, IsolateGroup* isolate_group) { ASSERT(size >= 0); set_external_size(size); if (SpaceForExternal() == Heap::kNew) { SetExternalNewSpaceBit(); } - isolate->heap()->AllocateExternal(raw()->GetClassIdMayBeSmi(), - external_size(), SpaceForExternal()); + isolate_group->heap()->AllocateExternal( + raw()->GetClassIdMayBeSmi(), external_size(), SpaceForExternal()); } // Called when the referent becomes unreachable. - void UpdateUnreachable(Isolate* isolate) { - EnsureFreeExternal(isolate); - Finalize(isolate, this); + void UpdateUnreachable(IsolateGroup* isolate_group) { + EnsureFreeExternal(isolate_group); + Finalize(isolate_group, this); } // Called when the referent has moved, potentially between generations. - void UpdateRelocated(Isolate* isolate) { + void UpdateRelocated(IsolateGroup* isolate_group) { if (IsSetNewSpaceBit() && (SpaceForExternal() == Heap::kOld)) { - isolate->heap()->PromoteExternal(raw()->GetClassIdMayBeSmi(), - external_size()); + isolate_group->heap()->PromoteExternal(raw()->GetClassIdMayBeSmi(), + external_size()); ClearExternalNewSpaceBit(); } } // Idempotent. Called when the handle is explicitly deleted or the // referent becomes unreachable. - void EnsureFreeExternal(Isolate* isolate) { - isolate->heap()->FreeExternal(external_size(), SpaceForExternal()); + void EnsureFreeExternal(IsolateGroup* isolate_group) { + isolate_group->heap()->FreeExternal(external_size(), SpaceForExternal()); set_external_size(0); } @@ -268,7 +268,8 @@ class FinalizablePersistentHandle { : raw_(NULL), peer_(NULL), external_data_(0), callback_(NULL) {} ~FinalizablePersistentHandle() {} - static void Finalize(Isolate* isolate, FinalizablePersistentHandle* handle); + static void Finalize(IsolateGroup* isolate_group, + FinalizablePersistentHandle* handle); // Overload the raw_ field as a next pointer when adding freed // handles to the free list. @@ -677,7 +678,7 @@ class ApiGrowableArray : public BaseGrowableArray { // Implementation of the API State used in dart api for maintaining // local scopes, persistent handles etc. These are setup on a per isolate -// basis and destroyed when the isolate is shutdown. +// group basis and destroyed when the isolate group is shutdown. class ApiState { public: ApiState() @@ -706,6 +707,8 @@ class ApiState { } } + void MergeOtherApiState(ApiState* api_state); + void VisitObjectPointersUnlocked(ObjectPointerVisitor* visitor) { persistent_handles_.VisitObjectPointers(visitor); if (visitor->visit_weak_persistent_handles()) { @@ -817,14 +820,14 @@ inline FinalizablePersistentHandle* FinalizablePersistentHandle::New( void* peer, Dart_WeakPersistentHandleFinalizer callback, intptr_t external_size) { - ApiState* state = isolate->api_state(); + ApiState* state = isolate->group()->api_state(); ASSERT(state != NULL); FinalizablePersistentHandle* ref = state->AllocateWeakPersistentHandle(); ref->set_raw(object); ref->set_peer(peer); ref->set_callback(callback); // This may trigger GC, so it must be called last. - ref->SetExternalSize(external_size, isolate); + ref->SetExternalSize(external_size, isolate->group()); return ref; } diff --git a/runtime/vm/gdb_helpers.cc b/runtime/vm/gdb_helpers.cc index 6cb98f04c71..1260263d0ae 100644 --- a/runtime/vm/gdb_helpers.cc +++ b/runtime/vm/gdb_helpers.cc @@ -75,7 +75,8 @@ void _printInterpreterStackTrace(RawObject** fp, class PrintObjectPointersVisitor : public ObjectPointerVisitor { public: - PrintObjectPointersVisitor() : ObjectPointerVisitor(Isolate::Current()) {} + PrintObjectPointersVisitor() + : ObjectPointerVisitor(IsolateGroup::Current()) {} void VisitPointers(RawObject** first, RawObject** last) { for (RawObject** p = first; p <= last; p++) { diff --git a/runtime/vm/heap/become.cc b/runtime/vm/heap/become.cc index 936cbc52abd..5f5ffbaaeec 100644 --- a/runtime/vm/heap/become.cc +++ b/runtime/vm/heap/become.cc @@ -79,7 +79,7 @@ static void ForwardObjectTo(RawObject* before_obj, RawObject* after_obj) { class ForwardPointersVisitor : public ObjectPointerVisitor { public: explicit ForwardPointersVisitor(Thread* thread) - : ObjectPointerVisitor(thread->isolate()), + : ObjectPointerVisitor(thread->isolate_group()), thread_(thread), visiting_object_(NULL) {} @@ -217,8 +217,7 @@ void Become::CrashDump(RawObject* before_obj, RawObject* after_obj) { void Become::ElementsForwardIdentity(const Array& before, const Array& after) { Thread* thread = Thread::Current(); - Isolate* isolate = thread->isolate(); - Heap* heap = isolate->heap(); + auto heap = thread->isolate_group()->heap(); TIMELINE_FUNCTION_GC_DURATION(thread, "Become::ElementsForwardIdentity"); HeapIterationScope his(thread); @@ -273,12 +272,12 @@ void Become::ElementsForwardIdentity(const Array& before, const Array& after) { void Become::FollowForwardingPointers(Thread* thread) { // N.B.: We forward the heap before forwarding the stack. This limits the // amount of following of forwarding pointers needed to get at stack maps. - Isolate* isolate = thread->isolate(); - Heap* heap = isolate->heap(); + auto isolate_group = thread->isolate_group(); + Heap* heap = isolate_group->heap(); // Clear the store buffer; will be rebuilt as we forward the heap. - isolate->ReleaseStoreBuffers(); - isolate->store_buffer()->Reset(); + isolate_group->ReleaseStoreBuffers(); + isolate_group->store_buffer()->Reset(); ForwardPointersVisitor pointer_visitor(thread); @@ -291,17 +290,21 @@ void Become::FollowForwardingPointers(Thread* thread) { } // C++ pointers. - isolate->VisitObjectPointers(&pointer_visitor, - ValidationPolicy::kValidateFrames); + isolate_group->VisitObjectPointers(&pointer_visitor, + ValidationPolicy::kValidateFrames); #ifndef PRODUCT - ObjectIdRing* ring = isolate->object_id_ring(); - ASSERT(ring != NULL); - ring->VisitPointers(&pointer_visitor); + isolate_group->ForEachIsolate( + [&](Isolate* isolate) { + ObjectIdRing* ring = isolate->object_id_ring(); + ASSERT(ring != NULL); + ring->VisitPointers(&pointer_visitor); + }, + /*at_safepoint=*/true); #endif // !PRODUCT // Weak persistent handles. ForwardHeapPointersHandleVisitor handle_visitor(thread); - isolate->VisitWeakPersistentHandles(&handle_visitor); + isolate_group->VisitWeakPersistentHandles(&handle_visitor); } } // namespace dart diff --git a/runtime/vm/heap/compactor.cc b/runtime/vm/heap/compactor.cc index 06b9529736a..0ba365ac13c 100644 --- a/runtime/vm/heap/compactor.cc +++ b/runtime/vm/heap/compactor.cc @@ -119,14 +119,14 @@ void HeapPage::AllocateForwardingPage() { class CompactorTask : public ThreadPool::Task { public: - CompactorTask(Isolate* isolate, + CompactorTask(IsolateGroup* isolate_group, GCCompactor* compactor, ThreadBarrier* barrier, RelaxedAtomic* next_forwarding_task, HeapPage* head, HeapPage** tail, FreeList* freelist) - : isolate_(isolate), + : isolate_group_(isolate_group), compactor_(compactor), barrier_(barrier), next_forwarding_task_(next_forwarding_task), @@ -145,7 +145,7 @@ class CompactorTask : public ThreadPool::Task { uword SlideBlock(uword first_object, ForwardingPage* forwarding_page); void PlanMoveToContiguousSize(intptr_t size); - Isolate* isolate_; + IsolateGroup* isolate_group_; GCCompactor* compactor_; ThreadBarrier* barrier_; RelaxedAtomic* next_forwarding_task_; @@ -243,7 +243,7 @@ void GCCompactor::Compact(HeapPage* pages, for (intptr_t task_index = 0; task_index < num_tasks; task_index++) { Dart::thread_pool()->Run( - thread()->isolate(), this, &barrier, &next_forwarding_task, + thread()->isolate_group(), this, &barrier, &next_forwarding_task, heads[task_index], &tails[task_index], freelist); } @@ -320,8 +320,8 @@ void GCCompactor::Compact(HeapPage* pages, } void CompactorTask::Run() { - bool result = - Thread::EnterIsolateAsHelper(isolate_, Thread::kCompactorTask, true); + bool result = Thread::EnterIsolateGroupAsHelper(isolate_group_, + Thread::kCompactorTask, true); ASSERT(result); #ifdef SUPPORT_TIMELINE Thread* thread = Thread::Current(); @@ -371,7 +371,7 @@ void CompactorTask::Run() { case 0: { TIMELINE_FUNCTION_GC_DURATION(thread, "ForwardLargePages"); for (HeapPage* large_page = - isolate_->heap()->old_space()->large_pages_; + isolate_group_->heap()->old_space()->large_pages_; large_page != NULL; large_page = large_page->next()) { large_page->VisitObjectPointers(compactor_); } @@ -379,28 +379,32 @@ void CompactorTask::Run() { } case 1: { TIMELINE_FUNCTION_GC_DURATION(thread, "ForwardNewSpace"); - isolate_->heap()->new_space()->VisitObjectPointers(compactor_); + isolate_group_->heap()->new_space()->VisitObjectPointers(compactor_); break; } case 2: { TIMELINE_FUNCTION_GC_DURATION(thread, "ForwardRememberedSet"); - isolate_->store_buffer()->VisitObjectPointers(compactor_); + isolate_group_->store_buffer()->VisitObjectPointers(compactor_); break; } case 3: { TIMELINE_FUNCTION_GC_DURATION(thread, "ForwardWeakTables"); - isolate_->heap()->ForwardWeakTables(compactor_); + isolate_group_->heap()->ForwardWeakTables(compactor_); break; } case 4: { TIMELINE_FUNCTION_GC_DURATION(thread, "ForwardWeakHandles"); - isolate_->VisitWeakPersistentHandles(compactor_); + isolate_group_->VisitWeakPersistentHandles(compactor_); break; } #ifndef PRODUCT case 5: { TIMELINE_FUNCTION_GC_DURATION(thread, "ForwardObjectIdRing"); - isolate_->object_id_ring()->VisitPointers(compactor_); + isolate_group_->ForEachIsolate( + [&](Isolate* isolate) { + isolate->object_id_ring()->VisitPointers(compactor_); + }, + /*at_safepoint=*/true); break; } #endif // !PRODUCT @@ -411,7 +415,7 @@ void CompactorTask::Run() { barrier_->Sync(); } - Thread::ExitIsolateAsHelper(true); + Thread::ExitIsolateGroupAsHelper(true); // This task is done. Notify the original thread. barrier_->Exit(); @@ -653,7 +657,8 @@ void GCCompactor::ForwardStackPointers() { // N.B.: Heap pointers have already been forwarded. We forward the heap before // forwarding the stack to limit the number of places that need to be aware of // forwarding when reading stack maps. - isolate()->VisitObjectPointers(this, ValidationPolicy::kDontValidateFrames); + isolate_group()->VisitObjectPointers(this, + ValidationPolicy::kDontValidateFrames); } } // namespace dart diff --git a/runtime/vm/heap/compactor.h b/runtime/vm/heap/compactor.h index 6bc036c081d..cd6b29c271c 100644 --- a/runtime/vm/heap/compactor.h +++ b/runtime/vm/heap/compactor.h @@ -27,7 +27,7 @@ class GCCompactor : public ValueObject, public: GCCompactor(Thread* thread, Heap* heap) : HandleVisitor(thread), - ObjectPointerVisitor(thread->isolate()), + ObjectPointerVisitor(thread->isolate_group()), heap_(heap) {} ~GCCompactor() {} diff --git a/runtime/vm/heap/freelist.cc b/runtime/vm/heap/freelist.cc index e77f199cf83..d08ec12a7b1 100644 --- a/runtime/vm/heap/freelist.cc +++ b/runtime/vm/heap/freelist.cc @@ -378,4 +378,41 @@ FreeListElement* FreeList::TryAllocateLargeLocked(intptr_t minimum_size) { return NULL; } +void FreeList::MergeOtherFreelist(FreeList* other, bool is_protected) { + // The [other] free list is from a dying isolate. There are no other threads + // accessing it, so there is no need to lock here. + MutexLocker ml(&mutex_); + for (intptr_t i = 0; i < (kNumLists + 1); ++i) { + FreeListElement* other_head = other->free_lists_[i]; + if (other_head != nullptr) { + // If we didn't have a freelist element before we have to set the bit now, + // since we will get 1+ elements from [other]. + FreeListElement* old_head = free_lists_[i]; + if (old_head == nullptr && i != kNumLists) { + free_map_.Set(i, true); + } + + // Chain other's list in. + FreeListElement* last = other_head; + while (last->next() != nullptr) { + last = last->next(); + } + + if (is_protected) { + VirtualMemory::Protect(reinterpret_cast(last), sizeof(*last), + VirtualMemory::kReadWrite); + } + last->set_next(old_head); + if (is_protected) { + VirtualMemory::Protect(reinterpret_cast(last), sizeof(*last), + VirtualMemory::kReadExecute); + } + free_lists_[i] = other_head; + } + } + + last_free_small_size_ = + Utils::Maximum(last_free_small_size_, other->last_free_small_size_); +} + } // namespace dart diff --git a/runtime/vm/heap/freelist.h b/runtime/vm/heap/freelist.h index 09ad0063803..7c885cf1147 100644 --- a/runtime/vm/heap/freelist.h +++ b/runtime/vm/heap/freelist.h @@ -117,6 +117,8 @@ class FreeList { return 0; } + void MergeOtherFreelist(FreeList* freelist, bool is_protected); + private: static const int kNumLists = 128; static const intptr_t kInitialFreeListSearchBudget = 1000; diff --git a/runtime/vm/heap/heap.cc b/runtime/vm/heap/heap.cc index 03be4362976..9c6f2066bde 100644 --- a/runtime/vm/heap/heap.cc +++ b/runtime/vm/heap/heap.cc @@ -2,6 +2,9 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +#include +#include + #include "vm/heap/heap.h" #include "platform/assert.h" @@ -33,10 +36,27 @@ namespace dart { DEFINE_FLAG(bool, write_protect_vm_isolate, true, "Write protect vm_isolate."); -Heap::Heap(Isolate* isolate, +// We ensure that the GC does not use the current isolate. +class NoActiveIsolateScope { + public: + NoActiveIsolateScope() : thread_(Thread::Current()) { + saved_isolate_ = thread_->isolate_; + thread_->isolate_ = nullptr; + } + ~NoActiveIsolateScope() { + ASSERT(thread_->isolate_ == nullptr); + thread_->isolate_ = saved_isolate_; + } + + private: + Thread* thread_; + Isolate* saved_isolate_; +}; + +Heap::Heap(IsolateGroup* isolate_group, intptr_t max_new_gen_semi_words, intptr_t max_old_gen_words) - : isolate_(isolate), + : isolate_group_(isolate_group), new_space_(this, max_new_gen_semi_words), old_space_(this, max_old_gen_words), barrier_(), @@ -178,7 +198,7 @@ uword Heap::AllocateOld(intptr_t size, HeapPage::PageType type) { void Heap::AllocateExternal(intptr_t cid, intptr_t size, Space space) { ASSERT(Thread::Current()->no_safepoint_scope_depth() == 0); if (space == kNew) { - isolate()->AssertCurrentThreadIsMutator(); + Isolate::Current()->AssertCurrentThreadIsMutator(); new_space_.AllocateExternal(cid, size); if (new_space_.ExternalInWords() <= (4 * new_space_.CapacityInWords())) { return; @@ -331,13 +351,13 @@ void HeapIterationScope::IterateVMIsolateObjects(ObjectVisitor* visitor) const { void HeapIterationScope::IterateObjectPointers( ObjectPointerVisitor* visitor, ValidationPolicy validate_frames) { - isolate()->VisitObjectPointers(visitor, validate_frames); + isolate_group()->VisitObjectPointers(visitor, validate_frames); } void HeapIterationScope::IterateStackPointers( ObjectPointerVisitor* visitor, ValidationPolicy validate_frames) { - isolate()->VisitStackPointers(visitor, validate_frames); + isolate_group()->VisitStackPointers(visitor, validate_frames); } void Heap::VisitObjectPointers(ObjectPointerVisitor* visitor) const { @@ -462,7 +482,7 @@ void Heap::NotifyLowMemory() { void Heap::EvacuateNewSpace(Thread* thread, GCReason reason) { ASSERT((reason != kOldSpace) && (reason != kPromotion)); - if (thread->isolate() == Dart::vm_isolate()) { + if (thread->isolate_group() == Dart::vm_isolate()->group()) { // The vm isolate cannot safely collect garbage due to unvisited read-only // handles and slots bootstrapped with RAW_NULL. Ignore GC requests to // trigger a nice out-of-memory message instead of a crash in the middle of @@ -483,8 +503,9 @@ void Heap::EvacuateNewSpace(Thread* thread, GCReason reason) { } void Heap::CollectNewSpaceGarbage(Thread* thread, GCReason reason) { + NoActiveIsolateScope no_active_isolate_scope; ASSERT((reason != kOldSpace) && (reason != kPromotion)); - if (thread->isolate() == Dart::vm_isolate()) { + if (thread->isolate_group() == Dart::vm_isolate()->group()) { // The vm isolate cannot safely collect garbage due to unvisited read-only // handles and slots bootstrapped with RAW_NULL. Ignore GC requests to // trigger a nice out-of-memory message instead of a crash in the middle of @@ -516,12 +537,14 @@ void Heap::CollectNewSpaceGarbage(Thread* thread, GCReason reason) { void Heap::CollectOldSpaceGarbage(Thread* thread, GCType type, GCReason reason) { + NoActiveIsolateScope no_active_isolate_scope; + ASSERT(reason != kNewSpace); ASSERT(type != kScavenge); if (FLAG_use_compactor) { type = kMarkCompact; } - if (thread->isolate() == Dart::vm_isolate()) { + if (thread->isolate_group() == Dart::vm_isolate()->group()) { // The vm isolate cannot safely collect garbage due to unvisited read-only // handles and slots bootstrapped with RAW_NULL. Ignore GC requests to // trigger a nice out-of-memory message instead of a crash in the middle of @@ -537,9 +560,12 @@ void Heap::CollectOldSpaceGarbage(Thread* thread, RecordAfterGC(type); PrintStats(); NOT_IN_PRODUCT(PrintStatsToTimeline(&tbes, reason)); + // Some Code objects may have been collected so invalidate handler cache. - thread->isolate()->handler_info_cache()->Clear(); - thread->isolate()->catch_entry_moves_cache()->Clear(); + thread->isolate_group()->ForEachIsolate([&](Isolate* isolate) { + isolate->handler_info_cache()->Clear(); + isolate->catch_entry_moves_cache()->Clear(); + }); EndOldSpaceGC(); } } @@ -649,10 +675,10 @@ void Heap::WaitForSweeperTasks(Thread* thread) { void Heap::UpdateGlobalMaxUsed() { #if !defined(PRODUCT) - ASSERT(isolate_ != NULL); + ASSERT(isolate_group_ != NULL); // We are accessing the used in words count for both new and old space // without synchronizing. The value of this metric is approximate. - isolate_->GetHeapGlobalUsedMaxMetric()->SetValue( + isolate_group_->GetHeapGlobalUsedMaxMetric()->SetValue( (UsedInWords(Heap::kNew) * kWordSize) + (UsedInWords(Heap::kOld) * kWordSize)); #endif // !defined(PRODUCT) @@ -676,12 +702,13 @@ void Heap::WriteProtect(bool read_only) { old_space_.WriteProtect(read_only); } -void Heap::Init(Isolate* isolate, +void Heap::Init(IsolateGroup* isolate_group, intptr_t max_new_gen_words, intptr_t max_old_gen_words) { - ASSERT(isolate->heap() == NULL); - Heap* heap = new Heap(isolate, max_new_gen_words, max_old_gen_words); - isolate->set_heap(heap); + ASSERT(isolate_group->heap() == nullptr); + std::unique_ptr heap( + new Heap(isolate_group, max_new_gen_words, max_old_gen_words)); + isolate_group->set_heap(std::move(heap)); } const char* Heap::RegionName(Space space) { @@ -708,6 +735,22 @@ void Heap::CollectOnNthAllocation(intptr_t num_allocations) { gc_on_nth_allocation_ = num_allocations; } +void Heap::MergeOtherHeap(Heap* other) { + ASSERT(!other->gc_new_space_in_progress_); + ASSERT(!other->gc_old_space_in_progress_); + ASSERT(!other->read_only_); + ASSERT(other->new_space()->UsedInWords() == 0); + ASSERT(other->old_space()->tasks() == 0); + + old_space_.MergeOtherPageSpace(other->old_space()); + + for (intptr_t i = 0; i < kNumWeakSelectors; ++i) { + // The new space rehashing should not be necessary. + new_weak_tables_[i]->MergeOtherWeakTable(other->new_weak_tables_[i]); + old_weak_tables_[i]->MergeOtherWeakTable(other->old_weak_tables_[i]); + } +} + void Heap::CollectForDebugging() { if (gc_on_nth_allocation_ == kNoForcedGarbageCollection) return; gc_on_nth_allocation_--; @@ -727,12 +770,12 @@ ObjectSet* Heap::CreateAllocatedObjectSet( this->AddRegionsToObjectSet(allocated_set); { - VerifyObjectVisitor object_visitor(isolate(), allocated_set, + VerifyObjectVisitor object_visitor(isolate_group(), allocated_set, mark_expectation); this->VisitObjectsNoImagePages(&object_visitor); } { - VerifyObjectVisitor object_visitor(isolate(), allocated_set, + VerifyObjectVisitor object_visitor(isolate_group(), allocated_set, kRequireMarked); this->VisitObjectsImagePages(&object_visitor); } @@ -741,7 +784,7 @@ ObjectSet* Heap::CreateAllocatedObjectSet( vm_isolate->heap()->AddRegionsToObjectSet(allocated_set); { // VM isolate heap is premarked. - VerifyObjectVisitor vm_object_visitor(isolate(), allocated_set, + VerifyObjectVisitor vm_object_visitor(isolate_group(), allocated_set, kRequireMarked); vm_isolate->heap()->VisitObjects(&vm_object_visitor); } @@ -762,7 +805,7 @@ bool Heap::VerifyGC(MarkExpectation mark_expectation) const { ObjectSet* allocated_set = CreateAllocatedObjectSet(stack_zone.GetZone(), mark_expectation); - VerifyPointersVisitor visitor(isolate(), allocated_set); + VerifyPointersVisitor visitor(isolate_group(), allocated_set); VisitObjectPointers(&visitor); // Only returning a value so that Heap::Validate can be called from an ASSERT. @@ -908,8 +951,12 @@ void Heap::ForwardWeakEntries(RawObject* before_object, // We only come here during hot reload, in which case we assume that none of // the isolates is in the middle of sending messages. - RELEASE_ASSERT(isolate()->forward_table_new() == nullptr); - RELEASE_ASSERT(isolate()->forward_table_old() == nullptr); + isolate_group()->ForEachIsolate( + [&](Isolate* isolate) { + RELEASE_ASSERT(isolate->forward_table_new() == nullptr); + RELEASE_ASSERT(isolate->forward_table_old() == nullptr); + }, + /*at_safepoint=*/true); } void Heap::ForwardWeakTables(ObjectPointerVisitor* visitor) { @@ -922,8 +969,12 @@ void Heap::ForwardWeakTables(ObjectPointerVisitor* visitor) { // Isolates might have forwarding tables (used for during snapshoting in // isolate communication). - auto table_old = isolate()->forward_table_old(); - if (table_old != nullptr) table_old->Forward(visitor); + isolate_group()->ForEachIsolate( + [&](Isolate* isolate) { + auto table_old = isolate->forward_table_old(); + if (table_old != nullptr) table_old->Forward(visitor); + }, + /*at_safepoint=*/true); } #ifndef PRODUCT @@ -980,11 +1031,15 @@ void Heap::RecordAfterGC(GCType type) { (type == kMarkSweep && gc_old_space_in_progress_) || (type == kMarkCompact && gc_old_space_in_progress_)); #ifndef PRODUCT - if (Service::gc_stream.enabled() && - !Isolate::IsVMInternalIsolate(isolate())) { - ServiceEvent event(isolate(), ServiceEvent::kGC); - event.set_gc_stats(&stats_); - Service::HandleEvent(&event); + // For now we'll emit the same GC events on all isolates. + if (Service::gc_stream.enabled()) { + isolate_group_->ForEachIsolate([&](Isolate* isolate) { + if (!Isolate::IsVMInternalIsolate(isolate)) { + ServiceEvent event(isolate, ServiceEvent::kGC); + event.set_gc_stats(&stats_); + Service::HandleEvent(&event); + } + }); } #endif // !PRODUCT } @@ -1012,7 +1067,7 @@ void Heap::PrintStats() { // clang-format off OS::PrintErr( - "[ %-13.13s, %10s(%9s), " // GC(isolate), type(reason) + "[ %-13.13s, %10s(%9s), " // GC(isolate-group), type(reason) "%4" Pd ", " // count "%6.2f, " // start time "%5.1f, " // total time @@ -1025,11 +1080,11 @@ void Heap::PrintStats() { "%6.2f, %6.2f, %6.2f, %6.2f, %6.2f, %6.2f, " // times "%" Pd ", %" Pd ", %" Pd ", %" Pd ", " // data "]\n", // End with a comma to make it easier to import in spreadsheets. - isolate()->name(), + isolate_group()->source()->name, GCTypeToString(stats_.type_), GCReasonToString(stats_.reason_), stats_.num_, - MicrosecondsToSeconds(isolate()->UptimeMicros()), + MicrosecondsToSeconds(isolate_group_->UptimeMicros()), MicrosecondsToMilliseconds(stats_.after_.micros_ - stats_.before_.micros_), RoundWordsToKB(stats_.before_.new_.used_in_words), diff --git a/runtime/vm/heap/heap.h b/runtime/vm/heap/heap.h index a9e2c05e8f2..d2e4b5fea4e 100644 --- a/runtime/vm/heap/heap.h +++ b/runtime/vm/heap/heap.h @@ -23,6 +23,7 @@ namespace dart { // Forward declarations. class Isolate; +class IsolateGroup; class ObjectPointerVisitor; class ObjectSet; class ServiceEvent; @@ -151,7 +152,6 @@ class Heap { // Enables growth control on the page space heaps. This should be // called before any user code is executed. void InitGrowthControl(); - void EnableGrowthControl() { SetGrowthControlState(true); } void DisableGrowthControl() { SetGrowthControlState(false); } void SetGrowthControlState(bool state); bool GrowthControlState(); @@ -164,7 +164,7 @@ class Heap { } // Initialize the heap and register it with the isolate. - static void Init(Isolate* isolate, + static void Init(IsolateGroup* isolate_group, intptr_t max_new_gen_words, intptr_t max_old_gen_words); @@ -293,7 +293,7 @@ class Heap { } #endif // PRODUCT - Isolate* isolate() const { return isolate_; } + IsolateGroup* isolate_group() const { return isolate_group_; } Monitor* barrier() const { return &barrier_; } Monitor* barrier_done() const { return &barrier_done_; } @@ -317,6 +317,8 @@ class Heap { void CollectOnNthAllocation(intptr_t num_allocations); + void MergeOtherHeap(Heap* other); + private: class GCStats : public ValueObject { public: @@ -348,7 +350,7 @@ class Heap { DISALLOW_COPY_AND_ASSIGN(GCStats); }; - Heap(Isolate* isolate, + Heap(IsolateGroup* isolate_group, intptr_t max_new_gen_semi_words, // Max capacity of new semi-space. intptr_t max_old_gen_words); @@ -391,7 +393,7 @@ class Heap { // Trigger major GC if 'gc_on_nth_allocation_' is set. void CollectForDebugging(); - Isolate* isolate_; + IsolateGroup* isolate_group_; // The different spaces used for allocation. Scavenger new_space_; diff --git a/runtime/vm/heap/heap_test.cc b/runtime/vm/heap/heap_test.cc index 0bbe2fa63aa..948d6f3f057 100644 --- a/runtime/vm/heap/heap_test.cc +++ b/runtime/vm/heap/heap_test.cc @@ -122,7 +122,7 @@ TEST_CASE(ClassHeapStats) { CountObjectsVisitor visitor(thread, class_table->NumCids()); HeapIterationScope iter(thread); iter.IterateObjects(&visitor); - isolate->VisitWeakPersistentHandles(&visitor); + isolate->group()->VisitWeakPersistentHandles(&visitor); EXPECT_EQ(2, visitor.new_count_[cid]); EXPECT_EQ(0, visitor.old_count_[cid]); } @@ -135,7 +135,7 @@ TEST_CASE(ClassHeapStats) { CountObjectsVisitor visitor(thread, class_table->NumCids()); HeapIterationScope iter(thread); iter.IterateObjects(&visitor); - isolate->VisitWeakPersistentHandles(&visitor); + isolate->group()->VisitWeakPersistentHandles(&visitor); EXPECT_EQ(1, visitor.new_count_[cid]); EXPECT_EQ(0, visitor.old_count_[cid]); } @@ -149,7 +149,7 @@ TEST_CASE(ClassHeapStats) { CountObjectsVisitor visitor(thread, class_table->NumCids()); HeapIterationScope iter(thread); iter.IterateObjects(&visitor); - isolate->VisitWeakPersistentHandles(&visitor); + isolate->group()->VisitWeakPersistentHandles(&visitor); EXPECT_EQ(0, visitor.new_count_[cid]); EXPECT_EQ(1, visitor.old_count_[cid]); } @@ -162,7 +162,7 @@ TEST_CASE(ClassHeapStats) { CountObjectsVisitor visitor(thread, class_table->NumCids()); HeapIterationScope iter(thread); iter.IterateObjects(&visitor); - isolate->VisitWeakPersistentHandles(&visitor); + isolate->group()->VisitWeakPersistentHandles(&visitor); EXPECT_EQ(0, visitor.new_count_[cid]); EXPECT_EQ(1, visitor.old_count_[cid]); } @@ -174,7 +174,7 @@ TEST_CASE(ClassHeapStats) { CountObjectsVisitor visitor(thread, class_table->NumCids()); HeapIterationScope iter(thread); iter.IterateObjects(&visitor); - isolate->VisitWeakPersistentHandles(&visitor); + isolate->group()->VisitWeakPersistentHandles(&visitor); EXPECT_EQ(0, visitor.new_count_[cid]); EXPECT_EQ(1, visitor.old_count_[cid]); } @@ -190,7 +190,7 @@ TEST_CASE(ClassHeapStats) { CountObjectsVisitor visitor(thread, class_table->NumCids()); HeapIterationScope iter(thread); iter.IterateObjects(&visitor); - isolate->VisitWeakPersistentHandles(&visitor); + isolate->group()->VisitWeakPersistentHandles(&visitor); EXPECT_EQ(0, visitor.new_count_[cid]); EXPECT_EQ(0, visitor.old_count_[cid]); } @@ -627,7 +627,7 @@ ISOLATE_UNIT_TEST_CASE(ExternalAllocationStats) { CountObjectsVisitor visitor(thread, isolate->class_table()->NumCids()); HeapIterationScope iter(thread); iter.IterateObjects(&visitor); - isolate->VisitWeakPersistentHandles(&visitor); + isolate->group()->VisitWeakPersistentHandles(&visitor); EXPECT_LE(visitor.old_external_size_[kArrayCid], heap->old_space()->ExternalInWords() * kWordSize); EXPECT_LE(visitor.new_external_size_[kArrayCid], diff --git a/runtime/vm/heap/marker.cc b/runtime/vm/heap/marker.cc index 87d9c080acc..2075d322bcc 100644 --- a/runtime/vm/heap/marker.cc +++ b/runtime/vm/heap/marker.cc @@ -84,11 +84,11 @@ class MarkerWorkList : public ValueObject { template class MarkingVisitorBase : public ObjectPointerVisitor { public: - MarkingVisitorBase(Isolate* isolate, + MarkingVisitorBase(IsolateGroup* isolate_group, PageSpace* page_space, MarkingStack* marking_stack, MarkingStack* deferred_marking_stack) - : ObjectPointerVisitor(isolate), + : ObjectPointerVisitor(isolate_group), thread_(Thread::Current()), page_space_(page_space), work_list_(marking_stack), @@ -96,8 +96,9 @@ class MarkingVisitorBase : public ObjectPointerVisitor { delayed_weak_properties_(NULL), marked_bytes_(0), marked_micros_(0) { - ASSERT(thread_->isolate() == isolate); + ASSERT(thread_->isolate_group() == isolate_group); } + ~MarkingVisitorBase() {} uintptr_t marked_bytes() const { return marked_bytes_; } int64_t marked_micros() const { return marked_micros_; } @@ -347,14 +348,14 @@ class MarkingWeakVisitor : public HandleVisitor { public: explicit MarkingWeakVisitor(Thread* thread) : HandleVisitor(thread), - class_table_(thread->isolate()->shared_class_table()) {} + class_table_(thread->isolate_group()->class_table()) {} void VisitHandle(uword addr) { FinalizablePersistentHandle* handle = reinterpret_cast(addr); RawObject* raw_obj = handle->raw(); if (IsUnreachable(raw_obj)) { - handle->UpdateUnreachable(thread()->isolate()); + handle->UpdateUnreachable(thread()->isolate_group()); } } @@ -365,16 +366,20 @@ class MarkingWeakVisitor : public HandleVisitor { }; void GCMarker::Prologue() { - isolate_->ReleaseStoreBuffers(); + isolate_group_->ReleaseStoreBuffers(); #ifndef DART_PRECOMPILED_RUNTIME - Thread* mutator_thread = isolate_->mutator_thread(); - if (mutator_thread != NULL) { - Interpreter* interpreter = mutator_thread->interpreter(); - if (interpreter != NULL) { - interpreter->ClearLookupCache(); - } - } + isolate_group_->ForEachIsolate( + [&](Isolate* isolate) { + Thread* mutator_thread = isolate->mutator_thread(); + if (mutator_thread != NULL) { + Interpreter* interpreter = mutator_thread->interpreter(); + if (interpreter != NULL) { + interpreter->ClearLookupCache(); + } + } + }, + /*at_safepoint=*/true); #endif } @@ -401,9 +406,10 @@ void GCMarker::IterateRoots(ObjectPointerVisitor* visitor) { switch (slice) { case kIsolate: { - TIMELINE_FUNCTION_GC_DURATION(Thread::Current(), "ProcessIsolate"); - isolate_->VisitObjectPointers(visitor, - ValidationPolicy::kDontValidateFrames); + TIMELINE_FUNCTION_GC_DURATION(Thread::Current(), + "ProcessIsolateGroupRoots"); + isolate_group_->VisitObjectPointers( + visitor, ValidationPolicy::kDontValidateFrames); break; } case kNewSpace: { @@ -460,9 +466,9 @@ void GCMarker::IterateWeakRoots(Thread* thread) { void GCMarker::ProcessWeakHandles(Thread* thread) { TIMELINE_FUNCTION_GC_DURATION(thread, "ProcessWeakHandles"); MarkingWeakVisitor visitor(thread); - ApiState* state = isolate_->api_state(); + ApiState* state = isolate_group_->api_state(); ASSERT(state != NULL); - isolate_->VisitWeakPersistentHandles(&visitor); + isolate_group_->VisitWeakPersistentHandles(&visitor); } void GCMarker::ProcessWeakTables(Thread* thread) { @@ -486,7 +492,7 @@ void GCMarker::ProcessWeakTables(Thread* thread) { void GCMarker::ProcessRememberedSet(Thread* thread) { TIMELINE_FUNCTION_GC_DURATION(thread, "ProcessRememberedSet"); // Filter collected objects from the remembered set. - StoreBuffer* store_buffer = isolate_->store_buffer(); + StoreBuffer* store_buffer = isolate_group_->store_buffer(); StoreBufferBlock* reading = store_buffer->Blocks(); StoreBufferBlock* writing = store_buffer->PopNonFullBlock(); while (reading != NULL) { @@ -515,8 +521,8 @@ void GCMarker::ProcessRememberedSet(Thread* thread) { class ObjectIdRingClearPointerVisitor : public ObjectPointerVisitor { public: - explicit ObjectIdRingClearPointerVisitor(Isolate* isolate) - : ObjectPointerVisitor(isolate) {} + explicit ObjectIdRingClearPointerVisitor(IsolateGroup* isolate_group) + : ObjectPointerVisitor(isolate_group) {} void VisitPointers(RawObject** first, RawObject** last) { for (RawObject** current = first; current <= last; current++) { @@ -533,31 +539,35 @@ class ObjectIdRingClearPointerVisitor : public ObjectPointerVisitor { void GCMarker::ProcessObjectIdTable(Thread* thread) { #ifndef PRODUCT TIMELINE_FUNCTION_GC_DURATION(thread, "ProcessObjectIdTable"); - ObjectIdRingClearPointerVisitor visitor(isolate_); - ObjectIdRing* ring = isolate_->object_id_ring(); - ASSERT(ring != NULL); - ring->VisitPointers(&visitor); + ObjectIdRingClearPointerVisitor visitor(isolate_group_); + isolate_group_->ForEachIsolate( + [&](Isolate* isolate) { + ObjectIdRing* ring = isolate->object_id_ring(); + ASSERT(ring != NULL); + ring->VisitPointers(&visitor); + }, + /*at_safepoint=*/true); #endif // !PRODUCT } class ParallelMarkTask : public ThreadPool::Task { public: ParallelMarkTask(GCMarker* marker, - Isolate* isolate, + IsolateGroup* isolate_group, MarkingStack* marking_stack, ThreadBarrier* barrier, SyncMarkingVisitor* visitor, RelaxedAtomic* num_busy) : marker_(marker), - isolate_(isolate), + isolate_group_(isolate_group), marking_stack_(marking_stack), barrier_(barrier), visitor_(visitor), num_busy_(num_busy) {} virtual void Run() { - bool result = - Thread::EnterIsolateAsHelper(isolate_, Thread::kMarkerTask, true); + bool result = Thread::EnterIsolateGroupAsHelper( + isolate_group_, Thread::kMarkerTask, /*bypass_safepoint=*/true); ASSERT(result); { Thread* thread = Thread::Current(); @@ -640,7 +650,7 @@ class ParallelMarkTask : public ThreadPool::Task { delete visitor_; } - Thread::ExitIsolateAsHelper(true); + Thread::ExitIsolateGroupAsHelper(/*bypass_safepoint=*/true); // This task is done. Notify the original thread. barrier_->Exit(); @@ -648,7 +658,7 @@ class ParallelMarkTask : public ThreadPool::Task { private: GCMarker* marker_; - Isolate* isolate_; + IsolateGroup* isolate_group_; MarkingStack* marking_stack_; ThreadBarrier* barrier_; SyncMarkingVisitor* visitor_; @@ -660,11 +670,11 @@ class ParallelMarkTask : public ThreadPool::Task { class ConcurrentMarkTask : public ThreadPool::Task { public: ConcurrentMarkTask(GCMarker* marker, - Isolate* isolate, + IsolateGroup* isolate_group, PageSpace* page_space, SyncMarkingVisitor* visitor) : marker_(marker), - isolate_(isolate), + isolate_group_(isolate_group), page_space_(page_space), visitor_(visitor) { #if defined(DEBUG) @@ -674,8 +684,8 @@ class ConcurrentMarkTask : public ThreadPool::Task { } virtual void Run() { - bool result = - Thread::EnterIsolateAsHelper(isolate_, Thread::kMarkerTask, true); + bool result = Thread::EnterIsolateGroupAsHelper( + isolate_group_, Thread::kMarkerTask, /*bypass_safepoint=*/true); ASSERT(result); { TIMELINE_FUNCTION_GC_DURATION(Thread::Current(), "ConcurrentMark"); @@ -693,7 +703,7 @@ class ConcurrentMarkTask : public ThreadPool::Task { } // Exit isolate cleanly *before* notifying it, to avoid shutdown race. - Thread::ExitIsolateAsHelper(true); + Thread::ExitIsolateGroupAsHelper(/*bypass_safepoint=*/true); // This marker task is done. Notify the original isolate. { MonitorLocker ml(page_space_->tasks_lock()); @@ -710,7 +720,7 @@ class ConcurrentMarkTask : public ThreadPool::Task { private: GCMarker* marker_; - Isolate* isolate_; + IsolateGroup* isolate_group_; PageSpace* page_space_; SyncMarkingVisitor* visitor_; @@ -744,8 +754,8 @@ intptr_t GCMarker::MarkedWordsPerMicro() const { return marked_words_per_job_micro * jobs; } -GCMarker::GCMarker(Isolate* isolate, Heap* heap) - : isolate_(isolate), +GCMarker::GCMarker(IsolateGroup* isolate_group, Heap* heap) + : isolate_group_(isolate_group), heap_(heap), marking_stack_(), visitors_(), @@ -760,8 +770,8 @@ GCMarker::GCMarker(Isolate* isolate, Heap* heap) GCMarker::~GCMarker() { // Cleanup in case isolate shutdown happens after starting the concurrent // marker and before finalizing. - if (isolate_->marking_stack() != NULL) { - isolate_->DisableIncrementalBarrier(); + if (isolate_group_->marking_stack() != NULL) { + isolate_group_->DisableIncrementalBarrier(); for (intptr_t i = 0; i < FLAG_marker_tasks; i++) { visitors_[i]->AbandonWork(); delete visitors_[i]; @@ -771,7 +781,8 @@ GCMarker::~GCMarker() { } void GCMarker::StartConcurrentMark(PageSpace* page_space) { - isolate_->EnableIncrementalBarrier(&marking_stack_, &deferred_marking_stack_); + isolate_group_->EnableIncrementalBarrier(&marking_stack_, + &deferred_marking_stack_); const intptr_t num_tasks = FLAG_marker_tasks; @@ -790,12 +801,12 @@ void GCMarker::StartConcurrentMark(PageSpace* page_space) { ResetSlices(); for (intptr_t i = 0; i < num_tasks; i++) { ASSERT(visitors_[i] == NULL); - visitors_[i] = new SyncMarkingVisitor(isolate_, page_space, &marking_stack_, - &deferred_marking_stack_); + visitors_[i] = new SyncMarkingVisitor( + isolate_group_, page_space, &marking_stack_, &deferred_marking_stack_); // Begin marking on a helper thread. bool result = Dart::thread_pool()->Run( - this, isolate_, page_space, visitors_[i]); + this, isolate_group_, page_space, visitors_[i]); ASSERT(result); } @@ -807,8 +818,8 @@ void GCMarker::StartConcurrentMark(PageSpace* page_space) { } void GCMarker::MarkObjects(PageSpace* page_space) { - if (isolate_->marking_stack() != NULL) { - isolate_->DisableIncrementalBarrier(); + if (isolate_group_->marking_stack() != NULL) { + isolate_group_->DisableIncrementalBarrier(); } Prologue(); @@ -819,7 +830,7 @@ void GCMarker::MarkObjects(PageSpace* page_space) { TIMELINE_FUNCTION_GC_DURATION(thread, "Mark"); int64_t start = OS::GetCurrentMonotonicMicros(); // Mark everything on main thread. - UnsyncMarkingVisitor mark(isolate_, page_space, &marking_stack_, + UnsyncMarkingVisitor mark(isolate_group_, page_space, &marking_stack_, &deferred_marking_stack_); ResetSlices(); IterateRoots(&mark); @@ -844,12 +855,14 @@ void GCMarker::MarkObjects(PageSpace* page_space) { visitor = visitors_[i]; visitors_[i] = NULL; } else { - visitor = new SyncMarkingVisitor( - isolate_, page_space, &marking_stack_, &deferred_marking_stack_); + visitor = + new SyncMarkingVisitor(isolate_group_, page_space, + &marking_stack_, &deferred_marking_stack_); } bool result = Dart::thread_pool()->Run( - this, isolate_, &marking_stack_, &barrier, visitor, &num_busy); + this, isolate_group_, &marking_stack_, &barrier, visitor, + &num_busy); ASSERT(result); } bool more_to_mark = false; diff --git a/runtime/vm/heap/marker.h b/runtime/vm/heap/marker.h index 35ebea90146..a4ad4c0b402 100644 --- a/runtime/vm/heap/marker.h +++ b/runtime/vm/heap/marker.h @@ -14,7 +14,7 @@ namespace dart { // Forward declarations. class HandleVisitor; class Heap; -class Isolate; +class IsolateGroup; class ObjectPointerVisitor; class PageSpace; class RawWeakProperty; @@ -30,7 +30,7 @@ class Thread; // is exited during concurrent marking. class GCMarker { public: - GCMarker(Isolate* isolate, Heap* heap); + GCMarker(IsolateGroup* isolate_group, Heap* heap); ~GCMarker(); // Mark roots synchronously, then spawn tasks to concurrently drain the @@ -60,7 +60,7 @@ class GCMarker { template void FinalizeResultsFrom(MarkingVisitorType* visitor); - Isolate* const isolate_; + IsolateGroup* const isolate_group_; Heap* const heap_; MarkingStack marking_stack_; MarkingStack deferred_marking_stack_; diff --git a/runtime/vm/heap/pages.cc b/runtime/vm/heap/pages.cc index cf646cba19f..a5ecaab87a2 100644 --- a/runtime/vm/heap/pages.cc +++ b/runtime/vm/heap/pages.cc @@ -361,7 +361,7 @@ HeapPage* PageSpace::AllocatePage(HeapPage::PageType type, bool link) { page->set_object_end(page->memory_->end()); if ((type != HeapPage::kExecutable) && (heap_ != nullptr) && - (heap_->isolate() != Dart::vm_isolate())) { + (heap_->isolate_group() != Dart::vm_isolate()->group())) { page->AllocateForwardingPage(); } return page; @@ -680,7 +680,7 @@ class ExclusiveCodePageIterator : ValueObject { void PageSpace::MakeIterable() const { // Assert not called from concurrent sweeper task. // TODO(koda): Use thread/task identity when implemented. - ASSERT(Isolate::Current()->heap() != NULL); + ASSERT(IsolateGroup::Current()->heap() != NULL); if (bump_top_ < bump_end_) { FreeListElement::AsElement(bump_top_, bump_end_ - bump_top_); } @@ -706,9 +706,9 @@ void PageSpace::UpdateMaxCapacityLocked() { return; } ASSERT(heap_ != NULL); - ASSERT(heap_->isolate() != NULL); - Isolate* isolate = heap_->isolate(); - isolate->GetHeapOldCapacityMaxMetric()->SetValue( + ASSERT(heap_->isolate_group() != NULL); + auto isolate_group = heap_->isolate_group(); + isolate_group->GetHeapOldCapacityMaxMetric()->SetValue( static_cast(usage_.capacity_in_words) * kWordSize); #endif // !defined(PRODUCT) } @@ -720,9 +720,9 @@ void PageSpace::UpdateMaxUsed() { return; } ASSERT(heap_ != NULL); - ASSERT(heap_->isolate() != NULL); - Isolate* isolate = heap_->isolate(); - isolate->GetHeapOldUsedMaxMetric()->SetValue(UsedInWords() * kWordSize); + ASSERT(heap_->isolate_group() != NULL); + auto isolate_group = heap_->isolate_group(); + isolate_group->GetHeapOldUsedMaxMetric()->SetValue(UsedInWords() * kWordSize); #endif // !defined(PRODUCT) } @@ -859,8 +859,8 @@ void PageSpace::WriteProtect(bool read_only) { #ifndef PRODUCT void PageSpace::PrintToJSONObject(JSONObject* object) const { - Isolate* isolate = Isolate::Current(); - ASSERT(isolate != NULL); + auto isolate_group = IsolateGroup::Current(); + ASSERT(isolate_group != nullptr); JSONObject space(object, "old"); space.AddProperty("type", "HeapSpace"); space.AddProperty("name", "old"); @@ -871,7 +871,7 @@ void PageSpace::PrintToJSONObject(JSONObject* object) const { space.AddProperty64("external", ExternalInWords() * kWordSize); space.AddProperty("time", MicrosecondsToSeconds(gc_time_micros())); if (collections() > 0) { - int64_t run_time = isolate->UptimeMicros(); + int64_t run_time = isolate_group->UptimeMicros(); run_time = Utils::Maximum(run_time, static_cast(0)); double run_time_millis = MicrosecondsToMilliseconds(run_time); double avg_time_between_collections = @@ -1079,14 +1079,16 @@ void PageSpace::CollectGarbageAtSafepoint(bool compact, int64_t pre_safe_point) { Thread* thread = Thread::Current(); ASSERT(thread->IsAtSafepoint()); - Isolate* isolate = heap_->isolate(); - ASSERT(isolate == Isolate::Current()); + auto isolate_group = heap_->isolate_group(); + ASSERT(isolate_group == IsolateGroup::Current()); const int64_t start = OS::GetCurrentMonotonicMicros(); // Perform various cleanup that relies on no tasks interfering. - isolate->class_table()->FreeOldTables(); - isolate->field_table()->FreeOldTables(); + isolate_group->class_table()->FreeOldTables(); + isolate_group->ForEachIsolate( + [&](Isolate* isolate) { isolate->field_table()->FreeOldTables(); }, + /*at_safepoint=*/true); NoSafepointScope no_safepoints; @@ -1112,7 +1114,7 @@ void PageSpace::CollectGarbageAtSafepoint(bool compact, // Mark all reachable old-gen objects. if (marker_ == NULL) { ASSERT(phase() == kDone); - marker_ = new GCMarker(isolate, heap_); + marker_ = new GCMarker(isolate_group, heap_); } else { ASSERT(phase() == kAwaitingFinalization); } @@ -1177,7 +1179,7 @@ void PageSpace::CollectGarbageAtSafepoint(bool compact, Compact(thread); set_phase(kDone); } else if (FLAG_concurrent_sweep) { - ConcurrentSweep(isolate); + ConcurrentSweep(isolate_group); } else { SweepLarge(); Sweep(); @@ -1261,17 +1263,17 @@ void PageSpace::Sweep() { } } -void PageSpace::ConcurrentSweep(Isolate* isolate) { +void PageSpace::ConcurrentSweep(IsolateGroup* isolate_group) { // Start the concurrent sweeper task now. - GCSweeper::SweepConcurrent(isolate, pages_, pages_tail_, large_pages_, + GCSweeper::SweepConcurrent(isolate_group, pages_, pages_tail_, large_pages_, large_pages_tail_, &freelist_[HeapPage::kData]); } void PageSpace::Compact(Thread* thread) { - thread->isolate()->set_compaction_in_progress(true); + thread->isolate_group()->set_compaction_in_progress(true); GCCompactor compactor(thread, heap_); compactor.Compact(pages_, &freelist_[HeapPage::kData], &pages_lock_); - thread->isolate()->set_compaction_in_progress(false); + thread->isolate_group()->set_compaction_in_progress(false); if (FLAG_verify_after_gc) { OS::PrintErr("Verifying after compacting..."); @@ -1381,6 +1383,92 @@ bool PageSpace::IsObjectFromImagePages(dart::RawObject* object) { return false; } +static void AppendList(HeapPage** pages, + HeapPage** pages_tail, + HeapPage** other_pages, + HeapPage** other_pages_tail) { + ASSERT((*pages == nullptr) == (*pages_tail == nullptr)); + ASSERT((*other_pages == nullptr) == (*other_pages_tail == nullptr)); + + if (*other_pages != nullptr) { + if (*pages_tail == nullptr) { + *pages = *other_pages; + *pages_tail = *other_pages_tail; + } else { + const bool is_execute = FLAG_write_protect_code && + (*pages_tail)->type() == HeapPage::kExecutable; + if (is_execute) { + (*pages_tail)->WriteProtect(false); + } + (*pages_tail)->set_next(*other_pages); + if (is_execute) { + (*pages_tail)->WriteProtect(true); + } + *pages_tail = *other_pages_tail; + } + *other_pages = nullptr; + *other_pages_tail = nullptr; + } +} + +static void EnsureEqualImagePages(HeapPage* pages, HeapPage* other_pages) { +#if defined(DEBUG) + while (pages != nullptr) { + ASSERT((pages == nullptr) == (other_pages == nullptr)); + ASSERT(pages->object_start() == other_pages->object_start()); + ASSERT(pages->object_end() == other_pages->object_end()); + pages = pages->next(); + other_pages = other_pages->next(); + } +#endif +} + +void PageSpace::MergeOtherPageSpace(PageSpace* other) { + MutexLocker ml(&pages_lock_); + MutexLocker ml2(&other->pages_lock_); + + other->AbandonBumpAllocation(); + + ASSERT(other->bump_top_ == 0 && other->bump_end_ == 0); + ASSERT(other->tasks_ == 0); + ASSERT(other->concurrent_marker_tasks_ == 0); + ASSERT(other->phase_ == kDone); + DEBUG_ASSERT(other->iterating_thread_ == nullptr); + ASSERT(other->marker_ == nullptr); + + for (intptr_t i = 0; i < HeapPage::kNumPageTypes; ++i) { + const bool is_protected = + FLAG_write_protect_code && i == HeapPage::kExecutable; + freelist_[i].MergeOtherFreelist(&other->freelist_[i], is_protected); + other->freelist_[i].Reset(); + } + + AppendList(&pages_, &pages_tail_, &other->pages_, &other->pages_tail_); + AppendList(&exec_pages_, &exec_pages_tail_, &other->exec_pages_, + &other->exec_pages_tail_); + AppendList(&large_pages_, &large_pages_tail_, &other->large_pages_, + &other->large_pages_tail_); + // We intentionall do not merge [image_pages_] beause [this] and [other] have + // the same mmap()ed image page areas. + EnsureEqualImagePages(image_pages_, other->image_pages_); + + // We intentionaly do not increase [max_capacity_in_words_] because this can + // lead [max_capacity_in_words_] to become larger and larger and eventually + // wrap-around and become negative. + allocated_black_in_words_ += other->allocated_black_in_words_; + gc_time_micros_ += other->gc_time_micros_; + collections_ += other->collections_; + + usage_.capacity_in_words += other->usage_.capacity_in_words; + usage_.used_in_words += other->usage_.used_in_words; + usage_.external_in_words += other->usage_.external_in_words; + + page_space_controller_.MergeOtherPageSpaceController( + &other->page_space_controller_); + + ASSERT(FLAG_concurrent_mark || other->enable_concurrent_mark_ == false); +} + PageSpaceController::PageSpaceController(Heap* heap, int heap_growth_ratio, int heap_growth_max, @@ -1576,11 +1664,19 @@ void PageSpaceController::RecordUpdate(SpaceUsage before, if (FLAG_log_growth) { THR_Print("%s: threshold=%" Pd "kB, idle_threshold=%" Pd "kB, reason=%s\n", - heap_->isolate()->name(), gc_threshold_in_words_ / KBInWords, + heap_->isolate_group()->source()->name, + gc_threshold_in_words_ / KBInWords, idle_gc_threshold_in_words_ / KBInWords, reason); } } +void PageSpaceController::MergeOtherPageSpaceController( + PageSpaceController* other) { + last_usage_.capacity_in_words += other->last_usage_.capacity_in_words; + last_usage_.used_in_words += other->last_usage_.used_in_words; + last_usage_.external_in_words += other->last_usage_.external_in_words; +} + void PageSpaceGarbageCollectionHistory::AddGarbageCollectionTime(int64_t start, int64_t end) { Entry entry; diff --git a/runtime/vm/heap/pages.h b/runtime/vm/heap/pages.h index d8ee95901b5..a8ef135d503 100644 --- a/runtime/vm/heap/pages.h +++ b/runtime/vm/heap/pages.h @@ -237,7 +237,10 @@ class PageSpaceController { bool is_enabled() { return is_enabled_; } private: + friend class PageSpace; // For MergeOtherPageSpaceController + void RecordUpdate(SpaceUsage before, SpaceUsage after, const char* reason); + void MergeOtherPageSpaceController(PageSpaceController* other); Heap* heap_; @@ -261,7 +264,7 @@ class PageSpaceController { // we grow the heap more aggressively. const int garbage_collection_time_ratio_; - // Perform a synchronous GC when capacity exceeds this amount. + // Perform a GC when capacity exceeds this amount. intptr_t gc_threshold_in_words_; // Start considering idle GC when capacity exceeds this amount. @@ -450,6 +453,8 @@ class PageSpace { bool IsObjectFromImagePages(RawObject* object); + void MergeOtherPageSpace(PageSpace* other); + private: // Ids for time and data records in Heap::GCStats. enum { @@ -506,7 +511,7 @@ class PageSpace { int64_t pre_safe_point); void SweepLarge(); void Sweep(); - void ConcurrentSweep(Isolate* isolate); + void ConcurrentSweep(IsolateGroup* isolate_group); void Compact(Thread* thread); static intptr_t LargePageSizeInWordsFor(intptr_t size); diff --git a/runtime/vm/heap/pointer_block.cc b/runtime/vm/heap/pointer_block.cc index 19428f7003e..69bc1f515cf 100644 --- a/runtime/vm/heap/pointer_block.cc +++ b/runtime/vm/heap/pointer_block.cc @@ -105,10 +105,11 @@ void StoreBuffer::PushBlock(Block* block, ThresholdPolicy policy) { if ((policy == kCheckThreshold) && Overflowed()) { MutexLocker ml(&mutex_); Thread* thread = Thread::Current(); - // Sanity check: it makes no sense to schedule the GC in another isolate. + // Sanity check: it makes no sense to schedule the GC in another isolate + // group. // (If Isolate ever gets multiple store buffers, we should avoid this // coupling by passing in an explicit callback+parameter at construction.) - ASSERT(thread->isolate()->store_buffer() == this); + ASSERT(thread->isolate_group()->store_buffer() == this); thread->ScheduleInterrupts(Thread::kVMInterrupt); } } diff --git a/runtime/vm/heap/safepoint.cc b/runtime/vm/heap/safepoint.cc index 3bf198baeca..2773fe80169 100644 --- a/runtime/vm/heap/safepoint.cc +++ b/runtime/vm/heap/safepoint.cc @@ -14,11 +14,9 @@ DEFINE_FLAG(bool, trace_safepoint, false, "Trace Safepoint logic."); SafepointOperationScope::SafepointOperationScope(Thread* T) : ThreadStackResource(T) { - ASSERT(T != NULL); - Isolate* I = T->isolate(); - ASSERT(I != NULL); + ASSERT(T != nullptr && T->isolate_group() != nullptr); - SafepointHandler* handler = I->group()->safepoint_handler(); + SafepointHandler* handler = T->isolate_group()->safepoint_handler(); ASSERT(handler != NULL); // Signal all threads to get to a safepoint and wait for them to @@ -28,12 +26,10 @@ SafepointOperationScope::SafepointOperationScope(Thread* T) SafepointOperationScope::~SafepointOperationScope() { Thread* T = thread(); - ASSERT(T != NULL); - Isolate* I = T->isolate(); - ASSERT(I != NULL); + ASSERT(T != nullptr && T->isolate_group() != nullptr); // Resume all threads which are blocked for the safepoint operation. - SafepointHandler* handler = I->safepoint_handler(); + SafepointHandler* handler = T->isolate_group()->safepoint_handler(); ASSERT(handler != NULL); handler->ResumeThreads(T); } @@ -42,10 +38,10 @@ ForceGrowthSafepointOperationScope::ForceGrowthSafepointOperationScope( Thread* T) : ThreadStackResource(T) { ASSERT(T != NULL); - Isolate* I = T->isolate(); - ASSERT(I != NULL); + IsolateGroup* IG = T->isolate_group(); + ASSERT(IG != NULL); - SafepointHandler* handler = I->group()->safepoint_handler(); + SafepointHandler* handler = IG->safepoint_handler(); ASSERT(handler != NULL); // Signal all threads to get to a safepoint and wait for them to @@ -53,7 +49,7 @@ ForceGrowthSafepointOperationScope::ForceGrowthSafepointOperationScope( handler->SafepointThreads(T); // N.B.: Change growth policy inside the safepoint to prevent racy access. - Heap* heap = I->heap(); + Heap* heap = IG->heap(); current_growth_controller_state_ = heap->GrowthControlState(); heap->DisableGrowthControl(); } @@ -61,15 +57,15 @@ ForceGrowthSafepointOperationScope::ForceGrowthSafepointOperationScope( ForceGrowthSafepointOperationScope::~ForceGrowthSafepointOperationScope() { Thread* T = thread(); ASSERT(T != NULL); - Isolate* I = T->isolate(); - ASSERT(I != NULL); + IsolateGroup* IG = T->isolate_group(); + ASSERT(IG != NULL); // N.B.: Change growth policy inside the safepoint to prevent racy access. - Heap* heap = I->heap(); + Heap* heap = IG->heap(); heap->SetGrowthControlState(current_growth_controller_state_); // Resume all threads which are blocked for the safepoint operation. - SafepointHandler* handler = I->safepoint_handler(); + SafepointHandler* handler = IG->safepoint_handler(); ASSERT(handler != NULL); handler->ResumeThreads(T); @@ -138,7 +134,6 @@ void SafepointHandler::SafepointThreads(Thread* T) { // Thread is not already at a safepoint so try to // get it to a safepoint and wait for it to check in. if (current->IsMutatorThread()) { - ASSERT(T->isolate() != NULL); current->ScheduleInterruptsLocked(Thread::kVMInterrupt); } MonitorLocker sl(&safepoint_lock_); diff --git a/runtime/vm/heap/scavenger.cc b/runtime/vm/heap/scavenger.cc index e90fc3c92d1..4dd199e55e1 100644 --- a/runtime/vm/heap/scavenger.cc +++ b/runtime/vm/heap/scavenger.cc @@ -87,10 +87,10 @@ static inline void objcpy(void* dst, const void* src, size_t size) { class ScavengerVisitor : public ObjectPointerVisitor { public: - explicit ScavengerVisitor(Isolate* isolate, + explicit ScavengerVisitor(IsolateGroup* isolate_group, Scavenger* scavenger, SemiSpace* from) - : ObjectPointerVisitor(isolate), + : ObjectPointerVisitor(isolate_group), thread_(Thread::Current()), scavenger_(scavenger), from_(from), @@ -280,8 +280,8 @@ class ScavengerWeakVisitor : public HandleVisitor { ScavengerWeakVisitor(Thread* thread, Scavenger* scavenger) : HandleVisitor(thread), scavenger_(scavenger), - class_table_(thread->isolate()->shared_class_table()) { - ASSERT(scavenger->heap_->isolate() == thread->isolate()); + class_table_(thread->isolate_group()->class_table()) { + ASSERT(scavenger->heap_->isolate_group() == thread->isolate_group()); } void VisitHandle(uword addr) { @@ -289,9 +289,9 @@ class ScavengerWeakVisitor : public HandleVisitor { reinterpret_cast(addr); RawObject** p = handle->raw_addr(); if (scavenger_->IsUnreachable(p)) { - handle->UpdateUnreachable(thread()->isolate()); + handle->UpdateUnreachable(thread()->isolate_group()); } else { - handle->UpdateRelocated(thread()->isolate()); + handle->UpdateRelocated(thread()->isolate_group()); } } @@ -306,8 +306,9 @@ class ScavengerWeakVisitor : public HandleVisitor { // StoreBuffers. class VerifyStoreBufferPointerVisitor : public ObjectPointerVisitor { public: - VerifyStoreBufferPointerVisitor(Isolate* isolate, const SemiSpace* to) - : ObjectPointerVisitor(isolate), to_(to) {} + VerifyStoreBufferPointerVisitor(IsolateGroup* isolate_group, + const SemiSpace* to) + : ObjectPointerVisitor(isolate_group), to_(to) {} void VisitPointers(RawObject** first, RawObject** last) { for (RawObject** current = first; current <= last; current++) { @@ -475,8 +476,8 @@ intptr_t Scavenger::NewSizeInWords(intptr_t old_size_in_words) const { } } -SemiSpace* Scavenger::Prologue(Isolate* isolate) { - isolate->ReleaseStoreBuffers(); +SemiSpace* Scavenger::Prologue(IsolateGroup* isolate_group) { + isolate_group->ReleaseStoreBuffers(); // Flip the two semi-spaces so that to_ is always the space for allocating // objects. @@ -497,14 +498,18 @@ SemiSpace* Scavenger::Prologue(Isolate* isolate) { return from; } -void Scavenger::Epilogue(Isolate* isolate, SemiSpace* from) { +void Scavenger::Epilogue(IsolateGroup* isolate_group, SemiSpace* from) { // All objects in the to space have been copied from the from space at this // moment. // Ensure the mutator thread will fail the next allocation. This will force // mutator to allocate a new TLAB - Thread* mutator_thread = isolate->mutator_thread(); - ASSERT((mutator_thread == NULL) || (!mutator_thread->HasActiveTLAB())); + isolate_group->ForEachIsolate( + [&](Isolate* isolate) { + Thread* mutator_thread = isolate->mutator_thread(); + ASSERT((mutator_thread == NULL) || (!mutator_thread->HasActiveTLAB())); + }, + /*at_safepoint=*/true); double avg_frac = stats_history_.Get(0).PromoCandidatesSuccessFraction(); if (stats_history_.Size() >= 2) { @@ -567,7 +572,8 @@ void Scavenger::Epilogue(Isolate* isolate, SemiSpace* from) { PageSpace* page_space = heap_->old_space(); MonitorLocker ml(page_space->tasks_lock()); if (page_space->tasks() == 0) { - VerifyStoreBufferPointerVisitor verify_store_buffer_visitor(isolate, to_); + VerifyStoreBufferPointerVisitor verify_store_buffer_visitor(isolate_group, + to_); heap_->old_space()->VisitObjectPointers(&verify_store_buffer_visitor); } } @@ -595,11 +601,11 @@ bool Scavenger::ShouldPerformIdleScavenge(int64_t deadline) { return estimated_scavenge_completion <= deadline; } -void Scavenger::IterateStoreBuffers(Isolate* isolate, +void Scavenger::IterateStoreBuffers(IsolateGroup* isolate_group, ScavengerVisitor* visitor) { // Iterating through the store buffers. // Grab the deduplication sets out of the isolate's consolidated store buffer. - StoreBufferBlock* pending = isolate->store_buffer()->Blocks(); + StoreBufferBlock* pending = isolate_group->store_buffer()->Blocks(); intptr_t total_count = 0; while (pending != NULL) { StoreBufferBlock* next = pending->next(); @@ -617,7 +623,8 @@ void Scavenger::IterateStoreBuffers(Isolate* isolate, } pending->Reset(); // Return the emptied block for recycling (no need to check threshold). - isolate->store_buffer()->PushBlock(pending, StoreBuffer::kIgnoreThreshold); + isolate_group->store_buffer()->PushBlock(pending, + StoreBuffer::kIgnoreThreshold); pending = next; } @@ -631,29 +638,34 @@ void Scavenger::IterateStoreBuffers(Isolate* isolate, visitor->VisitingOldObject(NULL); } -void Scavenger::IterateObjectIdTable(Isolate* isolate, +void Scavenger::IterateObjectIdTable(IsolateGroup* isolate_group, ScavengerVisitor* visitor) { #ifndef PRODUCT - isolate->object_id_ring()->VisitPointers(visitor); + isolate_group->ForEachIsolate( + [&](Isolate* isolate) { + isolate->object_id_ring()->VisitPointers(visitor); + }, + /*at_safepoint=*/true); #endif // !PRODUCT } -void Scavenger::IterateRoots(Isolate* isolate, ScavengerVisitor* visitor) { +void Scavenger::IterateRoots(IsolateGroup* isolate_group, + ScavengerVisitor* visitor) { #ifdef SUPPORT_TIMELINE Thread* thread = Thread::Current(); #endif int64_t start = OS::GetCurrentMonotonicMicros(); { TIMELINE_FUNCTION_GC_DURATION(thread, "ProcessRoots"); - isolate->VisitObjectPointers(visitor, - ValidationPolicy::kDontValidateFrames); + isolate_group->VisitObjectPointers(visitor, + ValidationPolicy::kDontValidateFrames); } int64_t middle = OS::GetCurrentMonotonicMicros(); { TIMELINE_FUNCTION_GC_DURATION(thread, "ProcessRememberedSet"); - IterateStoreBuffers(isolate, visitor); + IterateStoreBuffers(isolate_group, visitor); } - IterateObjectIdTable(isolate, visitor); + IterateObjectIdTable(isolate_group, visitor); int64_t end = OS::GetCurrentMonotonicMicros(); heap_->RecordData(kToKBAfterStoreBuffer, RoundWordsToKB(UsedInWords())); heap_->RecordTime(kVisitIsolateRoots, middle - start); @@ -682,8 +694,9 @@ bool Scavenger::IsUnreachable(RawObject** p) { return true; } -void Scavenger::IterateWeakRoots(Isolate* isolate, HandleVisitor* visitor) { - isolate->VisitWeakPersistentHandles(visitor); +void Scavenger::IterateWeakRoots(IsolateGroup* isolate_group, + HandleVisitor* visitor) { + isolate_group->VisitWeakPersistentHandles(visitor); } void Scavenger::ProcessToSpace(ScavengerVisitor* visitor) { @@ -767,10 +780,10 @@ void Scavenger::UpdateMaxHeapCapacity() { } ASSERT(to_ != NULL); ASSERT(heap_ != NULL); - Isolate* isolate = heap_->isolate(); - ASSERT(isolate != NULL); - isolate->GetHeapNewCapacityMaxMetric()->SetValue(to_->size_in_words() * - kWordSize); + auto isolate_group = heap_->isolate_group(); + ASSERT(isolate_group != NULL); + isolate_group->GetHeapNewCapacityMaxMetric()->SetValue(to_->size_in_words() * + kWordSize); #endif // !defined(PRODUCT) } @@ -782,9 +795,9 @@ void Scavenger::UpdateMaxHeapUsage() { } ASSERT(to_ != NULL); ASSERT(heap_ != NULL); - Isolate* isolate = heap_->isolate(); - ASSERT(isolate != NULL); - isolate->GetHeapNewUsedMaxMetric()->SetValue(UsedInWords() * kWordSize); + auto isolate_group = heap_->isolate_group(); + ASSERT(isolate_group != NULL); + isolate_group->GetHeapNewUsedMaxMetric()->SetValue(UsedInWords() * kWordSize); #endif // !defined(PRODUCT) } @@ -859,13 +872,16 @@ void Scavenger::ProcessWeakReferences() { // Each isolate might have a weak table used for fast snapshot writing (i.e. // isolate communication). Rehash those tables if need be. - auto isolate = heap_->isolate(); - auto table = isolate->forward_table_new(); - if (table != NULL) { - auto replacement = WeakTable::NewFrom(table); - rehash_weak_table(table, replacement, isolate->forward_table_old()); - isolate->set_forward_table_new(replacement); - } + heap_->isolate_group()->ForEachIsolate( + [&](Isolate* isolate) { + auto table = isolate->forward_table_new(); + if (table != nullptr) { + auto replacement = WeakTable::NewFrom(table); + rehash_weak_table(table, replacement, isolate->forward_table_old()); + isolate->set_forward_table_new(replacement); + } + }, + /*at_safepoint=*/true); // The queued weak properties at this point do not refer to reachable keys, // so we clear their key and value fields. @@ -898,45 +914,41 @@ void Scavenger::MakeNewSpaceIterable() const { ASSERT(Thread::Current()->IsAtSafepoint() || (Thread::Current()->task_kind() == Thread::kMarkerTask) || (Thread::Current()->task_kind() == Thread::kCompactorTask)); - Isolate* isolate = heap_->isolate(); - MonitorLocker ml(isolate->threads_lock(), false); - Thread* current = heap_->isolate()->thread_registry()->active_list(); + auto isolate_group = heap_->isolate_group(); + MonitorLocker ml(isolate_group->threads_lock(), false); + Thread* current = heap_->isolate_group()->thread_registry()->active_list(); while (current != NULL) { - // NOTE: During the transition period all isolates within an isolate group - // share the thread registry, but have their own heap. - // So we explicitly filter those threads which belong to the isolate of - // interest (once we have a shared heap this needs to change). - if (current->isolate() == isolate) { - if (current->HasActiveTLAB()) { - heap_->MakeTLABIterable(current); - } + if (current->HasActiveTLAB()) { + heap_->MakeTLABIterable(current); } current = current->next(); } - Thread* mutator_thread = isolate->mutator_thread(); - if (mutator_thread != NULL) { - heap_->MakeTLABIterable(mutator_thread); - } + isolate_group->ForEachIsolate( + [&](Isolate* isolate) { + Thread* mutator_thread = isolate->mutator_thread(); + if (mutator_thread != NULL) { + heap_->MakeTLABIterable(mutator_thread); + } + }, + /*at_safepoint=*/true); } -void Scavenger::AbandonTLABs(Isolate* isolate) { +void Scavenger::AbandonTLABs(IsolateGroup* isolate_group) { ASSERT(Thread::Current()->IsAtSafepoint()); - MonitorLocker ml(isolate->threads_lock(), false); - Thread* current = isolate->thread_registry()->active_list(); + MonitorLocker ml(isolate_group->threads_lock(), false); + Thread* current = isolate_group->thread_registry()->active_list(); while (current != NULL) { - // NOTE: During the transition period all isolates within an isolate group - // share the thread registry, but have their own heap. - // So we explicitly filter those threads which belong to the isolate of - // interest (once we have a shared heap this needs to change). - if (current->isolate() == isolate) { - heap_->AbandonRemainingTLAB(current); - } + heap_->AbandonRemainingTLAB(current); current = current->next(); } - Thread* mutator_thread = isolate->mutator_thread(); - if (mutator_thread != NULL) { - heap_->AbandonRemainingTLAB(mutator_thread); - } + isolate_group->ForEachIsolate( + [&](Isolate* isolate) { + Thread* mutator_thread = isolate->mutator_thread(); + if (mutator_thread != NULL) { + heap_->AbandonRemainingTLAB(mutator_thread); + } + }, + /*at_safepoint=*/true); } void Scavenger::VisitObjectPointers(ObjectPointerVisitor* visitor) const { @@ -1015,7 +1027,7 @@ uword Scavenger::TryAllocateNewTLAB(Thread* thread, intptr_t size) { } void Scavenger::Scavenge() { - Isolate* isolate = heap_->isolate(); + auto isolate_group = heap_->isolate_group(); // Ensure that all threads for this isolate are at a safepoint (either stopped // or in native code). If two threads are racing at this point, the loser // will continue with its scavenge after waiting for the winner to complete. @@ -1047,21 +1059,21 @@ void Scavenger::Scavenge() { } // Prepare for a scavenge. - AbandonTLABs(isolate); + AbandonTLABs(isolate_group); intptr_t abandoned_bytes = GetAndResetAbandonedInBytes(); SpaceUsage usage_before = GetCurrentUsage(); intptr_t promo_candidate_words = (survivor_end_ - FirstObjectStart()) / kWordSize; - SemiSpace* from = Prologue(isolate); + SemiSpace* from = Prologue(isolate_group); // The API prologue/epilogue may create/destroy zones, so we must not // depend on zone allocations surviving beyond the epilogue callback. { StackZone zone(thread); // Setup the visitor and run the scavenge. - ScavengerVisitor visitor(isolate, this, from); + ScavengerVisitor visitor(isolate_group, this, from); page_space->AcquireDataLock(); - IterateRoots(isolate, &visitor); + IterateRoots(isolate_group, &visitor); int64_t iterate_roots = OS::GetCurrentMonotonicMicros(); { TIMELINE_FUNCTION_GC_DURATION(thread, "ProcessToSpace"); @@ -1071,7 +1083,7 @@ void Scavenger::Scavenge() { { TIMELINE_FUNCTION_GC_DURATION(thread, "ProcessWeakHandles"); ScavengerWeakVisitor weak_visitor(thread, this); - IterateWeakRoots(isolate, &weak_visitor); + IterateWeakRoots(isolate_group, &weak_visitor); } ProcessWeakReferences(); page_space->ReleaseDataLock(); @@ -1085,7 +1097,7 @@ void Scavenger::Scavenge() { visitor.bytes_promoted() >> kWordSizeLog2, abandoned_bytes >> kWordSizeLog2)); } - Epilogue(isolate, from); + Epilogue(isolate_group, from); // TODO(koda): Make verification more compatible with concurrent sweep. if (FLAG_verify_after_gc && !FLAG_concurrent_sweep) { @@ -1106,15 +1118,15 @@ void Scavenger::WriteProtect(bool read_only) { #ifndef PRODUCT void Scavenger::PrintToJSONObject(JSONObject* object) const { - Isolate* isolate = Isolate::Current(); - ASSERT(isolate != NULL); + auto isolate_group = IsolateGroup::Current(); + ASSERT(isolate_group != nullptr); JSONObject space(object, "new"); space.AddProperty("type", "HeapSpace"); space.AddProperty("name", "new"); space.AddProperty("vmName", "Scavenger"); space.AddProperty("collections", collections()); if (collections() > 0) { - int64_t run_time = isolate->UptimeMicros(); + int64_t run_time = isolate_group->UptimeMicros(); run_time = Utils::Maximum(run_time, static_cast(0)); double run_time_millis = MicrosecondsToMilliseconds(run_time); double avg_time_between_collections = diff --git a/runtime/vm/heap/scavenger.h b/runtime/vm/heap/scavenger.h index e3144098871..2171b84024a 100644 --- a/runtime/vm/heap/scavenger.h +++ b/runtime/vm/heap/scavenger.h @@ -229,7 +229,7 @@ class Scavenger { void MakeNewSpaceIterable() const; int64_t FreeSpaceInWords(Isolate* isolate) const; - void AbandonTLABs(Isolate* isolate); + void AbandonTLABs(IsolateGroup* isolate_group); private: // Ids for time and data records in Heap::GCStats. @@ -251,18 +251,22 @@ class Scavenger { uword FirstObjectStart() const { return to_->start() + kNewObjectAlignmentOffset; } - SemiSpace* Prologue(Isolate* isolate); - void IterateStoreBuffers(Isolate* isolate, ScavengerVisitor* visitor); - void IterateObjectIdTable(Isolate* isolate, ScavengerVisitor* visitor); - void IterateRoots(Isolate* isolate, ScavengerVisitor* visitor); - void IterateWeakProperties(Isolate* isolate, ScavengerVisitor* visitor); - void IterateWeakReferences(Isolate* isolate, ScavengerVisitor* visitor); - void IterateWeakRoots(Isolate* isolate, HandleVisitor* visitor); + SemiSpace* Prologue(IsolateGroup* isolate_group); + void IterateStoreBuffers(IsolateGroup* isolate_group, + ScavengerVisitor* visitor); + void IterateObjectIdTable(IsolateGroup* isolate_group, + ScavengerVisitor* visitor); + void IterateRoots(IsolateGroup* isolate_group, ScavengerVisitor* visitor); + void IterateWeakProperties(IsolateGroup* isolate_group, + ScavengerVisitor* visitor); + void IterateWeakReferences(IsolateGroup* isolate_group, + ScavengerVisitor* visitor); + void IterateWeakRoots(IsolateGroup* isolate_group, HandleVisitor* visitor); void ProcessToSpace(ScavengerVisitor* visitor); void EnqueueWeakProperty(RawWeakProperty* raw_weak); uword ProcessWeakProperty(RawWeakProperty* raw_weak, ScavengerVisitor* visitor); - void Epilogue(Isolate* isolate, SemiSpace* from); + void Epilogue(IsolateGroup* isolate_group, SemiSpace* from); bool IsUnreachable(RawObject** p); diff --git a/runtime/vm/heap/sweeper.cc b/runtime/vm/heap/sweeper.cc index 8f23e7fbbb5..405357d521b 100644 --- a/runtime/vm/heap/sweeper.cc +++ b/runtime/vm/heap/sweeper.cc @@ -106,21 +106,21 @@ intptr_t GCSweeper::SweepLargePage(HeapPage* page) { class ConcurrentSweeperTask : public ThreadPool::Task { public: - ConcurrentSweeperTask(Isolate* isolate, + ConcurrentSweeperTask(IsolateGroup* isolate_group, PageSpace* old_space, HeapPage* first, HeapPage* last, HeapPage* large_first, HeapPage* large_last, FreeList* freelist) - : task_isolate_(isolate), + : task_isolate_group_(isolate_group), old_space_(old_space), first_(first), last_(last), large_first_(large_first), large_last_(large_last), freelist_(freelist) { - ASSERT(task_isolate_ != NULL); + ASSERT(task_isolate_group_ != NULL); ASSERT(first_ != NULL); ASSERT(old_space_ != NULL); ASSERT(last_ != NULL); @@ -131,8 +131,8 @@ class ConcurrentSweeperTask : public ThreadPool::Task { } virtual void Run() { - bool result = - Thread::EnterIsolateAsHelper(task_isolate_, Thread::kSweeperTask, true); + bool result = Thread::EnterIsolateGroupAsHelper( + task_isolate_group_, Thread::kSweeperTask, /*bypass_safepoint=*/true); ASSERT(result); { Thread* thread = Thread::Current(); @@ -197,7 +197,7 @@ class ConcurrentSweeperTask : public ThreadPool::Task { } } // Exit isolate cleanly *before* notifying it, to avoid shutdown race. - Thread::ExitIsolateAsHelper(true); + Thread::ExitIsolateGroupAsHelper(/*bypass_safepoint=*/true); // This sweeper task is done. Notify the original isolate. { MonitorLocker ml(old_space_->tasks_lock()); @@ -209,7 +209,7 @@ class ConcurrentSweeperTask : public ThreadPool::Task { } private: - Isolate* task_isolate_; + IsolateGroup* task_isolate_group_; PageSpace* old_space_; HeapPage* first_; HeapPage* last_; @@ -218,15 +218,15 @@ class ConcurrentSweeperTask : public ThreadPool::Task { FreeList* freelist_; }; -void GCSweeper::SweepConcurrent(Isolate* isolate, +void GCSweeper::SweepConcurrent(IsolateGroup* isolate_group, HeapPage* first, HeapPage* last, HeapPage* large_first, HeapPage* large_last, FreeList* freelist) { bool result = Dart::thread_pool()->Run( - isolate, isolate->heap()->old_space(), first, last, large_first, - large_last, freelist); + isolate_group, isolate_group->heap()->old_space(), first, last, + large_first, large_last, freelist); ASSERT(result); } diff --git a/runtime/vm/heap/sweeper.h b/runtime/vm/heap/sweeper.h index 068a30d0e98..60d3d3c62b8 100644 --- a/runtime/vm/heap/sweeper.h +++ b/runtime/vm/heap/sweeper.h @@ -13,7 +13,7 @@ namespace dart { class FreeList; class Heap; class HeapPage; -class Isolate; +class IsolateGroup; class PageSpace; // The class GCSweeper is used to visit the heap after marking to reclaim unused @@ -35,7 +35,7 @@ class GCSweeper { intptr_t SweepLargePage(HeapPage* page); // Sweep the regular sized data pages between first and last inclusive. - static void SweepConcurrent(Isolate* isolate, + static void SweepConcurrent(IsolateGroup* isolate_group, HeapPage* first, HeapPage* last, HeapPage* large_first, diff --git a/runtime/vm/heap/verifier.cc b/runtime/vm/heap/verifier.cc index 237777e440c..12762ec51dc 100644 --- a/runtime/vm/heap/verifier.cc +++ b/runtime/vm/heap/verifier.cc @@ -41,7 +41,7 @@ void VerifyObjectVisitor::VisitObject(RawObject* raw_obj) { } } allocated_set_->Add(raw_obj); - raw_obj->Validate(isolate_); + raw_obj->Validate(isolate_group_); } void VerifyPointersVisitor::VisitPointers(RawObject** first, RawObject** last) { @@ -69,18 +69,19 @@ void VerifyWeakPointersVisitor::VisitHandle(uword addr) { void VerifyPointersVisitor::VerifyPointers(MarkExpectation mark_expectation) { Thread* thread = Thread::Current(); - Isolate* isolate = thread->isolate(); + auto isolate_group = thread->isolate_group(); HeapIterationScope iteration(thread); StackZone stack_zone(thread); - ObjectSet* allocated_set = isolate->heap()->CreateAllocatedObjectSet( + ObjectSet* allocated_set = isolate_group->heap()->CreateAllocatedObjectSet( stack_zone.GetZone(), mark_expectation); - VerifyPointersVisitor visitor(isolate, allocated_set); + VerifyPointersVisitor visitor(isolate_group, allocated_set); // Visit all strongly reachable objects. iteration.IterateObjectPointers(&visitor, ValidationPolicy::kValidateFrames); VerifyWeakPointersVisitor weak_visitor(&visitor); + // Visit weak handles and prologue weak handles. - isolate->VisitWeakPersistentHandles(&weak_visitor); + isolate_group->VisitWeakPersistentHandles(&weak_visitor); } #if defined(DEBUG) @@ -88,16 +89,22 @@ VerifyCanonicalVisitor::VerifyCanonicalVisitor(Thread* thread) : thread_(thread), instanceHandle_(Instance::Handle(thread->zone())) {} void VerifyCanonicalVisitor::VisitObject(RawObject* obj) { - if ((obj->GetClassId() >= kInstanceCid) && - (obj->GetClassId() != kTypeArgumentsCid)) { - if (obj->IsCanonical()) { - instanceHandle_ ^= obj; - const bool is_canonical = instanceHandle_.CheckIsCanonical(thread_); - if (!is_canonical) { - OS::PrintErr("Instance `%s` is not canonical!\n", - instanceHandle_.ToCString()); + // TODO(dartbug.com/36097): The heap walk can encounter canonical objects of + // other isolates. We should either scan live objects from the roots of each + // individual isolate, or wait until we are ready to share constants across + // isolates. + if (!FLAG_enable_isolate_groups) { + if ((obj->GetClassId() >= kInstanceCid) && + (obj->GetClassId() != kTypeArgumentsCid)) { + if (obj->IsCanonical()) { + instanceHandle_ ^= obj; + const bool is_canonical = instanceHandle_.CheckIsCanonical(thread_); + if (!is_canonical) { + OS::PrintErr("Instance `%s` is not canonical!\n", + instanceHandle_.ToCString()); + } + ASSERT(is_canonical); } - ASSERT(is_canonical); } } } diff --git a/runtime/vm/heap/verifier.h b/runtime/vm/heap/verifier.h index 23ca6b4647e..ae13e90f16c 100644 --- a/runtime/vm/heap/verifier.h +++ b/runtime/vm/heap/verifier.h @@ -15,7 +15,7 @@ namespace dart { // Forward declarations. -class Isolate; +class IsolateGroup; class ObjectSet; class RawObject; @@ -23,17 +23,17 @@ enum MarkExpectation { kForbidMarked, kAllowMarked, kRequireMarked }; class VerifyObjectVisitor : public ObjectVisitor { public: - VerifyObjectVisitor(Isolate* isolate, + VerifyObjectVisitor(IsolateGroup* isolate_group, ObjectSet* allocated_set, MarkExpectation mark_expectation) - : isolate_(isolate), + : isolate_group_(isolate_group), allocated_set_(allocated_set), mark_expectation_(mark_expectation) {} virtual void VisitObject(RawObject* obj); private: - Isolate* isolate_; + IsolateGroup* isolate_group_; ObjectSet* allocated_set_; MarkExpectation mark_expectation_; @@ -44,8 +44,9 @@ class VerifyObjectVisitor : public ObjectVisitor { // the pointers visited are contained in the isolate heap. class VerifyPointersVisitor : public ObjectPointerVisitor { public: - explicit VerifyPointersVisitor(Isolate* isolate, ObjectSet* allocated_set) - : ObjectPointerVisitor(isolate), allocated_set_(allocated_set) {} + explicit VerifyPointersVisitor(IsolateGroup* isolate_group, + ObjectSet* allocated_set) + : ObjectPointerVisitor(isolate_group), allocated_set_(allocated_set) {} virtual void VisitPointers(RawObject** first, RawObject** last); diff --git a/runtime/vm/heap/weak_table.cc b/runtime/vm/heap/weak_table.cc index 95acc864fbc..b4cf28d820d 100644 --- a/runtime/vm/heap/weak_table.cc +++ b/runtime/vm/heap/weak_table.cc @@ -132,4 +132,12 @@ void WeakTable::Rehash() { free(old_data); } +void WeakTable::MergeOtherWeakTable(WeakTable* other) { + for (intptr_t i = 0; i < other->size(); i++) { + if (other->IsValidEntryAtExclusive(i)) { + SetValue(other->ObjectAtExclusive(i), ValueIndex(i)); + } + } +} + } // namespace dart diff --git a/runtime/vm/heap/weak_table.h b/runtime/vm/heap/weak_table.h index c1487b1d037..051051f1dc8 100644 --- a/runtime/vm/heap/weak_table.h +++ b/runtime/vm/heap/weak_table.h @@ -130,6 +130,8 @@ class WeakTable { void Reset(); + void MergeOtherWeakTable(WeakTable* other); + private: enum { kObjectOffset = 0, diff --git a/runtime/vm/interpreter.cc b/runtime/vm/interpreter.cc index d59dbfec409..208920c90b8 100644 --- a/runtime/vm/interpreter.cc +++ b/runtime/vm/interpreter.cc @@ -282,7 +282,7 @@ DART_FORCE_INLINE static bool TryAllocate(Thread* thread, const uword start = thread->top(); #ifndef PRODUCT - auto table = thread->isolate()->shared_class_table(); + auto table = thread->isolate_group()->class_table(); if (UNLIKELY(table->TraceAllocationFor(class_id))) { return false; } diff --git a/runtime/vm/isolate.cc b/runtime/vm/isolate.cc index 6121c42e4f2..cffe4dff0f0 100644 --- a/runtime/vm/isolate.cc +++ b/runtime/vm/isolate.cc @@ -203,48 +203,133 @@ DisableIdleTimerScope::~DisableIdleTimerScope() { } } -IsolateGroup::IsolateGroup(std::unique_ptr source, +class FinalizeWeakPersistentHandlesVisitor : public HandleVisitor { + public: + explicit FinalizeWeakPersistentHandlesVisitor(IsolateGroup* isolate_group) + : HandleVisitor(Thread::Current()), isolate_group_(isolate_group) {} + + void VisitHandle(uword addr) { + auto handle = reinterpret_cast(addr); + handle->UpdateUnreachable(isolate_group_); + } + + private: + IsolateGroup* isolate_group_; + + DISALLOW_COPY_AND_ASSIGN(FinalizeWeakPersistentHandlesVisitor); +}; + +IsolateGroup::IsolateGroup(std::shared_ptr source, void* embedder_data) : embedder_data_(embedder_data), - isolates_rwlock_(new RwLock()), + isolates_lock_(new SafepointRwLock()), isolates_(), + start_time_micros_(OS::GetCurrentMonotonicMicros()), #if !defined(PRODUCT) && !defined(DART_PRECOMPILED_RUNTIME) last_reload_timestamp_(OS::GetCurrentTimeMillis()), #endif source_(std::move(source)), + api_state_(new ApiState()), thread_registry_(new ThreadRegistry()), - safepoint_handler_(new SafepointHandler(this)) { + safepoint_handler_(new SafepointHandler(this)), + shared_class_table_(new SharedClassTable()), + store_buffer_(new StoreBuffer()), + heap_(nullptr) { } -IsolateGroup::~IsolateGroup() {} +IsolateGroup::~IsolateGroup() { + // Finalize any weak persistent handles with a non-null referent. + FinalizeWeakPersistentHandlesVisitor visitor(this); + api_state()->VisitWeakHandlesUnlocked(&visitor); + + // Ensure we destroy the heap before the other members. + heap_ = nullptr; + ASSERT(marking_stack_ == nullptr); +} void IsolateGroup::RegisterIsolate(Isolate* isolate) { - WriteRwLocker wl(ThreadState::Current(), isolates_rwlock_.get()); + SafepointWriteRwLocker ml(Thread::Current(), isolates_lock_.get()); + RegisterIsolateLocked(isolate); +} + +void IsolateGroup::RegisterIsolateLocked(Isolate* isolate) { isolates_.Append(isolate); isolate_count_++; } +bool IsolateGroup::ContainsOnlyOneIsolate() { + SafepointWriteRwLocker ml(Thread::Current(), isolates_lock_.get()); + return isolate_count_ == 0; +} + +void IsolateGroup::RunWithLockedGroup(std::function fun) { + SafepointReadRwLocker ml(Thread::Current(), isolates_lock_.get()); + fun(); +} + void IsolateGroup::UnregisterIsolate(Isolate* isolate) { - bool is_last_isolate = false; - { - WriteRwLocker wl(ThreadState::Current(), isolates_rwlock_.get()); - isolates_.Remove(isolate); - isolate_count_--; - is_last_isolate = isolate_count_ == 0; - } - if (is_last_isolate) { - // If the creation of the isolate group (or the first isolate within the - // isolate group) failed, we do not invoke the cleanup callback (the - // embedder is responsible for handling the creation error). - if (initial_spawn_successful_) { - auto group_shutdown_callback = Isolate::GroupCleanupCallback(); - if (group_shutdown_callback != nullptr) { - group_shutdown_callback(embedder_data()); - } + SafepointWriteRwLocker ml(Thread::Current(), isolates_lock_.get()); + isolates_.Remove(isolate); +} + +bool IsolateGroup::UnregisterIsolateDecrementCount(Isolate* isolate) { + SafepointWriteRwLocker ml(Thread::Current(), isolates_lock_.get()); + isolate_count_--; + return isolate_count_ == 0; +} + +void IsolateGroup::CreateHeap(bool is_vm_isolate, + bool is_service_or_kernel_isolate) { + Heap::Init(this, + is_vm_isolate + ? 0 // New gen size 0; VM isolate should only allocate in old. + : FLAG_new_gen_semi_max_size * MBInWords, + (is_service_or_kernel_isolate ? kDefaultMaxOldGenHeapSize + : FLAG_old_gen_heap_size) * + MBInWords); + + is_vm_isolate_heap_ = is_vm_isolate; + +#if !defined(PRODUCT) +#define ISOLATE_METRIC_CONSTRUCTORS(type, variable, name, unit) \ + metric_##variable##_.InitInstance(this, name, nullptr, Metric::unit); + ISOLATE_GROUP_METRIC_LIST(ISOLATE_METRIC_CONSTRUCTORS) +#undef ISOLATE_METRIC_CONSTRUCTORS +#endif +} + +void IsolateGroup::Shutdown() { + if (heap_ != nullptr) { + // Wait for any concurrent GC tasks to finish before shutting down. + // TODO(rmacnak): Interrupt tasks for faster shutdown. + PageSpace* old_space = heap_->old_space(); + MonitorLocker ml(old_space->tasks_lock()); + while (old_space->tasks() > 0) { + ml.Wait(); } - UnregisterIsolateGroup(this); - delete this; + // Needs to happen before ~PageSpace so TLS and the thread registery are + // still valid. + old_space->AbandonMarkingForShutdown(); } + + UnregisterIsolateGroup(this); + + // If the creation of the isolate group (or the first isolate within the + // isolate group) failed, we do not invoke the cleanup callback (the + // embedder is responsible for handling the creation error). + if (initial_spawn_successful_) { + auto group_shutdown_callback = Isolate::GroupCleanupCallback(); + if (group_shutdown_callback != nullptr) { + group_shutdown_callback(embedder_data()); + } + } + + delete this; +} + +void IsolateGroup::set_heap(std::unique_ptr heap) { + idle_time_handler_.InitializeWithHeap(heap.get()); + heap_ = std::move(heap); } Thread* IsolateGroup::ScheduleThreadLocked(MonitorLocker* ml, @@ -289,6 +374,8 @@ Thread* IsolateGroup::ScheduleThreadLocked(MonitorLocker* ml, thread->isolate_ = nullptr; thread->isolate_group_ = this; thread->field_table_values_ = nullptr; + ASSERT(heap() != nullptr); + thread->heap_ = heap(); thread->set_os_thread(os_thread); ASSERT(thread->execution_state() == Thread::kThreadInNative); thread->set_execution_state(Thread::kThreadInVM); @@ -307,6 +394,13 @@ void IsolateGroup::UnscheduleThreadLocked(MonitorLocker* ml, Thread* thread, bool is_mutator, bool bypass_safepoint) { + // Clear since GC will not visit the thread once it is unscheduled. Do this + // under the thread lock to prevent races with the GC visiting thread roots. + if (!is_mutator) { + thread->heap()->AbandonRemainingTLAB(thread); + thread->ClearReusableHandles(); + } + // Disassociate the 'Thread' structure and unschedule the thread // from this isolate group. if (!is_mutator) { @@ -1226,10 +1320,9 @@ Isolate::Isolate(IsolateGroup* isolate_group, current_tag_(UserTag::null()), default_tag_(UserTag::null()), ic_miss_code_(Code::null()), - shared_class_table_(new SharedClassTable()), - class_table_(shared_class_table_.get()), + class_table_(isolate_group->class_table()), field_table_(new FieldTable()), - store_buffer_(new StoreBuffer()), + isolate_group_(isolate_group), #if !defined(DART_PRECOMPILED_RUNTIME) native_callback_trampolines_(), #endif @@ -1244,6 +1337,9 @@ Isolate::Isolate(IsolateGroup* isolate_group, #undef ISOLATE_METRIC_CONSTRUCTORS reload_every_n_stack_overflow_checks_(FLAG_reload_every), #endif // !defined(PRODUCT) +#if !defined(PRODUCT) + object_id_ring_(new ObjectIdRing()), +#endif start_time_micros_(OS::GetCurrentMonotonicMicros()), random_(), mutex_(NOT_IN_PRODUCT("Isolate::mutex_")), @@ -1289,9 +1385,6 @@ Isolate::Isolate(IsolateGroup* isolate_group, } NOT_IN_PRECOMPILED(optimizing_background_compiler_ = new BackgroundCompiler(this, /* optimizing = */ true)); - - isolate_group->RegisterIsolate(this); - isolate_group_ = isolate_group; } #undef REUSABLE_HANDLE_SCOPE_INIT @@ -1303,11 +1396,6 @@ Isolate::~Isolate() { // RELEASE_ASSERT(reload_context_ == NULL); #endif // !defined(PRODUCT) && !defined(DART_PRECOMPILED_RUNTIME) - // Run isolate group specific cleanup function if the last isolate in an - // isolate group died. - isolate_group_->UnregisterIsolate(this); - isolate_group_ = nullptr; - delete reverse_pc_lookup_cache_; reverse_pc_lookup_cache_ = nullptr; @@ -1332,12 +1420,8 @@ Isolate::~Isolate() { #endif // !defined(PRODUCT) free(name_); - delete store_buffer_; - delete heap_; - ASSERT(marking_stack_ == nullptr); delete object_store_; delete field_table_; - delete api_state_; #if defined(USING_SIMULATOR) delete simulator_; #endif @@ -1403,52 +1487,37 @@ Isolate* Isolate::InitIsolate(const char* name_prefix, #undef ISOLATE_METRIC_INIT #endif // !defined(PRODUCT) - bool is_service_or_kernel_isolate = false; - if (ServiceIsolate::NameEquals(name_prefix)) { - ASSERT(!ServiceIsolate::Exists()); - is_service_or_kernel_isolate = true; - } -#if !defined(DART_PRECOMPILED_RUNTIME) - if (KernelIsolate::NameEquals(name_prefix)) { - ASSERT(!KernelIsolate::Exists()); - KernelIsolate::SetKernelIsolate(result); - is_service_or_kernel_isolate = true; - } -#endif // !defined(DART_PRECOMPILED_RUNTIME) - - Heap::Init(result, - is_vm_isolate - ? 0 // New gen size 0; VM isolate should only allocate in old. - : FLAG_new_gen_semi_max_size * MBInWords, - (is_service_or_kernel_isolate ? kDefaultMaxOldGenHeapSize - : FLAG_old_gen_heap_size) * - MBInWords); - - // TODO(5411455): For now just set the recently created isolate as - // the current isolate. + // First we ensure we enter the isolate. This will ensure we're participating + // in any safepointing requests from this point on. Other threads requesting a + // safepoint operation will therefore wait until we've stopped. + // + // Though the [result] isolate is still in a state where no memory has been + // allocated, which means it's safe to GC the isolate group until here. if (!Thread::EnterIsolate(result)) { - // We failed to enter the isolate, it is possible the VM is shutting down, - // return back a NULL so that CreateIsolate reports back an error. - if (KernelIsolate::IsKernelIsolate(result)) { - KernelIsolate::SetKernelIsolate(nullptr); - } - if (ServiceIsolate::IsServiceIsolate(result)) { - ServiceIsolate::SetServiceIsolate(nullptr); - } delete result; return nullptr; } + // Now we register the isolate in the group. From this point on any GC would + // traverse the isolate roots (before this point, the roots are only pointing + // to vm-isolate objects, e.g. null) + isolate_group->RegisterIsolate(result); + + if (ServiceIsolate::NameEquals(name_prefix)) { + ASSERT(!ServiceIsolate::Exists()); + ServiceIsolate::SetServiceIsolate(result); +#if !defined(DART_PRECOMPILED_RUNTIME) + } else if (KernelIsolate::NameEquals(name_prefix)) { + ASSERT(!KernelIsolate::Exists()); + KernelIsolate::SetKernelIsolate(result); +#endif // !defined(DART_PRECOMPILED_RUNTIME) + } + // Setup the isolate message handler. MessageHandler* handler = new IsolateMessageHandler(result); ASSERT(handler != nullptr); result->set_message_handler(handler); - // Setup the Dart API state. - ApiState* state = new ApiState(); - ASSERT(state != nullptr); - result->set_api_state(state); - result->set_main_port(PortMap::CreatePort(result->message_handler())); #if defined(DEBUG) // Verify that we are never reusing a live origin id. @@ -1471,22 +1540,10 @@ Isolate* Isolate::InitIsolate(const char* name_prefix, } } -#ifndef PRODUCT - ObjectIdRing::Init(result); -#endif // !PRODUCT - // Add to isolate list. Shutdown and delete the isolate on failure. if (!TryMarkIsolateReady(result)) { result->LowLevelShutdown(); - Thread::ExitIsolate(); - if (KernelIsolate::IsKernelIsolate(result)) { - KernelIsolate::SetKernelIsolate(nullptr); - } - if (ServiceIsolate::IsServiceIsolate(result)) { - ServiceIsolate::SetServiceIsolate(nullptr); - } - - delete result; + Isolate::LowLevelCleanup(result); return nullptr; } @@ -1516,14 +1573,14 @@ RawObject* Isolate::CallTagHandler(Dart_LibraryTag tag, void Isolate::SetupImagePage(const uint8_t* image_buffer, bool is_executable) { Image image(image_buffer); - heap_->SetupImagePage(image.object_start(), image.object_size(), - is_executable); + heap()->SetupImagePage(image.object_start(), image.object_size(), + is_executable); } void Isolate::ScheduleInterrupts(uword interrupt_bits) { // We take the threads lock here to ensure that the mutator thread does not // exit the isolate while we are trying to schedule interrupts on it. - MonitorLocker ml(threads_lock()); + MonitorLocker ml(group()->threads_lock()); Thread* mthread = mutator_thread(); if (mthread != nullptr) { mthread->ScheduleInterrupts(interrupt_bits); @@ -1535,6 +1592,10 @@ void Isolate::set_name(const char* name) { name_ = strdup(name); } +int64_t IsolateGroup::UptimeMicros() const { + return OS::GetCurrentMonotonicMicros() - start_time_micros_; +} + int64_t Isolate::UptimeMicros() const { return OS::GetCurrentMonotonicMicros() - start_time_micros_; } @@ -1595,7 +1656,7 @@ bool IsolateGroup::ReloadSources(JSONStream* js, RELEASE_ASSERT(isolates_.First() == isolates_.Last()); RELEASE_ASSERT(isolates_.First() == Isolate::Current()); - auto shared_class_table = Isolate::Current()->shared_class_table(); + auto shared_class_table = IsolateGroup::Current()->class_table(); std::shared_ptr group_reload_context( new IsolateGroupReloadContext(this, shared_class_table, js)); group_reload_context_ = group_reload_context; @@ -1628,7 +1689,7 @@ bool IsolateGroup::ReloadKernel(JSONStream* js, RELEASE_ASSERT(isolates_.First() == isolates_.Last()); RELEASE_ASSERT(isolates_.First() == Isolate::Current()); - auto shared_class_table = Isolate::Current()->shared_class_table(); + auto shared_class_table = IsolateGroup::Current()->class_table(); std::shared_ptr group_reload_context( new IsolateGroupReloadContext(this, shared_class_table, js)); group_reload_context_ = group_reload_context; @@ -2098,20 +2159,6 @@ RawFunction* Isolate::ClosureFunctionFromIndex(intptr_t idx) const { return Function::RawCast(closures_array.At(idx)); } -class FinalizeWeakPersistentHandlesVisitor : public HandleVisitor { - public: - FinalizeWeakPersistentHandlesVisitor() : HandleVisitor(Thread::Current()) {} - - void VisitHandle(uword addr) { - FinalizablePersistentHandle* handle = - reinterpret_cast(addr); - handle->UpdateUnreachable(thread()->isolate()); - } - - private: - DISALLOW_COPY_AND_ASSIGN(FinalizeWeakPersistentHandlesVisitor); -}; - // static void Isolate::NotifyLowMemory() { Isolate::KillAllIsolates(Isolate::kLowMemoryMsg); @@ -2154,10 +2201,6 @@ void Isolate::LowLevelShutdown() { } #endif // !PRODUCT - // Finalize any weak persistent handles with a non-null referent. - FinalizeWeakPersistentHandlesVisitor visitor; - api_state()->VisitWeakHandlesUnlocked(&visitor); - #if !defined(PRODUCT) if (FLAG_dump_megamorphic_stats) { MegamorphicCacheTable::PrintSizes(this); @@ -2234,19 +2277,6 @@ void Isolate::Shutdown() { #endif } - if (heap_ != nullptr) { - // Wait for any concurrent GC tasks to finish before shutting down. - // TODO(rmacnak): Interrupt tasks for faster shutdown. - PageSpace* old_space = heap_->old_space(); - MonitorLocker ml(old_space->tasks_lock()); - while (old_space->tasks() > 0) { - ml.Wait(); - } - // Needs to happen before ~PageSpace so TLS and the thread registery are - // still valid. - old_space->AbandonMarkingForShutdown(); - } - #if !defined(PRODUCT) && !defined(DART_PRECOMPILED_RUNTIME) if (FLAG_check_reloaded && is_runnable() && !Isolate::IsVMInternalIsolate(this)) { @@ -2263,26 +2293,67 @@ void Isolate::Shutdown() { Isolate::UnMarkIsolateReady(this); LowLevelShutdown(); -#if defined(DEBUG) - // No concurrent sweeper tasks should be running at this point. - if (heap_ != nullptr) { - PageSpace* old_space = heap_->old_space(); - MonitorLocker ml(old_space->tasks_lock()); - ASSERT(old_space->tasks() == 0); - } -#endif + // Now we can unregister from the thread, invoke cleanup callback, delete the + // isolate (and possibly the isolate group). + Isolate::LowLevelCleanup(this); +} - // TODO(5411455): For now just make sure there are no current isolates - // as we are shutting down the isolate. +void Isolate::LowLevelCleanup(Isolate* isolate) { + const bool is_application_isolate = !Isolate::IsVMInternalIsolate(isolate); +#if !defined(DART_PECOMPILED_RUNTIME) + if (KernelIsolate::IsKernelIsolate(isolate)) { + KernelIsolate::SetKernelIsolate(nullptr); +#endif + } else if (ServiceIsolate::IsServiceIsolate(isolate)) { + ServiceIsolate::SetServiceIsolate(nullptr); + } + + // Cache these two fields, since they are no longer available after the + // `delete this` further down. + IsolateGroup* isolate_group = isolate->isolate_group_; + Dart_IsolateCleanupCallback cleanup = Isolate::CleanupCallback(); + auto callback_data = isolate->init_callback_data_; + + // From this point on the isolate is no longer visited by GC (which is ok, + // since we're just going to delete it anyway). + isolate_group->UnregisterIsolate(isolate); + + // Since the death of this isolate is not the death of the heap, we have to + // leave the new space iterable (e.g. for old space marking) by abanoning the + // TLAB. + isolate->group()->heap()->AbandonRemainingTLAB(Thread::Current()); + + // From this point on the isolate doesn't participate in safepointing + // requests anymore. Thread::ExitIsolate(); + // Now it's safe to delete the isolate. + delete isolate; + // Run isolate specific cleanup function for all non "vm-isolate's. - if (Dart::vm_isolate() != this) { - Dart_IsolateCleanupCallback cleanup = Isolate::CleanupCallback(); + if (Dart::vm_isolate() != isolate) { if (cleanup != nullptr) { - cleanup(isolate_group_->embedder_data(), init_callback_data()); + cleanup(isolate_group->embedder_data(), callback_data); } } + + const bool shutdown_group = + isolate_group->UnregisterIsolateDecrementCount(isolate); + if (shutdown_group) { + isolate_group->Shutdown(); + } else { + if (FLAG_enable_isolate_groups) { + // TODO(dartbug.com/36097): An isolate just died. A significant amount of + // memory might have become unreachable. We should evaluate how to best + // inform the GC about this situation. + } + } + + // After deleting the isolate we know that all it's resources have been freed. + // We still delay the notification to a possible call to `Dart::Cleanup()` to + // after a potential shutdown of the group, which would turn down any pending + // GC tasks as well as the heap. + Isolate::MarkIsolateDead(is_application_isolate); } Dart_InitializeIsolateCallback Isolate::initialize_callback_ = nullptr; @@ -2305,7 +2376,9 @@ void Isolate::VisitObjectPointers(ObjectPointerVisitor* visitor, ASSERT(visitor != nullptr); // Visit objects in the object store. - object_store()->VisitObjectPointers(visitor); + if (object_store() != nullptr) { + object_store()->VisitObjectPointers(visitor); + } // Visit objects in the class table. class_table()->VisitObjectPointers(visitor); @@ -2313,11 +2386,6 @@ void Isolate::VisitObjectPointers(ObjectPointerVisitor* visitor, // Visit objects in the field table. field_table()->VisitObjectPointers(visitor); - // Visit the dart api state for all local and persistent handles. - if (api_state() != nullptr) { - api_state()->VisitObjectPointersUnlocked(visitor); - } - visitor->clear_gc_root_type(); // Visit the objects directly referenced from the isolate structure. visitor->VisitPointer(reinterpret_cast(¤t_tag_)); @@ -2348,7 +2416,9 @@ void Isolate::VisitObjectPointers(ObjectPointerVisitor* visitor, #if !defined(PRODUCT) // Visit objects in the debugger. - debugger()->VisitObjectPointers(visitor); + if (debugger() != nullptr) { + debugger()->VisitObjectPointers(visitor); + } #if !defined(DART_PRECOMPILED_RUNTIME) // Visit objects that are being used for isolate reload. if (reload_context() != nullptr) { @@ -2367,56 +2437,45 @@ void Isolate::VisitObjectPointers(ObjectPointerVisitor* visitor, deopt_context()->VisitObjectPointers(visitor); } #endif // !defined(DART_PRECOMPILED_RUNTIME) - - VisitStackPointers(visitor, validate_frames); } -void Isolate::VisitStackPointers(ObjectPointerVisitor* visitor, - ValidationPolicy validate_frames) { - visitor->set_gc_root_type("stack"); - // Visit objects in all threads (e.g., Dart stack, handles in zones). - thread_registry()->VisitObjectPointers(this, visitor, validate_frames); - - // Visit mutator thread, even if the isolate isn't entered/scheduled (there - // might be live API handles to visit). - if (mutator_thread_ != nullptr) { - mutator_thread_->VisitObjectPointers(visitor, validate_frames); - } - visitor->clear_gc_root_type(); +void IsolateGroup::ReleaseStoreBuffers() { + thread_registry()->ReleaseStoreBuffers(); } -void Isolate::VisitWeakPersistentHandles(HandleVisitor* visitor) { - if (api_state() != nullptr) { - api_state()->VisitWeakHandlesUnlocked(visitor); - } -} - -void Isolate::ReleaseStoreBuffers() { - thread_registry()->ReleaseStoreBuffers(this); -} - -void Isolate::EnableIncrementalBarrier(MarkingStack* marking_stack, - MarkingStack* deferred_marking_stack) { +void IsolateGroup::EnableIncrementalBarrier( + MarkingStack* marking_stack, + MarkingStack* deferred_marking_stack) { ASSERT(marking_stack_ == nullptr); marking_stack_ = marking_stack; deferred_marking_stack_ = deferred_marking_stack; - thread_registry()->AcquireMarkingStacks(this); + thread_registry()->AcquireMarkingStacks(); ASSERT(Thread::Current()->is_marking()); } -void Isolate::DisableIncrementalBarrier() { - thread_registry()->ReleaseMarkingStacks(this); +void IsolateGroup::DisableIncrementalBarrier() { + thread_registry()->ReleaseMarkingStacks(); ASSERT(marking_stack_ != nullptr); marking_stack_ = nullptr; deferred_marking_stack_ = nullptr; - ASSERT(!Thread::Current()->is_marking()); } void IsolateGroup::ForEachIsolate( - std::function function) { - ReadRwLocker wl(ThreadState::Current(), isolates_rwlock_.get()); - for (Isolate* isolate : isolates_) { - function(isolate); + std::function function, + bool at_safepoint) { + if (at_safepoint) { + ASSERT(Thread::Current()->IsAtSafepoint() || + (Thread::Current()->task_kind() == Thread::kMutatorTask) || + (Thread::Current()->task_kind() == Thread::kMarkerTask) || + (Thread::Current()->task_kind() == Thread::kCompactorTask)); + for (Isolate* isolate : isolates_) { + function(isolate); + } + } else { + SafepointReadRwLocker ml(Thread::Current(), isolates_lock_.get()); + for (Isolate* isolate : isolates_) { + function(isolate); + } } } @@ -2426,22 +2485,79 @@ void IsolateGroup::RunWithStoppedMutators( bool use_force_growth_in_otherwise) { auto thread = Thread::Current(); - ReadRwLocker wl(thread, isolates_rwlock_.get()); - const bool only_one_isolate = isolates_.First() == isolates_.Last(); - if (thread->IsMutatorThread() && only_one_isolate) { + if (thread->IsMutatorThread() && !FLAG_enable_isolate_groups) { single_current_mutator(); - } else { - // We use the more strict safepoint operation scope here (which ensures that - // all other threads, including auxiliary threads are at a safepoint), even - // though we only need to ensure that the mutator threads are stopped. - if (use_force_growth_in_otherwise) { - ForceGrowthSafepointOperationScope safepoint_scope(thread); - otherwise(); - } else { - SafepointOperationScope safepoint_scope(thread); - otherwise(); + return; + } + + { + SafepointReadRwLocker ml(thread, isolates_lock_.get()); + const bool only_one_isolate = isolates_.First() == isolates_.Last(); + if (thread->IsMutatorThread() && only_one_isolate) { + single_current_mutator(); + return; } } + + // We use the more strict safepoint operation scope here (which ensures that + // all other threads, including auxiliary threads are at a safepoint), even + // though we only need to ensure that the mutator threads are stopped. + if (use_force_growth_in_otherwise) { + ForceGrowthSafepointOperationScope safepoint_scope(thread); + otherwise(); + } else { + SafepointOperationScope safepoint_scope(thread); + otherwise(); + } +} + +void IsolateGroup::VisitObjectPointers(ObjectPointerVisitor* visitor, + ValidationPolicy validate_frames) { + ForEachIsolate( + [&](Isolate* isolate) { + isolate->VisitObjectPointers(visitor, validate_frames); + }, + /*at_safepoint=*/true); + api_state()->VisitObjectPointersUnlocked(visitor); + VisitStackPointers(visitor, validate_frames); +} + +void IsolateGroup::VisitStackPointers(ObjectPointerVisitor* visitor, + ValidationPolicy validate_frames) { + visitor->set_gc_root_type("stack"); + + // Visit objects in all threads (e.g. Dart stack, handles in zones), except + // for the mutator threads themselves. + thread_registry()->VisitObjectPointers(this, visitor, validate_frames); + + ForEachIsolate( + [&](Isolate* isolate) { + // Visit mutator thread, even if the isolate isn't entered/scheduled + // (there might be live API handles to visit). + if (isolate->mutator_thread_ != nullptr) { + isolate->mutator_thread_->VisitObjectPointers(visitor, + validate_frames); + } + }, + /*at_safepoint=*/true); + + visitor->clear_gc_root_type(); +} + +void IsolateGroup::VisitWeakPersistentHandles(HandleVisitor* visitor) { + api_state()->VisitWeakHandlesUnlocked(visitor); +} + +uword IsolateGroup::FindPendingDeoptAtSafepoint(uword fp) { + for (Isolate* isolate : isolates_) { + for (intptr_t i = 0; i < isolate->pending_deopts_->length(); i++) { + if ((*isolate->pending_deopts_)[i].fp() == fp) { + return (*isolate->pending_deopts_)[i].pc(); + } + } + } + FATAL("Missing pending deopt entry"); + return 0; } RawClass* Isolate::GetClassForHeapWalkAt(intptr_t cid) { @@ -2460,10 +2576,10 @@ RawClass* Isolate::GetClassForHeapWalkAt(intptr_t cid) { return raw_class; } -intptr_t Isolate::GetClassSizeForHeapWalkAt(intptr_t cid) { +intptr_t IsolateGroup::GetClassSizeForHeapWalkAt(intptr_t cid) { #if !defined(PRODUCT) && !defined(DART_PRECOMPILED_RUNTIME) - if (group()->IsReloading()) { - return group()->reload_context()->GetClassSizeForHeapWalkAt(cid); + if (IsReloading()) { + return group_reload_context_->GetClassSizeForHeapWalkAt(cid); } else { return class_table()->SizeAt(cid); } @@ -3064,20 +3180,19 @@ std::unique_ptr Isolate::LookupIsolateNameByPort(Dart_Port port) { bool Isolate::TryMarkIsolateReady(Isolate* isolate) { MonitorLocker ml(isolate_creation_monitor_); - if (!creation_enabled_) { - return false; - } total_isolates_count_++; if (!Isolate::IsVMInternalIsolate(isolate)) { application_isolates_count_++; } + if (!creation_enabled_) { + return false; + } isolate->accepts_messages_ = true; return true; } void Isolate::UnMarkIsolateReady(Isolate* isolate) { MonitorLocker ml(isolate_creation_monitor_); - ASSERT(total_isolates_count_ > 0); isolate->accepts_messages_ = false; } @@ -3171,6 +3286,7 @@ class IsolateKillerVisitor : public IsolateVisitor { virtual ~IsolateKillerVisitor() {} void VisitIsolate(Isolate* isolate) { + MonitorLocker ml(Isolate::isolate_creation_monitor_); ASSERT(isolate != nullptr); if (ShouldKill(isolate)) { if (isolate->AcceptsMessagesLocked()) { @@ -3192,15 +3308,11 @@ class IsolateKillerVisitor : public IsolateVisitor { }; void Isolate::KillAllIsolates(LibMsgId msg_id) { - MonitorLocker ml(isolate_creation_monitor_); - IsolateKillerVisitor visitor(msg_id); VisitIsolates(&visitor); } void Isolate::KillIfExists(Isolate* isolate, LibMsgId msg_id) { - MonitorLocker ml(isolate_creation_monitor_); - IsolateKillerVisitor visitor(isolate, msg_id); VisitIsolates(&visitor); } @@ -3251,6 +3363,9 @@ Thread* Isolate::ScheduleThread(bool is_mutator, bool bypass_safepoint) { // We lazily create a [Thread] structure for the mutator thread, but we'll // reuse it until the death of the isolate. Thread* existing_mutator_thread = is_mutator ? mutator_thread_ : nullptr; + if (existing_mutator_thread != nullptr) { + ASSERT(existing_mutator_thread->is_mutator_thread_); + } // Schedule the thread into the isolate by associating a 'Thread' structure // with it (this is done while we are holding the thread registry lock). @@ -3261,13 +3376,11 @@ Thread* Isolate::ScheduleThread(bool is_mutator, bool bypass_safepoint) { ASSERT(mutator_thread_ == nullptr || mutator_thread_ == thread); mutator_thread_ = thread; scheduled_mutator_thread_ = thread; + thread->is_mutator_thread_ = true; } thread->isolate_ = this; thread->field_table_values_ = field_table_->table(); - ASSERT(heap() != nullptr); - thread->heap_ = heap(); - return thread; } @@ -3283,13 +3396,6 @@ void Isolate::UnscheduleThread(Thread* thread, // no_safepoint_scope_depth increments/decrements. MonitorLocker ml(group()->threads_lock(), false); - // Clear since GC will not visit the thread once it is unscheduled. Do this - // under the thread lock to prevent races with the GC visiting thread roots. - thread->ClearReusableHandles(); - if (!is_mutator) { - thread->heap()->AbandonRemainingTLAB(thread); - } - if (is_mutator) { if (thread->sticky_error() != Error::null()) { ASSERT(sticky_error_ == Error::null()); diff --git a/runtime/vm/isolate.h b/runtime/vm/isolate.h index d5784c0fdf5..6f7a3eee6f6 100644 --- a/runtime/vm/isolate.h +++ b/runtime/vm/isolate.h @@ -82,6 +82,7 @@ class RawInt32x4; class RawUserTag; class ReversePcLookupCache; class RwLock; +class SafepointRwLock; class SafepointHandler; class SampleBuffer; class SendPort; @@ -213,6 +214,11 @@ class IsolateGroupSource { // The kernel buffer used in `Dart_LoadScriptFromKernel`. const uint8_t* script_kernel_buffer; intptr_t script_kernel_size; + + // During AppJit training we perform a permutation of the class ids before + // invoking the "main" script. + // Any newly spawned isolates need to use this permutation map. + std::unique_ptr cid_permutation_map; }; // Tracks idle time and notifies heap when idle time expired. @@ -262,21 +268,67 @@ class DisableIdleTimerScope : public ValueObject { // Represents an isolate group and is shared among all isolates within a group. class IsolateGroup : public IntrusiveDListEntry { public: - IsolateGroup(std::unique_ptr source, void* embedder_data); + IsolateGroup(std::shared_ptr source, void* embedder_data); ~IsolateGroup(); IsolateGroupSource* source() const { return source_.get(); } + std::shared_ptr shareable_source() const { + return source_; + } void* embedder_data() const { return embedder_data_; } + bool initial_spawn_successful() { return initial_spawn_successful_; } void set_initial_spawn_successful() { initial_spawn_successful_ = true; } + Heap* heap() const { return heap_.get(); } + + IdleTimeHandler* idle_time_handler() { return &idle_time_handler_; } + void RegisterIsolate(Isolate* isolate); + void RegisterIsolateLocked(Isolate* isolate); void UnregisterIsolate(Isolate* isolate); + // Returns `true` if this was the last isolate and the caller is responsible + // for deleting the isolate group. + bool UnregisterIsolateDecrementCount(Isolate* isolate); + + bool ContainsOnlyOneIsolate(); + + void RunWithLockedGroup(std::function fun); Monitor* threads_lock() const; ThreadRegistry* thread_registry() const { return thread_registry_.get(); } SafepointHandler* safepoint_handler() { return safepoint_handler_.get(); } + void CreateHeap(bool is_vm_isolate, bool is_service_or_kernel_isolate); + void Shutdown(); + +#if !defined(PRODUCT) +#define ISOLATE_METRIC_ACCESSOR(type, variable, name, unit) \ + type* Get##variable##Metric() { return &metric_##variable##_; } + ISOLATE_GROUP_METRIC_LIST(ISOLATE_METRIC_ACCESSOR); +#undef ISOLATE_METRIC_ACCESSOR + + void UpdateLastAllocationProfileAccumulatorResetTimestamp() { + last_allocationprofile_accumulator_reset_timestamp_ = + OS::GetCurrentTimeMillis(); + } + + int64_t last_allocationprofile_accumulator_reset_timestamp() const { + return last_allocationprofile_accumulator_reset_timestamp_; + } + + void UpdateLastAllocationProfileGCTimestamp() { + last_allocationprofile_gc_timestamp_ = OS::GetCurrentTimeMillis(); + } + + int64_t last_allocationprofile_gc_timestamp() const { + return last_allocationprofile_gc_timestamp_; + } +#endif // !defined(PRODUCT) + + SharedClassTable* class_table() const { return shared_class_table_.get(); } + StoreBuffer* store_buffer() const { return store_buffer_.get(); } + static inline IsolateGroup* Current() { Thread* thread = Thread::Current(); return thread == nullptr ? nullptr : thread->isolate_group(); @@ -304,11 +356,29 @@ class IsolateGroup : public IntrusiveDListEntry { library_tag_handler_ = handler; } + intptr_t GetClassSizeForHeapWalkAt(intptr_t cid); + + // Prepares all threads in an isolate for Garbage Collection. + void ReleaseStoreBuffers(); + void EnableIncrementalBarrier(MarkingStack* marking_stack, + MarkingStack* deferred_marking_stack); + void DisableIncrementalBarrier(); + + MarkingStack* marking_stack() const { return marking_stack_; } + MarkingStack* deferred_marking_stack() const { + return deferred_marking_stack_; + } + // Runs the given [function] on every isolate in the isolate group. // - // During the duration of this function, no new isolates can be added to the - // isolate group. - void ForEachIsolate(std::function function); + // During the duration of this function, no new isolates can be added or + // removed. + // + // If [at_safepoint] is `true`, then the entire isolate group must be in a + // safepoint. There is therefore no reason to guard against other threads + // adding/removing isolates, so no locks will be held. + void ForEachIsolate(std::function function, + bool at_safepoint = false); // Ensures mutators are stopped during execution of the provided function. // @@ -381,21 +451,79 @@ class IsolateGroup : public IntrusiveDListEntry { static void RegisterIsolateGroup(IsolateGroup* isolate_group); static void UnregisterIsolateGroup(IsolateGroup* isolate_group); + int64_t UptimeMicros() const; + + ApiState* api_state() const { return api_state_.get(); } + + // Visit all object pointers. Caller must ensure concurrent sweeper is not + // running, and the visitor must not allocate. + void VisitObjectPointers(ObjectPointerVisitor* visitor, + ValidationPolicy validate_frames); + void VisitStackPointers(ObjectPointerVisitor* visitor, + ValidationPolicy validate_frames); + void VisitWeakPersistentHandles(HandleVisitor* visitor); + + bool compaction_in_progress() const { + return CompactionInProgressBit::decode(isolate_group_flags_); + } + void set_compaction_in_progress(bool value) { + isolate_group_flags_ = + CompactionInProgressBit::update(value, isolate_group_flags_); + } + + uword FindPendingDeoptAtSafepoint(uword fp); + private: + friend class Heap; + friend class StackFrame; // For `[isolates_].First()`. + +#define ISOLATE_GROUP_FLAG_BITS(V) V(CompactionInProgress) + + // Isolate specific flags. + enum FlagBits { +#define DECLARE_BIT(Name) k##Name##Bit, + ISOLATE_GROUP_FLAG_BITS(DECLARE_BIT) +#undef DECLARE_BIT + }; + +#define DECLARE_BITFIELD(Name) \ + class Name##Bit : public BitField {}; + ISOLATE_GROUP_FLAG_BITS(DECLARE_BITFIELD) +#undef DECLARE_BITFIELD + + void set_heap(std::unique_ptr value); + + bool is_vm_isolate_heap_ = false; void* embedder_data_ = nullptr; - std::unique_ptr isolates_rwlock_; + std::unique_ptr isolates_lock_; IntrusiveDList isolates_; intptr_t isolate_count_ = 0; bool initial_spawn_successful_ = false; Dart_LibraryTagHandler library_tag_handler_ = nullptr; + int64_t start_time_micros_; #if !defined(PRODUCT) && !defined(DART_PRECOMPILED_RUNTIME) int64_t last_reload_timestamp_; std::shared_ptr group_reload_context_; #endif - std::unique_ptr source_; +#if !defined(PRODUCT) +#define ISOLATE_METRIC_VARIABLE(type, variable, name, unit) \ + type metric_##variable##_; + ISOLATE_GROUP_METRIC_LIST(ISOLATE_METRIC_VARIABLE); +#undef ISOLATE_METRIC_VARIABLE + + // Timestamps of last operation via service. + int64_t last_allocationprofile_accumulator_reset_timestamp_ = 0; + int64_t last_allocationprofile_gc_timestamp_ = 0; + +#endif // !defined(PRODUCT) + + MarkingStack* marking_stack_ = nullptr; + MarkingStack* deferred_marking_stack_ = nullptr; + std::shared_ptr source_; + std::unique_ptr api_state_; std::unique_ptr thread_registry_; std::unique_ptr safepoint_handler_; @@ -404,6 +532,12 @@ class IsolateGroup : public IntrusiveDListEntry { static Random isolate_group_random_; uint64_t id_ = isolate_group_random_.NextUInt64(); + + std::unique_ptr shared_class_table_; + std::unique_ptr store_buffer_; + std::unique_ptr heap_; + IdleTimeHandler idle_time_handler_; + uint32_t isolate_group_flags_ = 0; }; class Isolate : public BaseIsolate, public IntrusiveDListEntry { @@ -454,29 +588,12 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry { void ValidateConstants(); #endif - // Visits weak object pointers. - void VisitWeakPersistentHandles(HandleVisitor* visitor); - - // Prepares all threads in an isolate for Garbage Collection. - void ReleaseStoreBuffers(); - void EnableIncrementalBarrier(MarkingStack* marking_stack, - MarkingStack* deferred_marking_stack); - void DisableIncrementalBarrier(); - - StoreBuffer* store_buffer() const { return store_buffer_; } - MarkingStack* marking_stack() const { return marking_stack_; } - MarkingStack* deferred_marking_stack() const { - return deferred_marking_stack_; - } - ThreadRegistry* thread_registry() const { return group()->thread_registry(); } SafepointHandler* safepoint_handler() const { return group()->safepoint_handler(); } - SharedClassTable* shared_class_table() { return shared_class_table_.get(); } - ClassTable* class_table() { return &class_table_; } static intptr_t class_table_offset() { return OFFSET_OF(Isolate, class_table_); @@ -486,7 +603,6 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry { // Prefers old classes when we are in the middle of a reload. RawClass* GetClassForHeapWalkAt(intptr_t cid); - intptr_t GetClassSizeForHeapWalkAt(intptr_t cid); static intptr_t ic_miss_code_offset() { return OFFSET_OF(Isolate, ic_miss_code_); @@ -531,13 +647,7 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry { void SendInternalLibMessage(LibMsgId msg_id, uint64_t capability); - IdleTimeHandler* idle_time_handler() { return &idle_time_handler_; } - - Heap* heap() const { return heap_; } - void set_heap(Heap* value) { - idle_time_handler_.InitializeWithHeap(value); - heap_ = value; - } + Heap* heap() const { return isolate_group_->heap(); } ObjectStore* object_store() const { return object_store_; } void set_object_store(ObjectStore* value) { object_store_ = value; } @@ -545,9 +655,6 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry { return OFFSET_OF(Isolate, object_store_); } - ApiState* api_state() const { return api_state_; } - void set_api_state(ApiState* value) { api_state_ = value; } - void set_init_callback_data(void* value) { init_callback_data_ = value; } void* init_callback_data() const { return init_callback_data_; } @@ -591,13 +698,6 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry { #endif } - bool compaction_in_progress() const { - return CompactionInProgressBit::decode(isolate_flags_); - } - void set_compaction_in_progress(bool value) { - isolate_flags_ = CompactionInProgressBit::update(value, isolate_flags_); - } - IsolateSpawnState* spawn_state() const { return spawn_state_.get(); } void set_spawn_state(std::unique_ptr value) { spawn_state_ = std::move(value); @@ -724,7 +824,6 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry { } #if !defined(PRODUCT) - void set_object_id_ring(ObjectIdRing* ring) { object_id_ring_ = ring; } ObjectIdRing* object_id_ring() { return object_id_ring_; } #endif // !defined(PRODUCT) @@ -749,25 +848,6 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry { return optimizing_background_compiler_; } -#if !defined(PRODUCT) - void UpdateLastAllocationProfileAccumulatorResetTimestamp() { - last_allocationprofile_accumulator_reset_timestamp_ = - OS::GetCurrentTimeMillis(); - } - - int64_t last_allocationprofile_accumulator_reset_timestamp() const { - return last_allocationprofile_accumulator_reset_timestamp_; - } - - void UpdateLastAllocationProfileGCTimestamp() { - last_allocationprofile_gc_timestamp_ = OS::GetCurrentTimeMillis(); - } - - int64_t last_allocationprofile_gc_timestamp() const { - return last_allocationprofile_gc_timestamp_; - } -#endif // !defined(PRODUCT) - intptr_t BlockClassFinalization() { ASSERT(defer_finalization_count_ >= 0); return defer_finalization_count_++; @@ -857,11 +937,6 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry { void set_ic_miss_code(const Code& code); -#if !defined(PRODUCT) - Metric* metrics_list_head() { return metrics_list_head_; } - void set_metrics_list_head(Metric* metric) { metrics_list_head_ = metric; } -#endif // !defined(PRODUCT) - RawGrowableObjectArray* deoptimized_code_array() const { return deoptimized_code_array_; } @@ -1078,6 +1153,9 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry { private: friend class Dart; // Init, InitOnce, Shutdown. friend class IsolateKillerVisitor; // Kill(). + friend Isolate* CreateWithinExistingIsolateGroup(IsolateGroup* g, + const char* n, + char** e); Isolate(IsolateGroup* group, const Dart_IsolateFlags& api_flags); @@ -1090,8 +1168,13 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry { // The isolate_creation_monitor_ should be held when calling Kill(). void KillLocked(LibMsgId msg_id); - void LowLevelShutdown(); void Shutdown(); + void LowLevelShutdown(); + + // Unregister the [isolate] from the thread, remove it from the isolate group, + // invoke the cleanup function (if any), delete the isolate and possibly + // delete the isolate group (if it's the last isolate in the group). + static void LowLevelCleanup(Isolate* isolate); void BuildName(const char* name_prefix); @@ -1119,7 +1202,6 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry { const GrowableObjectArray& value); #endif // !defined(PRODUCT) - Monitor* threads_lock() { return isolate_group_->threads_lock(); } Thread* ScheduleThread(bool is_mutator, bool bypass_safepoint = false); void UnscheduleThread(Thread* thread, bool is_mutator, @@ -1142,18 +1224,13 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry { RawUserTag* current_tag_; RawUserTag* default_tag_; RawCode* ic_miss_code_; - std::unique_ptr shared_class_table_; ObjectStore* object_store_ = nullptr; ClassTable class_table_; FieldTable* field_table_ = nullptr; bool single_step_ = false; // End accessed from generated code. - StoreBuffer* store_buffer_ = nullptr; - MarkingStack* marking_stack_ = nullptr; - MarkingStack* deferred_marking_stack_ = nullptr; - Heap* heap_ = nullptr; - IsolateGroup* isolate_group_ = nullptr; + IsolateGroup* isolate_group_; IdleTimeHandler idle_time_handler_; #if !defined(DART_PRECOMPILED_RUNTIME) @@ -1176,7 +1253,6 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry { V(UseFieldGuards) \ V(UseOsr) \ V(Obfuscate) \ - V(CompactionInProgress) \ V(ShouldLoadVmService) \ V(UnsafeTrustStrongModeTypes) @@ -1206,10 +1282,6 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry { Debugger* debugger_ = nullptr; int64_t last_resume_timestamp_; - // Timestamps of last operation via service. - int64_t last_allocationprofile_accumulator_reset_timestamp_ = 0; - int64_t last_allocationprofile_gc_timestamp_ = 0; - VMTagCounters vm_tag_counters_; // We use 6 list entries for each pending service extension calls. @@ -1223,8 +1295,6 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry { kRegisteredEntrySize}; RawGrowableObjectArray* registered_service_extension_handlers_; - Metric* metrics_list_head_ = nullptr; - // Used to wake the isolate when it is in the pause event loop. Monitor* pause_loop_monitor_ = nullptr; @@ -1253,7 +1323,6 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry { uint64_t terminate_capability_ = 0; void* init_callback_data_ = nullptr; Dart_EnvironmentCallback environment_callback_ = nullptr; - ApiState* api_state_ = nullptr; Random random_; Simulator* simulator_ = nullptr; Mutex mutex_; // Protects compiler stats. diff --git a/runtime/vm/isolate_reload.h b/runtime/vm/isolate_reload.h index ba6ba598b0a..5bcf2a38d44 100644 --- a/runtime/vm/isolate_reload.h +++ b/runtime/vm/isolate_reload.h @@ -294,7 +294,8 @@ class IsolateGroupReloadContext { friend class MarkFunctionsForRecompilation; // IsDirty. friend class ReasonForCancelling; friend class IsolateReloadContext; - friend class Instance; // GetClassSizeForHeapWalkAt + friend class IsolateGroup; // GetClassSizeForHeapWalkAt + friend class RawObject; // GetClassSizeForHeapWalkAt static Dart_FileModifiedCallback file_modified_callback_; }; diff --git a/runtime/vm/lockers.cc b/runtime/vm/lockers.cc index b5bd727afd2..7ad731dfc46 100644 --- a/runtime/vm/lockers.cc +++ b/runtime/vm/lockers.cc @@ -5,6 +5,7 @@ #include "vm/lockers.h" #include "platform/assert.h" #include "vm/heap/safepoint.h" +#include "vm/isolate.h" namespace dart { @@ -26,7 +27,7 @@ Monitor::WaitResult MonitorLocker::WaitWithSafepointCheck(Thread* thread, // Fast update failed which means we could potentially be in the middle // of a safepoint operation and need to block for it. monitor_->Exit(); - SafepointHandler* handler = thread->isolate()->safepoint_handler(); + SafepointHandler* handler = thread->isolate_group()->safepoint_handler(); handler->ExitSafepointUsingLock(thread); monitor_->Enter(); } @@ -84,22 +85,4 @@ Monitor::WaitResult SafepointMonitorLocker::Wait(int64_t millis) { } } -ReadRwLocker::ReadRwLocker(ThreadState* thread_state, RwLock* rw_lock) - : StackResource(thread_state), rw_lock_(rw_lock) { - rw_lock_->EnterRead(); -} - -ReadRwLocker::~ReadRwLocker() { - rw_lock_->LeaveRead(); -} - -WriteRwLocker::WriteRwLocker(ThreadState* thread_state, RwLock* rw_lock) - : StackResource(thread_state), rw_lock_(rw_lock) { - rw_lock_->EnterWrite(); -} - -WriteRwLocker::~WriteRwLocker() { - rw_lock_->LeaveWrite(); -} - } // namespace dart diff --git a/runtime/vm/lockers.h b/runtime/vm/lockers.h index 5a4706ec55e..16c6d93feb1 100644 --- a/runtime/vm/lockers.h +++ b/runtime/vm/lockers.h @@ -245,6 +245,8 @@ class SafepointMonitorLocker : public ValueObject { Monitor::WaitResult Wait(int64_t millis = Monitor::kNoTimeout); + void NotifyAll() { monitor_->NotifyAll(); } + private: Monitor* const monitor_; @@ -296,6 +298,51 @@ class RwLock { intptr_t state_ = 0; }; +class SafepointRwLock { + public: + SafepointRwLock() {} + ~SafepointRwLock() {} + + private: + friend class SafepointReadRwLocker; + friend class SafepointWriteRwLocker; + + void EnterRead() { + SafepointMonitorLocker ml(&monitor_); + while (state_ == -1) { + ml.Wait(); + } + ++state_; + } + void LeaveRead() { + SafepointMonitorLocker ml(&monitor_); + ASSERT(state_ > 0); + if (--state_ == 0) { + ml.NotifyAll(); + } + } + + void EnterWrite() { + SafepointMonitorLocker ml(&monitor_); + while (state_ != 0) { + ml.Wait(); + } + state_ = -1; + } + void LeaveWrite() { + SafepointMonitorLocker ml(&monitor_); + ASSERT(state_ == -1); + state_ = 0; + ml.NotifyAll(); + } + + Monitor monitor_; + // [state_] > 0 : The lock is held by multiple readers. + // [state_] == 0 : The lock is free (no readers/writers). + // [state_] == -1: The lock is held by a single writer. + intptr_t state_ = 0; +}; + /* * Locks a given [RwLock] for reading purposes. * @@ -309,13 +356,32 @@ class RwLock { */ class ReadRwLocker : public StackResource { public: - ReadRwLocker(ThreadState* thread_state, RwLock* rw_lock); - ~ReadRwLocker(); + ReadRwLocker(ThreadState* thread_state, RwLock* rw_lock) + : StackResource(thread_state), rw_lock_(rw_lock) { + rw_lock_->EnterRead(); + } + ~ReadRwLocker() { rw_lock_->LeaveRead(); } private: RwLock* rw_lock_; }; +/* + * In addition to what [ReadRwLocker] does, this implementation also gets into a + * safepoint if necessary. + */ +class SafepointReadRwLocker : public StackResource { + public: + SafepointReadRwLocker(ThreadState* thread_state, SafepointRwLock* rw_lock) + : StackResource(thread_state), rw_lock_(rw_lock) { + rw_lock_->EnterRead(); + } + ~SafepointReadRwLocker() { rw_lock_->LeaveRead(); } + + private: + SafepointRwLock* rw_lock_; +}; + /* * Locks a given [RwLock] for writing purposes. * @@ -329,13 +395,34 @@ class ReadRwLocker : public StackResource { */ class WriteRwLocker : public StackResource { public: - WriteRwLocker(ThreadState* thread_state, RwLock* rw_lock); - ~WriteRwLocker(); + WriteRwLocker(ThreadState* thread_state, RwLock* rw_lock) + : StackResource(thread_state), rw_lock_(rw_lock) { + rw_lock_->EnterWrite(); + } + + ~WriteRwLocker() { rw_lock_->LeaveWrite(); } private: RwLock* rw_lock_; }; +/* + * In addition to what [WriteRwLocker] does, this implementation also gets into a + * safepoint if necessary. + */ +class SafepointWriteRwLocker : public StackResource { + public: + SafepointWriteRwLocker(ThreadState* thread_state, SafepointRwLock* rw_lock) + : StackResource(thread_state), rw_lock_(rw_lock) { + rw_lock_->EnterWrite(); + } + + ~SafepointWriteRwLocker() { rw_lock_->LeaveWrite(); } + + private: + SafepointRwLock* rw_lock_; +}; + } // namespace dart #endif // RUNTIME_VM_LOCKERS_H_ diff --git a/runtime/vm/message_handler.cc b/runtime/vm/message_handler.cc index 4b37bc09264..10126303004 100644 --- a/runtime/vm/message_handler.cc +++ b/runtime/vm/message_handler.cc @@ -8,6 +8,7 @@ #include "vm/dart.h" #include "vm/heap/safepoint.h" +#include "vm/isolate.h" #include "vm/lockers.h" #include "vm/object.h" #include "vm/object_store.h" @@ -202,7 +203,7 @@ MessageHandler::MessageStatus MessageHandler::HandleMessages( ml->Enter(); auto idle_time_handler = - isolate() != nullptr ? isolate()->idle_time_handler() : nullptr; + isolate() != nullptr ? isolate()->group()->idle_time_handler() : nullptr; MessageStatus max_status = kOK; Message::Priority min_priority = @@ -523,12 +524,17 @@ void MessageHandler::TaskCallback() { bool MessageHandler::CheckIfIdleLocked(MonitorLocker* ml) { if (isolate() == nullptr || - !isolate()->idle_time_handler()->ShouldCheckForIdle()) { + !isolate()->group()->idle_time_handler()->ShouldCheckForIdle()) { // No idle task to schedule. return false; } + if (!isolate()->group()->initial_spawn_successful()) { + // The isolate has not started running application code yet. + return false; + } int64_t idle_expirary = 0; - if (isolate()->idle_time_handler()->ShouldNotifyIdle(&idle_expirary)) { + if (isolate()->group()->idle_time_handler()->ShouldNotifyIdle( + &idle_expirary)) { // We've been without a message long enough to hope we can do some // cleanup before the next message arrives. RunIdleTaskLocked(ml); @@ -553,7 +559,7 @@ void MessageHandler::RunIdleTaskLocked(MonitorLocker* ml) { ml->Exit(); { StartIsolateScope start_isolate(isolate()); - isolate()->idle_time_handler()->NotifyIdleUsingDefaultDeadline(); + isolate()->group()->idle_time_handler()->NotifyIdleUsingDefaultDeadline(); } ml->Enter(); } diff --git a/runtime/vm/metrics.cc b/runtime/vm/metrics.cc index 6194760cd2b..3a592727ae2 100644 --- a/runtime/vm/metrics.cc +++ b/runtime/vm/metrics.cc @@ -22,54 +22,41 @@ DEFINE_FLAG(bool, Metric* Metric::vm_list_head_ = NULL; -Metric::Metric() - : isolate_(NULL), - name_(NULL), - description_(NULL), - unit_(kCounter), - value_(0), - next_(NULL) {} +Metric::Metric() : unit_(kCounter), value_(0) {} +Metric::~Metric() {} + +void Metric::InitInstance(IsolateGroup* isolate_group, + const char* name, + const char* description, + Unit unit) { + // Only called once. + ASSERT(name != NULL); + isolate_group_ = isolate_group; + name_ = name; + description_ = description; + unit_ = unit; +} void Metric::InitInstance(Isolate* isolate, const char* name, const char* description, Unit unit) { // Only called once. - ASSERT(next_ == NULL); ASSERT(name != NULL); isolate_ = isolate; name_ = name; description_ = description; unit_ = unit; - RegisterWithIsolate(); } void Metric::InitInstance(const char* name, const char* description, Unit unit) { // Only called once. - ASSERT(next_ == NULL); ASSERT(name != NULL); name_ = name; description_ = description; unit_ = unit; - RegisterWithVM(); -} - -void Metric::CleanupInstance() { - // Only deregister metrics which had been registered. Metrics without a name - // are from shallow copy isolates. - if (name_ != NULL) { - if (isolate_ == NULL) { - DeregisterWithVM(); - } else { - DeregisterWithIsolate(); - } - } -} - -Metric::~Metric() { - CleanupInstance(); } #ifndef PRODUCT @@ -94,7 +81,7 @@ void Metric::PrintJSON(JSONStream* stream) { obj.AddProperty("name", name_); obj.AddProperty("description", description_); obj.AddProperty("unit", UnitString(unit())); - if (isolate_ == NULL) { + if (isolate_ == nullptr && isolate_group_ == nullptr) { obj.AddFixedServiceId("vm/metrics/%s", name_); } else { obj.AddFixedServiceId("metrics/native/%s", name_); @@ -156,136 +143,40 @@ char* Metric::ToString() { return zone->PrintToString("%s %s", name(), ValueToString(Value(), unit())); } -bool Metric::NameExists(Metric* head, const char* name) { - ASSERT(name != NULL); - while (head != NULL) { - const char* metric_name = head->name(); - ASSERT(metric_name != NULL); - if (strcmp(metric_name, name) == 0) { - return true; - } - head = head->next(); - } - return false; -} - -void Metric::RegisterWithIsolate() { - ASSERT(isolate_ != NULL); - ASSERT(next_ == NULL); - // No duplicate names allowed. - ASSERT(!NameExists(isolate_->metrics_list_head(), name())); - Metric* head = isolate_->metrics_list_head(); - if (head != NULL) { - set_next(head); - } - isolate_->set_metrics_list_head(this); -} - -void Metric::DeregisterWithIsolate() { - Metric* head = isolate_->metrics_list_head(); - ASSERT(head != NULL); - // Handle head of list case. - if (head == this) { - isolate_->set_metrics_list_head(next()); - set_next(NULL); - return; - } - Metric* previous = NULL; - while (true) { - previous = head; - ASSERT(previous != NULL); - head = head->next(); - if (head == NULL) { - break; - } - if (head == this) { - // Remove this from list. - previous->set_next(head->next()); - set_next(NULL); - return; - } - ASSERT(head != NULL); - } - UNREACHABLE(); -} - -void Metric::RegisterWithVM() { - ASSERT(isolate_ == NULL); - ASSERT(next_ == NULL); - // No duplicate names allowed. - ASSERT(!NameExists(vm_list_head_, name())); - Metric* head = vm_list_head_; - if (head != NULL) { - set_next(head); - } - vm_list_head_ = this; -} - -void Metric::DeregisterWithVM() { - ASSERT(isolate_ == NULL); - Metric* head = vm_list_head_; - if (head == NULL) { - return; - } - // Handle head of list case. - if (head == this) { - vm_list_head_ = next(); - set_next(NULL); - return; - } - Metric* previous = NULL; - while (true) { - previous = head; - ASSERT(previous != NULL); - head = head->next(); - if (head == NULL) { - break; - } - if (head == this) { - // Remove this from list. - previous->set_next(head->next()); - set_next(NULL); - return; - } - ASSERT(head != NULL); - } - UNREACHABLE(); -} - int64_t MetricHeapOldUsed::Value() const { - ASSERT(isolate() == Isolate::Current()); - return isolate()->heap()->UsedInWords(Heap::kOld) * kWordSize; + ASSERT(isolate_group() == IsolateGroup::Current()); + return isolate_group()->heap()->UsedInWords(Heap::kOld) * kWordSize; } int64_t MetricHeapOldCapacity::Value() const { - ASSERT(isolate() == Isolate::Current()); - return isolate()->heap()->CapacityInWords(Heap::kOld) * kWordSize; + ASSERT(isolate_group() == IsolateGroup::Current()); + return isolate_group()->heap()->CapacityInWords(Heap::kOld) * kWordSize; } int64_t MetricHeapOldExternal::Value() const { - ASSERT(isolate() == Isolate::Current()); - return isolate()->heap()->ExternalInWords(Heap::kOld) * kWordSize; + ASSERT(isolate_group() == IsolateGroup::Current()); + return isolate_group()->heap()->ExternalInWords(Heap::kOld) * kWordSize; } int64_t MetricHeapNewUsed::Value() const { - ASSERT(isolate() == Isolate::Current()); - return isolate()->heap()->UsedInWords(Heap::kNew) * kWordSize; + ASSERT(isolate_group() == IsolateGroup::Current()); + return isolate_group()->heap()->UsedInWords(Heap::kNew) * kWordSize; } int64_t MetricHeapNewCapacity::Value() const { - ASSERT(isolate() == Isolate::Current()); - return isolate()->heap()->CapacityInWords(Heap::kNew) * kWordSize; + ASSERT(isolate_group() == IsolateGroup::Current()); + return isolate_group()->heap()->CapacityInWords(Heap::kNew) * kWordSize; } int64_t MetricHeapNewExternal::Value() const { - ASSERT(isolate() == Isolate::Current()); - return isolate()->heap()->ExternalInWords(Heap::kNew) * kWordSize; + ASSERT(isolate_group() == IsolateGroup::Current()); + return isolate_group()->heap()->ExternalInWords(Heap::kNew) * kWordSize; } int64_t MetricHeapUsed::Value() const { - ASSERT(isolate() == Isolate::Current()); - return isolate()->heap()->UsedInWords(Heap::kNew) * kWordSize + - isolate()->heap()->UsedInWords(Heap::kOld) * kWordSize; + ASSERT(isolate_group() == IsolateGroup::Current()); + return isolate_group()->heap()->UsedInWords(Heap::kNew) * kWordSize + + isolate_group()->heap()->UsedInWords(Heap::kOld) * kWordSize; } int64_t MetricIsolateCount::Value() const { @@ -317,17 +208,13 @@ void Metric::Cleanup() { // Create a zone to allocate temporary strings in. StackZone sz(Thread::Current()); OS::PrintErr("Printing metrics for VM\n"); - Metric* current = Metric::vm_head(); - while (current != NULL) { - OS::PrintErr("%s\n", current->ToString()); - current = current->next(); - } + +#define VM_METRIC_INIT(type, variable, name, unit) \ + OS::PrintErr("%s\n", vm_metric_##variable.ToString()); + VM_METRIC_LIST(VM_METRIC_INIT); +#undef VM_METRIC_INIT OS::PrintErr("\n"); } -#define VM_METRIC_CLEANUP(type, variable, name, unit) \ - vm_metric_##variable.CleanupInstance(); - VM_METRIC_LIST(VM_METRIC_CLEANUP); -#undef VM_METRIC_CLEANUP } MaxMetric::MaxMetric() : Metric() { diff --git a/runtime/vm/metrics.h b/runtime/vm/metrics.h index 36d9e4bdcbf..ea17ad43d20 100644 --- a/runtime/vm/metrics.h +++ b/runtime/vm/metrics.h @@ -10,10 +10,11 @@ namespace dart { class Isolate; +class IsolateGroup; class JSONStream; -// Metrics for each isolate. -#define ISOLATE_METRIC_LIST(V) \ +// Metrics for each isolate group. +#define ISOLATE_GROUP_METRIC_LIST(V) \ V(MetricHeapOldUsed, HeapOldUsed, "heap.old.used", kByte) \ V(MaxMetric, HeapOldUsedMax, "heap.old.used.max", kByte) \ V(MetricHeapOldCapacity, HeapOldCapacity, "heap.old.capacity", kByte) \ @@ -25,7 +26,10 @@ class JSONStream; V(MaxMetric, HeapNewCapacityMax, "heap.new.capacity.max", kByte) \ V(MetricHeapNewExternal, HeapNewExternal, "heap.new.external", kByte) \ V(MetricHeapUsed, HeapGlobalUsed, "heap.global.used", kByte) \ - V(MaxMetric, HeapGlobalUsedMax, "heap.global.used.max", kByte) \ + V(MaxMetric, HeapGlobalUsedMax, "heap.global.used.max", kByte) + +// Metrics for each isolate. +#define ISOLATE_METRIC_LIST(V) \ V(Metric, RunnableLatency, "isolate.runnable.latency", kMicrosecond) \ V(Metric, RunnableHeapSize, "isolate.runnable.heap", kByte) @@ -48,16 +52,20 @@ class Metric { static void Cleanup(); - // Initialize and register a metric for an isolate. + // Initialize a metric for an isolate. void InitInstance(Isolate* isolate, const char* name, const char* description, Unit unit); - // Initialize and register a metric for the VM. - void InitInstance(const char* name, const char* description, Unit unit); + // Initialize a metric for an isolate group. + void InitInstance(IsolateGroup* isolate_group, + const char* name, + const char* description, + Unit unit); - void CleanupInstance(); + // Initialize a metric for the VM. + void InitInstance(const char* name, const char* description, Unit unit); virtual ~Metric(); @@ -76,16 +84,16 @@ class Metric { void increment() { value_++; } - Metric* next() const { return next_; } - void set_next(Metric* next) { next_ = next; } - const char* name() const { return name_; } const char* description() const { return description_; } Unit unit() const { return unit_; } - // Will be NULL for Metric that is VM-global. + // Only non-null for isolate specific metrics. Isolate* isolate() const { return isolate_; } + // Only non-null for isolate group specific metrics. + IsolateGroup* isolate_group() const { return isolate_group_; } + static Metric* vm_head() { return vm_list_head_; } // Override to get a callback when value is serialized to JSON. @@ -93,19 +101,12 @@ class Metric { virtual int64_t Value() const { return value(); } private: - Isolate* isolate_; - const char* name_; - const char* description_; + Isolate* isolate_ = nullptr; + IsolateGroup* isolate_group_ = nullptr; + const char* name_ = nullptr; + const char* description_ = nullptr; Unit unit_; int64_t value_; - Metric* next_; - - static bool NameExists(Metric* head, const char* name); - - void RegisterWithIsolate(); - void DeregisterWithIsolate(); - void RegisterWithVM(); - void DeregisterWithVM(); static Metric* vm_list_head_; DISALLOW_COPY_AND_ASSIGN(Metric); diff --git a/runtime/vm/native_entry.cc b/runtime/vm/native_entry.cc index 8cb80016c98..1b3c546f309 100644 --- a/runtime/vm/native_entry.cc +++ b/runtime/vm/native_entry.cc @@ -206,7 +206,7 @@ void NativeEntry::AutoScopeNativeCallWrapperNoStackCheck( ASSERT(thread->execution_state() == Thread::kThreadInGenerated); { Isolate* isolate = thread->isolate(); - ApiState* state = isolate->api_state(); + ApiState* state = isolate->group()->api_state(); ASSERT(state != NULL); TRACE_NATIVE_CALL("0x%" Px "", reinterpret_cast(func)); thread->EnterApiScope(); diff --git a/runtime/vm/object.cc b/runtime/vm/object.cc index dbd7e8b230b..50c1907b9d5 100644 --- a/runtime/vm/object.cc +++ b/runtime/vm/object.cc @@ -2540,8 +2540,7 @@ void Object::CheckHandle() const { } ASSERT(vtable() == builtin_vtables_[cid]); if (FLAG_verify_handles && raw_->IsHeapObject()) { - Isolate* isolate = Isolate::Current(); - Heap* isolate_heap = isolate->heap(); + Heap* isolate_heap = IsolateGroup::Current()->heap(); Heap* vm_isolate_heap = Dart::vm_isolate()->heap(); uword addr = RawObject::ToAddr(raw_); if (!isolate_heap->Contains(addr) && !vm_isolate_heap->Contains(addr)) { @@ -2577,7 +2576,7 @@ RawObject* Object::Allocate(intptr_t cls_id, intptr_t size, Heap::Space space) { } } #ifndef PRODUCT - auto class_table = thread->isolate()->shared_class_table(); + auto class_table = thread->isolate_group()->class_table(); if (class_table->TraceAllocationFor(cls_id)) { Profiler::SampleAllocation(thread, cls_id); } @@ -2604,7 +2603,7 @@ RawObject* Object::Allocate(intptr_t cls_id, intptr_t size, Heap::Space space) { class WriteBarrierUpdateVisitor : public ObjectPointerVisitor { public: explicit WriteBarrierUpdateVisitor(Thread* thread, RawObject* obj) - : ObjectPointerVisitor(thread->isolate()), + : ObjectPointerVisitor(thread->isolate()->group()), thread_(thread), old_obj_(obj) { ASSERT(old_obj_->IsOldObject()); @@ -3235,7 +3234,7 @@ UnboxedFieldBitmap Class::CalculateFieldOffsets() const { if (FLAG_precompiled_mode) { host_bitmap = - Isolate::Current()->shared_class_table()->GetUnboxedFieldsMapAt( + Isolate::Current()->group()->class_table()->GetUnboxedFieldsMapAt( super.id()); } } @@ -3701,7 +3700,8 @@ void Class::Finalize() const { // Sets the new size in the class table. isolate->class_table()->SetAt(id(), raw()); if (FLAG_precompiled_mode) { - isolate->shared_class_table()->SetUnboxedFieldsMapAt(id(), host_bitmap); + isolate->group()->class_table()->SetUnboxedFieldsMapAt(id(), + host_bitmap); } } } @@ -3797,7 +3797,7 @@ void Class::DisableAllCHAOptimizedCode() { bool Class::TraceAllocation(Isolate* isolate) const { #ifndef PRODUCT - auto class_table = isolate->shared_class_table(); + auto class_table = isolate->group()->class_table(); return class_table->TraceAllocationFor(id()); #else return false; @@ -3809,7 +3809,7 @@ void Class::SetTraceAllocation(bool trace_allocation) const { Isolate* isolate = Isolate::Current(); const bool changed = trace_allocation != this->TraceAllocation(isolate); if (changed) { - auto class_table = isolate->shared_class_table(); + auto class_table = isolate->group()->class_table(); class_table->SetTraceAllocationFor(id(), trace_allocation); DisableAllocationStub(); } @@ -17169,8 +17169,9 @@ uint32_t Instance::CanonicalizeHash() const { hash = instance_size / kWordSize; uword this_addr = reinterpret_cast(this->raw_ptr()); Instance& member = Instance::Handle(); + const auto unboxed_fields_bitmap = - thread->isolate()->shared_class_table()->GetUnboxedFieldsMapAt( + thread->isolate()->group()->class_table()->GetUnboxedFieldsMapAt( GetClassId()); for (intptr_t offset = Instance::NextFieldOffset(); offset < instance_size; @@ -17191,8 +17192,8 @@ uint32_t Instance::CanonicalizeHash() const { #if defined(DEBUG) class CheckForPointers : public ObjectPointerVisitor { public: - explicit CheckForPointers(Isolate* isolate) - : ObjectPointerVisitor(isolate), has_pointers_(false) {} + explicit CheckForPointers(IsolateGroup* isolate_group) + : ObjectPointerVisitor(isolate_group), has_pointers_(false) {} bool has_pointers() const { return has_pointers_; } @@ -17222,7 +17223,7 @@ bool Instance::CheckAndCanonicalizeFields(Thread* thread, const intptr_t instance_size = SizeFromClass(); ASSERT(instance_size != 0); const auto unboxed_fields_bitmap = - thread->isolate()->shared_class_table()->GetUnboxedFieldsMapAt( + thread->isolate()->group()->class_table()->GetUnboxedFieldsMapAt( GetClassId()); for (intptr_t offset = Instance::NextFieldOffset(); offset < instance_size; offset += kWordSize) { @@ -17249,7 +17250,7 @@ bool Instance::CheckAndCanonicalizeFields(Thread* thread, } else { #if defined(DEBUG) // Make sure that we are not missing any fields. - CheckForPointers has_pointers(Isolate::Current()); + CheckForPointers has_pointers(Isolate::Current()->group()); this->raw()->VisitPointers(&has_pointers); ASSERT(!has_pointers.has_pointers()); #endif // DEBUG diff --git a/runtime/vm/object.h b/runtime/vm/object.h index 6a973c9eabf..b7d29f8242a 100644 --- a/runtime/vm/object.h +++ b/runtime/vm/object.h @@ -801,7 +801,7 @@ class Object { #undef DECLARE_SHARED_READONLY_HANDLE friend void ClassTable::Register(const Class& cls); - friend void RawObject::Validate(Isolate* isolate) const; + friend void RawObject::Validate(IsolateGroup* isolate_group) const; friend class Closure; friend class SnapshotReader; friend class InstanceDeserializationCluster; @@ -10518,7 +10518,7 @@ RawClass* Object::clazz() const { if ((raw_value & kSmiTagMask) == kSmiTag) { return Smi::Class(); } - ASSERT(!Isolate::Current()->compaction_in_progress()); + ASSERT(!IsolateGroup::Current()->compaction_in_progress()); return Isolate::Current()->class_table()->At(raw()->GetClassId()); } diff --git a/runtime/vm/object_graph.cc b/runtime/vm/object_graph.cc index 98b112fa4c0..a982bb6226e 100644 --- a/runtime/vm/object_graph.cc +++ b/runtime/vm/object_graph.cc @@ -37,24 +37,28 @@ static bool IsUserClass(intptr_t cid) { // - Use tag bits for compact Node and sentinel representations. class ObjectGraph::Stack : public ObjectPointerVisitor { public: - explicit Stack(Isolate* isolate) - : ObjectPointerVisitor(isolate), + explicit Stack(IsolateGroup* isolate_group) + : ObjectPointerVisitor(isolate_group), include_vm_objects_(true), - data_(kInitialCapacity) {} + data_(kInitialCapacity) { + object_ids_ = new WeakTable(); + } + ~Stack() { + delete object_ids_; + object_ids_ = nullptr; + } // Marks and pushes. Used to initialize this stack with roots. // We can use ObjectIdTable normally used by serializers because it // won't be in use while handling a service request (ObjectGraph's only use). virtual void VisitPointers(RawObject** first, RawObject** last) { - Heap* heap = isolate()->heap(); for (RawObject** current = first; current <= last; ++current) { - if ((*current)->IsHeapObject() && - !(*current)->InVMIsolateHeap() && - heap->GetObjectId(*current) == 0) { // not visited yet + if ((*current)->IsHeapObject() && !(*current)->InVMIsolateHeap() && + object_ids_->GetValueExclusive(*current) == 0) { // not visited yet if (!include_vm_objects_ && !IsUserClass((*current)->GetClassId())) { continue; } - heap->SetObjectId(*current, 1); + object_ids_->SetValueExclusive(*current, 1); Node node; node.ptr = current; node.obj = *current; @@ -91,7 +95,6 @@ class ObjectGraph::Stack : public ObjectPointerVisitor { clear_gc_root_type(); } } - isolate()->heap()->ResetObjectIdTable(); } virtual bool visit_weak_persistent_handles() const { @@ -126,6 +129,10 @@ class ObjectGraph::Stack : public ObjectPointerVisitor { return kNoParent; } + // During the iteration of the heap we are already at a safepoint, so there is + // no need to let the GC know about [object_ids_] (i.e. GC cannot run while we + // use [object_ids]). + WeakTable* object_ids_ = nullptr; GrowableArray data_; friend class StackIterator; DISALLOW_COPY_AND_ASSIGN(Stack); @@ -214,15 +221,16 @@ ObjectGraph::ObjectGraph(Thread* thread) : ThreadStackResource(thread) { ObjectGraph::~ObjectGraph() {} void ObjectGraph::IterateObjects(ObjectGraph::Visitor* visitor) { - Stack stack(isolate()); + Stack stack(isolate_group()); stack.set_visit_weak_persistent_handles( visitor->visit_weak_persistent_handles()); - isolate()->VisitObjectPointers(&stack, ValidationPolicy::kDontValidateFrames); + isolate_group()->VisitObjectPointers(&stack, + ValidationPolicy::kDontValidateFrames); stack.TraverseGraph(visitor); } void ObjectGraph::IterateUserObjects(ObjectGraph::Visitor* visitor) { - Stack stack(isolate()); + Stack stack(isolate_group()); stack.set_visit_weak_persistent_handles( visitor->visit_weak_persistent_handles()); IterateUserFields(&stack); @@ -232,7 +240,7 @@ void ObjectGraph::IterateUserObjects(ObjectGraph::Visitor* visitor) { void ObjectGraph::IterateObjectsFrom(const Object& root, ObjectGraph::Visitor* visitor) { - Stack stack(isolate()); + Stack stack(isolate_group()); stack.set_visit_weak_persistent_handles( visitor->visit_weak_persistent_handles()); RawObject* root_raw = root.raw(); @@ -262,7 +270,7 @@ class InstanceAccumulator : public ObjectVisitor { void ObjectGraph::IterateObjectsFrom(intptr_t class_id, ObjectGraph::Visitor* visitor) { HeapIterationScope iteration(thread()); - Stack stack(isolate()); + Stack stack(isolate_group()); InstanceAccumulator accumulator(&stack, class_id); iteration.IterateObjectsNoImagePages(&accumulator); @@ -455,7 +463,7 @@ class InboundReferencesVisitor : public ObjectVisitor, RawObject* target, const Array& references, Object* scratch) - : ObjectPointerVisitor(isolate), + : ObjectPointerVisitor(isolate->group()), source_(NULL), target_(target), references_(references), @@ -753,7 +761,7 @@ class Pass1Visitor : public ObjectVisitor, public: explicit Pass1Visitor(HeapSnapshotWriter* writer) : ObjectVisitor(), - ObjectPointerVisitor(Isolate::Current()), + ObjectPointerVisitor(IsolateGroup::Current()), HandleVisitor(Thread::Current()), writer_(writer) {} @@ -806,8 +814,9 @@ class Pass2Visitor : public ObjectVisitor, public: explicit Pass2Visitor(HeapSnapshotWriter* writer) : ObjectVisitor(), - ObjectPointerVisitor(Isolate::Current()), + ObjectPointerVisitor(IsolateGroup::Current()), HandleVisitor(Thread::Current()), + isolate_(thread()->isolate()), writer_(writer) {} void VisitObject(RawObject* obj) { @@ -920,9 +929,9 @@ class Pass2Visitor : public ObjectVisitor, } DoCount(); - obj->VisitPointersPrecise(this); + obj->VisitPointersPrecise(isolate_, this); DoWrite(); - obj->VisitPointersPrecise(this); + obj->VisitPointersPrecise(isolate_, this); } void ScrubAndWriteUtf8(RawString* str) { @@ -983,6 +992,10 @@ class Pass2Visitor : public ObjectVisitor, } private: + // TODO(dartbug.com/36097): Once the shared class table contains more + // information than just the size (i.e. includes an immutable class + // descriptor), we can remove this dependency on the current isolate. + Isolate* isolate_; HeapSnapshotWriter* const writer_; bool writing_ = false; intptr_t counted_ = 0; @@ -1129,7 +1142,7 @@ void HeapSnapshotWriter::Write() { iteration.IterateObjects(&visitor); // External properties. - isolate()->VisitWeakPersistentHandles(&visitor); + isolate()->group()->VisitWeakPersistentHandles(&visitor); } { @@ -1157,7 +1170,7 @@ void HeapSnapshotWriter::Write() { // External properties. WriteUnsigned(external_property_count_); - isolate()->VisitWeakPersistentHandles(&visitor); + isolate()->group()->VisitWeakPersistentHandles(&visitor); } { diff --git a/runtime/vm/object_id_ring.cc b/runtime/vm/object_id_ring.cc index aba613abc58..2652db36de2 100644 --- a/runtime/vm/object_id_ring.cc +++ b/runtime/vm/object_id_ring.cc @@ -12,19 +12,10 @@ namespace dart { #ifndef PRODUCT -void ObjectIdRing::Init(Isolate* isolate, int32_t capacity) { - ObjectIdRing* ring = new ObjectIdRing(isolate, capacity); - isolate->set_object_id_ring(ring); -} - ObjectIdRing::~ObjectIdRing() { ASSERT(table_ != NULL); free(table_); table_ = NULL; - if (isolate_ != NULL) { - isolate_->set_object_id_ring(NULL); - isolate_ = NULL; - } } int32_t ObjectIdRing::GetIdForObject(RawObject* object, IdPolicy policy) { @@ -95,9 +86,8 @@ void ObjectIdRing::PrintJSON(JSONStream* js) { } } -ObjectIdRing::ObjectIdRing(Isolate* isolate, int32_t capacity) { +ObjectIdRing::ObjectIdRing(int32_t capacity) { ASSERT(capacity > 0); - isolate_ = isolate; serial_num_ = 0; wrapped_ = false; table_ = NULL; diff --git a/runtime/vm/object_id_ring.h b/runtime/vm/object_id_ring.h index faa25b0cbf5..2fe809006e3 100644 --- a/runtime/vm/object_id_ring.h +++ b/runtime/vm/object_id_ring.h @@ -11,7 +11,6 @@ namespace dart { // Forward declarations. class RawObject; -class Isolate; class ObjectPointerVisitor; class JSONStream; @@ -40,8 +39,7 @@ class ObjectIdRing { static const int32_t kInvalidId = -1; static const int32_t kDefaultCapacity = 8192; - static void Init(Isolate* isolate, int32_t capacity = kDefaultCapacity); - + explicit ObjectIdRing(int32_t capacity = kDefaultCapacity); ~ObjectIdRing(); // Adds the argument to the ring and returns its id. Note we do not allow @@ -61,8 +59,6 @@ class ObjectIdRing { void SetCapacityAndMaxSerial(int32_t capacity, int32_t max_serial); int32_t FindExistingIdForObject(RawObject* raw_obj); - ObjectIdRing(Isolate* isolate, int32_t capacity); - Isolate* isolate_; RawObject** table_; int32_t max_serial_; int32_t capacity_; diff --git a/runtime/vm/object_reload.cc b/runtime/vm/object_reload.cc index 70e520d0e7e..8199db7b085 100644 --- a/runtime/vm/object_reload.cc +++ b/runtime/vm/object_reload.cc @@ -848,6 +848,7 @@ bool Class::CanReloadFinalized(const Class& replacement, // Make sure the declaration types argument count matches for the two classes. // ex. class A {} cannot be replace with class A {}. auto group_context = context->group_reload_context(); + auto shared_class_table = group_context->isolate_group()->class_table(); if (NumTypeArguments() != replacement.NumTypeArguments()) { group_context->AddReasonForCancelling( new (context->zone()) @@ -857,12 +858,10 @@ bool Class::CanReloadFinalized(const Class& replacement, if (RequiresInstanceMorphing(replacement)) { ASSERT(id() == replacement.id()); const classid_t cid = id(); - // We unconditionally create an instance morpher. As a side effect of // building the morpher, we will mark all new fields as late. auto instance_morpher = InstanceMorpher::CreateFromClassDescriptors( - context->zone(), context->isolate()->shared_class_table(), *this, - replacement); + context->zone(), shared_class_table, *this, replacement); group_context->EnsureHasInstanceMorpherFor(cid, instance_morpher); } return true; diff --git a/runtime/vm/profiler.cc b/runtime/vm/profiler.cc index 3aaf191b557..4b65a025097 100644 --- a/runtime/vm/profiler.cc +++ b/runtime/vm/profiler.cc @@ -1376,7 +1376,7 @@ void Profiler::SampleThread(Thread* thread, SampleThreadSingleFrame(thread, pc); return; } - if (isolate->compaction_in_progress()) { + if (isolate->group()->compaction_in_progress()) { // The Dart stack isn't fully walkable. SampleThreadSingleFrame(thread, pc); return; diff --git a/runtime/vm/raw_object.cc b/runtime/vm/raw_object.cc index 7bf1bb92995..9cf9ab3f399 100644 --- a/runtime/vm/raw_object.cc +++ b/runtime/vm/raw_object.cc @@ -26,7 +26,7 @@ bool RawObject::InVMIsolateHeap() const { return heap->old_space()->ContainsUnsafe(ToAddr(this)); } -void RawObject::Validate(Isolate* isolate) const { +void RawObject::Validate(IsolateGroup* isolate_group) const { if (Object::void_class_ == reinterpret_cast(kHeapObjectTag)) { // Validation relies on properly initialized class classes. Skip if the // VM is still being initialized. @@ -63,12 +63,12 @@ void RawObject::Validate(Isolate* isolate) const { FATAL1("Old object missing kOldBit: %x\n", tags); } } - intptr_t class_id = ClassIdTag::decode(tags); - if (!isolate->shared_class_table()->IsValidIndex(class_id)) { + const intptr_t class_id = ClassIdTag::decode(tags); + if (!isolate_group->class_table()->IsValidIndex(class_id)) { FATAL1("Invalid class id encountered %" Pd "\n", class_id); } if (class_id == kNullCid && - isolate->shared_class_table()->HasValidClassAt(class_id)) { + isolate_group->class_table()->HasValidClassAt(class_id)) { // Null class not yet initialized; skip. return; } @@ -225,11 +225,10 @@ intptr_t RawObject::HeapSizeFromClass() const { default: { // Get the (constant) instance size out of the class object. // TODO(koda): Add Size(ClassTable*) interface to allow caching in loops. - Isolate* isolate = Isolate::Current(); + auto isolate_group = IsolateGroup::Current(); #if defined(DEBUG) - auto class_table = isolate->shared_class_table(); #if !defined(DART_PRECOMPILED_RUNTIME) - auto reload_context = isolate->group()->reload_context(); + auto reload_context = isolate_group->reload_context(); const bool use_saved_class_table = reload_context != nullptr ? reload_context->UseSavedSizeTableForGC() : false; @@ -237,6 +236,7 @@ intptr_t RawObject::HeapSizeFromClass() const { const bool use_saved_class_table = false; #endif + auto class_table = isolate_group->class_table(); ASSERT(use_saved_class_table || class_table->SizeAt(class_id) > 0); if (!class_table->IsValidIndex(class_id) || (!class_table->HasValidClassAt(class_id) && !use_saved_class_table)) { @@ -244,7 +244,7 @@ intptr_t RawObject::HeapSizeFromClass() const { class_id, this, static_cast(ptr()->tags_)); } #endif // DEBUG - instance_size = isolate->GetClassSizeForHeapWalkAt(class_id); + instance_size = isolate_group->GetClassSizeForHeapWalkAt(class_id); } } ASSERT(instance_size != 0); @@ -379,7 +379,8 @@ intptr_t RawObject::VisitPointersPredefined(ObjectPointerVisitor* visitor, #endif } -void RawObject::VisitPointersPrecise(ObjectPointerVisitor* visitor) { +void RawObject::VisitPointersPrecise(Isolate* isolate, + ObjectPointerVisitor* visitor) { intptr_t class_id = GetClassId(); if (class_id < kNumPredefinedCids) { VisitPointersPredefined(visitor, class_id); @@ -387,8 +388,7 @@ void RawObject::VisitPointersPrecise(ObjectPointerVisitor* visitor) { } // N.B.: Not using the heap size! - uword next_field_offset = visitor->isolate() - ->GetClassForHeapWalkAt(class_id) + uword next_field_offset = isolate->GetClassForHeapWalkAt(class_id) ->ptr() ->host_next_field_offset_in_words_ << kWordSizeLog2; @@ -657,8 +657,8 @@ intptr_t RawInstance::VisitInstancePointers(RawInstance* raw_obj, uint32_t tags = raw_obj->ptr()->tags_; intptr_t instance_size = SizeTag::decode(tags); if (instance_size == 0) { - instance_size = - visitor->isolate()->GetClassSizeForHeapWalkAt(raw_obj->GetClassId()); + instance_size = visitor->isolate_group()->GetClassSizeForHeapWalkAt( + raw_obj->GetClassId()); } // Calculate the first and last raw object pointer fields. diff --git a/runtime/vm/raw_object.h b/runtime/vm/raw_object.h index 288c36d6b95..025ea510072 100644 --- a/runtime/vm/raw_object.h +++ b/runtime/vm/raw_object.h @@ -29,6 +29,7 @@ typedef RawObject* RawCompressed; // Forward declarations. class Isolate; +class IsolateGroup; #define DEFINE_FORWARD_DECLARATION(clazz) class Raw##clazz; CLASS_LIST(DEFINE_FORWARD_DECLARATION) #undef DEFINE_FORWARD_DECLARATION @@ -456,7 +457,7 @@ class RawObject { return (addr >= this_addr) && (addr < (this_addr + this_size)); } - void Validate(Isolate* isolate) const; + void Validate(IsolateGroup* isolate_group) const; bool FindObject(FindObjectVisitor* visitor); // This function may access the class-ID in the header, but it cannot access @@ -539,7 +540,7 @@ class RawObject { // This variant ensures that we do not visit the extra slot created from // rounding up instance sizes up to the allocation unit. - void VisitPointersPrecise(ObjectPointerVisitor* visitor); + void VisitPointersPrecise(Isolate* isolate, ObjectPointerVisitor* visitor); static RawObject* FromAddr(uword addr) { // We expect the untagged address here. diff --git a/runtime/vm/raw_object_snapshot.cc b/runtime/vm/raw_object_snapshot.cc index 9216283987d..df261cd4e7d 100644 --- a/runtime/vm/raw_object_snapshot.cc +++ b/runtime/vm/raw_object_snapshot.cc @@ -1713,7 +1713,7 @@ void RawTransferableTypedData::WriteTo(SnapshotWriter* writer, [](void* data, Dart_WeakPersistentHandle handle, void* peer) { TransferableTypedDataPeer* tpeer = reinterpret_cast(peer); - tpeer->handle()->EnsureFreeExternal(Isolate::Current()); + tpeer->handle()->EnsureFreeExternal(IsolateGroup::Current()); tpeer->ClearData(); }); } diff --git a/runtime/vm/service.cc b/runtime/vm/service.cc index 38559b6c96d..1e183ae2ce3 100644 --- a/runtime/vm/service.cc +++ b/runtime/vm/service.cc @@ -3440,26 +3440,41 @@ static bool HandleNativeMetricsList(Thread* thread, JSONStream* js) { obj.AddProperty("type", "MetricList"); { JSONArray metrics(&obj, "metrics"); - Metric* current = thread->isolate()->metrics_list_head(); - while (current != NULL) { - metrics.AddValue(current); - current = current->next(); - } + + auto isolate = thread->isolate(); +#define ADD_METRIC(type, variable, name, unit) \ + metrics.AddValue(isolate->Get##variable##Metric()); + ISOLATE_METRIC_LIST(ADD_METRIC); +#undef ADD_METRIC + + auto isolate_group = thread->isolate_group(); +#define ADD_METRIC(type, variable, name, unit) \ + metrics.AddValue(isolate_group->Get##variable##Metric()); + ISOLATE_GROUP_METRIC_LIST(ADD_METRIC); +#undef ADD_METRIC } return true; } static bool HandleNativeMetric(Thread* thread, JSONStream* js, const char* id) { - Metric* current = thread->isolate()->metrics_list_head(); - while (current != NULL) { - const char* name = current->name(); - ASSERT(name != NULL); - if (strcmp(name, id) == 0) { - current->PrintJSON(js); - return true; - } - current = current->next(); + auto isolate = thread->isolate(); +#define ADD_METRIC(type, variable, name, unit) \ + if (strcmp(id, name) == 0) { \ + isolate->Get##variable##Metric()->PrintJSON(js); \ + return true; \ } + ISOLATE_METRIC_LIST(ADD_METRIC); +#undef ADD_METRIC + + auto isolate_group = thread->isolate_group(); +#define ADD_METRIC(type, variable, name, unit) \ + if (strcmp(id, name) == 0) { \ + isolate_group->Get##variable##Metric()->PrintJSON(js); \ + return true; \ + } + ISOLATE_GROUP_METRIC_LIST(ADD_METRIC); +#undef ADD_METRIC + PrintInvalidParamError(js, "metricId"); return true; } @@ -3965,13 +3980,14 @@ static bool GetAllocationProfileImpl(Thread* thread, return true; } } - Isolate* isolate = thread->isolate(); + auto isolate = thread->isolate(); + auto isolate_group = thread->isolate_group(); if (should_reset_accumulator) { - isolate->UpdateLastAllocationProfileAccumulatorResetTimestamp(); + isolate_group->UpdateLastAllocationProfileAccumulatorResetTimestamp(); } if (should_collect) { - isolate->UpdateLastAllocationProfileGCTimestamp(); - isolate->heap()->CollectAllGarbage(); + isolate_group->UpdateLastAllocationProfileGCTimestamp(); + isolate_group->heap()->CollectAllGarbage(); } isolate->class_table()->AllocationProfilePrintJSON(js, internal); return true; @@ -4161,7 +4177,7 @@ static bool GetPersistentHandles(Thread* thread, JSONStream* js) { Isolate* isolate = thread->isolate(); ASSERT(isolate != NULL); - ApiState* api_state = isolate->api_state(); + ApiState* api_state = isolate->group()->api_state(); ASSERT(api_state != NULL); { diff --git a/runtime/vm/snapshot.cc b/runtime/vm/snapshot.cc index 0eb8130fee1..8968b17568a 100644 --- a/runtime/vm/snapshot.cc +++ b/runtime/vm/snapshot.cc @@ -621,7 +621,7 @@ RawObject* SnapshotReader::ReadInstance(intptr_t object_id, intptr_t result_cid = result->GetClassId(); const auto unboxed_fields = - isolate()->shared_class_table()->GetUnboxedFieldsMapAt(result_cid); + isolate()->group()->class_table()->GetUnboxedFieldsMapAt(result_cid); while (offset < next_field_offset) { if (unboxed_fields.Get(offset / kWordSize)) { @@ -1476,7 +1476,8 @@ void SnapshotWriter::WriteInstance(RawObject* raw, WriteObjectImpl(cls, kAsInlinedObject); const auto unboxed_fields = - isolate()->shared_class_table()->GetUnboxedFieldsMapAt(cls->ptr()->id_); + isolate()->group()->class_table()->GetUnboxedFieldsMapAt( + cls->ptr()->id_); // Write out all the fields for the object. // Instance::NextFieldOffset() returns the offset of the first field in diff --git a/runtime/vm/snapshot.h b/runtime/vm/snapshot.h index 3c528e8ec4c..e38f4bfda84 100644 --- a/runtime/vm/snapshot.h +++ b/runtime/vm/snapshot.h @@ -780,11 +780,11 @@ class MessageWriter : public SnapshotWriter { }; // An object pointer visitor implementation which writes out -// objects to a snap shot. +// objects to a snapshot. class SnapshotWriterVisitor : public ObjectPointerVisitor { public: SnapshotWriterVisitor(SnapshotWriter* writer, bool as_references) - : ObjectPointerVisitor(Isolate::Current()), + : ObjectPointerVisitor(Isolate::Current()->group()), writer_(writer), as_references_(as_references) {} diff --git a/runtime/vm/stack_frame.cc b/runtime/vm/stack_frame.cc index 564d610c23e..7f554a1abe4 100644 --- a/runtime/vm/stack_frame.cc +++ b/runtime/vm/stack_frame.cc @@ -103,15 +103,25 @@ void FrameLayout::Init() { #endif } -Isolate* StackFrame::IsolateOfBareInstructionsFrame() const { - auto isolate = this->isolate(); - +Isolate* StackFrame::IsolateOfBareInstructionsFrame(bool needed_for_gc) const { + Isolate* isolate = Dart::vm_isolate(); if (isolate->object_store()->code_order_table() != Object::null()) { auto rct = isolate->reverse_pc_lookup_cache(); if (rct->Contains(pc())) return isolate; } - isolate = Dart::vm_isolate(); + isolate = this->isolate(); + // The active isolate is null only during GC, in which case it does not matter + // which isolate we use for the reverse-pc lookup table, since the metadata + // is the same across all isolates. + // TODO(dartbug.com/36097): Avoid having the [ReversePcLookupTable] + // per-isolate. Right now we still need it per-isolate for non-GC cases, e.g. + // for stack walking code which relies on finding owner functions of code + // objects. + if (isolate == nullptr) { + ASSERT(needed_for_gc); + isolate = isolate_group()->isolates_.First(); + } if (isolate->object_store()->code_order_table() != Object::null()) { auto rct = isolate->reverse_pc_lookup_cache(); if (rct->Contains(pc())) return isolate; @@ -123,7 +133,7 @@ Isolate* StackFrame::IsolateOfBareInstructionsFrame() const { bool StackFrame::IsBareInstructionsDartFrame() const { NoSafepointScope no_safepoint; - if (auto isolate = IsolateOfBareInstructionsFrame()) { + if (auto isolate = IsolateOfBareInstructionsFrame(/*needed_for_gc=*/true)) { Code code; auto rct = isolate->reverse_pc_lookup_cache(); code = rct->Lookup(pc(), /*is_return_address=*/true); @@ -138,7 +148,7 @@ bool StackFrame::IsBareInstructionsDartFrame() const { bool StackFrame::IsBareInstructionsStubFrame() const { NoSafepointScope no_safepoint; - if (auto isolate = IsolateOfBareInstructionsFrame()) { + if (auto isolate = IsolateOfBareInstructionsFrame(/*needed_for_gc=*/true)) { Code code; auto rct = isolate->reverse_pc_lookup_cache(); code = rct->Lookup(pc(), /*is_return_address=*/true); @@ -252,7 +262,7 @@ void StackFrame::VisitObjectPointers(ObjectPointerVisitor* visitor) { NoSafepointScope no_safepoint; Code code; - if (auto isolate = IsolateOfBareInstructionsFrame()) { + if (auto isolate = IsolateOfBareInstructionsFrame(/*needed_for_gc=*/true)) { auto const rct = isolate->reverse_pc_lookup_cache(); code = rct->Lookup(pc(), /*is_return_address=*/true); } else { @@ -278,8 +288,16 @@ void StackFrame::VisitObjectPointers(ObjectPointerVisitor* visitor) { CompressedStackMaps maps; maps = code.compressed_stackmaps(); CompressedStackMaps global_table; - global_table = - this->isolate()->object_store()->canonicalized_stack_map_entries(); + + // The GC does not have an active isolate, only an active isolate group, + // yet the global compressed stack map table is only stored in the object + // store. It has the same contents for all isolates, so we just pick the + // one from the first isolate here. + // TODO(dartbug.com/36097): Avoid having this per-isolate and instead store + // it per isolate group. + auto isolate = isolate_group()->isolates_.First(); + + global_table = isolate->object_store()->canonicalized_stack_map_entries(); CompressedStackMapsIterator it(maps, global_table); const uword start = code.PayloadStart(); const uint32_t pc_offset = pc() - start; @@ -387,7 +405,7 @@ RawCode* StackFrame::LookupDartCode() const { // where Thread::Current() is NULL, so we cannot create a NoSafepointScope. NoSafepointScope no_safepoint; #endif - if (auto isolate = IsolateOfBareInstructionsFrame()) { + if (auto isolate = IsolateOfBareInstructionsFrame(/*needed_for_gc=*/false)) { auto const rct = isolate->reverse_pc_lookup_cache(); return rct->Lookup(pc(), /*is_return_address=*/true); } @@ -402,7 +420,7 @@ RawCode* StackFrame::LookupDartCode() const { RawCode* StackFrame::GetCodeObject() const { ASSERT(!is_interpreted()); - if (auto isolate = IsolateOfBareInstructionsFrame()) { + if (auto isolate = IsolateOfBareInstructionsFrame(/*needed_for_gc=*/false)) { auto const rct = isolate->reverse_pc_lookup_cache(); return rct->Lookup(pc(), /*is_return_address=*/true); } else { diff --git a/runtime/vm/stack_frame.h b/runtime/vm/stack_frame.h index 00ed7c91a11..39af4d576ea 100644 --- a/runtime/vm/stack_frame.h +++ b/runtime/vm/stack_frame.h @@ -100,7 +100,10 @@ class StackFrame : public ValueObject { // // If the frame does not belong to a bare instructions snapshot, it will // return nullptr. - Isolate* IsolateOfBareInstructionsFrame() const; + // + // [needed_for_gc] has to be set to `true` if the caller needs only GC + // relevant information. + Isolate* IsolateOfBareInstructionsFrame(bool needed_for_gc) const; // Returns true iff the current frame is a bare instructions dart frame. bool IsBareInstructionsDartFrame() const; @@ -150,6 +153,7 @@ class StackFrame : public ValueObject { } Isolate* isolate() const { return thread_->isolate(); } + IsolateGroup* isolate_group() const { return thread_->isolate_group(); } Thread* thread() const { return thread_; } @@ -172,7 +176,7 @@ class StackFrame : public ValueObject { kWordSize))); ASSERT(raw_pc != StubCode::DeoptimizeLazyFromThrow().EntryPoint()); if (raw_pc == StubCode::DeoptimizeLazyFromReturn().EntryPoint()) { - return isolate()->FindPendingDeopt(GetCallerFp()); + return isolate_group()->FindPendingDeoptAtSafepoint(GetCallerFp()); } return raw_pc; } diff --git a/runtime/vm/tags.cc b/runtime/vm/tags.cc index e756a8fdb8c..8636df76d5a 100644 --- a/runtime/vm/tags.cc +++ b/runtime/vm/tags.cc @@ -78,7 +78,7 @@ VMTag::TagEntry VMTag::entries_[] = { VMTagScope::VMTagScope(Thread* thread, uword tag, bool conditional_set) : ThreadStackResource(thread) { - ASSERT(isolate() != NULL); + ASSERT(isolate_group() != NULL); previous_tag_ = thread->vm_tag(); if (conditional_set) { thread->set_vm_tag(tag); @@ -86,7 +86,7 @@ VMTagScope::VMTagScope(Thread* thread, uword tag, bool conditional_set) } VMTagScope::~VMTagScope() { - ASSERT(isolate() != NULL); + ASSERT(isolate_group() != NULL); thread()->set_vm_tag(previous_tag_); } diff --git a/runtime/vm/thread.cc b/runtime/vm/thread.cc index 57d8dc89991..022688abb39 100644 --- a/runtime/vm/thread.cc +++ b/runtime/vm/thread.cc @@ -342,11 +342,9 @@ bool Thread::EnterIsolateAsHelper(Isolate* isolate, TaskKind kind, bool bypass_safepoint) { ASSERT(kind != kMutatorTask); - const bool kIsNotMutatorThread = false; - Thread* thread = - isolate->ScheduleThread(kIsNotMutatorThread, bypass_safepoint); + const bool kIsMutatorThread = false; + Thread* thread = isolate->ScheduleThread(kIsMutatorThread, bypass_safepoint); if (thread != NULL) { - ASSERT(thread->store_buffer_block_ == NULL); ASSERT(!thread->IsMutatorThread()); ASSERT(thread->isolate() == isolate); ASSERT(thread->isolate_group() == isolate->group()); @@ -367,8 +365,37 @@ void Thread::ExitIsolateAsHelper(bool bypass_safepoint) { Isolate* isolate = thread->isolate(); ASSERT(isolate != NULL); - const bool kIsNotMutatorThread = false; - isolate->UnscheduleThread(thread, kIsNotMutatorThread, bypass_safepoint); + const bool kIsMutatorThread = false; + isolate->UnscheduleThread(thread, kIsMutatorThread, bypass_safepoint); +} + +bool Thread::EnterIsolateGroupAsHelper(IsolateGroup* isolate_group, + TaskKind kind, + bool bypass_safepoint) { + ASSERT(kind != kMutatorTask); + Thread* thread = isolate_group->ScheduleThread(bypass_safepoint); + if (thread != NULL) { + ASSERT(!thread->IsMutatorThread()); + ASSERT(thread->isolate() == nullptr); + ASSERT(thread->isolate_group() == isolate_group); + thread->FinishEntering(kind); + return true; + } + return false; +} + +void Thread::ExitIsolateGroupAsHelper(bool bypass_safepoint) { + Thread* thread = Thread::Current(); + ASSERT(thread != nullptr); + ASSERT(!thread->IsMutatorThread()); + ASSERT(thread->isolate() == nullptr); + ASSERT(thread->isolate_group() != nullptr); + + thread->PrepareLeaving(); + + const bool kIsMutatorThread = false; + thread->isolate_group()->UnscheduleThread(thread, kIsMutatorThread, + bypass_safepoint); } void Thread::ReleaseStoreBuffer() { @@ -379,7 +406,7 @@ void Thread::ReleaseStoreBuffer() { // Make sure to get an *empty* block; the isolate needs all entries // at GC time. // TODO(koda): Replace with an epilogue (PrepareAfterGC) that acquires. - store_buffer_block_ = isolate()->store_buffer()->PopEmptyBlock(); + store_buffer_block_ = isolate_group()->store_buffer()->PopEmptyBlock(); } void Thread::SetStackLimit(uword limit) { @@ -490,7 +517,7 @@ RawError* Thread::HandleInterrupts() { uword interrupt_bits = GetAndClearInterrupts(); if ((interrupt_bits & kVMInterrupt) != 0) { CheckForSafepoint(); - if (isolate()->store_buffer()->Overflowed()) { + if (isolate_group()->store_buffer()->Overflowed()) { if (FLAG_verbose_gc) { OS::PrintErr("Scavenge scheduled by store buffer overflow.\n"); } @@ -546,11 +573,11 @@ void Thread::StoreBufferAddObjectGC(RawObject* obj) { void Thread::StoreBufferRelease(StoreBuffer::ThresholdPolicy policy) { StoreBufferBlock* block = store_buffer_block_; store_buffer_block_ = NULL; - isolate()->store_buffer()->PushBlock(block, policy); + isolate_group()->store_buffer()->PushBlock(block, policy); } void Thread::StoreBufferAcquire() { - store_buffer_block_ = isolate()->store_buffer()->PopNonFullBlock(); + store_buffer_block_ = isolate_group()->store_buffer()->PopNonFullBlock(); } void Thread::MarkingStackBlockProcess() { @@ -581,11 +608,11 @@ void Thread::MarkingStackRelease() { MarkingStackBlock* block = marking_stack_block_; marking_stack_block_ = NULL; write_barrier_mask_ = RawObject::kGenerationalBarrierMask; - isolate()->marking_stack()->PushBlock(block); + isolate_group()->marking_stack()->PushBlock(block); } void Thread::MarkingStackAcquire() { - marking_stack_block_ = isolate()->marking_stack()->PopEmptyBlock(); + marking_stack_block_ = isolate_group()->marking_stack()->PopEmptyBlock(); write_barrier_mask_ = RawObject::kGenerationalBarrierMask | RawObject::kIncrementalBarrierMask; } @@ -593,16 +620,19 @@ void Thread::MarkingStackAcquire() { void Thread::DeferredMarkingStackRelease() { MarkingStackBlock* block = deferred_marking_stack_block_; deferred_marking_stack_block_ = NULL; - isolate()->deferred_marking_stack()->PushBlock(block); + isolate_group()->deferred_marking_stack()->PushBlock(block); } void Thread::DeferredMarkingStackAcquire() { deferred_marking_stack_block_ = - isolate()->deferred_marking_stack()->PopEmptyBlock(); + isolate_group()->deferred_marking_stack()->PopEmptyBlock(); } bool Thread::IsMutatorThread() const { - return ((isolate_ != NULL) && (isolate_->mutator_thread() == this)); + if (isolate_ != nullptr) { + ASSERT(is_mutator_thread_ == (isolate_->mutator_thread() == this)); + } + return is_mutator_thread_; } bool Thread::CanCollectGarbage() const { @@ -894,22 +924,22 @@ void Thread::UnwindScopes(uword stack_marker) { } void Thread::EnterSafepointUsingLock() { - isolate()->safepoint_handler()->EnterSafepointUsingLock(this); + isolate_group()->safepoint_handler()->EnterSafepointUsingLock(this); } void Thread::ExitSafepointUsingLock() { - isolate()->safepoint_handler()->ExitSafepointUsingLock(this); + isolate_group()->safepoint_handler()->ExitSafepointUsingLock(this); } void Thread::BlockForSafepoint() { - isolate()->safepoint_handler()->BlockForSafepoint(this); + isolate_group()->safepoint_handler()->BlockForSafepoint(this); } void Thread::FinishEntering(TaskKind kind) { ASSERT(store_buffer_block_ == nullptr); task_kind_ = kind; - if (isolate()->marking_stack() != NULL) { + if (isolate_group()->marking_stack() != NULL) { // Concurrent mark in progress. Enable barrier for this thread. MarkingStackAcquire(); DeferredMarkingStackAcquire(); @@ -920,7 +950,7 @@ void Thread::FinishEntering(TaskKind kind) { if (kind == kMutatorTask) { StoreBufferAcquire(); } else { - store_buffer_block_ = isolate()->store_buffer()->PopEmptyBlock(); + store_buffer_block_ = isolate_group()->store_buffer()->PopEmptyBlock(); } } diff --git a/runtime/vm/thread.h b/runtime/vm/thread.h index 845db7a3740..9117de40ff5 100644 --- a/runtime/vm/thread.h +++ b/runtime/vm/thread.h @@ -271,6 +271,11 @@ class Thread : public ThreadState { bool bypass_safepoint = false); static void ExitIsolateAsHelper(bool bypass_safepoint = false); + static bool EnterIsolateGroupAsHelper(IsolateGroup* isolate_group, + TaskKind kind, + bool bypass_safepoint); + static void ExitIsolateGroupAsHelper(bool bypass_safepoint); + // Empties the store buffer block into the isolate. void ReleaseStoreBuffer(); void AcquireMarkingStack(); @@ -981,6 +986,7 @@ class Thread : public ThreadState { #endif Thread* next_; // Used to chain the thread structures in an isolate. + bool is_mutator_thread_ = false; explicit Thread(bool is_vm_isolate); @@ -1021,6 +1027,7 @@ class Thread : public ThreadState { friend class Simulator; friend class StackZone; friend class ThreadRegistry; + friend class NoActiveIsolateScope; friend class CompilerState; friend class compiler::target::Thread; friend class FieldTable; diff --git a/runtime/vm/thread_registry.cc b/runtime/vm/thread_registry.cc index efbd8926c6e..8ef9227c793 100644 --- a/runtime/vm/thread_registry.cc +++ b/runtime/vm/thread_registry.cc @@ -41,15 +41,16 @@ void ThreadRegistry::ReturnThreadLocked(Thread* thread) { ReturnToFreelistLocked(thread); } -void ThreadRegistry::VisitObjectPointers(Isolate* isolate_of_interest, - ObjectPointerVisitor* visitor, - ValidationPolicy validate_frames) { +void ThreadRegistry::VisitObjectPointers( + IsolateGroup* isolate_group_of_interest, + ObjectPointerVisitor* visitor, + ValidationPolicy validate_frames) { MonitorLocker ml(threads_lock()); Thread* thread = active_list_; while (thread != NULL) { - if (thread->isolate() == isolate_of_interest) { + if (thread->isolate_group() == isolate_group_of_interest) { // The mutator thread is visited by the isolate itself (see - // [Isolate::VisitStackPointers]). + // [IsolateGroup::VisitStackPointers]). if (!thread->IsMutatorThread()) { thread->VisitObjectPointers(visitor, validate_frames); } @@ -58,42 +59,37 @@ void ThreadRegistry::VisitObjectPointers(Isolate* isolate_of_interest, } } -void ThreadRegistry::ReleaseStoreBuffers(Isolate* isolate_of_interest) { +void ThreadRegistry::ReleaseStoreBuffers() { MonitorLocker ml(threads_lock()); Thread* thread = active_list_; while (thread != NULL) { - if (thread->isolate() == isolate_of_interest) { - if (!thread->BypassSafepoints()) { - thread->ReleaseStoreBuffer(); - } + if (!thread->BypassSafepoints()) { + thread->ReleaseStoreBuffer(); } thread = thread->next_; } } -void ThreadRegistry::AcquireMarkingStacks(Isolate* isolate_of_interest) { +void ThreadRegistry::AcquireMarkingStacks() { MonitorLocker ml(threads_lock()); Thread* thread = active_list_; while (thread != NULL) { - if (thread->isolate() == isolate_of_interest) { - if (!thread->BypassSafepoints()) { - thread->MarkingStackAcquire(); - thread->DeferredMarkingStackAcquire(); - } + if (!thread->BypassSafepoints()) { + thread->MarkingStackAcquire(); + thread->DeferredMarkingStackAcquire(); } thread = thread->next_; } } -void ThreadRegistry::ReleaseMarkingStacks(Isolate* isolate_of_interest) { +void ThreadRegistry::ReleaseMarkingStacks() { MonitorLocker ml(threads_lock()); Thread* thread = active_list_; while (thread != NULL) { - if (thread->isolate() == isolate_of_interest) { - if (!thread->BypassSafepoints()) { - thread->MarkingStackRelease(); - thread->DeferredMarkingStackRelease(); - } + if (!thread->BypassSafepoints()) { + thread->MarkingStackRelease(); + thread->DeferredMarkingStackRelease(); + ASSERT(!thread->is_marking()); } thread = thread->next_; } diff --git a/runtime/vm/thread_registry.h b/runtime/vm/thread_registry.h index a9059a5db50..d8dc02e8676 100644 --- a/runtime/vm/thread_registry.h +++ b/runtime/vm/thread_registry.h @@ -25,13 +25,13 @@ class ThreadRegistry { ThreadRegistry() : threads_lock_(), active_list_(NULL), free_list_(NULL) {} ~ThreadRegistry(); - void VisitObjectPointers(Isolate* isolate_of_interest, + void VisitObjectPointers(IsolateGroup* isolate_group_of_interest, ObjectPointerVisitor* visitor, ValidationPolicy validate_frames); - void ReleaseStoreBuffers(Isolate* isolate_of_interest); - void AcquireMarkingStacks(Isolate* isolate_of_interest); - void ReleaseMarkingStacks(Isolate* isolate_of_interest); + void ReleaseStoreBuffers(); + void AcquireMarkingStacks(); + void ReleaseMarkingStacks(); #ifndef PRODUCT void PrintJSON(JSONStream* stream) const; diff --git a/runtime/vm/thread_stack_resource.cc b/runtime/vm/thread_stack_resource.cc index 69faf9471f4..8f1dd907bb2 100644 --- a/runtime/vm/thread_stack_resource.cc +++ b/runtime/vm/thread_stack_resource.cc @@ -23,4 +23,8 @@ Isolate* ThreadStackResource::isolate() const { return thread()->isolate(); } +IsolateGroup* ThreadStackResource::isolate_group() const { + return thread()->isolate_group(); +} + } // namespace dart diff --git a/runtime/vm/thread_stack_resource.h b/runtime/vm/thread_stack_resource.h index abbf5215491..fa9dd312e7c 100644 --- a/runtime/vm/thread_stack_resource.h +++ b/runtime/vm/thread_stack_resource.h @@ -11,6 +11,7 @@ namespace dart { class Isolate; +class IsolateGroup; class ThreadState; class Thread; @@ -25,6 +26,7 @@ class ThreadStackResource : public StackResource { return reinterpret_cast(StackResource::thread()); } Isolate* isolate() const; + IsolateGroup* isolate_group() const; }; } // namespace dart diff --git a/runtime/vm/thread_test.cc b/runtime/vm/thread_test.cc index b11f6695459..748103ef590 100644 --- a/runtime/vm/thread_test.cc +++ b/runtime/vm/thread_test.cc @@ -82,8 +82,8 @@ VM_UNIT_TEST_CASE(Monitor) { class ObjectCounter : public ObjectPointerVisitor { public: - explicit ObjectCounter(Isolate* isolate, const Object* obj) - : ObjectPointerVisitor(isolate), obj_(obj), count_(0) {} + explicit ObjectCounter(IsolateGroup* isolate_group, const Object* obj) + : ObjectPointerVisitor(isolate_group), obj_(obj), count_(0) {} virtual void VisitPointers(RawObject** first, RawObject** last) { for (RawObject** current = first; current <= last; ++current) { @@ -131,7 +131,7 @@ class TaskWithZoneAllocation : public ThreadPool::Task { EXPECT(smi.Value() == unique_smi); { HeapIterationScope iteration(thread); - ObjectCounter counter(isolate_, &smi); + ObjectCounter counter(isolate_->group(), &smi); // Ensure that our particular zone is visited. iteration.IterateStackPointers(&counter, ValidationPolicy::kValidateFrames); @@ -148,7 +148,7 @@ class TaskWithZoneAllocation : public ThreadPool::Task { EXPECT(unique_str.Equals(unique_chars)); { HeapIterationScope iteration(thread); - ObjectCounter str_counter(isolate_, &unique_str); + ObjectCounter str_counter(isolate_->group(), &unique_str); // Ensure that our particular zone is visited. iteration.IterateStackPointers(&str_counter, ValidationPolicy::kValidateFrames); @@ -568,7 +568,7 @@ class SafepointTestTask : public ThreadPool::Task { // But occasionally, organize a rendezvous. HeapIterationScope iteration(thread); // Establishes a safepoint. ASSERT(thread->IsAtSafepoint()); - ObjectCounter counter(isolate_, &smi); + ObjectCounter counter(isolate_->group(), &smi); iteration.IterateStackPointers(&counter, ValidationPolicy::kValidateFrames); { diff --git a/runtime/vm/timeline.cc b/runtime/vm/timeline.cc index d2e88e4dc8b..48d4884e84a 100644 --- a/runtime/vm/timeline.cc +++ b/runtime/vm/timeline.cc @@ -395,7 +395,8 @@ TimelineEvent::TimelineEvent() label_(NULL), stream_(NULL), thread_(OSThread::kInvalidThreadId), - isolate_id_(ILLEGAL_PORT) {} + isolate_id_(ILLEGAL_PORT), + isolate_group_id_(0) {} TimelineEvent::~TimelineEvent() { Reset(); @@ -408,6 +409,7 @@ void TimelineEvent::Reset() { state_ = 0; thread_ = OSThread::kInvalidThreadId; isolate_id_ = ILLEGAL_PORT; + isolate_group_id_ = 0; stream_ = NULL; label_ = NULL; arguments_.Free(); @@ -560,12 +562,11 @@ void TimelineEvent::Init(EventType event_type, const char* label) { OSThread* os_thread = OSThread::Current(); ASSERT(os_thread != NULL); thread_ = os_thread->trace_id(); - Isolate* isolate = Isolate::Current(); - if (isolate != NULL) { - isolate_id_ = isolate->main_port(); - } else { - isolate_id_ = ILLEGAL_PORT; - } + auto thread = Thread::Current(); + auto isolate = thread != nullptr ? thread->isolate() : nullptr; + auto isolate_group = thread != nullptr ? thread->isolate_group() : nullptr; + isolate_id_ = (isolate != nullptr) ? isolate->main_port() : ILLEGAL_PORT; + isolate_group_id_ = (isolate_group != nullptr) ? isolate_group->id() : 0; label_ = label; arguments_.Free(); set_event_type(event_type); @@ -664,12 +665,20 @@ void TimelineEvent::PrintJSON(JSONStream* stream) const { ASSERT(arguments_.length() == 1); stream->AppendSerializedObject("args", arguments_[0].value); if (isolate_id_ != ILLEGAL_PORT) { - // If we have one, append the isolate id. stream->UncloseObject(); stream->PrintfProperty("isolateId", ISOLATE_SERVICE_ID_FORMAT_STRING, static_cast(isolate_id_)); stream->CloseObject(); } + if (isolate_group_id_ != 0) { + stream->UncloseObject(); + stream->PrintfProperty("isolateGroupId", + ISOLATE_GROUP_SERVICE_ID_FORMAT_STRING, + isolate_group_id_); + stream->CloseObject(); + } else { + ASSERT(isolate_group_id_ == ILLEGAL_PORT); + } } else { JSONObject args(&obj, "args"); for (intptr_t i = 0; i < arguments_.length(); i++) { @@ -677,10 +686,16 @@ void TimelineEvent::PrintJSON(JSONStream* stream) const { args.AddProperty(arg.name, arg.value); } if (isolate_id_ != ILLEGAL_PORT) { - // If we have one, append the isolate id. args.AddPropertyF("isolateId", ISOLATE_SERVICE_ID_FORMAT_STRING, static_cast(isolate_id_)); } + if (isolate_group_id_ != 0) { + args.AddPropertyF("isolateGroupId", + ISOLATE_GROUP_SERVICE_ID_FORMAT_STRING, + isolate_group_id_); + } else { + ASSERT(isolate_group_id_ == ILLEGAL_PORT); + } } } #endif diff --git a/runtime/vm/timeline.h b/runtime/vm/timeline.h index 0ecdefa7a95..98491763cd8 100644 --- a/runtime/vm/timeline.h +++ b/runtime/vm/timeline.h @@ -344,6 +344,8 @@ class TimelineEvent { Dart_Port isolate_id() const { return isolate_id_; } + uint64_t isolate_group_id() const { return isolate_group_id_; } + const char* label() const { return label_; } // Does this duration end before |micros| ? @@ -460,6 +462,7 @@ class TimelineEvent { TimelineStream* stream_; ThreadId thread_; Dart_Port isolate_id_; + uint64_t isolate_group_id_; friend class TimelineEventRecorder; friend class TimelineEventEndlessRecorder; diff --git a/runtime/vm/virtual_memory_fuchsia.cc b/runtime/vm/virtual_memory_fuchsia.cc index 4f2aa7845d2..ec747b06d9a 100644 --- a/runtime/vm/virtual_memory_fuchsia.cc +++ b/runtime/vm/virtual_memory_fuchsia.cc @@ -183,7 +183,8 @@ void VirtualMemory::FreeSubSegment(void* address, intptr_t size) { void VirtualMemory::Protect(void* address, intptr_t size, Protection mode) { #if defined(DEBUG) Thread* thread = Thread::Current(); - ASSERT((thread == nullptr) || thread->IsMutatorThread() || + ASSERT(thread == nullptr || thread->IsMutatorThread() || + thread->isolate() == nullptr || thread->isolate()->mutator_thread()->IsAtSafepoint()); #endif const uword start_address = reinterpret_cast(address); diff --git a/runtime/vm/virtual_memory_posix.cc b/runtime/vm/virtual_memory_posix.cc index 1c1c5e1cbf5..7010b264f19 100644 --- a/runtime/vm/virtual_memory_posix.cc +++ b/runtime/vm/virtual_memory_posix.cc @@ -288,7 +288,8 @@ void VirtualMemory::FreeSubSegment(void* address, void VirtualMemory::Protect(void* address, intptr_t size, Protection mode) { #if defined(DEBUG) Thread* thread = Thread::Current(); - ASSERT((thread == nullptr) || thread->IsMutatorThread() || + ASSERT(thread == nullptr || thread->IsMutatorThread() || + thread->isolate() == nullptr || thread->isolate()->mutator_thread()->IsAtSafepoint()); #endif uword start_address = reinterpret_cast(address); diff --git a/runtime/vm/virtual_memory_win.cc b/runtime/vm/virtual_memory_win.cc index d52ea07c7f8..fd7c5d11773 100644 --- a/runtime/vm/virtual_memory_win.cc +++ b/runtime/vm/virtual_memory_win.cc @@ -90,7 +90,8 @@ void VirtualMemory::FreeSubSegment(void* address, void VirtualMemory::Protect(void* address, intptr_t size, Protection mode) { #if defined(DEBUG) Thread* thread = Thread::Current(); - ASSERT((thread == nullptr) || thread->IsMutatorThread() || + ASSERT(thread == nullptr || thread->IsMutatorThread() || + thread->isolate() == nullptr || thread->isolate()->mutator_thread()->IsAtSafepoint()); #endif uword start_address = reinterpret_cast(address); diff --git a/runtime/vm/visitor.cc b/runtime/vm/visitor.cc index 0fd28d43857..8274cc86bc6 100644 --- a/runtime/vm/visitor.cc +++ b/runtime/vm/visitor.cc @@ -8,9 +8,9 @@ namespace dart { -ObjectPointerVisitor::ObjectPointerVisitor(Isolate* isolate) - : isolate_(isolate), +ObjectPointerVisitor::ObjectPointerVisitor(IsolateGroup* isolate_group) + : isolate_group_(isolate_group), gc_root_type_("unknown"), - shared_class_table_(isolate->shared_class_table()) {} + shared_class_table_(isolate_group->class_table()) {} } // namespace dart diff --git a/runtime/vm/visitor.h b/runtime/vm/visitor.h index 9d5bf750d82..2dd68f49774 100644 --- a/runtime/vm/visitor.h +++ b/runtime/vm/visitor.h @@ -14,6 +14,7 @@ namespace dart { // Forward declarations. class Isolate; +class IsolateGroup; class RawObject; class RawFunction; class RawTypedDataView; @@ -21,10 +22,10 @@ class RawTypedDataView; // An object pointer visitor interface. class ObjectPointerVisitor { public: - explicit ObjectPointerVisitor(Isolate* isolate); + explicit ObjectPointerVisitor(IsolateGroup* isolate_group); virtual ~ObjectPointerVisitor() {} - Isolate* isolate() const { return isolate_; } + IsolateGroup* isolate_group() const { return isolate_group_; } // Visit pointers inside the given typed data [view]. // @@ -59,7 +60,7 @@ class ObjectPointerVisitor { } private: - Isolate* isolate_; + IsolateGroup* isolate_group_; const char* gc_root_type_; SharedClassTable* shared_class_table_; diff --git a/tests/standalone/io/socket_finalizer_test.dart b/tests/standalone/io/socket_finalizer_test.dart index d25ac1c6f0e..5718559f8b4 100644 --- a/tests/standalone/io/socket_finalizer_test.dart +++ b/tests/standalone/io/socket_finalizer_test.dart @@ -36,8 +36,18 @@ main() async { Expect.fail("Socket error $e"); }); isolate.kill(); + + // Cause a GC to collect the [socket] from [connectorIsolate]. + for (int i = 0; i < 100000; ++i) { + produceGarbage(); + } }); await completer.future; await server.close(); asyncEnd(); } + +@pragma('vm:never-inline') +produceGarbage() => all.add(List(1024)); + +final all = []; diff --git a/tests/standalone_2/io/socket_finalizer_test.dart b/tests/standalone_2/io/socket_finalizer_test.dart index 84ade9e6c0a..cbaae2cccf2 100644 --- a/tests/standalone_2/io/socket_finalizer_test.dart +++ b/tests/standalone_2/io/socket_finalizer_test.dart @@ -16,7 +16,7 @@ import 'dart:isolate'; import "package:async_helper/async_helper.dart"; import "package:expect/expect.dart"; -ConnectorIsolate(Object portObj) async { +connectorIsolate(Object portObj) async { int port = portObj; Socket socket = await Socket.connect("127.0.0.1", port); socket.listen((_) {}); @@ -25,7 +25,7 @@ ConnectorIsolate(Object portObj) async { main() async { asyncStart(); ServerSocket server = await ServerSocket.bind("127.0.0.1", 0); - Isolate isolate = await Isolate.spawn(ConnectorIsolate, server.port); + Isolate isolate = await Isolate.spawn(connectorIsolate, server.port); Completer completer = new Completer(); server.listen((Socket socket) { socket.listen((_) {}, onDone: () { @@ -36,8 +36,18 @@ main() async { Expect.fail("Socket error $e"); }); isolate.kill(); + + // Cause a GC to collect the [socket] from [connectorIsolate]. + for (int i = 0; i < 100000; ++i) { + produceGarbage(); + } }); await completer.future; await server.close(); asyncEnd(); } + +@pragma('vm:never-inline') +produceGarbage() => all.add(List(1024)); + +final all = [];