dart-lang/sdk
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity
Files
ed2899ea4ac32c59caf8a34e6c9b015d8f7de0e2
sdk/runtime/vm/thread.cc
T
Ben Konyi 8caeaf7a6c [ VM ] Add support for heap sampling profiler 
This CL introduces new embedding APIs for supporting heap sample
profiling. A registered sampling callback is invoked approximately every
N bytes based on an exponential distribution, providing information
about the isolate group the allocation occurred in, the user visible
name of the allocated object type, a weak persistent handle to the
allocated object, and the size of the allocation.

Sampling is triggered using artificial TLAB boundaries to cause
allocations to be sampled to take the allocation slow path where the
registered callback can be invoked with the allocation information.

Only new space allocations are currently traced, with old space
allocation support to be added in a future CL.

TEST=Dart_HeapSampling

Change-Id: I22bcdeec6e823bc1ab44898d4c596fbed7169fa1
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/264520
Commit-Queue: Ben Konyi <bkonyi@google.com>
Reviewed-by: Siva Annamalai <asiva@google.com>
Reviewed-by: Ryan Macnak <rmacnak@google.com>
2022-11-22 18:05:24 +00:00

1159 lines
38 KiB
C++
Raw Blame History

// Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#include "vm/thread.h"
#include "vm/cpu.h"
#include "vm/dart_api_state.h"
#include "vm/growable_array.h"
#include "vm/heap/safepoint.h"
#include "vm/isolate.h"
#include "vm/json_stream.h"
#include "vm/lockers.h"
#include "vm/log.h"
#include "vm/message_handler.h"
#include "vm/native_entry.h"
#include "vm/object.h"
#include "vm/object_store.h"
#include "vm/os_thread.h"
#include "vm/profiler.h"
#include "vm/runtime_entry.h"
#include "vm/service.h"
#include "vm/stub_code.h"
#include "vm/symbols.h"
#include "vm/thread_interrupter.h"
#include "vm/thread_registry.h"
#include "vm/timeline.h"
#include "vm/zone.h"
#if !defined(DART_PRECOMPILED_RUNTIME)
#include "vm/ffi_callback_trampolines.h"
#endif // !defined(DART_PRECOMPILED_RUNTIME)
namespace dart {
#if !defined(PRODUCT)
DECLARE_FLAG(bool, trace_service);
DECLARE_FLAG(bool, trace_service_verbose);
#endif // !defined(PRODUCT)
Thread::~Thread() {
// We should cleanly exit any isolate before destruction.
ASSERT(isolate_ == nullptr);
ASSERT(store_buffer_block_ == nullptr);
ASSERT(marking_stack_block_ == nullptr);
// There should be no top api scopes at this point.
ASSERT(api_top_scope() == nullptr);
// Delete the resusable api scope if there is one.
if (api_reusable_scope_ != nullptr) {
delete api_reusable_scope_;
api_reusable_scope_ = nullptr;
}
DO_IF_TSAN(delete tsan_utils_);
}
#if defined(DEBUG)
#define REUSABLE_HANDLE_SCOPE_INIT(object) \
reusable_##object##_handle_scope_active_(false),
#else
#define REUSABLE_HANDLE_SCOPE_INIT(object)
#endif // defined(DEBUG)
#define REUSABLE_HANDLE_INITIALIZERS(object) object##_handle_(nullptr),
Thread::Thread(bool is_vm_isolate)
: ThreadState(false),
stack_limit_(0),
write_barrier_mask_(UntaggedObject::kGenerationalBarrierMask),
heap_base_(0),
isolate_(nullptr),
dispatch_table_array_(nullptr),
saved_stack_limit_(0),
stack_overflow_flags_(0),
heap_(nullptr),
top_exit_frame_info_(0),
store_buffer_block_(nullptr),
marking_stack_block_(nullptr),
vm_tag_(0),
active_exception_(Object::null()),
active_stacktrace_(Object::null()),
global_object_pool_(ObjectPool::null()),
resume_pc_(0),
execution_state_(kThreadInNative),
safepoint_state_(0),
ffi_callback_code_(GrowableObjectArray::null()),
ffi_callback_stack_return_(TypedData::null()),
api_top_scope_(nullptr),
double_truncate_round_supported_(
TargetCPUFeatures::double_truncate_round_supported() ? 1 : 0),
tsan_utils_(DO_IF_TSAN(new TsanUtils()) DO_IF_NOT_TSAN(nullptr)),
task_kind_(kUnknownTask),
dart_stream_(nullptr),
service_extension_stream_(nullptr),
thread_lock_(),
api_reusable_scope_(nullptr),
no_callback_scope_depth_(0),
#if defined(DEBUG)
no_safepoint_scope_depth_(0),
#endif
reusable_handles_(),
stack_overflow_count_(0),
hierarchy_info_(nullptr),
type_usage_info_(nullptr),
sticky_error_(Error::null()),
REUSABLE_HANDLE_LIST(REUSABLE_HANDLE_INITIALIZERS)
REUSABLE_HANDLE_LIST(REUSABLE_HANDLE_SCOPE_INIT)
#if defined(USING_SAFE_STACK)
saved_safestack_limit_(0),
#endif
#if !defined(PRODUCT)
next_(nullptr),
heap_sampler_(this) {
#else
next_(nullptr) {
#endif
#if defined(SUPPORT_TIMELINE)
dart_stream_ = Timeline::GetDartStream();
ASSERT(dart_stream_ != nullptr);
#endif
#ifndef PRODUCT
service_extension_stream_ = &Service::extension_stream;
ASSERT(service_extension_stream_ != nullptr);
#endif
#define DEFAULT_INIT(type_name, member_name, init_expr, default_init_value) \
member_name = default_init_value;
CACHED_CONSTANTS_LIST(DEFAULT_INIT)
#undef DEFAULT_INIT
for (intptr_t i = 0; i < kNumberOfDartAvailableCpuRegs; ++i) {
write_barrier_wrappers_entry_points_[i] = 0;
}
#define DEFAULT_INIT(name) name##_entry_point_ = 0;
RUNTIME_ENTRY_LIST(DEFAULT_INIT)
#undef DEFAULT_INIT
#define DEFAULT_INIT(returntype, name, ...) name##_entry_point_ = 0;
LEAF_RUNTIME_ENTRY_LIST(DEFAULT_INIT)
#undef DEFAULT_INIT
// We cannot initialize the VM constants here for the vm isolate thread
// due to boot strapping issues.
if (!is_vm_isolate) {
InitVMConstants();
}
#if defined(DART_HOST_OS_FUCHSIA)
next_task_id_ = trace_generate_nonce();
#else
next_task_id_ = Random::GlobalNextUInt64();
#endif
memset(&unboxed_runtime_arg_, 0, sizeof(simd128_value_t));
}
static const double double_nan_constant = NAN;
static const struct ALIGN16 {
uint64_t a;
uint64_t b;
} double_negate_constant = {0x8000000000000000ULL, 0x8000000000000000ULL};
static const struct ALIGN16 {
uint64_t a;
uint64_t b;
} double_abs_constant = {0x7FFFFFFFFFFFFFFFULL, 0x7FFFFFFFFFFFFFFFULL};
static const struct ALIGN16 {
uint32_t a;
uint32_t b;
uint32_t c;
uint32_t d;
} float_not_constant = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF};
static const struct ALIGN16 {
uint32_t a;
uint32_t b;
uint32_t c;
uint32_t d;
} float_negate_constant = {0x80000000, 0x80000000, 0x80000000, 0x80000000};
static const struct ALIGN16 {
uint32_t a;
uint32_t b;
uint32_t c;
uint32_t d;
} float_absolute_constant = {0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF};
static const struct ALIGN16 {
uint32_t a;
uint32_t b;
uint32_t c;
uint32_t d;
} float_zerow_constant = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000};
void Thread::InitVMConstants() {
heap_base_ = Object::null()->heap_base();
#define ASSERT_VM_HEAP(type_name, member_name, init_expr, default_init_value) \
ASSERT((init_expr)->IsOldObject());
CACHED_VM_OBJECTS_LIST(ASSERT_VM_HEAP)
#undef ASSERT_VM_HEAP
#define INIT_VALUE(type_name, member_name, init_expr, default_init_value) \
ASSERT(member_name == default_init_value); \
member_name = (init_expr);
CACHED_CONSTANTS_LIST(INIT_VALUE)
#undef INIT_VALUE
for (intptr_t i = 0; i < kNumberOfDartAvailableCpuRegs; ++i) {
write_barrier_wrappers_entry_points_[i] =
StubCode::WriteBarrierWrappers().EntryPoint() +
i * kStoreBufferWrapperSize;
}
#define INIT_VALUE(name) \
ASSERT(name##_entry_point_ == 0); \
name##_entry_point_ = k##name##RuntimeEntry.GetEntryPoint();
RUNTIME_ENTRY_LIST(INIT_VALUE)
#undef INIT_VALUE
#define INIT_VALUE(returntype, name, ...) \
ASSERT(name##_entry_point_ == 0); \
name##_entry_point_ = k##name##RuntimeEntry.GetEntryPoint();
LEAF_RUNTIME_ENTRY_LIST(INIT_VALUE)
#undef INIT_VALUE
// Setup the thread specific reusable handles.
#define REUSABLE_HANDLE_ALLOCATION(object) \
this->object##_handle_ = this->AllocateReusableHandle<object>();
REUSABLE_HANDLE_LIST(REUSABLE_HANDLE_ALLOCATION)
#undef REUSABLE_HANDLE_ALLOCATION
}
void Thread::set_active_exception(const Object& value) {
active_exception_ = value.ptr();
}
void Thread::set_active_stacktrace(const Object& value) {
active_stacktrace_ = value.ptr();
}
ErrorPtr Thread::sticky_error() const {
return sticky_error_;
}
void Thread::set_sticky_error(const Error& value) {
ASSERT(!value.IsNull());
sticky_error_ = value.ptr();
}
void Thread::ClearStickyError() {
sticky_error_ = Error::null();
}
ErrorPtr Thread::StealStickyError() {
NoSafepointScope no_safepoint;
ErrorPtr return_value = sticky_error_;
sticky_error_ = Error::null();
return return_value;
}
const char* Thread::TaskKindToCString(TaskKind kind) {
switch (kind) {
case kUnknownTask:
return "kUnknownTask";
case kMutatorTask:
return "kMutatorTask";
case kCompilerTask:
return "kCompilerTask";
case kSweeperTask:
return "kSweeperTask";
case kMarkerTask:
return "kMarkerTask";
default:
UNREACHABLE();
return "";
}
}
bool Thread::EnterIsolate(Isolate* isolate, bool is_nested_reenter) {
const bool kIsMutatorThread = true;
const bool kBypassSafepoint = false;
is_nested_reenter = is_nested_reenter ||
(isolate->mutator_thread() != nullptr &&
isolate->mutator_thread()->top_exit_frame_info() != 0);
Thread* thread = isolate->ScheduleThread(kIsMutatorThread, is_nested_reenter,
kBypassSafepoint);
if (thread != nullptr) {
ASSERT(thread->store_buffer_block_ == nullptr);
ASSERT(thread->isolate() == isolate);
ASSERT(thread->isolate_group() == isolate->group());
thread->FinishEntering(kMutatorTask);
return true;
}
return false;
}
void Thread::ExitIsolate(bool is_nested_exit) {
Thread* thread = Thread::Current();
ASSERT(thread != nullptr);
ASSERT(thread->IsMutatorThread());
ASSERT(thread->isolate() != nullptr);
ASSERT(thread->isolate_group() != nullptr);
DEBUG_ASSERT(!thread->IsAnyReusableHandleScopeActive());
thread->PrepareLeaving();
Isolate* isolate = thread->isolate();
thread->set_vm_tag(isolate->is_runnable() ? VMTag::kIdleTagId
: VMTag::kLoadWaitTagId);
const bool kIsMutatorThread = true;
const bool kBypassSafepoint = false;
is_nested_exit =
is_nested_exit || (isolate->mutator_thread() != nullptr &&
isolate->mutator_thread()->top_exit_frame_info() != 0);
isolate->UnscheduleThread(thread, kIsMutatorThread, is_nested_exit,
kBypassSafepoint);
}
bool Thread::EnterIsolateAsHelper(Isolate* isolate,
TaskKind kind,
bool bypass_safepoint) {
ASSERT(kind != kMutatorTask);
const bool kIsMutatorThread = false;
const bool kIsNestedReenter = false;
Thread* thread = isolate->ScheduleThread(kIsMutatorThread, kIsNestedReenter,
bypass_safepoint);
if (thread != nullptr) {
ASSERT(!thread->IsMutatorThread());
ASSERT(thread->isolate() == isolate);
ASSERT(thread->isolate_group() == isolate->group());
thread->FinishEntering(kind);
return true;
}
return false;
}
void Thread::ExitIsolateAsHelper(bool bypass_safepoint) {
Thread* thread = Thread::Current();
ASSERT(thread != nullptr);
ASSERT(!thread->IsMutatorThread());
ASSERT(thread->isolate() != nullptr);
ASSERT(thread->isolate_group() != nullptr);
thread->PrepareLeaving();
Isolate* isolate = thread->isolate();
ASSERT(isolate != nullptr);
const bool kIsMutatorThread = false;
const bool kIsNestedExit = false;
isolate->UnscheduleThread(thread, kIsMutatorThread, kIsNestedExit,
bypass_safepoint);
}
bool Thread::EnterIsolateGroupAsHelper(IsolateGroup* isolate_group,
TaskKind kind,
bool bypass_safepoint) {
ASSERT(kind != kMutatorTask);
Thread* thread = isolate_group->ScheduleThread(bypass_safepoint);
if (thread != nullptr) {
ASSERT(!thread->IsMutatorThread());
ASSERT(thread->isolate() == nullptr);
ASSERT(thread->isolate_group() == isolate_group);
thread->FinishEntering(kind);
return true;
}
return false;
}
void Thread::ExitIsolateGroupAsHelper(bool bypass_safepoint) {
Thread* thread = Thread::Current();
ASSERT(thread != nullptr);
ASSERT(!thread->IsMutatorThread());
ASSERT(thread->isolate() == nullptr);
ASSERT(thread->isolate_group() != nullptr);
thread->PrepareLeaving();
const bool kIsMutatorThread = false;
thread->isolate_group()->UnscheduleThread(thread, kIsMutatorThread,
bypass_safepoint);
}
void Thread::ReleaseStoreBuffer() {
ASSERT(IsAtSafepoint());
if (store_buffer_block_ == nullptr || store_buffer_block_->IsEmpty()) {
return; // Nothing to release.
}
// Prevent scheduling another GC by ignoring the threshold.
StoreBufferRelease(StoreBuffer::kIgnoreThreshold);
// Make sure to get an *empty* block; the isolate needs all entries
// at GC time.
// TODO(koda): Replace with an epilogue (PrepareAfterGC) that acquires.
store_buffer_block_ = isolate_group()->store_buffer()->PopEmptyBlock();
}
void Thread::SetStackLimit(uword limit) {
// The thread setting the stack limit is not necessarily the thread which
// the stack limit is being set on.
MonitorLocker ml(&thread_lock_);
if (!HasScheduledInterrupts()) {
// No interrupt pending, set stack_limit_ too.
stack_limit_.store(limit);
}
saved_stack_limit_ = limit;
}
void Thread::ClearStackLimit() {
SetStackLimit(~static_cast<uword>(0));
}
static bool IsInterruptLimit(uword limit) {
return (limit & ~Thread::kInterruptsMask) ==
(kInterruptStackLimit & ~Thread::kInterruptsMask);
}
void Thread::ScheduleInterrupts(uword interrupt_bits) {
ASSERT((interrupt_bits & ~kInterruptsMask) == 0); // Must fit in mask.
uword old_limit = stack_limit_.load();
uword new_limit;
do {
if (IsInterruptLimit(old_limit)) {
new_limit = old_limit | interrupt_bits;
} else {
new_limit = (kInterruptStackLimit & ~kInterruptsMask) | interrupt_bits;
}
} while (!stack_limit_.compare_exchange_weak(old_limit, new_limit));
}
uword Thread::GetAndClearInterrupts() {
uword interrupt_bits = 0;
uword old_limit = stack_limit_.load();
uword new_limit = saved_stack_limit_;
do {
if (IsInterruptLimit(old_limit)) {
interrupt_bits = interrupt_bits | (old_limit & kInterruptsMask);
} else {
return interrupt_bits;
}
} while (!stack_limit_.compare_exchange_weak(old_limit, new_limit));
return interrupt_bits;
}
ErrorPtr Thread::HandleInterrupts() {
uword interrupt_bits = GetAndClearInterrupts();
if ((interrupt_bits & kVMInterrupt) != 0) {
CheckForSafepoint();
if (isolate_group()->store_buffer()->Overflowed()) {
// Evacuate: If the popular store buffer targets are copied instead of
// promoted, the store buffer won't shrink and a second scavenge will
// occur that does promote them.
heap()->CollectGarbage(this, GCType::kEvacuate, GCReason::kStoreBuffer);
}
#if !defined(PRODUCT)
// Don't block system isolates to process CPU samples to avoid blocking
// them during critical tasks (e.g., initial compilation).
if (!Isolate::IsSystemIsolate(isolate())) {
// Processes completed SampleBlocks and sends CPU sample events over the
// service protocol when applicable.
SampleBlockBuffer* sample_buffer = Profiler::sample_block_buffer();
if (sample_buffer != nullptr && sample_buffer->process_blocks()) {
sample_buffer->ProcessCompletedBlocks();
}
}
#endif // !defined(PRODUCT)
}
if ((interrupt_bits & kMessageInterrupt) != 0) {
MessageHandler::MessageStatus status =
isolate()->message_handler()->HandleOOBMessages();
if (status != MessageHandler::kOK) {
// False result from HandleOOBMessages signals that the isolate should
// be terminating.
if (FLAG_trace_isolates) {
OS::PrintErr(
"[!] Terminating isolate due to OOB message:\n"
"\tisolate: %s\n",
isolate()->name());
}
return StealStickyError();
}
}
return Error::null();
}
uword Thread::GetAndClearStackOverflowFlags() {
uword stack_overflow_flags = stack_overflow_flags_;
stack_overflow_flags_ = 0;
return stack_overflow_flags;
}
void Thread::StoreBufferBlockProcess(StoreBuffer::ThresholdPolicy policy) {
StoreBufferRelease(policy);
StoreBufferAcquire();
}
void Thread::StoreBufferAddObject(ObjectPtr obj) {
ASSERT(this == Thread::Current());
store_buffer_block_->Push(obj);
if (store_buffer_block_->IsFull()) {
StoreBufferBlockProcess(StoreBuffer::kCheckThreshold);
}
}
void Thread::StoreBufferAddObjectGC(ObjectPtr obj) {
store_buffer_block_->Push(obj);
if (store_buffer_block_->IsFull()) {
StoreBufferBlockProcess(StoreBuffer::kIgnoreThreshold);
}
}
void Thread::StoreBufferRelease(StoreBuffer::ThresholdPolicy policy) {
StoreBufferBlock* block = store_buffer_block_;
store_buffer_block_ = nullptr;
isolate_group()->store_buffer()->PushBlock(block, policy);
}
void Thread::StoreBufferAcquire() {
store_buffer_block_ = isolate_group()->store_buffer()->PopNonFullBlock();
}
void Thread::MarkingStackBlockProcess() {
MarkingStackRelease();
MarkingStackAcquire();
}
void Thread::DeferredMarkingStackBlockProcess() {
DeferredMarkingStackRelease();
DeferredMarkingStackAcquire();
}
void Thread::MarkingStackAddObject(ObjectPtr obj) {
marking_stack_block_->Push(obj);
if (marking_stack_block_->IsFull()) {
MarkingStackBlockProcess();
}
}
void Thread::DeferredMarkingStackAddObject(ObjectPtr obj) {
deferred_marking_stack_block_->Push(obj);
if (deferred_marking_stack_block_->IsFull()) {
DeferredMarkingStackBlockProcess();
}
}
void Thread::MarkingStackRelease() {
MarkingStackBlock* block = marking_stack_block_;
marking_stack_block_ = nullptr;
write_barrier_mask_ = UntaggedObject::kGenerationalBarrierMask;
isolate_group()->marking_stack()->PushBlock(block);
}
void Thread::MarkingStackAcquire() {
marking_stack_block_ = isolate_group()->marking_stack()->PopEmptyBlock();
write_barrier_mask_ = UntaggedObject::kGenerationalBarrierMask |
UntaggedObject::kIncrementalBarrierMask;
}
void Thread::DeferredMarkingStackRelease() {
MarkingStackBlock* block = deferred_marking_stack_block_;
deferred_marking_stack_block_ = nullptr;
isolate_group()->deferred_marking_stack()->PushBlock(block);
}
void Thread::DeferredMarkingStackAcquire() {
deferred_marking_stack_block_ =
isolate_group()->deferred_marking_stack()->PopEmptyBlock();
}
bool Thread::CanCollectGarbage() const {
// We grow the heap instead of triggering a garbage collection when a
// thread is at a safepoint in the following situations :
// - background compiler thread finalizing and installing code
// - disassembly of the generated code is done after compilation
// So essentially we state that garbage collection is possible only
// when we are not at a safepoint.
return !IsAtSafepoint();
}
bool Thread::IsExecutingDartCode() const {
return (top_exit_frame_info() == 0) && VMTag::IsDartTag(vm_tag());
}
bool Thread::HasExitedDartCode() const {
return (top_exit_frame_info() != 0) && !VMTag::IsDartTag(vm_tag());
}
template <class C>
C* Thread::AllocateReusableHandle() {
C* handle = reinterpret_cast<C*>(reusable_handles_.AllocateScopedHandle());
C::initializeHandle(handle, C::null());
return handle;
}
void Thread::ClearReusableHandles() {
#define CLEAR_REUSABLE_HANDLE(object) *object##_handle_ = object::null();
REUSABLE_HANDLE_LIST(CLEAR_REUSABLE_HANDLE)
#undef CLEAR_REUSABLE_HANDLE
}
void Thread::VisitObjectPointers(ObjectPointerVisitor* visitor,
ValidationPolicy validation_policy) {
ASSERT(visitor != nullptr);
if (zone() != nullptr) {
zone()->VisitObjectPointers(visitor);
}
// Visit objects in thread specific handles area.
reusable_handles_.VisitObjectPointers(visitor);
visitor->VisitPointer(reinterpret_cast<ObjectPtr*>(&global_object_pool_));
visitor->VisitPointer(reinterpret_cast<ObjectPtr*>(&active_exception_));
visitor->VisitPointer(reinterpret_cast<ObjectPtr*>(&active_stacktrace_));
visitor->VisitPointer(reinterpret_cast<ObjectPtr*>(&sticky_error_));
visitor->VisitPointer(reinterpret_cast<ObjectPtr*>(&ffi_callback_code_));
visitor->VisitPointer(
reinterpret_cast<ObjectPtr*>(&ffi_callback_stack_return_));
// Visit the api local scope as it has all the api local handles.
ApiLocalScope* scope = api_top_scope_;
while (scope != nullptr) {
scope->local_handles()->VisitObjectPointers(visitor);
scope = scope->previous();
}
// Only the mutator thread can run Dart code.
if (IsMutatorThread()) {
// The MarkTask, which calls this method, can run on a different thread. We
// therefore assume the mutator is at a safepoint and we can iterate its
// stack.
// TODO(vm-team): It would be beneficial to be able to ask the mutator
// thread whether it is in fact blocked at the moment (at a "safepoint") so
// we can safely iterate its stack.
//
// Unfortunately we cannot use `this->IsAtSafepoint()` here because that
// will return `false` even though the mutator thread is waiting for mark
// tasks (which iterate its stack) to finish.
const StackFrameIterator::CrossThreadPolicy cross_thread_policy =
StackFrameIterator::kAllowCrossThreadIteration;
// Iterate over all the stack frames and visit objects on the stack.
StackFrameIterator frames_iterator(top_exit_frame_info(), validation_policy,
this, cross_thread_policy);
StackFrame* frame = frames_iterator.NextFrame();
visitor->set_gc_root_type("frame");
while (frame != nullptr) {
frame->VisitObjectPointers(visitor);
frame = frames_iterator.NextFrame();
}
visitor->clear_gc_root_type();
} else {
// We are not on the mutator thread.
RELEASE_ASSERT(top_exit_frame_info() == 0);
}
}
class RestoreWriteBarrierInvariantVisitor : public ObjectPointerVisitor {
public:
RestoreWriteBarrierInvariantVisitor(IsolateGroup* group,
Thread* thread,
Thread::RestoreWriteBarrierInvariantOp op)
: ObjectPointerVisitor(group),
thread_(thread),
current_(Thread::Current()),
op_(op) {}
void VisitPointers(ObjectPtr* first, ObjectPtr* last) {
for (; first != last + 1; first++) {
ObjectPtr obj = *first;
// Stores into new-space objects don't need a write barrier.
if (obj->IsSmiOrNewObject()) continue;
// To avoid adding too much work into the remembered set, skip large
// arrays. Write barrier elimination will not remove the barrier
// if we can trigger GC between array allocation and store.
if (obj->GetClassId() == kArrayCid) {
const auto length = Smi::Value(Array::RawCast(obj)->untag()->length());
if (length > Array::kMaxLengthForWriteBarrierElimination) {
continue;
}
}
// Dart code won't store into VM-internal objects except Contexts and
// UnhandledExceptions. This assumption is checked by an assertion in
// WriteBarrierElimination::UpdateVectorForBlock.
if (!obj->IsDartInstance() && !obj->IsContext() &&
!obj->IsUnhandledException())
continue;
// Dart code won't store into canonical instances.
if (obj->untag()->IsCanonical()) continue;
// Objects in the VM isolate heap are immutable and won't be
// stored into. Check this condition last because there's no bit
// in the header for it.
if (obj->untag()->InVMIsolateHeap()) continue;
switch (op_) {
case Thread::RestoreWriteBarrierInvariantOp::kAddToRememberedSet:
obj->untag()->EnsureInRememberedSet(current_);
if (current_->is_marking()) {
current_->DeferredMarkingStackAddObject(obj);
}
break;
case Thread::RestoreWriteBarrierInvariantOp::kAddToDeferredMarkingStack:
// Re-scan obj when finalizing marking.
current_->DeferredMarkingStackAddObject(obj);
break;
}
}
}
void VisitCompressedPointers(uword heap_base,
CompressedObjectPtr* first,
CompressedObjectPtr* last) {
UNREACHABLE(); // Stack slots are not compressed.
}
private:
Thread* const thread_;
Thread* const current_;
Thread::RestoreWriteBarrierInvariantOp op_;
};
// Write barrier elimination assumes that all live temporaries will be
// in the remembered set after a scavenge triggered by a non-Dart-call
// instruction (see Instruction::CanCallDart()), and additionally they will be
// in the deferred marking stack if concurrent marking started. Specifically,
// this includes any instruction which will always create an exit frame
// below the current frame before any other Dart frames.
//
// Therefore, to support this assumption, we scan the stack after a scavenge
// or when concurrent marking begins and add all live temporaries in
// Dart frames preceding an exit frame to the store buffer or deferred
// marking stack.
void Thread::RestoreWriteBarrierInvariant(RestoreWriteBarrierInvariantOp op) {
ASSERT(IsAtSafepoint());
ASSERT(IsMutatorThread());
const StackFrameIterator::CrossThreadPolicy cross_thread_policy =
StackFrameIterator::kAllowCrossThreadIteration;
StackFrameIterator frames_iterator(top_exit_frame_info(),
ValidationPolicy::kDontValidateFrames,
this, cross_thread_policy);
RestoreWriteBarrierInvariantVisitor visitor(isolate_group(), this, op);
ObjectStore* object_store = isolate_group()->object_store();
bool scan_next_dart_frame = false;
for (StackFrame* frame = frames_iterator.NextFrame(); frame != nullptr;
frame = frames_iterator.NextFrame()) {
if (frame->IsExitFrame()) {
scan_next_dart_frame = true;
} else if (frame->IsEntryFrame()) {
/* Continue searching. */
} else if (frame->IsStubFrame()) {
const uword pc = frame->pc();
if (Code::ContainsInstructionAt(
object_store->init_late_static_field_stub(), pc) ||
Code::ContainsInstructionAt(
object_store->init_late_final_static_field_stub(), pc) ||
Code::ContainsInstructionAt(
object_store->init_late_instance_field_stub(), pc) ||
Code::ContainsInstructionAt(
object_store->init_late_final_instance_field_stub(), pc)) {
scan_next_dart_frame = true;
}
} else {
ASSERT(frame->IsDartFrame(/*validate=*/false));
if (scan_next_dart_frame) {
frame->VisitObjectPointers(&visitor);
}
scan_next_dart_frame = false;
}
}
}
void Thread::DeferredMarkLiveTemporaries() {
RestoreWriteBarrierInvariant(
RestoreWriteBarrierInvariantOp::kAddToDeferredMarkingStack);
}
void Thread::RememberLiveTemporaries() {
RestoreWriteBarrierInvariant(
RestoreWriteBarrierInvariantOp::kAddToRememberedSet);
}
bool Thread::CanLoadFromThread(const Object& object) {
// In order to allow us to use assembler helper routines with non-[Code]
// objects *before* stubs are initialized, we only loop ver the stubs if the
// [object] is in fact a [Code] object.
if (object.IsCode()) {
#define CHECK_OBJECT(type_name, member_name, expr, default_init_value) \
if (object.ptr() == expr) { \
return true; \
}
CACHED_VM_STUBS_LIST(CHECK_OBJECT)
#undef CHECK_OBJECT
}
// For non [Code] objects we check if the object equals to any of the cached
// non-stub entries.
#define CHECK_OBJECT(type_name, member_name, expr, default_init_value) \
if (object.ptr() == expr) { \
return true; \
}
CACHED_NON_VM_STUB_LIST(CHECK_OBJECT)
#undef CHECK_OBJECT
return false;
}
intptr_t Thread::OffsetFromThread(const Object& object) {
// In order to allow us to use assembler helper routines with non-[Code]
// objects *before* stubs are initialized, we only loop ver the stubs if the
// [object] is in fact a [Code] object.
if (object.IsCode()) {
#define COMPUTE_OFFSET(type_name, member_name, expr, default_init_value) \
ASSERT((expr)->untag()->InVMIsolateHeap()); \
if (object.ptr() == expr) { \
return Thread::member_name##offset(); \
}
CACHED_VM_STUBS_LIST(COMPUTE_OFFSET)
#undef COMPUTE_OFFSET
}
// For non [Code] objects we check if the object equals to any of the cached
// non-stub entries.
#define COMPUTE_OFFSET(type_name, member_name, expr, default_init_value) \
if (object.ptr() == expr) { \
return Thread::member_name##offset(); \
}
CACHED_NON_VM_STUB_LIST(COMPUTE_OFFSET)
#undef COMPUTE_OFFSET
UNREACHABLE();
return -1;
}
bool Thread::ObjectAtOffset(intptr_t offset, Object* object) {
if (Isolate::Current() == Dart::vm_isolate()) {
// --disassemble-stubs runs before all the references through
// thread have targets
return false;
}
#define COMPUTE_OFFSET(type_name, member_name, expr, default_init_value) \
if (Thread::member_name##offset() == offset) { \
*object = expr; \
return true; \
}
CACHED_VM_OBJECTS_LIST(COMPUTE_OFFSET)
#undef COMPUTE_OFFSET
return false;
}
intptr_t Thread::OffsetFromThread(const RuntimeEntry* runtime_entry) {
#define COMPUTE_OFFSET(name) \
if (runtime_entry->function() == k##name##RuntimeEntry.function()) { \
return Thread::name##_entry_point_offset(); \
}
RUNTIME_ENTRY_LIST(COMPUTE_OFFSET)
#undef COMPUTE_OFFSET
#define COMPUTE_OFFSET(returntype, name, ...) \
if (runtime_entry->function() == k##name##RuntimeEntry.function()) { \
return Thread::name##_entry_point_offset(); \
}
LEAF_RUNTIME_ENTRY_LIST(COMPUTE_OFFSET)
#undef COMPUTE_OFFSET
UNREACHABLE();
return -1;
}
#if defined(DEBUG)
bool Thread::TopErrorHandlerIsSetJump() const {
if (long_jump_base() == nullptr) return false;
if (top_exit_frame_info_ == 0) return true;
#if defined(USING_SIMULATOR) || defined(USING_SAFE_STACK)
// False positives: simulator stack and native stack are unordered.
return true;
#else
return reinterpret_cast<uword>(long_jump_base()) < top_exit_frame_info_;
#endif
}
bool Thread::TopErrorHandlerIsExitFrame() const {
if (top_exit_frame_info_ == 0) return false;
if (long_jump_base() == nullptr) return true;
#if defined(USING_SIMULATOR) || defined(USING_SAFE_STACK)
// False positives: simulator stack and native stack are unordered.
return true;
#else
return top_exit_frame_info_ < reinterpret_cast<uword>(long_jump_base());
#endif
}
#endif // defined(DEBUG)
bool Thread::IsValidHandle(Dart_Handle object) const {
return IsValidLocalHandle(object) || IsValidZoneHandle(object) ||
IsValidScopedHandle(object);
}
bool Thread::IsValidLocalHandle(Dart_Handle object) const {
ApiLocalScope* scope = api_top_scope_;
while (scope != nullptr) {
if (scope->local_handles()->IsValidHandle(object)) {
return true;
}
scope = scope->previous();
}
return false;
}
intptr_t Thread::CountLocalHandles() const {
intptr_t total = 0;
ApiLocalScope* scope = api_top_scope_;
while (scope != nullptr) {
total += scope->local_handles()->CountHandles();
scope = scope->previous();
}
return total;
}
int Thread::ZoneSizeInBytes() const {
int total = 0;
ApiLocalScope* scope = api_top_scope_;
while (scope != nullptr) {
total += scope->zone()->SizeInBytes();
scope = scope->previous();
}
return total;
}
void Thread::EnterApiScope() {
ASSERT(MayAllocateHandles());
ApiLocalScope* new_scope = api_reusable_scope();
if (new_scope == nullptr) {
new_scope = new ApiLocalScope(api_top_scope(), top_exit_frame_info());
ASSERT(new_scope != nullptr);
} else {
new_scope->Reinit(this, api_top_scope(), top_exit_frame_info());
set_api_reusable_scope(nullptr);
}
set_api_top_scope(new_scope); // New scope is now the top scope.
}
void Thread::ExitApiScope() {
ASSERT(MayAllocateHandles());
ApiLocalScope* scope = api_top_scope();
ApiLocalScope* reusable_scope = api_reusable_scope();
set_api_top_scope(scope->previous()); // Reset top scope to previous.
if (reusable_scope == nullptr) {
scope->Reset(this); // Reset the old scope which we just exited.
set_api_reusable_scope(scope);
} else {
ASSERT(reusable_scope != scope);
delete scope;
}
}
void Thread::UnwindScopes(uword stack_marker) {
// Unwind all scopes using the same stack_marker, i.e. all scopes allocated
// under the same top_exit_frame_info.
ApiLocalScope* scope = api_top_scope_;
while (scope != nullptr && scope->stack_marker() != 0 &&
scope->stack_marker() == stack_marker) {
api_top_scope_ = scope->previous();
delete scope;
scope = api_top_scope_;
}
}
void Thread::EnterSafepointUsingLock() {
isolate_group()->safepoint_handler()->EnterSafepointUsingLock(this);
}
void Thread::ExitSafepointUsingLock() {
isolate_group()->safepoint_handler()->ExitSafepointUsingLock(this);
}
void Thread::BlockForSafepoint() {
isolate_group()->safepoint_handler()->BlockForSafepoint(this);
}
void Thread::FinishEntering(TaskKind kind) {
ASSERT(store_buffer_block_ == nullptr);
task_kind_ = kind;
if (isolate_group()->marking_stack() != nullptr) {
// Concurrent mark in progress. Enable barrier for this thread.
MarkingStackAcquire();
DeferredMarkingStackAcquire();
}
// TODO(koda): Use StoreBufferAcquire once we properly flush
// before Scavenge.
if (kind == kMutatorTask) {
StoreBufferAcquire();
} else {
store_buffer_block_ = isolate_group()->store_buffer()->PopEmptyBlock();
}
}
void Thread::PrepareLeaving() {
ASSERT(store_buffer_block_ != nullptr);
ASSERT(execution_state() == Thread::kThreadInVM);
task_kind_ = kUnknownTask;
if (is_marking()) {
MarkingStackRelease();
DeferredMarkingStackRelease();
}
StoreBufferRelease();
}
DisableThreadInterruptsScope::DisableThreadInterruptsScope(Thread* thread)
: StackResource(thread) {
if (thread != nullptr) {
OSThread* os_thread = thread->os_thread();
ASSERT(os_thread != nullptr);
os_thread->DisableThreadInterrupts();
}
}
DisableThreadInterruptsScope::~DisableThreadInterruptsScope() {
if (thread() != nullptr) {
OSThread* os_thread = thread()->os_thread();
ASSERT(os_thread != nullptr);
os_thread->EnableThreadInterrupts();
}
}
const intptr_t kInitialCallbackIdsReserved = 16;
int32_t Thread::AllocateFfiCallbackId() {
Zone* Z = Thread::Current()->zone();
if (ffi_callback_code_ == GrowableObjectArray::null()) {
ffi_callback_code_ = GrowableObjectArray::New(kInitialCallbackIdsReserved);
}
const auto& array = GrowableObjectArray::Handle(Z, ffi_callback_code_);
array.Add(Code::Handle(Z, Code::null()));
const int32_t id = array.Length() - 1;
// Allocate a native callback trampoline if necessary.
#if !defined(DART_PRECOMPILED_RUNTIME)
if (NativeCallbackTrampolines::Enabled()) {
auto* const tramps = isolate()->native_callback_trampolines();
ASSERT(tramps->next_callback_id() == id);
tramps->AllocateTrampoline();
}
#endif
return id;
}
void Thread::SetFfiCallbackCode(int32_t callback_id, const Code& code) {
Zone* Z = Thread::Current()->zone();
/// In AOT the callback ID might have been allocated during compilation but
/// 'ffi_callback_code_' is initialized to empty again when the program
/// starts. Therefore we may need to initialize or expand it to accomodate
/// the callback ID.
if (ffi_callback_code_ == GrowableObjectArray::null()) {
ffi_callback_code_ = GrowableObjectArray::New(kInitialCallbackIdsReserved);
}
const auto& array = GrowableObjectArray::Handle(Z, ffi_callback_code_);
if (callback_id >= array.Length()) {
const int32_t capacity = array.Capacity();
if (callback_id >= capacity) {
// Ensure both that we grow enough and an exponential growth strategy.
const int32_t new_capacity =
Utils::Maximum(callback_id + 1, capacity * 2);
array.Grow(new_capacity);
}
array.SetLength(callback_id + 1);
}
array.SetAt(callback_id, code);
}
void Thread::SetFfiCallbackStackReturn(int32_t callback_id,
intptr_t stack_return_delta) {
#if defined(TARGET_ARCH_IA32)
#else
UNREACHABLE();
#endif
Zone* Z = Thread::Current()->zone();
/// In AOT the callback ID might have been allocated during compilation but
/// 'ffi_callback_code_' is initialized to empty again when the program
/// starts. Therefore we may need to initialize or expand it to accomodate
/// the callback ID.
if (ffi_callback_stack_return_ == TypedData::null()) {
ffi_callback_stack_return_ = TypedData::New(
kTypedDataInt8ArrayCid, kInitialCallbackIdsReserved, Heap::kOld);
}
auto& array = TypedData::Handle(Z, ffi_callback_stack_return_);
if (callback_id >= array.Length()) {
const int32_t capacity = array.Length();
if (callback_id >= capacity) {
// Ensure both that we grow enough and an exponential growth strategy.
const int32_t new_capacity =
Utils::Maximum(callback_id + 1, capacity * 2);
const auto& new_array = TypedData::Handle(
Z, TypedData::New(kTypedDataUint8ArrayCid, new_capacity, Heap::kOld));
for (intptr_t i = 0; i < capacity; i++) {
new_array.SetUint8(i, array.GetUint8(i));
}
array ^= new_array.ptr();
ffi_callback_stack_return_ = new_array.ptr();
}
}
ASSERT(callback_id < array.Length());
array.SetUint8(callback_id, stack_return_delta);
}
void Thread::VerifyCallbackIsolate(int32_t callback_id, uword entry) {
NoSafepointScope _;
const GrowableObjectArrayPtr array = ffi_callback_code_;
if (array == GrowableObjectArray::null()) {
FATAL("Cannot invoke callback on incorrect isolate.");
}
const SmiPtr length_smi = GrowableObjectArray::NoSafepointLength(array);
const intptr_t length = Smi::Value(length_smi);
if (callback_id < 0 || callback_id >= length) {
FATAL("Cannot invoke callback on incorrect isolate.");
}
if (entry != 0) {
CompressedObjectPtr* const code_array =
Array::DataOf(GrowableObjectArray::NoSafepointData(array));
// RawCast allocates handles in ASSERTs.
const CodePtr code = static_cast<CodePtr>(
code_array[callback_id].Decompress(array.heap_base()));
if (!Code::ContainsInstructionAt(code, entry)) {
FATAL("Cannot invoke callback on incorrect isolate.");
}
}
}
} // namespace dart
Reference in New Issue View Git Blame Copy Permalink