dart-lang/sdk
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[vm] Default to non-atomic for heap accesses.

Browse Source 
Avoids inadvertently covering up data races.

TEST=tsan
Change-Id: Icb001391638b9f467145cf889363d47527933cd9
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/446988
Reviewed-by: Alexander Aprelev <aam@google.com>
Commit-Queue: Ryan Macnak <rmacnak@google.com>
This commit is contained in:
Ryan Macnak
2025-09-02 11:09:19 -07:00
committed by Commit Queue
parent 0f8250c036
commit eb3fdd7683
12 changed files with 266 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/vm/code_patcher_arm64.cc
+1 -1
View File
@@ -29,7 +29,7 @@ class PoolPointerCall : public ValueObject {
}
void SetTarget(const Code& target) const {
object_pool_.SetObjectAt(pp_index(), target);
object_pool_.SetObjectAt<std::memory_order_release>(pp_index(), target);
// No need to flush the instruction cache, since the code is not modified.
}
runtime/vm/code_patcher_riscv.cc
+1 -1
View File
@@ -39,7 +39,7 @@ class PoolPointerCall : public ValueObject {
}
void SetTarget(const Code& target) const {
object_pool_.SetObjectAt(pp_index(), target);
object_pool_.SetObjectAt<std::memory_order_release>(pp_index(), target);
// No need to flush the instruction cache, since the code is not modified.
}
runtime/vm/code_patcher_x64.cc
+10 -8
View File
@@ -147,7 +147,7 @@ class UnoptimizedCall : public ValueObject {
}
void set_target(const Code& target) const {
object_pool_.SetObjectAt(code_index_, target);
object_pool_.SetObjectAt<std::memory_order_release>(code_index_, target);
// No need to flush the instruction cache, since the code is not modified.
}
@@ -181,7 +181,8 @@ class NativeCall : public ValueObject {
}
void set_native_function(NativeFunction func) const {
object_pool_.SetRawValueAt(argument_index(), reinterpret_cast<uword>(func));
object_pool_.SetRawValueAt<std::memory_order_relaxed>(
argument_index(), reinterpret_cast<uword>(func));
}
CodePtr target() const {
@@ -191,7 +192,7 @@ class NativeCall : public ValueObject {
}
void set_target(const Code& target) const {
object_pool_.SetObjectAt(code_index_, target);
object_pool_.SetObjectAt<std::memory_order_release>(code_index_, target);
// No need to flush the instruction cache, since the code is not modified.
}
@@ -220,7 +221,7 @@ class InstanceCall : public UnoptimizedCall {
ObjectPtr data() const { return object_pool_.ObjectAt(argument_index()); }
void set_data(const Object& data) const {
ASSERT(data.IsArray() || data.IsICData() || data.IsMegamorphicCache());
object_pool_.SetObjectAt(argument_index(), data);
object_pool_.SetObjectAt<std::memory_order_release>(argument_index(), data);
}
private:
@@ -265,7 +266,7 @@ class PoolPointerCall : public ValueObject {
}
void SetTarget(const Code& target) const {
object_pool_.SetObjectAt(code_index_, target);
object_pool_.SetObjectAt<std::memory_order_release>(code_index_, target);
// No need to flush the instruction cache, since the code is not modified.
}
@@ -294,7 +295,7 @@ class SwitchableCallBase : public ValueObject {
void SetData(const Object& data) const {
ASSERT(!Object::Handle(object_pool_.ObjectAt(data_index())).IsCode());
object_pool_.SetObjectAt(data_index(), data);
object_pool_.SetObjectAt<std::memory_order_release>(data_index(), data);
// No need to flush the instruction cache, since the code is not modified.
}
@@ -353,7 +354,7 @@ class SwitchableCall : public SwitchableCallBase {
void SetTarget(const Code& target) const {
ASSERT(Object::Handle(object_pool_.ObjectAt(target_index())).IsCode());
object_pool_.SetObjectAt(target_index(), target);
object_pool_.SetObjectAt<std::memory_order_release>(target_index(), target);
// No need to flush the instruction cache, since the code is not modified.
}
@@ -424,7 +425,8 @@ class BareSwitchableCall : public SwitchableCallBase {
void SetTarget(const Code& target) const {
ASSERT(object_pool_.TypeAt(target_index()) ==
ObjectPool::EntryType::kImmediate);
object_pool_.SetRawValueAt(target_index(), target.MonomorphicEntryPoint());
object_pool_.SetRawValueAt<std::memory_order_relaxed>(
target_index(), target.MonomorphicEntryPoint());
}
uword target_entry() const { return object_pool_.RawValueAt(target_index()); }
runtime/vm/instructions_arm.cc
+14 -10
View File
@@ -75,7 +75,8 @@ CodePtr NativeCallPattern::target() const {
}
void NativeCallPattern::set_target(const Code& new_target) const {
object_pool_.SetObjectAt(target_code_pool_index_, new_target);
object_pool_.SetObjectAt<std::memory_order_release>(target_code_pool_index_,
new_target);
// No need to flush the instruction cache, since the code is not modified.
}
@@ -85,8 +86,8 @@ NativeFunction NativeCallPattern::native_function() const {
}
void NativeCallPattern::set_native_function(NativeFunction func) const {
object_pool_.SetRawValueAt(native_function_pool_index_,
reinterpret_cast<uword>(func));
object_pool_.SetRawValueAt<std::memory_order_relaxed>(
native_function_pool_index_, reinterpret_cast<uword>(func));
}
// Decodes a load sequence ending at 'end' (the last instruction of the load
@@ -216,7 +217,8 @@ CodePtr CallPattern::TargetCode() const {
}
void CallPattern::SetTargetCode(const Code& target_code) const {
object_pool_.SetObjectAt(target_code_pool_index_, target_code);
object_pool_.SetObjectAt<std::memory_order_release>(target_code_pool_index_,
target_code);
}
ObjectPtr ICCallPattern::Data() const {
@@ -225,7 +227,7 @@ ObjectPtr ICCallPattern::Data() const {
void ICCallPattern::SetData(const Object& data) const {
ASSERT(data.IsArray() || data.IsICData() || data.IsMegamorphicCache());
object_pool_.SetObjectAt(data_pool_index_, data);
object_pool_.SetObjectAt<std::memory_order_release>(data_pool_index_, data);
}
CodePtr ICCallPattern::TargetCode() const {
@@ -233,7 +235,8 @@ CodePtr ICCallPattern::TargetCode() const {
}
void ICCallPattern::SetTargetCode(const Code& target_code) const {
object_pool_.SetObjectAt(target_pool_index_, target_code);
object_pool_.SetObjectAt<std::memory_order_release>(target_pool_index_,
target_code);
}
SwitchableCallPatternBase::SwitchableCallPatternBase(
@@ -246,7 +249,7 @@ ObjectPtr SwitchableCallPatternBase::data() const {
void SwitchableCallPatternBase::SetData(const Object& data) const {
ASSERT(!Object::Handle(object_pool_.ObjectAt(data_pool_index_)).IsCode());
object_pool_.SetObjectAt(data_pool_index_, data);
object_pool_.SetObjectAt<std::memory_order_release>(data_pool_index_, data);
}
SwitchableCallPattern::SwitchableCallPattern(uword pc, const Code& code)
@@ -270,7 +273,8 @@ ObjectPtr SwitchableCallPattern::target() const {
void SwitchableCallPattern::SetTarget(const Code& target) const {
ASSERT(Object::Handle(object_pool_.ObjectAt(target_pool_index_)).IsCode());
object_pool_.SetObjectAt(target_pool_index_, target);
object_pool_.SetObjectAt<std::memory_order_release>(target_pool_index_,
target);
}
BareSwitchableCallPattern::BareSwitchableCallPattern(uword pc)
@@ -296,8 +300,8 @@ uword BareSwitchableCallPattern::target_entry() const {
void BareSwitchableCallPattern::SetTarget(const Code& target) const {
ASSERT(object_pool_.TypeAt(target_pool_index_) ==
ObjectPool::EntryType::kImmediate);
object_pool_.SetRawValueAt(target_pool_index_,
target.MonomorphicEntryPoint());
object_pool_.SetRawValueAt<std::memory_order_relaxed>(
target_pool_index_, target.MonomorphicEntryPoint());
}
ReturnPattern::ReturnPattern(uword pc) : pc_(pc) {}
runtime/vm/instructions_arm64.cc
+14 -10
View File
@@ -76,7 +76,8 @@ CodePtr NativeCallPattern::target() const {
}
void NativeCallPattern::set_target(const Code& target) const {
object_pool_.SetObjectAt(target_code_pool_index_, target);
object_pool_.SetObjectAt<std::memory_order_release>(target_code_pool_index_,
target);
// No need to flush the instruction cache, since the code is not modified.
}
@@ -86,8 +87,8 @@ NativeFunction NativeCallPattern::native_function() const {
}
void NativeCallPattern::set_native_function(NativeFunction func) const {
object_pool_.SetRawValueAt(native_function_pool_index_,
reinterpret_cast<uword>(func));
object_pool_.SetRawValueAt<std::memory_order_relaxed>(
native_function_pool_index_, reinterpret_cast<uword>(func));
}
// Decodes a load sequence ending at 'end' (the last instruction of the load
@@ -398,7 +399,8 @@ CodePtr CallPattern::TargetCode() const {
}
void CallPattern::SetTargetCode(const Code& target) const {
object_pool_.SetObjectAt(target_code_pool_index_, target);
object_pool_.SetObjectAt<std::memory_order_release>(target_code_pool_index_,
target);
// No need to flush the instruction cache, since the code is not modified.
}
@@ -408,7 +410,7 @@ ObjectPtr ICCallPattern::Data() const {
void ICCallPattern::SetData(const Object& data) const {
ASSERT(data.IsArray() || data.IsICData() || data.IsMegamorphicCache());
object_pool_.SetObjectAt(data_pool_index_, data);
object_pool_.SetObjectAt<std::memory_order_release>(data_pool_index_, data);
}
CodePtr ICCallPattern::TargetCode() const {
@@ -416,7 +418,8 @@ CodePtr ICCallPattern::TargetCode() const {
}
void ICCallPattern::SetTargetCode(const Code& target) const {
object_pool_.SetObjectAt(target_pool_index_, target);
object_pool_.SetObjectAt<std::memory_order_release>(target_pool_index_,
target);
// No need to flush the instruction cache, since the code is not modified.
}
@@ -430,7 +433,7 @@ ObjectPtr SwitchableCallPatternBase::data() const {
void SwitchableCallPatternBase::SetData(const Object& data) const {
ASSERT(!Object::Handle(object_pool_.ObjectAt(data_pool_index_)).IsCode());
object_pool_.SetObjectAt(data_pool_index_, data);
object_pool_.SetObjectAt<std::memory_order_release>(data_pool_index_, data);
}
SwitchableCallPattern::SwitchableCallPattern(uword pc, const Code& code)
@@ -456,7 +459,8 @@ ObjectPtr SwitchableCallPattern::target() const {
void SwitchableCallPattern::SetTarget(const Code& target) const {
ASSERT(Object::Handle(object_pool_.ObjectAt(target_pool_index_)).IsCode());
object_pool_.SetObjectAt(target_pool_index_, target);
object_pool_.SetObjectAt<std::memory_order_release>(target_pool_index_,
target);
}
BareSwitchableCallPattern::BareSwitchableCallPattern(uword pc)
@@ -483,8 +487,8 @@ uword BareSwitchableCallPattern::target_entry() const {
void BareSwitchableCallPattern::SetTarget(const Code& target) const {
ASSERT(object_pool_.TypeAt(target_pool_index_) ==
ObjectPool::EntryType::kImmediate);
object_pool_.SetRawValueAt(target_pool_index_,
target.MonomorphicEntryPoint());
object_pool_.SetRawValueAt<std::memory_order_relaxed>(
target_pool_index_, target.MonomorphicEntryPoint());
}
ReturnPattern::ReturnPattern(uword pc) : pc_(pc) {}
runtime/vm/instructions_riscv.cc
+13 -9
View File
@@ -90,7 +90,8 @@ CodePtr NativeCallPattern::target() const {
}
void NativeCallPattern::set_target(const Code& target) const {
object_pool_.SetObjectAt(target_code_pool_index_, target);
object_pool_.SetObjectAt<std::memory_order_release>(target_code_pool_index_,
target);
// No need to flush the instruction cache, since the code is not modified.
}
@@ -100,8 +101,8 @@ NativeFunction NativeCallPattern::native_function() const {
}
void NativeCallPattern::set_native_function(NativeFunction func) const {
object_pool_.SetRawValueAt(native_function_pool_index_,
reinterpret_cast<uword>(func));
object_pool_.SetRawValueAt<std::memory_order_relaxed>(
native_function_pool_index_, reinterpret_cast<uword>(func));
}
// Decodes a load sequence ending at 'end' (the last instruction of the load
@@ -297,7 +298,8 @@ CodePtr CallPattern::TargetCode() const {
}
void CallPattern::SetTargetCode(const Code& target) const {
object_pool_.SetObjectAt(target_code_pool_index_, target);
object_pool_.SetObjectAt<std::memory_order_release>(target_code_pool_index_,
target);
// No need to flush the instruction cache, since the code is not modified.
}
@@ -307,7 +309,7 @@ ObjectPtr ICCallPattern::Data() const {
void ICCallPattern::SetData(const Object& data) const {
ASSERT(data.IsArray() || data.IsICData() || data.IsMegamorphicCache());
object_pool_.SetObjectAt(data_pool_index_, data);
object_pool_.SetObjectAt<std::memory_order_release>(data_pool_index_, data);
}
CodePtr ICCallPattern::TargetCode() const {
@@ -315,7 +317,8 @@ CodePtr ICCallPattern::TargetCode() const {
}
void ICCallPattern::SetTargetCode(const Code& target) const {
object_pool_.SetObjectAt(target_pool_index_, target);
object_pool_.SetObjectAt<std::memory_order_release>(target_pool_index_,
target);
// No need to flush the instruction cache, since the code is not modified.
}
@@ -359,7 +362,8 @@ ObjectPtr SwitchableCallPattern::target() const {
void SwitchableCallPattern::SetTarget(const Code& target) const {
ASSERT(Object::Handle(object_pool_.ObjectAt(target_pool_index_)).IsCode());
object_pool_.SetObjectAt(target_pool_index_, target);
object_pool_.SetObjectAt<std::memory_order_release>(target_pool_index_,
target);
}
BareSwitchableCallPattern::BareSwitchableCallPattern(uword pc)
@@ -389,8 +393,8 @@ uword BareSwitchableCallPattern::target_entry() const {
void BareSwitchableCallPattern::SetTarget(const Code& target) const {
ASSERT(object_pool_.TypeAt(target_pool_index_) ==
ObjectPool::EntryType::kImmediate);
object_pool_.SetRawValueAt(target_pool_index_,
target.MonomorphicEntryPoint());
object_pool_.SetRawValueAt<std::memory_order_relaxed>(
target_pool_index_, target.MonomorphicEntryPoint());
}
ReturnPattern::ReturnPattern(uword pc) : pc_(pc) {}
runtime/vm/object.cc
+3 -4
View File
@@ -8396,8 +8396,6 @@ void Function::SetInstructionsSafe(const Code& value) const {
void Function::AttachCode(const Code& value) const {
ASSERT(IsolateGroup::Current()->program_lock()->IsCurrentThreadWriter());
// Finish setting up code before activating it.
value.set_owner(*this);
SetInstructions(value);
ASSERT(Function::Handle(value.function()).IsNull() ||
(value.function() == this->ptr()));
@@ -12275,7 +12273,7 @@ const char* FunctionType::ToCString() const {
}
void ClosureData::set_context_scope(const ContextScope& value) const {
untag()->set_context_scope(value.ptr());
untag()->set_context_scope<std::memory_order_release>(value.ptr());
}
void ClosureData::set_implicit_static_closure(const Closure& closure) const {
@@ -12824,7 +12822,8 @@ void Field::set_dependent_code(const WeakArray& array) const {
ASSERT(IsOriginal());
DEBUG_ASSERT(
IsolateGroup::Current()->program_lock()->IsCurrentThreadWriter());
untag()->set_dependent_code(array.ptr());
// relaxed: races with Field::Clone.
untag()->set_dependent_code<std::memory_order_relaxed>(array.ptr());
}
class FieldDependentArray : public WeakCodeReferences {
runtime/vm/object.h
+60 -16
View File
@@ -828,18 +828,28 @@ class Object {
// methods below or their counterparts in UntaggedObject, to ensure that the
// write barrier is correctly applied.
template <typename type, std::memory_order order = std::memory_order_relaxed>
template <typename type>
type LoadPointer(type const* addr) const {
return ptr()->untag()->LoadPointer<type>(addr);
}
template <typename type, std::memory_order order>
type LoadPointer(type const* addr) const {
return ptr()->untag()->LoadPointer<type, order>(addr);
}
template <typename type, std::memory_order order = std::memory_order_relaxed>
template <typename type>
void StorePointer(type const* addr, type value) const {
ptr()->untag()->StorePointer<type>(addr, value);
}
template <typename type, std::memory_order order>
void StorePointer(type const* addr, type value) const {
ptr()->untag()->StorePointer<type, order>(addr, value);
}
template <typename type,
typename compressed_type,
std::memory_order order = std::memory_order_relaxed>
template <typename type, typename compressed_type>
void StoreCompressedPointer(compressed_type const* addr, type value) const {
ptr()->untag()->StoreCompressedPointer<type, compressed_type>(addr, value);
}
template <typename type, typename compressed_type, std::memory_order order>
void StoreCompressedPointer(compressed_type const* addr, type value) const {
ptr()->untag()->StoreCompressedPointer<type, compressed_type, order>(addr,
value);
@@ -4386,7 +4396,9 @@ class ClosureData : public Object {
UntaggedClosureData::kNoAwaiterLinkDepth;
private:
ContextScopePtr context_scope() const { return untag()->context_scope(); }
ContextScopePtr context_scope() const {
return untag()->context_scope<std::memory_order_acquire>();
}
void set_context_scope(const ContextScope& value) const;
void set_packed_fields(uint32_t value) const {
@@ -5695,13 +5707,22 @@ class ObjectPool : public Object {
StoreNonPointer(&untag()->entry_bits()[index], bits);
}
template <std::memory_order order = std::memory_order_relaxed>
ObjectPtr ObjectAt(intptr_t index) const {
ASSERT(TypeAt(index) == EntryType::kTaggedObject);
return LoadPointer<ObjectPtr>(&(EntryAddr(index)->raw_obj_));
}
template <std::memory_order order>
ObjectPtr ObjectAt(intptr_t index) const {
ASSERT(TypeAt(index) == EntryType::kTaggedObject);
return LoadPointer<ObjectPtr, order>(&(EntryAddr(index)->raw_obj_));
}
template <std::memory_order order = std::memory_order_relaxed>
void SetObjectAt(intptr_t index, const Object& obj) const {
ASSERT((TypeAt(index) == EntryType::kTaggedObject) ||
(TypeAt(index) == EntryType::kImmediate && obj.IsSmi()));
StorePointer<ObjectPtr>(&EntryAddr(index)->raw_obj_, obj.ptr());
}
template <std::memory_order order>
void SetObjectAt(intptr_t index, const Object& obj) const {
ASSERT((TypeAt(index) == EntryType::kTaggedObject) ||
(TypeAt(index) == EntryType::kImmediate && obj.IsSmi()));
@@ -5716,6 +5737,12 @@ class ObjectPool : public Object {
ASSERT(TypeAt(index) != EntryType::kTaggedObject);
StoreNonPointer(&EntryAddr(index)->raw_value_, raw_value);
}
template <std::memory_order order>
void SetRawValueAt(intptr_t index, uword raw_value) const {
ASSERT(TypeAt(index) != EntryType::kTaggedObject);
StoreNonPointer<uword, uword, order>(&EntryAddr(index)->raw_value_,
raw_value);
}
static intptr_t InstanceSize() {
ASSERT(sizeof(UntaggedObjectPool) ==
@@ -9540,7 +9567,9 @@ class AbstractType : public Instance {
uword type_test_stub_entry_point() const {
return untag()->type_test_stub_entry_point_;
}
CodePtr type_test_stub() const { return untag()->type_test_stub(); }
CodePtr type_test_stub() const {
return untag()->type_test_stub<std::memory_order_acquire>();
}
// Sets the TTS to [stub].
//
@@ -11079,22 +11108,35 @@ class Array : public Instance {
return array->untag()->data();
}
template <std::memory_order order = std::memory_order_relaxed>
static ObjectPtr ElementAt(ArrayPtr array, intptr_t index) {
ASSERT((0 <= index) && (index < LengthOf(array)));
return array->untag()->element(index);
}
template <std::memory_order order>
static ObjectPtr ElementAt(ArrayPtr array, intptr_t index) {
ASSERT((0 <= index) && (index < LengthOf(array)));
return array->untag()->element<order>(index);
}
template <std::memory_order order = std::memory_order_relaxed>
ObjectPtr At(intptr_t index) const { return ElementAt(ptr(), index); }
template <std::memory_order order>
ObjectPtr At(intptr_t index) const {
return ElementAt<order>(ptr(), index);
}
template <std::memory_order order = std::memory_order_relaxed>
void SetAt(intptr_t index, const Object& value) const {
ASSERT((0 <= index) && (index < Length()));
untag()->set_element(index, value.ptr());
}
template <std::memory_order order>
void SetAt(intptr_t index, const Object& value) const {
ASSERT((0 <= index) && (index < Length()));
untag()->set_element<order>(index, value.ptr());
}
template <std::memory_order order = std::memory_order_relaxed>
void SetAt(intptr_t index, const Object& value, Thread* thread) const {
ASSERT((0 <= index) && (index < Length()));
untag()->set_element(index, value.ptr(), thread);
}
template <std::memory_order order>
void SetAt(intptr_t index, const Object& value, Thread* thread) const {
ASSERT((0 <= index) && (index < Length()));
untag()->set_element<order>(index, value.ptr(), thread);
@@ -11232,9 +11274,11 @@ class Array : public Instance {
untag()->set_length<std::memory_order_release>(Smi::New(value));
}
template <typename type,
std::memory_order order = std::memory_order_relaxed,
typename value_type>
template <typename type, typename value_type>
void StoreArrayPointer(type const* addr, value_type value) const {
ptr()->untag()->StoreArrayPointer<type, value_type>(addr, value);
}
template <typename type, std::memory_order order, typename value_type>
void StoreArrayPointer(type const* addr, value_type value) const {
ptr()->untag()->StoreArrayPointer<type, order, value_type>(addr, value);
}
runtime/vm/object_test.cc
+4
View File
@@ -2664,6 +2664,7 @@ ISOLATE_UNIT_TEST_CASE(Code) {
thread->isolate_group()->program_lock());
Code& code = Code::Handle(Code::FinalizeCodeAndNotify(
function, nullptr, &_assembler_, Code::PoolAttachment::kAttachPool));
code.set_owner(function);
function.AttachCode(code);
const Instructions& instructions = Instructions::Handle(code.instructions());
uword payload_start = instructions.PayloadStart();
@@ -2728,6 +2729,7 @@ ISOLATE_UNIT_TEST_CASE(EmbedStringInCode) {
thread->isolate_group()->program_lock());
const Code& code = Code::Handle(Code::FinalizeCodeAndNotify(
function, nullptr, &_assembler_, Code::PoolAttachment::kAttachPool));
code.set_owner(function);
function.AttachCode(code);
const Object& result =
Object::Handle(DartEntry::InvokeFunction(function, Array::empty_array()));
@@ -2754,6 +2756,7 @@ ISOLATE_UNIT_TEST_CASE(EmbedSmiInCode) {
thread->isolate_group()->program_lock());
const Code& code = Code::Handle(Code::FinalizeCodeAndNotify(
function, nullptr, &_assembler_, Code::PoolAttachment::kAttachPool));
code.set_owner(function);
function.AttachCode(code);
const Object& result =
Object::Handle(DartEntry::InvokeFunction(function, Array::empty_array()));
@@ -2775,6 +2778,7 @@ ISOLATE_UNIT_TEST_CASE(EmbedSmiIn64BitCode) {
thread->isolate_group()->program_lock());
const Code& code = Code::Handle(Code::FinalizeCodeAndNotify(
function, nullptr, &_assembler_, Code::PoolAttachment::kAttachPool));
code.set_owner(function);
function.AttachCode(code);
const Object& result =
Object::Handle(DartEntry::InvokeFunction(function, Array::empty_array()));
runtime/vm/raw_object.cc
+4 -1
View File
@@ -558,7 +558,10 @@ VARIABLE_COMPRESSED_VISITOR(TypeArguments,
VARIABLE_COMPRESSED_VISITOR(LocalVarDescriptors, raw_obj->untag()->num_entries_)
VARIABLE_COMPRESSED_VISITOR(ExceptionHandlers, raw_obj->untag()->num_entries())
VARIABLE_COMPRESSED_VISITOR(Context, raw_obj->untag()->num_variables_)
VARIABLE_COMPRESSED_VISITOR(Array, Smi::Value(raw_obj->untag()->length()))
// relaxed: for Array truncation
VARIABLE_COMPRESSED_VISITOR(
Array,
Smi::Value(raw_obj->untag()->length<std::memory_order_relaxed>()))
VARIABLE_COMPRESSED_VISITOR(
TypedData,
TypedData::ElementSizeInBytes(raw_obj->GetClassIdOfHeapObject()) *
runtime/vm/raw_object.h
+141 -40
View File
@@ -570,14 +570,21 @@ class UntaggedObject {
// All writes to heap objects should ultimately pass through one of the
// methods below or their counterparts in Object, to ensure that the
// write barrier is correctly applied.
template <typename type, std::memory_order order = std::memory_order_relaxed>
template <typename type>
type LoadPointer(type const* addr) const {
return *const_cast<type*>(addr);
}
template <typename type, std::memory_order order>
type LoadPointer(type const* addr) const {
return reinterpret_cast<std::atomic<type>*>(const_cast<type*>(addr))
->load(order);
}
template <typename type,
typename compressed_type,
std::memory_order order = std::memory_order_relaxed>
template <typename type, typename compressed_type>
type LoadCompressedPointer(compressed_type const* addr) const {
compressed_type v = *const_cast<compressed_type*>(addr);
return static_cast<type>(v.Decompress(heap_base()));
}
template <typename type, typename compressed_type, std::memory_order order>
type LoadCompressedPointer(compressed_type const* addr) const {
compressed_type v = reinterpret_cast<std::atomic<compressed_type>*>(
const_cast<compressed_type*>(addr))
@@ -596,7 +603,14 @@ class UntaggedObject {
return reinterpret_cast<uword>(this) & kHeapBaseMask;
}
template <typename type, std::memory_order order = std::memory_order_relaxed>
template <typename type>
void StorePointer(type const* addr, type value) {
*const_cast<type*>(addr) = value;
if (value.IsHeapObject()) {
CheckHeapPointerStore(value, Thread::Current());
}
}
template <typename type, std::memory_order order>
void StorePointer(type const* addr, type value) {
reinterpret_cast<std::atomic<type>*>(const_cast<type*>(addr))
->store(value, order);
@@ -605,9 +619,14 @@ class UntaggedObject {
}
}
template <typename type,
typename compressed_type,
std::memory_order order = std::memory_order_relaxed>
template <typename type, typename compressed_type>
void StoreCompressedPointer(compressed_type const* addr, type value) {
*const_cast<compressed_type*>(addr) = value;
if (value.IsHeapObject()) {
CheckHeapPointerStore(value, Thread::Current());
}
}
template <typename type, typename compressed_type, std::memory_order order>
void StoreCompressedPointer(compressed_type const* addr, type value) {
reinterpret_cast<std::atomic<compressed_type>*>(
const_cast<compressed_type*>(addr))
@@ -644,9 +663,14 @@ class UntaggedObject {
}
// Note: StoreArrayPointer won't work if value_type is a compressed pointer.
template <typename type,
std::memory_order order = std::memory_order_relaxed,
typename value_type = type>
template <typename type, typename value_type = type>
void StoreArrayPointer(type const* addr, value_type value) {
*const_cast<type*>(addr) = type(value);
if (value->IsHeapObject()) {
CheckArrayPointerStore(addr, value, Thread::Current());
}
}
template <typename type, std::memory_order order, typename value_type = type>
void StoreArrayPointer(type const* addr, value_type value) {
reinterpret_cast<std::atomic<type>*>(const_cast<type*>(addr))
->store(type(value), order);
@@ -663,6 +687,13 @@ class UntaggedObject {
}
}
template <typename type, typename compressed_type>
void StoreCompressedArrayPointer(compressed_type const* addr, type value) {
*const_cast<compressed_type*>(addr) = static_cast<compressed_type>(value);
if (value->IsHeapObject()) {
CheckArrayPointerStore(addr, value, Thread::Current());
}
}
template <typename type, typename compressed_type, std::memory_order order>
void StoreCompressedArrayPointer(compressed_type const* addr, type value) {
reinterpret_cast<std::atomic<compressed_type>*>(
@@ -709,12 +740,19 @@ class UntaggedObject {
return static_cast<type>(previous_value.Decompress(heap_base()));
}
template <std::memory_order order = std::memory_order_relaxed>
SmiPtr LoadSmi(SmiPtr const* addr) const {
return *const_cast<SmiPtr*>(addr);
}
template <std::memory_order order>
SmiPtr LoadSmi(SmiPtr const* addr) const {
return reinterpret_cast<std::atomic<SmiPtr>*>(const_cast<SmiPtr*>(addr))
->load(order);
}
template <std::memory_order order = std::memory_order_relaxed>
SmiPtr LoadCompressedSmi(CompressedSmiPtr const* addr) const {
return static_cast<SmiPtr>(
const_cast<CompressedSmiPtr*>(addr)->DecompressSmi());
}
template <std::memory_order order>
SmiPtr LoadCompressedSmi(CompressedSmiPtr const* addr) const {
return static_cast<SmiPtr>(reinterpret_cast<std::atomic<CompressedSmiPtr>*>(
const_cast<CompressedSmiPtr*>(addr))
@@ -724,17 +762,20 @@ class UntaggedObject {
// Use for storing into an explicitly Smi-typed field of an object
// (i.e., both the previous and new value are Smis).
template <typename type, std::memory_order order = std::memory_order_relaxed>
template <typename type>
void StoreSmi(type const* addr, type value) {
*const_cast<type*>(addr) = value;
}
template <typename type, std::memory_order order>
void StoreSmi(type const* addr, type value) {
// Can't use Contains, as array length is initialized through this method.
ASSERT(reinterpret_cast<uword>(addr) >= UntaggedObject::ToAddr(this));
reinterpret_cast<std::atomic<type>*>(const_cast<type*>(addr))
->store(value, order);
}
template <std::memory_order order = std::memory_order_relaxed>
void StoreCompressedSmi(CompressedSmiPtr const* addr, SmiPtr value) {
// Can't use Contains, as array length is initialized through this method.
ASSERT(reinterpret_cast<uword>(addr) >= UntaggedObject::ToAddr(this));
*const_cast<CompressedSmiPtr*>(addr) = value;
}
template <std::memory_order order>
void StoreCompressedSmi(CompressedSmiPtr const* addr, SmiPtr value) {
reinterpret_cast<std::atomic<CompressedSmiPtr>*>(
const_cast<CompressedSmiPtr*>(addr))
->store(static_cast<CompressedSmiPtr>(value), order);
@@ -896,11 +937,17 @@ inline intptr_t ObjectPtr::GetClassId() const {
#define POINTER_FIELD(type, name) \
public: \
template <std::memory_order order = std::memory_order_relaxed> \
type name() const { \
return LoadPointer<type>(&name##_); \
} \
template <std::memory_order order> \
type name() const { \
return LoadPointer<type, order>(&name##_); \
} \
template <std::memory_order order = std::memory_order_relaxed> \
void set_##name(type value) { \
StorePointer<type>(&name##_, value); \
} \
template <std::memory_order order> \
void set_##name(type value) { \
StorePointer<type, order>(&name##_, value); \
} \
@@ -910,14 +957,20 @@ inline intptr_t ObjectPtr::GetClassId() const {
#define COMPRESSED_POINTER_FIELD(type, name) \
public: \
template <std::memory_order order = std::memory_order_relaxed> \
type name() const { \
return LoadCompressedPointer<type, Compressed##type>(&name##_); \
} \
template <std::memory_order order> \
type name() const { \
return LoadCompressedPointer<type, Compressed##type, order>(&name##_); \
} \
type name##_ignore_race() const { \
return LoadCompressedPointerIgnoreRace<type, Compressed##type>(&name##_); \
} \
template <std::memory_order order = std::memory_order_relaxed> \
void set_##name(type value) { \
StoreCompressedPointer<type, Compressed##type>(&name##_, value); \
} \
template <std::memory_order order> \
void set_##name(type value) { \
StoreCompressedPointer<type, Compressed##type, order>(&name##_, value); \
} \
@@ -927,11 +980,17 @@ inline intptr_t ObjectPtr::GetClassId() const {
#define ARRAY_POINTER_FIELD(type, name) \
public: \
template <std::memory_order order = std::memory_order_relaxed> \
type name() const { \
return LoadPointer<type>(&name##_); \
} \
template <std::memory_order order> \
type name() const { \
return LoadPointer<type, order>(&name##_); \
} \
template <std::memory_order order = std::memory_order_relaxed> \
void set_##name(type value) { \
StoreArrayPointer<type>(&name##_, value); \
} \
template <std::memory_order order> \
void set_##name(type value) { \
StoreArrayPointer<type, order>(&name##_, value); \
} \
@@ -941,12 +1000,18 @@ inline intptr_t ObjectPtr::GetClassId() const {
#define COMPRESSED_ARRAY_POINTER_FIELD(type, name) \
public: \
template <std::memory_order order = std::memory_order_relaxed> \
type name() const { \
return LoadPointer<Compressed##type>(&name##_).Decompress(heap_base()); \
} \
template <std::memory_order order> \
type name() const { \
return LoadPointer<Compressed##type, order>(&name##_).Decompress( \
heap_base()); \
} \
template <std::memory_order order = std::memory_order_relaxed> \
void set_##name(type value) { \
StoreCompressedArrayPointer<type, Compressed##type>(&name##_, value); \
} \
template <std::memory_order order> \
void set_##name(type value) { \
StoreCompressedArrayPointer<type, Compressed##type, order>(&name##_, \
value); \
@@ -957,15 +1022,24 @@ inline intptr_t ObjectPtr::GetClassId() const {
#define VARIABLE_POINTER_FIELDS(type, accessor_name, array_name) \
public: \
template <std::memory_order order = std::memory_order_relaxed> \
type accessor_name(intptr_t index) const { \
return LoadPointer<type>(&array_name()[index]); \
} \
template <std::memory_order order> \
type accessor_name(intptr_t index) const { \
return LoadPointer<type, order>(&array_name()[index]); \
} \
template <std::memory_order order = std::memory_order_relaxed> \
void set_##accessor_name(intptr_t index, type value) { \
StoreArrayPointer<type>(&array_name()[index], value); \
} \
template <std::memory_order order> \
void set_##accessor_name(intptr_t index, type value) { \
StoreArrayPointer<type, order>(&array_name()[index], value); \
} \
template <std::memory_order order = std::memory_order_relaxed> \
void set_##accessor_name(intptr_t index, type value, Thread* thread) { \
StoreArrayPointer<type>(&array_name()[index], value, thread); \
} \
template <std::memory_order order> \
void set_##accessor_name(intptr_t index, type value, Thread* thread) { \
StoreArrayPointer<type, order>(&array_name()[index], value, thread); \
} \
@@ -981,17 +1055,29 @@ inline intptr_t ObjectPtr::GetClassId() const {
#define COMPRESSED_VARIABLE_POINTER_FIELDS(type, accessor_name, array_name) \
public: \
template <std::memory_order order = std::memory_order_relaxed> \
type accessor_name(intptr_t index) const { \
return LoadCompressedPointer<type, Compressed##type>( \
&array_name()[index]); \
} \
template <std::memory_order order> \
type accessor_name(intptr_t index) const { \
return LoadCompressedPointer<type, Compressed##type, order>( \
&array_name()[index]); \
} \
template <std::memory_order order = std::memory_order_relaxed> \
void set_##accessor_name(intptr_t index, type value) { \
StoreCompressedArrayPointer<type, Compressed##type>(&array_name()[index], \
value); \
} \
template <std::memory_order order> \
void set_##accessor_name(intptr_t index, type value) { \
StoreCompressedArrayPointer<type, Compressed##type, order>( \
&array_name()[index], value); \
} \
template <std::memory_order order = std::memory_order_relaxed> \
void set_##accessor_name(intptr_t index, type value, Thread* thread) { \
StoreCompressedArrayPointer<type, Compressed##type>(&array_name()[index], \
value, thread); \
} \
template <std::memory_order order> \
void set_##accessor_name(intptr_t index, type value, Thread* thread) { \
StoreCompressedArrayPointer<type, Compressed##type, order>( \
&array_name()[index], value, thread); \
@@ -1008,13 +1094,22 @@ inline intptr_t ObjectPtr::GetClassId() const {
#define SMI_FIELD(type, name) \
public: \
template <std::memory_order order = std::memory_order_relaxed> \
type name() const { \
type result = LoadSmi(&name##_); \
ASSERT(!result.IsHeapObject()); \
return result; \
} \
template <std::memory_order order> \
type name() const { \
type result = LoadSmi<order>(&name##_); \
ASSERT(!result.IsHeapObject()); \
return result; \
} \
template <std::memory_order order = std::memory_order_relaxed> \
void set_##name(type value) { \
ASSERT(!value.IsHeapObject()); \
StoreSmi<type>(&name##_, value); \
} \
template <std::memory_order order> \
void set_##name(type value) { \
ASSERT(!value.IsHeapObject()); \
StoreSmi<type, order>(&name##_, value); \
@@ -1025,16 +1120,25 @@ inline intptr_t ObjectPtr::GetClassId() const {
#define COMPRESSED_SMI_FIELD(type, name) \
public: \
template <std::memory_order order = std::memory_order_relaxed> \
type name() const { \
type result = LoadCompressedSmi(&name##_); \
ASSERT(!result.IsHeapObject()); \
return result; \
} \
template <std::memory_order order> \
type name() const { \
type result = LoadCompressedSmi<order>(&name##_); \
ASSERT(!result.IsHeapObject()); \
return result; \
} \
template <std::memory_order order = std::memory_order_relaxed> \
void set_##name(type value) { \
ASSERT(!value.IsHeapObject()); \
StoreCompressedSmi(&name##_, value); \
} \
template <std::memory_order order> \
void set_##name(type value) { \
ASSERT(!value.IsHeapObject()); \
StoreCompressedSmi<order>(&name##_, value); \
} \
\
protected: \
@@ -2403,9 +2507,6 @@ class UntaggedLocalVarDescriptors : public UntaggedObject {
COMPRESSED_VARIABLE_POINTER_FIELDS(StringPtr, name, names)
CompressedStringPtr* nameAddrAt(intptr_t i) { return &(names()[i]); }
void set_name(intptr_t i, StringPtr value) {
StoreCompressedPointer(nameAddrAt(i), value);
}
// Variable info with [num_entries_] entries.
VarInfo* data() {
runtime/vm/stub_code_test.cc
+1
View File
@@ -29,6 +29,7 @@ const Function& RegisterFakeFunction(const char* name, const Code& code) {
owner_class.SetFunctions(functions);
}
lib.AddClass(owner_class);
code.set_owner(function);
function.AttachCode(code);
return function;
}