diff --git a/runtime/vm/message_snapshot.cc b/runtime/vm/message_snapshot.cc index b48d5bfc815..c07eb6f628a 100644 --- a/runtime/vm/message_snapshot.cc +++ b/runtime/vm/message_snapshot.cc @@ -1918,20 +1918,41 @@ class Simd128MessageDeserializationCluster : public MessageDeserializationCluster { public: explicit Simd128MessageDeserializationCluster(intptr_t cid) - : MessageDeserializationCluster("Simd128"), cid_(cid) {} + : MessageDeserializationCluster("Simd128"), cid_(cid) { + ASSERT(cid_ == kInt32x4Cid || cid_ == kFloat32x4Cid || + cid_ == kFloat64x2Cid); +#if defined(DEBUG) + // If not for Int32x4, check that all the Int32x4-specific arguments used in + // ReadNodes match those for the actual class. + if (cid_ == kFloat32x4Cid) { + AssertSameStructure(); + } else if (cid_ == kFloat64x2Cid) { + AssertSameStructure(); + } +#endif + } ~Simd128MessageDeserializationCluster() {} +#if defined(DEBUG) + template + static void AssertSameStructure() { + ASSERT_EQUAL(Got::InstanceSize(), Expected::InstanceSize()); + ASSERT_EQUAL(Got::ContainsCompressedPointers(), + Expected::ContainsCompressedPointers()); + ASSERT_EQUAL(Object::from_offset(), Object::from_offset()); + ASSERT_EQUAL(Object::to_offset(), Object::to_offset()); + ASSERT_EQUAL(Got::value_offset(), Expected::value_offset()); + } +#endif + void ReadNodes(MessageDeserializer* d) { const intptr_t count = d->ReadUnsigned(); for (intptr_t i = 0; i < count; i++) { - ASSERT_EQUAL(Int32x4::InstanceSize(), Float32x4::InstanceSize()); - ASSERT_EQUAL(Int32x4::InstanceSize(), Float64x2::InstanceSize()); - ObjectPtr vector = - Object::Allocate(cid_, Int32x4::InstanceSize(), Heap::kNew, - Int32x4::ContainsCompressedPointers()); + ObjectPtr vector = Object::Allocate( + cid_, Int32x4::InstanceSize(), Heap::kNew, + Int32x4::ContainsCompressedPointers(), Object::from_offset(), + Object::to_offset()); d->AssignRef(vector); - ASSERT_EQUAL(Int32x4::value_offset(), Float32x4::value_offset()); - ASSERT_EQUAL(Int32x4::value_offset(), Float64x2::value_offset()); d->ReadBytes(&(static_cast(vector)->untag()->value_), sizeof(simd128_value_t)); } diff --git a/runtime/vm/object.cc b/runtime/vm/object.cc index 50400331b2b..694b201477e 100644 --- a/runtime/vm/object.cc +++ b/runtime/vm/object.cc @@ -568,8 +568,7 @@ void Object::InitNullAndBool(IsolateGroup* isolate_group) { heap->Allocate(thread, Instance::InstanceSize(), Heap::kOld); null_ = static_cast(address + kHeapObjectTag); // The call below is using 'null_' to initialize itself. - InitializeObject(address, kNullCid, Instance::InstanceSize(), - Instance::ContainsCompressedPointers()); + InitializeObjectVariant(address, kNullCid); null_->untag()->SetCanonical(); } @@ -580,16 +579,14 @@ void Object::InitNullAndBool(IsolateGroup* isolate_group) { { // Allocate a dummy bool object to give true the desired alignment. uword address = heap->Allocate(thread, Bool::InstanceSize(), Heap::kOld); - InitializeObject(address, kBoolCid, Bool::InstanceSize(), - Bool::ContainsCompressedPointers()); + InitializeObject(address); static_cast(address + kHeapObjectTag)->untag()->value_ = false; } { // Allocate true. uword address = heap->Allocate(thread, Bool::InstanceSize(), Heap::kOld); true_ = static_cast(address + kHeapObjectTag); - InitializeObject(address, kBoolCid, Bool::InstanceSize(), - Bool::ContainsCompressedPointers()); + InitializeObject(address); true_->untag()->value_ = true; true_->untag()->SetCanonical(); } @@ -597,8 +594,7 @@ void Object::InitNullAndBool(IsolateGroup* isolate_group) { // Allocate false. uword address = heap->Allocate(thread, Bool::InstanceSize(), Heap::kOld); false_ = static_cast(address + kHeapObjectTag); - InitializeObject(address, kBoolCid, Bool::InstanceSize(), - Bool::ContainsCompressedPointers()); + InitializeObject(address); false_->untag()->value_ = false; false_->untag()->SetCanonical(); } @@ -769,8 +765,7 @@ void Object::Init(IsolateGroup* isolate_group) { intptr_t size = Class::InstanceSize(); uword address = heap->Allocate(thread, size, Heap::kOld); class_class_ = static_cast(address + kHeapObjectTag); - InitializeObject(address, Class::kClassId, size, - Class::ContainsCompressedPointers()); + InitializeObject(address); Class fake; // Initialization from Class::New. @@ -1009,8 +1004,7 @@ void Object::Init(IsolateGroup* isolate_group) { // Allocate and initialize the empty_array instance. { uword address = heap->Allocate(thread, Array::InstanceSize(0), Heap::kOld); - InitializeObject(address, kImmutableArrayCid, Array::InstanceSize(0), - Array::ContainsCompressedPointers()); + InitializeObjectVariant(address, kImmutableArrayCid, 0); Array::initializeHandle(empty_array_, static_cast(address + kHeapObjectTag)); empty_array_->untag()->set_length(Smi::New(0)); @@ -1026,9 +1020,7 @@ void Object::Init(IsolateGroup* isolate_group) { TypeArguments::Cache::kHeaderSize + TypeArguments::Cache::kEntrySize; uword address = heap->Allocate(thread, Array::InstanceSize(array_size), Heap::kOld); - InitializeObject(address, kImmutableArrayCid, - Array::InstanceSize(array_size), - Array::ContainsCompressedPointers()); + InitializeObjectVariant(address, kImmutableArrayCid, array_size); Array::initializeHandle(empty_instantiations_cache_array_, static_cast(address + kHeapObjectTag)); empty_instantiations_cache_array_->untag()->set_length( @@ -1050,8 +1042,7 @@ void Object::Init(IsolateGroup* isolate_group) { { uword address = heap->Allocate(thread, ContextScope::InstanceSize(0), Heap::kOld); - InitializeObject(address, kContextScopeCid, ContextScope::InstanceSize(0), - ContextScope::ContainsCompressedPointers()); + InitializeObject(address, 0); ContextScope::initializeHandle( empty_context_scope_, static_cast(address + kHeapObjectTag)); @@ -1066,8 +1057,7 @@ void Object::Init(IsolateGroup* isolate_group) { { uword address = heap->Allocate(thread, ObjectPool::InstanceSize(0), Heap::kOld); - InitializeObject(address, kObjectPoolCid, ObjectPool::InstanceSize(0), - ObjectPool::ContainsCompressedPointers()); + InitializeObject(address, 0); ObjectPool::initializeHandle( empty_object_pool_, static_cast(address + kHeapObjectTag)); @@ -1080,8 +1070,7 @@ void Object::Init(IsolateGroup* isolate_group) { { const intptr_t instance_size = CompressedStackMaps::InstanceSize(0); uword address = heap->Allocate(thread, instance_size, Heap::kOld); - InitializeObject(address, kCompressedStackMapsCid, instance_size, - CompressedStackMaps::ContainsCompressedPointers()); + InitializeObject(address, 0); CompressedStackMaps::initializeHandle( empty_compressed_stackmaps_, static_cast(address + kHeapObjectTag)); @@ -1093,8 +1082,7 @@ void Object::Init(IsolateGroup* isolate_group) { { uword address = heap->Allocate(thread, PcDescriptors::InstanceSize(0), Heap::kOld); - InitializeObject(address, kPcDescriptorsCid, PcDescriptors::InstanceSize(0), - PcDescriptors::ContainsCompressedPointers()); + InitializeObject(address, 0); PcDescriptors::initializeHandle( empty_descriptors_, static_cast(address + kHeapObjectTag)); @@ -1107,9 +1095,7 @@ void Object::Init(IsolateGroup* isolate_group) { { uword address = heap->Allocate(thread, LocalVarDescriptors::InstanceSize(0), Heap::kOld); - InitializeObject(address, kLocalVarDescriptorsCid, - LocalVarDescriptors::InstanceSize(0), - LocalVarDescriptors::ContainsCompressedPointers()); + InitializeObject(address, 0); LocalVarDescriptors::initializeHandle( empty_var_descriptors_, static_cast(address + kHeapObjectTag)); @@ -1124,9 +1110,7 @@ void Object::Init(IsolateGroup* isolate_group) { { uword address = heap->Allocate(thread, ExceptionHandlers::InstanceSize(0), Heap::kOld); - InitializeObject(address, kExceptionHandlersCid, - ExceptionHandlers::InstanceSize(0), - ExceptionHandlers::ContainsCompressedPointers()); + InitializeObject(address, 0); ExceptionHandlers::initializeHandle( empty_exception_handlers_, static_cast(address + kHeapObjectTag)); @@ -1139,9 +1123,7 @@ void Object::Init(IsolateGroup* isolate_group) { { uword address = heap->Allocate(thread, ExceptionHandlers::InstanceSize(0), Heap::kOld); - InitializeObject(address, kExceptionHandlersCid, - ExceptionHandlers::InstanceSize(0), - ExceptionHandlers::ContainsCompressedPointers()); + InitializeObject(address, 0); ExceptionHandlers::initializeHandle( empty_async_exception_handlers_, static_cast(address + kHeapObjectTag)); @@ -1155,8 +1137,7 @@ void Object::Init(IsolateGroup* isolate_group) { { uword address = heap->Allocate(thread, TypeArguments::InstanceSize(0), Heap::kOld); - InitializeObject(address, kTypeArgumentsCid, TypeArguments::InstanceSize(0), - TypeArguments::ContainsCompressedPointers()); + InitializeObject(address, 0); TypeArguments::initializeHandle( empty_type_arguments_, static_cast(address + kHeapObjectTag)); @@ -2701,67 +2682,94 @@ StringPtr Object::DictionaryName() const { void Object::InitializeObject(uword address, intptr_t class_id, intptr_t size, - bool compressed) { + bool compressed, + uword ptr_field_start_offset, + uword ptr_field_end_offset) { // Note: we skip the header word here to avoid a racy read in the concurrent // marker from observing the null object when it reads into a heap page // allocated after marking started. uword cur = address + sizeof(UntaggedObject); + uword ptr_field_start = address + ptr_field_start_offset; + uword ptr_field_end = address + ptr_field_end_offset; uword end = address + size; - if (class_id == kInstructionsCid) { - compiler::target::uword initial_value = kBreakInstructionFiller; + // The start of pointer fields should always be past the object header, even + // if there are no pointer fields (ptr_field_end < ptr_field_start). + ASSERT(cur <= ptr_field_start); + // The start of pointer fields can be at the end for empty payload objects. + ASSERT(ptr_field_start <= end); + // The end of pointer fields should always be before the end, as the end of + // pointer fields is inclusive (the address of the last field to initialize). + ASSERT(ptr_field_end < end); + bool needs_init = true; + if (IsTypedDataBaseClassId(class_id) || class_id == kArrayCid) { + // If the size is greater than both kNewAllocatableSize and + // kAllocatablePageSize, the object must have been allocated to a new + // large page, which must already have been zero initialized by the OS. + // Note that zero is a GC-safe value. + // + // For arrays, the caller will then initialize the fields to null with + // safepoint checks to avoid blocking for the full duration of + // initializing this array. + needs_init = + IsAllocatableInNewSpace(size) || IsAllocatableViaFreeLists(size); + } + if (needs_init) { + // Initialize the memory prior to any pointer fields with 0. (This loop + // and the next will be a no-op if the object has no pointer fields.) + uword initial_value = 0; + while (cur < ptr_field_start) { + *reinterpret_cast(cur) = initial_value; + cur += kWordSize; + } + // Initialize any pointer fields with Object::null(). + initial_value = static_cast(null_); +#if defined(DART_COMPRESSED_POINTERS) + if (compressed) { + initial_value &= 0xFFFFFFFF; + initial_value |= initial_value << 32; + } + const bool has_pointer_fields = ptr_field_start <= ptr_field_end; + // If there are compressed pointer fields and the first compressed pointer + // field is not at a word start, then initialize it to Object::null(). + if (compressed && has_pointer_fields && + (ptr_field_start % kWordSize != 0)) { + *reinterpret_cast(ptr_field_start) = initial_value; + } +#endif + while (cur <= ptr_field_end) { + *reinterpret_cast(cur) = initial_value; + cur += kWordSize; + } + // Initialize the memory after any pointer fields with 0, unless this is + // an instructions object in which case we use the break instruction. + initial_value = class_id == kInstructionsCid ? kBreakInstructionFiller : 0; +#if defined(DART_COMPRESSED_POINTERS) + // If there are compressed pointer fields and the last compressed pointer + // field is the start of a word, then initialize the other part of the word + // to the new initial value. + // + // (We're guaranteed there's always space in the object after the last + // pointer field in this case since objects are allocated in multiples of + // the word size.) + if (compressed && has_pointer_fields && (ptr_field_end % kWordSize == 0)) { + *reinterpret_cast(ptr_field_end + + kCompressedWordSize) = initial_value; + } +#endif while (cur < end) { - *reinterpret_cast(cur) = initial_value; - cur += compiler::target::kWordSize; + *reinterpret_cast(cur) = initial_value; + cur += kWordSize; } } else { - uword initial_value; - bool needs_init; - if (IsTypedDataBaseClassId(class_id)) { - initial_value = 0; - // If the size is greater than both kNewAllocatableSize and - // kAllocatablePageSize, the object must have been allocated to a new - // large page, which must already have been zero initialized by the OS. - needs_init = - IsAllocatableInNewSpace(size) || IsAllocatableViaFreeLists(size); - } else { - initial_value = static_cast(null_); -#if defined(DART_COMPRESSED_POINTERS) - if (compressed) { - initial_value &= 0xFFFFFFFF; - initial_value |= initial_value << 32; - } -#endif - if (class_id == kArrayCid) { - // If the size is greater than both kNewAllocatableSize and - // kAllocatablePageSize, the object must have been allocated to a new - // large page, which must already have been zero initialized by the OS. - // Zero is a GC-safe value. The caller will initialize the fields to - // null with safepoint checks to avoid blocking for the full duration of - // initializing this array. - needs_init = - IsAllocatableInNewSpace(size) || IsAllocatableViaFreeLists(size); - if (!needs_init) { - initial_value = 0; // For ASSERT below. - } - } else { - needs_init = true; - } - } - if (needs_init) { - while (cur < end) { - *reinterpret_cast(cur) = initial_value; - cur += kWordSize; - } - } else { - // Check that MemorySanitizer understands this is initialized. - MSAN_CHECK_INITIALIZED(reinterpret_cast(address), size); + // Check that MemorySanitizer understands this is initialized. + MSAN_CHECK_INITIALIZED(reinterpret_cast(address), size); #if defined(DEBUG) - while (cur < end) { - ASSERT(*reinterpret_cast(cur) == initial_value); - cur += kWordSize; - } -#endif + const uword initial_value = 0; + while (cur < end) { + ASSERT_EQUAL(*reinterpret_cast(cur), initial_value); + cur += kWordSize; } +#endif } uword tags = 0; ASSERT(class_id != kIllegalCid); @@ -2796,7 +2804,9 @@ void Object::CheckHandle() const { ObjectPtr Object::Allocate(intptr_t cls_id, intptr_t size, Heap::Space space, - bool compressed) { + bool compressed, + uword ptr_field_start_offset, + uword ptr_field_end_offset) { ASSERT(Utils::IsAligned(size, kObjectAlignment)); Thread* thread = Thread::Current(); ASSERT(thread->execution_state() == Thread::kThreadInVM); @@ -2825,7 +2835,8 @@ ObjectPtr Object::Allocate(intptr_t cls_id, ObjectPtr raw_obj; NoSafepointScope no_safepoint(thread); - InitializeObject(address, cls_id, size, compressed); + InitializeObject(address, cls_id, size, compressed, ptr_field_start_offset, + ptr_field_end_offset); raw_obj = static_cast(address + kHeapObjectTag); ASSERT(cls_id == UntaggedObject::ClassIdTag::decode(raw_obj->untag()->tags_)); if (raw_obj->IsOldObject() && UNLIKELY(thread->is_marking())) { @@ -2937,8 +2948,12 @@ ObjectPtr Object::Clone(const Object& orig, ASSERT(!orig.IsFunctionType() || !FunctionType::Cast(orig).IsGeneric()); const Class& cls = Class::Handle(orig.clazz()); intptr_t size = orig.ptr()->untag()->HeapSize(); + // All fields (including non-SmiPtr fields) will be initialized with Smi 0, + // but the contents of the original object are copied over before the thread + // is allowed to reach a safepoint. ObjectPtr raw_clone = - Object::Allocate(cls.id(), size, space, cls.HasCompressedPointers()); + Object::Allocate(cls.id(), size, space, cls.HasCompressedPointers(), + from_offset(), to_offset()); NoSafepointScope no_safepoint; // Copy the body of the original into the clone. uword orig_addr = UntaggedObject::ToAddr(orig.ptr()); @@ -3088,11 +3103,9 @@ ClassPtr Class::New(IsolateGroup* isolate_group, bool register_class) { ASSERT(Object::class_class() != Class::null()); Class& result = Class::Handle(); { - ObjectPtr raw = - Object::Allocate(Class::kClassId, Class::InstanceSize(), Heap::kOld, - Class::ContainsCompressedPointers()); + auto raw = Object::Allocate(Heap::kOld); NoSafepointScope no_safepoint; - result ^= raw; + result = raw; } Object::VerifyBuiltinVtable(FakeObject::kClassId); NOT_IN_PRECOMPILED(result.set_token_pos(TokenPosition::kNoSource)); @@ -5003,11 +5016,9 @@ ClassPtr Class::NewCommon(intptr_t index) { ASSERT(Object::class_class() != Class::null()); Class& result = Class::Handle(); { - ObjectPtr raw = - Object::Allocate(Class::kClassId, Class::InstanceSize(), Heap::kOld, - Class::ContainsCompressedPointers()); + auto raw = Object::Allocate(Heap::kOld); NoSafepointScope no_safepoint; - result ^= raw; + result = raw; } // Here kIllegalCid means not-yet-assigned. Object::VerifyBuiltinVtable(index == kIllegalCid ? kInstanceCid @@ -6674,10 +6685,7 @@ const char* TypeParameters::ToCString() const { TypeParametersPtr TypeParameters::New(Heap::Space space) { ASSERT(Object::type_parameters_class() != Class::null()); - ObjectPtr ptr = - Object::Allocate(TypeParameters::kClassId, TypeParameters::InstanceSize(), - space, TypeParameters::ContainsCompressedPointers()); - return static_cast(ptr); + return Object::Allocate(space); } TypeParametersPtr TypeParameters::New(intptr_t count, Heap::Space space) { @@ -7634,11 +7642,9 @@ TypeArgumentsPtr TypeArguments::New(intptr_t len, Heap::Space space) { } TypeArguments& result = TypeArguments::Handle(); { - ObjectPtr raw = Object::Allocate( - TypeArguments::kClassId, TypeArguments::InstanceSize(len), space, - TypeArguments::ContainsCompressedPointers()); + auto raw = Object::Allocate(space, len); NoSafepointScope no_safepoint; - result ^= raw; + result = raw; // Length must be set before we start storing into the array. result.SetLength(len); result.SetHash(0); @@ -7813,10 +7819,7 @@ PatchClassPtr PatchClass::New(const Class& wrapped_class, PatchClassPtr PatchClass::New() { ASSERT(Object::patch_class_class() != Class::null()); - ObjectPtr raw = - Object::Allocate(PatchClass::kClassId, PatchClass::InstanceSize(), - Heap::kOld, PatchClass::ContainsCompressedPointers()); - return static_cast(raw); + return Object::Allocate(Heap::kOld); } void PatchClass::set_wrapped_class(const Class& value) const { @@ -9994,10 +9997,7 @@ bool Function::IsImplicitStaticClosureFunction(FunctionPtr func) { FunctionPtr Function::New(Heap::Space space) { ASSERT(Object::function_class() != Class::null()); - ObjectPtr raw = - Object::Allocate(Function::kClassId, Function::InstanceSize(), space, - Function::ContainsCompressedPointers()); - return static_cast(raw); + return Object::Allocate(space); } FunctionPtr Function::New(const FunctionType& signature, @@ -11331,12 +11331,7 @@ void ClosureData::set_awaiter_link(Function::AwaiterLink link) const { ClosureDataPtr ClosureData::New() { ASSERT(Object::closure_data_class() != Class::null()); - ClosureData& data = ClosureData::Handle(); - data ^= - Object::Allocate(ClosureData::kClassId, ClosureData::InstanceSize(), - Heap::kOld, ClosureData::ContainsCompressedPointers()); - data.set_packed_fields(0); - return data.ptr(); + return Object::Allocate(Heap::kOld); } const char* ClosureData::ToCString() const { @@ -11378,10 +11373,7 @@ void FunctionType::SetNumOptionalParameters( } FunctionTypePtr FunctionType::New(Heap::Space space) { - ObjectPtr raw = - Object::Allocate(FunctionType::kClassId, FunctionType::InstanceSize(), - space, FunctionType::ContainsCompressedPointers()); - return static_cast(raw); + return Object::Allocate(space); } FunctionTypePtr FunctionType::New(intptr_t num_parent_type_arguments, @@ -11479,13 +11471,8 @@ void FfiTrampolineData::set_callback_kind(FfiCallbackKind kind) const { FfiTrampolineDataPtr FfiTrampolineData::New() { ASSERT(Object::ffi_trampoline_data_class() != Class::null()); - ObjectPtr raw = Object::Allocate( - FfiTrampolineData::kClassId, FfiTrampolineData::InstanceSize(), - Heap::kOld, FfiTrampolineData::ContainsCompressedPointers()); - FfiTrampolineDataPtr data = static_cast(raw); + auto data = Object::Allocate(Heap::kOld); data->untag()->callback_id_ = -1; - data->untag()->is_leaf_ = false; - data->untag()->callback_kind_ = 0; return data; } @@ -11702,10 +11689,7 @@ void Field::SetFieldType(const AbstractType& value) const { FieldPtr Field::New() { ASSERT(Object::field_class() != Class::null()); - ObjectPtr raw = - Object::Allocate(Field::kClassId, Field::InstanceSize(), Heap::kOld, - Field::ContainsCompressedPointers()); - return static_cast(raw); + return Object::Allocate(Heap::kOld); } void Field::InitializeNew(const Field& result, @@ -13266,10 +13250,7 @@ StringPtr Script::GetSnippet(intptr_t from_line, ScriptPtr Script::New() { ASSERT(Object::script_class() != Class::null()); - ObjectPtr raw = - Object::Allocate(Script::kClassId, Script::InstanceSize(), Heap::kOld, - Script::ContainsCompressedPointers()); - return static_cast(raw); + return Object::Allocate