diff --git a/runtime/vm/compiler/assembler/assembler_base.cc b/runtime/vm/compiler/assembler/assembler_base.cc index 95c1e59070b..be9be2cccb1 100644 --- a/runtime/vm/compiler/assembler/assembler_base.cc +++ b/runtime/vm/compiler/assembler/assembler_base.cc @@ -32,7 +32,7 @@ AssemblerBase::~AssemblerBase() {} void AssemblerBase::LoadFromSlot(Register dst, Register base, const Slot& slot) { - if (slot.is_unboxed()) { + if (!slot.is_tagged()) { // The result cannot be a floating point or SIMD value. ASSERT(slot.representation() == kUntagged || RepresentationUtils::IsUnboxedInteger(slot.representation())); @@ -41,7 +41,15 @@ void AssemblerBase::LoadFromSlot(Register dst, ASSERT(RepresentationUtils::ValueSize(slot.representation()) <= compiler::target::kWordSize); auto const sz = RepresentationUtils::OperandSize(slot.representation()); - LoadFieldFromOffset(dst, base, slot.offset_in_bytes(), sz); + if (slot.has_untagged_instance()) { + LoadFromOffset(dst, base, slot.offset_in_bytes(), sz); + } else { + LoadFieldFromOffset(dst, base, slot.offset_in_bytes(), sz); + } + } else if (slot.has_untagged_instance()) { + // Non-Dart objects do not contain compressed pointers. + ASSERT(!slot.is_compressed()); + LoadFromOffset(dst, base, slot.offset_in_bytes()); } else if (!slot.is_guarded_field() && slot.type().ToCid() == kSmiCid) { if (slot.is_compressed()) { LoadCompressedSmiFieldFromOffset(dst, base, slot.offset_in_bytes()); @@ -60,10 +68,11 @@ void AssemblerBase::LoadFromSlot(Register dst, void AssemblerBase::StoreToSlot(Register src, Register base, const Slot& slot, - MemoryOrder memory_order) { + MemoryOrder memory_order, + Register scratch) { auto const can_be_smi = slot.type().CanBeSmi() ? kValueCanBeSmi : kValueIsNotSmi; - StoreToSlot(src, base, slot, can_be_smi, memory_order); + StoreToSlot(src, base, slot, can_be_smi, memory_order, scratch); } void AssemblerBase::StoreToSlot(Register src, @@ -72,7 +81,7 @@ void AssemblerBase::StoreToSlot(Register src, CanBeSmi can_be_smi, MemoryOrder memory_order, Register scratch) { - if (slot.is_unboxed()) { + if (!slot.is_tagged() || slot.has_untagged_instance()) { // Same as the no barrier case. StoreToSlotNoBarrier(src, base, slot, memory_order); } else if (slot.is_compressed()) { @@ -88,7 +97,7 @@ void AssemblerBase::StoreToSlotNoBarrier(Register src, Register base, const Slot& slot, MemoryOrder memory_order) { - if (slot.is_unboxed()) { + if (!slot.is_tagged()) { // The stored value cannot be a floating point or SIMD value. ASSERT(slot.representation() == kUntagged || RepresentationUtils::IsUnboxedInteger(slot.representation())); @@ -97,7 +106,15 @@ void AssemblerBase::StoreToSlotNoBarrier(Register src, ASSERT(RepresentationUtils::ValueSize(slot.representation()) <= compiler::target::kWordSize); auto const sz = RepresentationUtils::OperandSize(slot.representation()); - StoreFieldToOffset(src, base, slot.offset_in_bytes(), sz); + if (slot.has_untagged_instance()) { + StoreToOffset(src, base, slot.offset_in_bytes(), sz); + } else { + StoreFieldToOffset(src, base, slot.offset_in_bytes(), sz); + } + } else if (slot.has_untagged_instance()) { + // Non-Dart objects do not contain compressed pointers. + ASSERT(!slot.is_compressed()); + StoreToOffset(src, base, slot.offset_in_bytes()); } else if (slot.is_compressed()) { StoreCompressedIntoObjectOffsetNoBarrier(base, slot.offset_in_bytes(), src, memory_order); diff --git a/runtime/vm/compiler/assembler/assembler_base.h b/runtime/vm/compiler/assembler/assembler_base.h index d8ebe40bcb5..468b3017407 100644 --- a/runtime/vm/compiler/assembler/assembler_base.h +++ b/runtime/vm/compiler/assembler/assembler_base.h @@ -1063,7 +1063,8 @@ class AssemblerBase : public StackResource { void StoreToSlot(Register src, Register base, const Slot& slot, - MemoryOrder memory_order = kRelaxedNonAtomic); + MemoryOrder memory_order = kRelaxedNonAtomic, + Register scratch = TMP); // Truncates upper bits. virtual void LoadInt32FromBoxOrSmi(Register result, Register value) = 0; diff --git a/runtime/vm/compiler/backend/constant_propagator.cc b/runtime/vm/compiler/backend/constant_propagator.cc index cd0046359fd..101bd1b5265 100644 --- a/runtime/vm/compiler/backend/constant_propagator.cc +++ b/runtime/vm/compiler/backend/constant_propagator.cc @@ -780,10 +780,6 @@ void ConstantPropagator::VisitCCall(CCallInstr* instr) { SetValue(instr, non_constant_); } -void ConstantPropagator::VisitRawStoreField(RawStoreFieldInstr* instr) { - // Nothing to do. -} - void ConstantPropagator::VisitDebugStepCheck(DebugStepCheckInstr* instr) { // Nothing to do. } diff --git a/runtime/vm/compiler/backend/flow_graph_checker.cc b/runtime/vm/compiler/backend/flow_graph_checker.cc index cfe590219c4..1f39a381a8f 100644 --- a/runtime/vm/compiler/backend/flow_graph_checker.cc +++ b/runtime/vm/compiler/backend/flow_graph_checker.cc @@ -187,6 +187,8 @@ void FlowGraphChecker::VisitInstructions(BlockEntryInstr* block) { // Initial definitions are partially linked into graph. ASSERT1(def->next() == nullptr, def); ASSERT1(def->previous() == entry, def); + // No initial definition should contain an unsafe untagged pointer. + ASSERT1(!def->MayCreateUnsafeUntaggedPointer(), def); // Skip common constants as checking them could be slow. if (IsCommonConstant(def)) continue; // Visit the initial definition as instruction. @@ -460,7 +462,10 @@ void FlowGraphChecker::VisitDefUse(Definition* def, // We assume that all uses of a GC-movable untagged pointer are within the // same basic block as the definition. ASSERT2(def->GetBlock() == instruction->GetBlock(), def, instruction); - // Untagged pointers should not be returned from functions or FFI callbacks. + // Unsafe untagged pointers should not be used as inputs to Phi nodes in + // the same basic block. + ASSERT2(!instruction->IsPhi(), def, instruction); + // Unsafe untagged pointers should not be returned. ASSERT2(!instruction->IsReturnBase(), def, instruction); // Make sure no instruction between the definition and the use (including // the use) can trigger GC. diff --git a/runtime/vm/compiler/backend/il.cc b/runtime/vm/compiler/backend/il.cc index 8ff4d25c6d5..6e854311aa1 100644 --- a/runtime/vm/compiler/backend/il.cc +++ b/runtime/vm/compiler/backend/il.cc @@ -1015,11 +1015,10 @@ void AllocateTypedDataInstr::EmitNativeCode(FlowGraphCompiler* compiler) { Representation StoreFieldInstr::RequiredInputRepresentation( intptr_t index) const { - ASSERT((index == 0) || (index == 1)); if (index == 0) { - // The instance is always tagged. - return kTagged; + return slot_.has_untagged_instance() ? kUntagged : kTagged; } + ASSERT_EQUAL(index, 1); return slot().representation(); } @@ -4677,6 +4676,9 @@ LocationSummary* LoadFieldInstr::MakeLocationSummary(Zone* zone, void LoadFieldInstr::EmitNativeCode(FlowGraphCompiler* compiler) { const Register instance_reg = locs()->in(0).reg(); + ASSERT(OffsetInBytes() >= 0); // Field is finalized. + // For fields on Dart objects, the offset must point after the header. + ASSERT(OffsetInBytes() != 0 || slot().has_untagged_instance()); auto const rep = slot().representation(); if (calls_initializer()) { @@ -7181,7 +7183,7 @@ void MemoryCopyInstr::EmitUnrolledCopy(FlowGraphCompiler* compiler, #endif bool Utf8ScanInstr::IsScanFlagsUnboxed() const { - return scan_flags_field_.is_unboxed(); + return RepresentationUtils::IsUnboxed(scan_flags_field_.representation()); } InvokeMathCFunctionInstr::InvokeMathCFunctionInstr( @@ -7659,38 +7661,6 @@ void FfiCallInstr::EmitReturnMoves(FlowGraphCompiler* compiler, __ Comment("EmitReturnMovesEnd"); } -LocationSummary* RawStoreFieldInstr::MakeLocationSummary( - Zone* zone, - bool is_optimizing) const { - LocationSummary* summary = - new (zone) LocationSummary(zone, /*num_inputs=*/2, - /*num_temps=*/0, LocationSummary::kNoCall); - - summary->set_in(kBase, Location::RequiresRegister()); - summary->set_in(kValue, Location::RequiresRegister()); - - return summary; -} - -Representation RawStoreFieldInstr::RequiredInputRepresentation( - intptr_t idx) const { - switch (idx) { - case kBase: - return kUntagged; - case kValue: - return kTagged; - default: - break; - } - UNREACHABLE(); -} - -void RawStoreFieldInstr::EmitNativeCode(FlowGraphCompiler* compiler) { - const Register base_reg = locs()->in(kBase).reg(); - const Register value_reg = locs()->in(kValue).reg(); - compiler->assembler()->StoreMemoryValue(value_reg, base_reg, offset_); -} - LocationSummary* StoreFieldInstr::MakeLocationSummary(Zone* zone, bool opt) const { const intptr_t kNumInputs = 2; @@ -7756,8 +7726,9 @@ LocationSummary* StoreFieldInstr::MakeLocationSummary(Zone* zone, void StoreFieldInstr::EmitNativeCode(FlowGraphCompiler* compiler) { const Register instance_reg = locs()->in(kInstancePos).reg(); - const intptr_t offset_in_bytes = OffsetInBytes(); - ASSERT(offset_in_bytes > 0); // Field is finalized and points after header. + ASSERT(OffsetInBytes() >= 0); // Field is finalized. + // For fields on Dart objects, the offset must point after the header. + ASSERT(OffsetInBytes() != 0 || slot().has_untagged_instance()); const Representation rep = slot().representation(); if (rep == kUntagged) { @@ -7774,9 +7745,9 @@ void StoreFieldInstr::EmitNativeCode(FlowGraphCompiler* compiler) { auto const value_pair = locs()->in(kValuePos).AsPairLocation(); const Register value_lo = value_pair->At(0).reg(); const Register value_hi = value_pair->At(1).reg(); - __ StoreFieldToOffset(value_lo, instance_reg, offset_in_bytes); + __ StoreFieldToOffset(value_lo, instance_reg, OffsetInBytes()); __ StoreFieldToOffset(value_hi, instance_reg, - offset_in_bytes + compiler::target::kWordSize); + OffsetInBytes() + compiler::target::kWordSize); } } else if (RepresentationUtils::IsUnboxed(rep)) { ASSERT(slot().IsDartField()); @@ -7785,12 +7756,12 @@ void StoreFieldInstr::EmitNativeCode(FlowGraphCompiler* compiler) { switch (cid) { case kDoubleCid: __ StoreUnboxedDouble(value, instance_reg, - offset_in_bytes - kHeapObjectTag); + OffsetInBytes() - kHeapObjectTag); return; case kFloat32x4Cid: case kFloat64x2Cid: __ StoreUnboxedSimd128(value, instance_reg, - offset_in_bytes - kHeapObjectTag); + OffsetInBytes() - kHeapObjectTag); return; default: UNREACHABLE(); @@ -7804,7 +7775,7 @@ void StoreFieldInstr::EmitNativeCode(FlowGraphCompiler* compiler) { const auto& value = locs()->in(kValuePos).constant(); auto const size = slot().is_compressed() ? compiler::kObjectBytes : compiler::kWordBytes; - __ StoreObjectIntoObjectOffsetNoBarrier(instance_reg, offset_in_bytes, + __ StoreObjectIntoObjectOffsetNoBarrier(instance_reg, OffsetInBytes(), value, memory_order_, size); } else { __ StoreToSlotNoBarrier(locs()->in(kValuePos).reg(), instance_reg, slot(), @@ -7940,7 +7911,7 @@ Representation FfiCallInstr::representation() const { } if (marshaller_.IsHandle(compiler::ffi::kResultIndex)) { // The call returns a Dart_Handle, from which we need to extract the - // tagged pointer using RawLoadField. + // tagged pointer using LoadField with an appropriate slot. return kUntagged; } return marshaller_.RepInFfiCall(compiler::ffi::kResultIndex); diff --git a/runtime/vm/compiler/backend/il.h b/runtime/vm/compiler/backend/il.h index 4e6b3bfe20f..3559c308430 100644 --- a/runtime/vm/compiler/backend/il.h +++ b/runtime/vm/compiler/backend/il.h @@ -436,7 +436,6 @@ struct InstrAttrs { M(ClosureCall, _) \ M(FfiCall, _) \ M(CCall, kNoGC) \ - M(RawStoreField, kNoGC) \ M(InstanceCall, _) \ M(PolymorphicInstanceCall, _) \ M(DispatchTableCall, _) \ @@ -2809,6 +2808,13 @@ class PhiInstr : public VariadicDefinition { virtual Representation representation() const { return representation_; } + virtual bool MayCreateUnsafeUntaggedPointer() const { + // Unsafe untagged pointers should never escape the basic block in which + // they are defined, so they should never be the input to a Phi node. + // (This is checked in the FlowGraphChecker.) + return false; + } + virtual void set_representation(Representation r) { representation_ = r; } // Only Int32 phis in JIT mode are unboxed optimistically. @@ -6200,45 +6206,6 @@ class CCallInstr : public VariadicDefinition { DISALLOW_COPY_AND_ASSIGN(CCallInstr); }; -// Populates the untagged base + offset outside the heap with a tagged value. -// -// The store must be outside of the heap, does not emit a store barrier. -// For stores in the heap, use StoreIndexedInstr, which emits store barriers. -// -// Does not have a dual RawLoadFieldInstr, because for loads we do not have to -// distinguish between loading from within the heap or outside the heap. -// Use FlowGraphBuilder::RawLoadField. -class RawStoreFieldInstr : public TemplateInstruction<2, NoThrow> { - public: - RawStoreFieldInstr(Value* base, Value* value, int32_t offset) - : offset_(offset) { - SetInputAt(kBase, base); - SetInputAt(kValue, value); - } - - enum { kBase = 0, kValue = 1 }; - - DECLARE_INSTRUCTION(RawStoreField) - - virtual Representation RequiredInputRepresentation(intptr_t idx) const; - virtual bool ComputeCanDeoptimize() const { return false; } - virtual bool HasUnknownSideEffects() const { return false; } - - virtual bool CanEliminate(const BlockEntryInstr* block) const { - return false; - } - -#define FIELD_LIST(F) F(const int32_t, offset_) - - DECLARE_INSTRUCTION_SERIALIZABLE_FIELDS(RawStoreFieldInstr, - TemplateInstruction, - FIELD_LIST) -#undef FIELD_LIST - - private: - DISALLOW_COPY_AND_ASSIGN(RawStoreFieldInstr); -}; - class DebugStepCheckInstr : public TemplateInstruction<0, NoThrow> { public: DebugStepCheckInstr(const InstructionSource& source, @@ -6403,6 +6370,10 @@ class StoreFieldInstr : public TemplateInstruction<2, NoThrow> { bool is_initialization() const { return is_initialization_; } bool ShouldEmitStoreBarrier() const { + if (slot().has_untagged_instance()) { + // The instance is not a Dart object, so not traversed by the GC. + return false; + } if (slot().representation() != kTagged) { // The target field is native and unboxed, so not traversed by the GC. return false; @@ -8127,6 +8098,11 @@ class LoadFieldInstr : public TemplateLoadField<1> { loads_inner_pointer_ = value; } + virtual Representation RequiredInputRepresentation(intptr_t idx) const { + ASSERT_EQUAL(idx, 0); + return slot_.has_untagged_instance() ? kUntagged : kTagged; + } + virtual Representation representation() const; DECLARE_INSTRUCTION(LoadField) diff --git a/runtime/vm/compiler/backend/il_test.cc b/runtime/vm/compiler/backend/il_test.cc index 93b7cca75f9..742782f50b4 100644 --- a/runtime/vm/compiler/backend/il_test.cc +++ b/runtime/vm/compiler/backend/il_test.cc @@ -680,201 +680,6 @@ ISOLATE_UNIT_TEST_CASE(IRTest_DoubleEqualsSmi) { })); } -#ifdef DART_TARGET_OS_WINDOWS -const char* pointer_prefix = "0x"; -#else -const char* pointer_prefix = ""; -#endif - -ISOLATE_UNIT_TEST_CASE(IRTest_RawStoreField) { - InstancePtr ptr = Smi::New(100); - OS::Print("&ptr %p\n", &ptr); - - // clang-format off - auto kScript = Utils::CStringUniquePtr(OS::SCreate(nullptr, R"( - import 'dart:ffi'; - - void myFunction() { - final pointer = Pointer.fromAddress(%s%p); - anotherFunction(); - } - - void anotherFunction() {} - )", pointer_prefix, &ptr), std::free); - // clang-format on - - const auto& root_library = Library::Handle(LoadTestScript(kScript.get())); - Invoke(root_library, "myFunction"); - EXPECT_EQ(Smi::New(100), ptr); - - const auto& my_function = - Function::Handle(GetFunction(root_library, "myFunction")); - - TestPipeline pipeline(my_function, CompilerPass::kJIT); - FlowGraph* flow_graph = pipeline.RunPasses({ - CompilerPass::kComputeSSA, - }); - - Zone* const zone = Thread::Current()->zone(); - - StaticCallInstr* pointer = nullptr; - StaticCallInstr* another_function_call = nullptr; - { - ILMatcher cursor(flow_graph, flow_graph->graph_entry()->normal_entry()); - - EXPECT(cursor.TryMatch({ - kMoveGlob, - {kMatchAndMoveStaticCall, &pointer}, - {kMatchAndMoveStaticCall, &another_function_call}, - })); - } - auto pointer_value = Value(pointer); - auto* const load_field_instr = new (zone) LoadFieldInstr( - &pointer_value, Slot::PointerBase_data(), - InnerPointerAccess::kCannotBeInnerPointer, InstructionSource()); - flow_graph->InsertBefore(another_function_call, load_field_instr, nullptr, - FlowGraph::kValue); - auto load_field_value = Value(load_field_instr); - auto pointer_value2 = Value(pointer); - auto* const raw_store_field_instr = - new (zone) RawStoreFieldInstr(&load_field_value, &pointer_value2, 0); - flow_graph->InsertBefore(another_function_call, raw_store_field_instr, - nullptr, FlowGraph::kEffect); - another_function_call->RemoveFromGraph(); - - { - // Check we constructed the right graph. - ILMatcher cursor(flow_graph, flow_graph->graph_entry()->normal_entry()); - EXPECT(cursor.TryMatch({ - kMoveGlob, - kMatchAndMoveStaticCall, - kMatchAndMoveLoadField, - kMatchAndMoveRawStoreField, - })); - } - - pipeline.RunForcedOptimizedAfterSSAPasses(); - - { -#if !defined(PRODUCT) && !defined(USING_THREAD_SANITIZER) - SetFlagScope sfs(&FLAG_disassemble_optimized, true); -#endif - pipeline.CompileGraphAndAttachFunction(); - } - - // Ensure we can successfully invoke the function. - Invoke(root_library, "myFunction"); - - // Might be garbage if we ran a GC, but should never be a Smi. - EXPECT(!ptr.IsSmi()); -} - -// We do not have a RawLoadFieldInstr, instead we just use LoadIndexed for -// loading from outside the heap. -// -// This test constructs to instructions from FlowGraphBuilder::RawLoadField -// and exercises them to do a load from outside the heap. -ISOLATE_UNIT_TEST_CASE(IRTest_RawLoadField) { - InstancePtr ptr = Smi::New(100); - intptr_t ptr2 = 100; - OS::Print("&ptr %p &ptr2 %p\n", &ptr, &ptr2); - - // clang-format off - auto kScript = Utils::CStringUniquePtr(OS::SCreate(nullptr, R"( - import 'dart:ffi'; - - void myFunction() { - final pointer = Pointer.fromAddress(%s%p); - anotherFunction(); - final pointer2 = Pointer.fromAddress(%s%p); - pointer2.value = 3; - } - - void anotherFunction() {} - )", pointer_prefix, &ptr, pointer_prefix, &ptr2), std::free); - // clang-format on - - const auto& root_library = Library::Handle(LoadTestScript(kScript.get())); - Invoke(root_library, "myFunction"); - EXPECT_EQ(Smi::New(100), ptr); - EXPECT_EQ(3, ptr2); - - const auto& my_function = - Function::Handle(GetFunction(root_library, "myFunction")); - - TestPipeline pipeline(my_function, CompilerPass::kJIT); - FlowGraph* flow_graph = pipeline.RunPasses({ - CompilerPass::kComputeSSA, - }); - - Zone* const zone = Thread::Current()->zone(); - - StaticCallInstr* pointer = nullptr; - StaticCallInstr* another_function_call = nullptr; - StaticCallInstr* pointer2 = nullptr; - StaticCallInstr* pointer2_store = nullptr; - { - ILMatcher cursor(flow_graph, flow_graph->graph_entry()->normal_entry()); - - EXPECT(cursor.TryMatch({ - kMoveGlob, - {kMatchAndMoveStaticCall, &pointer}, - {kMatchAndMoveStaticCall, &another_function_call}, - {kMatchAndMoveStaticCall, &pointer2}, - {kMatchAndMoveStaticCall, &pointer2_store}, - })); - } - auto pointer_value = Value(pointer); - auto* const load_field_instr = new (zone) LoadFieldInstr( - &pointer_value, Slot::PointerBase_data(), - InnerPointerAccess::kCannotBeInnerPointer, InstructionSource()); - flow_graph->InsertBefore(another_function_call, load_field_instr, nullptr, - FlowGraph::kValue); - auto load_field_value = Value(load_field_instr); - auto* const constant_instr = new (zone) UnboxedConstantInstr( - Integer::ZoneHandle(zone, Integer::New(0, Heap::kOld)), kUnboxedIntPtr); - flow_graph->InsertBefore(another_function_call, constant_instr, nullptr, - FlowGraph::kValue); - auto constant_value = Value(constant_instr); - auto* const load_indexed_instr = new (zone) - LoadIndexedInstr(&load_field_value, &constant_value, - /*index_unboxed=*/true, /*index_scale=*/1, kArrayCid, - kAlignedAccess, DeoptId::kNone, InstructionSource()); - flow_graph->InsertBefore(another_function_call, load_indexed_instr, nullptr, - FlowGraph::kValue); - - another_function_call->RemoveFromGraph(); - pointer2_store->InputAt(2)->definition()->ReplaceUsesWith(load_indexed_instr); - - { - // Check we constructed the right graph. - ILMatcher cursor(flow_graph, flow_graph->graph_entry()->normal_entry()); - EXPECT(cursor.TryMatch({ - kMoveGlob, - kMatchAndMoveStaticCall, - kMatchAndMoveLoadField, - kMatchAndMoveUnboxedConstant, - kMatchAndMoveLoadIndexed, - kMatchAndMoveStaticCall, - kMatchAndMoveStaticCall, - })); - } - - pipeline.RunForcedOptimizedAfterSSAPasses(); - - { -#if !defined(PRODUCT) && !defined(USING_THREAD_SANITIZER) - SetFlagScope sfs(&FLAG_disassemble_optimized, true); -#endif - pipeline.CompileGraphAndAttachFunction(); - } - - // Ensure we can successfully invoke the function. - Invoke(root_library, "myFunction"); - EXPECT_EQ(Smi::New(100), ptr); - EXPECT_EQ(100, ptr2); -} - ISOLATE_UNIT_TEST_CASE(IRTest_LoadThread) { // clang-format off auto kScript = R"( diff --git a/runtime/vm/compiler/backend/memory_copy_test.cc b/runtime/vm/compiler/backend/memory_copy_test.cc index 7f84997c871..13f2ab788e6 100644 --- a/runtime/vm/compiler/backend/memory_copy_test.cc +++ b/runtime/vm/compiler/backend/memory_copy_test.cc @@ -14,7 +14,11 @@ namespace dart { -extern const char* pointer_prefix; +#ifdef DART_TARGET_OS_WINDOWS +const char* pointer_prefix = "0x"; +#else +const char* pointer_prefix = ""; +#endif static constexpr intptr_t kMemoryTestLength = 1024; static constexpr uint8_t kUnInitialized = 0xFE; diff --git a/runtime/vm/compiler/backend/range_analysis.cc b/runtime/vm/compiler/backend/range_analysis.cc index fccea17f747..ffe8f5b5d1b 100644 --- a/runtime/vm/compiler/backend/range_analysis.cc +++ b/runtime/vm/compiler/backend/range_analysis.cc @@ -2861,53 +2861,12 @@ void LoadFieldInstr::InferRange(RangeAnalysis* analysis, Range* range) { Definition::InferRange(analysis, range); break; - case Slot::Kind::kReceivePort_send_port: - case Slot::Kind::kReceivePort_handler: - case Slot::Kind::kLinkedHashBase_index: - case Slot::Kind::kImmutableLinkedHashBase_index: - case Slot::Kind::kLinkedHashBase_data: - case Slot::Kind::kImmutableLinkedHashBase_data: - case Slot::Kind::kGrowableObjectArray_data: - case Slot::Kind::kContext_parent: case Slot::Kind::kTypeArguments: - case Slot::Kind::kArray_type_arguments: - case Slot::Kind::kClosure_context: - case Slot::Kind::kClosure_delayed_type_arguments: - case Slot::Kind::kClosure_function: - case Slot::Kind::kClosure_function_type_arguments: - case Slot::Kind::kClosure_instantiator_type_arguments: - case Slot::Kind::kFinalizer_callback: - case Slot::Kind::kFinalizer_type_arguments: - case Slot::Kind::kFinalizerBase_all_entries: - case Slot::Kind::kFinalizerBase_detachments: - case Slot::Kind::kFinalizerBase_entries_collected: - case Slot::Kind::kFinalizerEntry_detach: - case Slot::Kind::kFinalizerEntry_finalizer: - case Slot::Kind::kFinalizerEntry_next: - case Slot::Kind::kFinalizerEntry_token: - case Slot::Kind::kFinalizerEntry_value: - case Slot::Kind::kNativeFinalizer_callback: - case Slot::Kind::kFunction_data: - case Slot::Kind::kFunction_signature: - case Slot::Kind::kFunctionType_named_parameter_names: - case Slot::Kind::kFunctionType_parameter_types: - case Slot::Kind::kFunctionType_type_parameters: - case Slot::Kind::kInstance_native_fields_array: - case Slot::Kind::kSuspendState_function_data: - case Slot::Kind::kSuspendState_then_callback: - case Slot::Kind::kSuspendState_error_callback: - case Slot::Kind::kTypedDataView_typed_data: case Slot::Kind::kTypeArgumentsIndex: - case Slot::Kind::kTypeParameters_names: - case Slot::Kind::kTypeParameters_flags: - case Slot::Kind::kTypeParameters_bounds: - case Slot::Kind::kTypeParameters_defaults: - case Slot::Kind::kUnhandledException_exception: - case Slot::Kind::kUnhandledException_stacktrace: - case Slot::Kind::kWeakProperty_key: - case Slot::Kind::kWeakProperty_value: - case Slot::Kind::kWeakReference_target: - case Slot::Kind::kWeakReference_type_arguments: +#define NATIVE_SLOT_CASE(ClassName, __, FieldName, ___, ____) \ + case Slot::Kind::k##ClassName##_##FieldName: + NOT_INT_NATIVE_SLOTS_LIST(NATIVE_SLOT_CASE) +#undef NATIVE_SLOT_CASE // Not an integer valued field. UNREACHABLE(); break; @@ -2917,22 +2876,13 @@ void LoadFieldInstr::InferRange(RangeAnalysis* analysis, Range* range) { UNREACHABLE(); break; -#define UNBOXED_NATIVE_NONADDRESS_SLOT_CASE(Class, Untagged, Field, Rep, \ - IsFinal) \ +#define UNBOXED_NATIVE_SLOT_CASE(Class, __, Field, ___, ____) \ case Slot::Kind::k##Class##_##Field: - UNBOXED_NATIVE_NONADDRESS_SLOTS_LIST(UNBOXED_NATIVE_NONADDRESS_SLOT_CASE) -#undef UNBOXED_NATIVE_NONADDRESS_SLOT_CASE + UNBOXED_NATIVE_SLOTS_LIST(UNBOXED_NATIVE_SLOT_CASE) +#undef UNBOXED_NATIVE_SLOT_CASE *range = Range::Full(slot().representation()); break; -#define UNBOXED_NATIVE_ADDRESS_SLOT_CASE(Class, Untagged, Field, MayMove, \ - IsFinal) \ - case Slot::Kind::k##Class##_##Field: - UNBOXED_NATIVE_ADDRESS_SLOTS_LIST(UNBOXED_NATIVE_ADDRESS_SLOT_CASE) -#undef UNBOXED_NATIVE_ADDRESS_SLOT_CASE - UNREACHABLE(); - break; - case Slot::Kind::kClosure_hash: case Slot::Kind::kLinkedHashBase_hash_mask: case Slot::Kind::kLinkedHashBase_used_data: diff --git a/runtime/vm/compiler/backend/redundancy_elimination.cc b/runtime/vm/compiler/backend/redundancy_elimination.cc index a409ab2d2c6..b650e4396a4 100644 --- a/runtime/vm/compiler/backend/redundancy_elimination.cc +++ b/runtime/vm/compiler/backend/redundancy_elimination.cc @@ -2217,8 +2217,8 @@ class LoadOptimizer : public ValueObject { const intptr_t pos = alloc->InputForSlot(*slot); if (pos != -1) { forward_def = alloc->InputAt(pos)->definition(); - } else if (slot->is_unboxed()) { - // Unboxed fields that are not provided as an input should not + } else if (!slot->is_tagged()) { + // Fields that do not contain tagged values should not // have a tagged null value forwarded for them, similar to // payloads of typed data arrays. continue; diff --git a/runtime/vm/compiler/backend/slot.cc b/runtime/vm/compiler/backend/slot.cc index 8f05413a71a..cd2e0eaa78a 100644 --- a/runtime/vm/compiler/backend/slot.cc +++ b/runtime/vm/compiler/backend/slot.cc @@ -70,8 +70,8 @@ Slot* SlotCache::CreateNativeSlot(Slot::Kind kind) { switch (kind) { #define FIELD_FINAL true #define FIELD_VAR false -#define DEFINE_NULLABLE_BOXED_NATIVE_FIELD(ClassName, UnderlyingType, \ - FieldName, cid, mutability) \ +#define DEFINE_NULLABLE_TAGGED_NATIVE_DART_FIELD(ClassName, UnderlyingType, \ + FieldName, cid, mutability) \ case Slot::Kind::k##ClassName##_##FieldName: \ return new (zone_) Slot( \ Slot::Kind::k##ClassName##_##FieldName, \ @@ -84,12 +84,13 @@ Slot* SlotCache::CreateNativeSlot(Slot::Kind kind) { k##cid##Cid, nullptr), \ kTagged); - NULLABLE_BOXED_NATIVE_SLOTS_LIST(DEFINE_NULLABLE_BOXED_NATIVE_FIELD) + NULLABLE_TAGGED_NATIVE_DART_SLOTS_LIST( + DEFINE_NULLABLE_TAGGED_NATIVE_DART_FIELD) -#undef DEFINE_NULLABLE_BOXED_NATIVE_FIELD +#undef DEFINE_NULLABLE_TAGGED_NATIVE_DART_FIELD -#define DEFINE_NONNULLABLE_BOXED_NATIVE_FIELD(ClassName, UnderlyingType, \ - FieldName, cid, mutability) \ +#define DEFINE_NONNULLABLE_TAGGED_NATIVE_DART_FIELD( \ + ClassName, UnderlyingType, FieldName, cid, mutability) \ case Slot::Kind::k##ClassName##_##FieldName: \ return new (zone_) Slot( \ Slot::Kind::k##ClassName##_##FieldName, \ @@ -102,40 +103,95 @@ Slot* SlotCache::CreateNativeSlot(Slot::Kind kind) { CompileType::kCannotBeSentinel, k##cid##Cid, nullptr), \ kTagged); - NONNULLABLE_BOXED_NATIVE_SLOTS_LIST(DEFINE_NONNULLABLE_BOXED_NATIVE_FIELD) + NONNULLABLE_INT_TAGGED_NATIVE_DART_SLOTS_LIST( + DEFINE_NONNULLABLE_TAGGED_NATIVE_DART_FIELD) + NONNULLABLE_NONINT_TAGGED_NATIVE_DART_SLOTS_LIST( + DEFINE_NONNULLABLE_TAGGED_NATIVE_DART_FIELD) -#undef DEFINE_NONNULLABLE_BOXED_NATIVE_FIELD +#undef DEFINE_NONNULLABLE_TAGGED_NATIVE_DART_FIELD -#define DEFINE_UNBOXED_NATIVE_NONADDRESS_FIELD( \ - ClassName, UnderlyingType, FieldName, representation, mutability) \ +#define DEFINE_UNBOXED_NATIVE_DART_FIELD(ClassName, UnderlyingType, FieldName, \ + representation, mutability) \ case Slot::Kind::k##ClassName##_##FieldName: \ return new (zone_) \ Slot(Slot::Kind::k##ClassName##_##FieldName, \ Slot::IsImmutableBit::encode(FIELD_##mutability) | \ - Slot::IsUnboxedBit::encode(true), \ + Slot::IsNonTaggedBit::encode(true), \ compiler::target::ClassName::FieldName##_offset(), \ #ClassName "." #FieldName, \ CompileType::FromUnboxedRepresentation(kUnboxed##representation), \ kUnboxed##representation); - UNBOXED_NATIVE_NONADDRESS_SLOTS_LIST(DEFINE_UNBOXED_NATIVE_NONADDRESS_FIELD) + UNBOXED_NATIVE_DART_SLOTS_LIST(DEFINE_UNBOXED_NATIVE_DART_FIELD) -#undef DEFINE_UNBOXED_NATIVE_NONADDRESS_FIELD +#undef DEFINE_UNBOXED_NATIVE_DART_FIELD -#define DEFINE_UNBOXED_NATIVE_ADDRESS_FIELD(ClassName, UnderlyingType, \ - FieldName, GcMayMove, mutability) \ +#define DEFINE_UNTAGGED_NATIVE_DART_FIELD(ClassName, UnderlyingType, \ + FieldName, GcMayMove, mutability) \ case Slot::Kind::k##ClassName##_##FieldName: \ return new (zone_) \ Slot(Slot::Kind::k##ClassName##_##FieldName, \ Slot::IsImmutableBit::encode(FIELD_##mutability) | \ Slot::MayContainInnerPointerBit::encode(GcMayMove) | \ - Slot::IsUnboxedBit::encode(true), \ + Slot::IsNonTaggedBit::encode(true), \ compiler::target::ClassName::FieldName##_offset(), \ #ClassName "." #FieldName, CompileType::Object(), kUntagged); - UNBOXED_NATIVE_ADDRESS_SLOTS_LIST(DEFINE_UNBOXED_NATIVE_ADDRESS_FIELD) + UNTAGGED_NATIVE_DART_SLOTS_LIST(DEFINE_UNTAGGED_NATIVE_DART_FIELD) + +#undef DEFINE_UNTAGGED_NATIVE_DART_FIELD + +#define DEFINE_NULLABLE_TAGGED_NATIVE_NONDART_FIELD(ClassName, __, FieldName, \ + cid, mutability) \ + case Slot::Kind::k##ClassName##_##FieldName: \ + return new (zone_) Slot( \ + Slot::Kind::k##ClassName##_##FieldName, \ + Slot::IsImmutableBit::encode(FIELD_##mutability) | \ + Slot::HasUntaggedInstanceBit::encode(true), \ + compiler::target::ClassName::FieldName##_offset(), \ + #ClassName "." #FieldName, \ + CompileType(CompileType::kCanBeNull, CompileType::kCannotBeSentinel, \ + k##cid##Cid, nullptr), \ + kTagged); + + NULLABLE_TAGGED_NATIVE_NONDART_SLOTS_LIST( + DEFINE_NULLABLE_TAGGED_NATIVE_NONDART_FIELD) + +#undef DEFINE_NULLABLE_TAGGED_NONDART_FIELD + +#define DEFINE_UNBOXED_NATIVE_NONDART_FIELD(ClassName, __, FieldName, \ + representation, mutability) \ + case Slot::Kind::k##ClassName##_##FieldName: \ + return new (zone_) \ + Slot(Slot::Kind::k##ClassName##_##FieldName, \ + Slot::IsImmutableBit::encode(FIELD_##mutability) | \ + Slot::IsNonTaggedBit::encode(true) | \ + Slot::HasUntaggedInstanceBit::encode(true), \ + compiler::target::ClassName::FieldName##_offset(), \ + #ClassName "." #FieldName, \ + CompileType::FromUnboxedRepresentation(kUnboxed##representation), \ + kUnboxed##representation); + + UNBOXED_NATIVE_NONDART_SLOTS_LIST(DEFINE_UNBOXED_NATIVE_NONDART_FIELD) + +#undef DEFINE_UNBOXED_NATIVE_NONDART_FIELD + +#define DEFINE_UNTAGGED_NATIVE_NONDART_FIELD(ClassName, __, FieldName, \ + gc_may_move, mutability) \ + case Slot::Kind::k##ClassName##_##FieldName: \ + return new (zone_) \ + Slot(Slot::Kind::k##ClassName##_##FieldName, \ + Slot::IsImmutableBit::encode(FIELD_##mutability) | \ + Slot::MayContainInnerPointerBit::encode(gc_may_move) | \ + Slot::IsNonTaggedBit::encode(true) | \ + Slot::HasUntaggedInstanceBit::encode(true), \ + compiler::target::ClassName::FieldName##_offset(), \ + #ClassName "." #FieldName, CompileType::Object(), kUntagged); + + UNTAGGED_NATIVE_NONDART_SLOTS_LIST(DEFINE_UNTAGGED_NATIVE_NONDART_FIELD) + +#undef DEFINE_UNTAGGED_NATIVE_NONDART_FIELD -#undef DEFINE_UNBOXED_NATIVE_NONADDRESS_FIELD #undef FIELD_VAR #undef FIELD_FINAL default: @@ -158,16 +214,17 @@ bool Slot::IsImmutableLengthSlot() const { return false; // Not length loads. -#define UNBOXED_NATIVE_SLOT_CASE(Class, Untagged, Field, Rep, IsFinal) \ + case Slot::Kind::kArrayElement: + case Slot::Kind::kCapturedVariable: + case Slot::Kind::kDartField: + case Slot::Kind::kRecordField: + case Slot::Kind::kTypeArguments: + case Slot::Kind::kTypeArgumentsIndex: +#define NOT_TAGGED_INT_NATIVE_SLOT_CASE(Class, __, Field, ___, ____) \ case Slot::Kind::k##Class##_##Field: - UNBOXED_NATIVE_SLOTS_LIST(UNBOXED_NATIVE_SLOT_CASE) -#undef UNBOXED_NATIVE_SLOT_CASE - case Slot::Kind::kReceivePort_send_port: - case Slot::Kind::kReceivePort_handler: - case Slot::Kind::kLinkedHashBase_index: - case Slot::Kind::kImmutableLinkedHashBase_index: - case Slot::Kind::kLinkedHashBase_data: - case Slot::Kind::kImmutableLinkedHashBase_data: + NOT_INT_NATIVE_SLOTS_LIST(NOT_TAGGED_INT_NATIVE_SLOT_CASE) + UNBOXED_NATIVE_SLOTS_LIST(NOT_TAGGED_INT_NATIVE_SLOT_CASE) +#undef NONTAGGED_NATIVE_DART_SLOT_CASE case Slot::Kind::kLinkedHashBase_hash_mask: case Slot::Kind::kLinkedHashBase_used_data: case Slot::Kind::kLinkedHashBase_deleted_keys: @@ -175,56 +232,11 @@ bool Slot::IsImmutableLengthSlot() const { case Slot::Kind::kArgumentsDescriptor_positional_count: case Slot::Kind::kArgumentsDescriptor_count: case Slot::Kind::kArgumentsDescriptor_size: - case Slot::Kind::kArrayElement: - case Slot::Kind::kInstance_native_fields_array: - case Slot::Kind::kTypeArguments: case Slot::Kind::kTypeArguments_hash: case Slot::Kind::kTypedDataView_offset_in_bytes: - case Slot::Kind::kTypedDataView_typed_data: - case Slot::Kind::kGrowableObjectArray_data: - case Slot::Kind::kArray_type_arguments: - case Slot::Kind::kContext_parent: - case Slot::Kind::kClosure_context: - case Slot::Kind::kClosure_delayed_type_arguments: - case Slot::Kind::kClosure_function: - case Slot::Kind::kClosure_function_type_arguments: - case Slot::Kind::kClosure_instantiator_type_arguments: case Slot::Kind::kClosure_hash: - case Slot::Kind::kCapturedVariable: - case Slot::Kind::kDartField: - case Slot::Kind::kFinalizer_callback: - case Slot::Kind::kFinalizer_type_arguments: - case Slot::Kind::kFinalizerBase_all_entries: - case Slot::Kind::kFinalizerBase_detachments: - case Slot::Kind::kFinalizerBase_entries_collected: - case Slot::Kind::kFinalizerEntry_detach: - case Slot::Kind::kFinalizerEntry_finalizer: - case Slot::Kind::kFinalizerEntry_next: - case Slot::Kind::kFinalizerEntry_token: - case Slot::Kind::kFinalizerEntry_value: - case Slot::Kind::kNativeFinalizer_callback: - case Slot::Kind::kFunction_data: - case Slot::Kind::kFunction_signature: - case Slot::Kind::kFunctionType_named_parameter_names: - case Slot::Kind::kFunctionType_parameter_types: - case Slot::Kind::kFunctionType_type_parameters: - case Slot::Kind::kRecordField: case Slot::Kind::kRecord_shape: - case Slot::Kind::kSuspendState_function_data: - case Slot::Kind::kSuspendState_then_callback: - case Slot::Kind::kSuspendState_error_callback: - case Slot::Kind::kTypeArgumentsIndex: case Slot::Kind::kAbstractType_hash: - case Slot::Kind::kTypeParameters_names: - case Slot::Kind::kTypeParameters_flags: - case Slot::Kind::kTypeParameters_bounds: - case Slot::Kind::kTypeParameters_defaults: - case Slot::Kind::kUnhandledException_exception: - case Slot::Kind::kUnhandledException_stacktrace: - case Slot::Kind::kWeakProperty_key: - case Slot::Kind::kWeakProperty_value: - case Slot::Kind::kWeakReference_target: - case Slot::Kind::kWeakReference_type_arguments: return false; } UNREACHABLE(); @@ -425,7 +437,7 @@ const Slot& Slot::Get(const Field& field, IsGuardedBit::encode(used_guarded_state) | IsCompressedBit::encode( compiler::target::Class::HasCompressedPointers(owner)) | - IsUnboxedBit::encode(is_unboxed), + IsNonTaggedBit::encode(is_unboxed), compiler::target::Field::OffsetOf(field), &field, type, rep, field_guard_state); diff --git a/runtime/vm/compiler/backend/slot.h b/runtime/vm/compiler/backend/slot.h index 6a0e879dfed..dba67446739 100644 --- a/runtime/vm/compiler/backend/slot.h +++ b/runtime/vm/compiler/backend/slot.h @@ -38,8 +38,8 @@ class LocalScope; class LocalVariable; class ParsedFunction; -// The list of slots that correspond to nullable boxed fields of native objects -// in the following format: +// The list of slots that correspond to nullable boxed fields of native +// Dart objects in the following format: // // V(class_name, underlying_type, field_name, exact_type, FINAL|VAR) // @@ -51,7 +51,7 @@ class ParsedFunction; // - the last component specifies whether field behaves like a final field // (i.e. initialized once at construction time and does not change after // that) or like a non-final field. -#define NULLABLE_BOXED_NATIVE_SLOTS_LIST(V) \ +#define NULLABLE_TAGGED_NATIVE_DART_SLOTS_LIST(V) \ V(Array, UntaggedArray, type_arguments, TypeArguments, FINAL) \ V(Finalizer, UntaggedFinalizer, type_arguments, TypeArguments, FINAL) \ V(FinalizerBase, UntaggedFinalizerBase, all_entries, Set, VAR) \ @@ -87,7 +87,7 @@ class ParsedFunction; V(WeakReference, UntaggedWeakReference, type_arguments, TypeArguments, FINAL) // The list of slots that correspond to non-nullable boxed fields of native -// objects in the following format: +// Dart objects that contain integers in the following format: // // V(class_name, underlying_type, field_name, exact_type, FINAL|VAR) // @@ -99,26 +99,13 @@ class ParsedFunction; // - the last component specifies whether field behaves like a final field // (i.e. initialized once at construction time and does not change after // that) or like a non-final field. -#define NONNULLABLE_BOXED_NATIVE_SLOTS_LIST(V) \ +#define NONNULLABLE_INT_TAGGED_NATIVE_DART_SLOTS_LIST(V) \ V(Array, UntaggedArray, length, Smi, FINAL) \ - V(Closure, UntaggedClosure, function, Function, FINAL) \ - V(Closure, UntaggedClosure, context, Dynamic, FINAL) \ - V(Closure, UntaggedClosure, hash, Context, VAR) \ - V(Finalizer, UntaggedFinalizer, callback, Closure, FINAL) \ - V(NativeFinalizer, UntaggedFinalizer, callback, Pointer, FINAL) \ - V(Function, UntaggedFunction, data, Dynamic, FINAL) \ - V(FunctionType, UntaggedFunctionType, named_parameter_names, Array, FINAL) \ - V(FunctionType, UntaggedFunctionType, parameter_types, Array, FINAL) \ + V(Closure, UntaggedClosure, hash, Smi, VAR) \ V(GrowableObjectArray, UntaggedGrowableObjectArray, length, Smi, VAR) \ - V(GrowableObjectArray, UntaggedGrowableObjectArray, data, Array, VAR) \ V(TypedDataBase, UntaggedTypedDataBase, length, Smi, FINAL) \ V(TypedDataView, UntaggedTypedDataView, offset_in_bytes, Smi, FINAL) \ - V(TypedDataView, UntaggedTypedDataView, typed_data, Dynamic, FINAL) \ V(String, UntaggedString, length, Smi, FINAL) \ - V(LinkedHashBase, UntaggedLinkedHashBase, index, TypedDataUint32Array, VAR) \ - V(LinkedHashBase, UntaggedLinkedHashBase, data, Array, VAR) \ - V(ImmutableLinkedHashBase, UntaggedLinkedHashBase, data, ImmutableArray, \ - FINAL) \ V(LinkedHashBase, UntaggedLinkedHashBase, hash_mask, Smi, VAR) \ V(LinkedHashBase, UntaggedLinkedHashBase, used_data, Smi, VAR) \ V(LinkedHashBase, UntaggedLinkedHashBase, deleted_keys, Smi, VAR) \ @@ -129,24 +116,41 @@ class ParsedFunction; V(Record, UntaggedRecord, shape, Smi, FINAL) \ V(TypeArguments, UntaggedTypeArguments, hash, Smi, VAR) \ V(TypeArguments, UntaggedTypeArguments, length, Smi, FINAL) \ - V(AbstractType, UntaggedTypeArguments, hash, Smi, VAR) \ + V(AbstractType, UntaggedTypeArguments, hash, Smi, VAR) + +// The list of slots that correspond to non-nullable boxed fields of native +// Dart objects that do not contain integers in the following format: +// +// V(class_name, underlying_type, field_name, exact_type, FINAL|VAR) +// +// - class_name and field_name specify the name of the host class and the name +// of the field respectively; +// - underlying_type: the Raw class which holds the field; +// - exact_type specifies exact type of the field (any load from this field +// would only yield instances of this type); +// - the last component specifies whether field behaves like a final field +// (i.e. initialized once at construction time and does not change after +// that) or like a non-final field. +#define NONNULLABLE_NONINT_TAGGED_NATIVE_DART_SLOTS_LIST(V) \ + V(Closure, UntaggedClosure, function, Function, FINAL) \ + V(Closure, UntaggedClosure, context, Dynamic, FINAL) \ + V(Finalizer, UntaggedFinalizer, callback, Closure, FINAL) \ + V(NativeFinalizer, UntaggedFinalizer, callback, Pointer, FINAL) \ + V(Function, UntaggedFunction, data, Dynamic, FINAL) \ + V(FunctionType, UntaggedFunctionType, named_parameter_names, Array, FINAL) \ + V(FunctionType, UntaggedFunctionType, parameter_types, Array, FINAL) \ + V(GrowableObjectArray, UntaggedGrowableObjectArray, data, Array, VAR) \ + V(TypedDataView, UntaggedTypedDataView, typed_data, Dynamic, FINAL) \ + V(LinkedHashBase, UntaggedLinkedHashBase, index, TypedDataUint32Array, VAR) \ + V(LinkedHashBase, UntaggedLinkedHashBase, data, Array, VAR) \ + V(ImmutableLinkedHashBase, UntaggedLinkedHashBase, data, ImmutableArray, \ + FINAL) \ V(TypeParameters, UntaggedTypeParameters, names, Array, FINAL) \ V(UnhandledException, UntaggedUnhandledException, exception, Dynamic, FINAL) \ V(UnhandledException, UntaggedUnhandledException, stacktrace, Dynamic, FINAL) -// Don't use Object or Instance, use Dynamic instead. The cid here should -// correspond to an exact type or Dynamic, not a static type. -// If we ever get a field of which the exact type is Instance (not a subtype), -// update the check below. -#define FOR_EACH_NATIVE_SLOT(_, __, ___, field_type, ____) \ - static_assert(k##field_type##Cid != kObjectCid); \ - static_assert(k##field_type##Cid != kInstanceCid); -NULLABLE_BOXED_NATIVE_SLOTS_LIST(FOR_EACH_NATIVE_SLOT) -NONNULLABLE_BOXED_NATIVE_SLOTS_LIST(FOR_EACH_NATIVE_SLOT) -#undef FOR_EACH_NATIVE_SLOT - -// List of slots that correspond to unboxed fields of native objects that -// do not contain untagged addresses in the following format: +// List of slots that correspond to fields of native objects that contain +// unboxed values in the following format: // // V(class_name, underlying_type, field_name, representation, FINAL|VAR) // @@ -163,7 +167,7 @@ NONNULLABLE_BOXED_NATIVE_SLOTS_LIST(FOR_EACH_NATIVE_SLOT) // // Note: Currently LoadFieldInstr::IsImmutableLengthLoad() assumes that no // unboxed slots represent length loads. -#define UNBOXED_NATIVE_NONADDRESS_SLOTS_LIST(V) \ +#define UNBOXED_NATIVE_DART_SLOTS_LIST(V) \ V(AbstractType, UntaggedAbstractType, flags, Uint32, FINAL) \ V(ClosureData, UntaggedClosureData, packed_fields, Uint32, FINAL) \ V(FinalizerEntry, UntaggedFinalizerEntry, external_size, IntPtr, VAR) \ @@ -174,16 +178,16 @@ NONNULLABLE_BOXED_NATIVE_SLOTS_LIST(FOR_EACH_NATIVE_SLOT) FINAL) \ V(SubtypeTestCache, UntaggedSubtypeTestCache, num_inputs, Uint32, FINAL) -// Unboxed native slots containing untagged addresses that do not exist -// in JIT mode. See UNBOXED_NATIVE_ADDRESS_SLOTS_LIST for the format. +// Native slots containing untagged addresses that do not exist in JIT mode. +// See UNTAGGED_NATIVE_DART_SLOTS_LIST for the format. #if defined(DART_PRECOMPILER) && !defined(TARGET_ARCH_IA32) -#define AOT_ONLY_UNBOXED_NATIVE_ADDRESS_SLOTS_LIST(V) \ +#define AOT_ONLY_UNTAGGED_NATIVE_DART_SLOTS_LIST(V) \ V(Closure, UntaggedClosure, entry_point, false, FINAL) #else -#define AOT_ONLY_UNBOXED_NATIVE_ADDRESS_SLOTS_LIST(V) +#define AOT_ONLY_UNTAGGED_NATIVE_DART_SLOTS_LIST(V) #endif -// List of slots that correspond to unboxed fields of native objects containing +// List of slots that correspond to fields of native Dart objects containing // untagged addresses in the following format: // // V(class_name, underlying_type, field_name, gc_may_move, FINAL|VAR) @@ -198,35 +202,209 @@ NONNULLABLE_BOXED_NATIVE_SLOTS_LIST(FOR_EACH_NATIVE_SLOT) // (i.e. initialized once at construction time and does not change after // that) or like a non-final field. // -// Note: As the underlying field is unboxed, these slots cannot be nullable. +// Note: As the underlying field is untagged, these slots cannot be nullable. // -// Note: All slots for unboxed fields that contain untagged addresses are given -// the kUntagged representation, and so a value loaded from these fields must -// be converted explicitly to an unboxed integer representation for any -// pointer arithmetic before use, and an unboxed integer must be converted -// explicitly to an untagged address before being stored to these fields. -// -// Note: Currently LoadFieldInstr::IsImmutableLengthLoad() assumes that no -// unboxed slots represent length loads. -#define UNBOXED_NATIVE_ADDRESS_SLOTS_LIST(V) \ - AOT_ONLY_UNBOXED_NATIVE_ADDRESS_SLOTS_LIST(V) \ +// Note: All slots for fields that contain untagged addresses are given +// the kUntagged representation. +#define UNTAGGED_NATIVE_DART_SLOTS_LIST(V) \ + AOT_ONLY_UNTAGGED_NATIVE_DART_SLOTS_LIST(V) \ V(Function, UntaggedFunction, entry_point, false, FINAL) \ V(FinalizerBase, UntaggedFinalizerBase, isolate, false, VAR) \ V(PointerBase, UntaggedPointerBase, data, true, VAR) -// For uses that do not need to know whether a given slot may contain an -// inner pointer to a GC-able object or not. (Generally, such users only need -// the class name, the underlying type, and/or the field name.) -#define UNBOXED_NATIVE_SLOTS_LIST(V) \ - UNBOXED_NATIVE_NONADDRESS_SLOTS_LIST(V) UNBOXED_NATIVE_ADDRESS_SLOTS_LIST(V) +// List of slots that correspond to fields of non-Dart objects containing +// tagged addresses of Dart objects in the following format: +// +// V(class_name, _, field_name, exact_type, FINAL|VAR) +// +// - class_name and field_name specify the name of the host class and the name +// of the field respectively; +// - exact_type specifies exact type of the field (any load from this field +// would only yield instances of this type); +// - the last component specifies whether field behaves like a final field +// (i.e. initialized once at construction time and does not change after +// that) or like a non-final field. +// +// Note: Currently LoadFieldInstr::IsImmutableLengthLoad() assumes that no +// slots of non-Dart values represent length loads. +#define NULLABLE_TAGGED_NATIVE_NONDART_SLOTS_LIST(V) \ + V(Isolate, _, finalizers, GrowableObjectArray, VAR) \ + V(LocalHandle, _, ptr, Dynamic, VAR) \ + V(ObjectStore, _, record_field_names, Array, VAR) \ + V(PersistentHandle, _, ptr, Dynamic, VAR) -// For uses that do not need the exact_type (boxed) or representation (unboxed) -// or whether a boxed native slot is nullable. (Generally, such users only need -// the class name, the underlying type, and/or the field name.) +// List of slots that correspond to fields of non-Dart objects containing +// unboxed values in the following format: +// +// V(class_name, _, field_name, representation, FINAL|VAR) +// +// - class_name and field_name specify the name of the host class and the name +// of the field respectively; +// - representation specifies the representation of the bits stored within +// the unboxed field (minus the kUnboxed prefix); +// - the last component specifies whether field behaves like a final field +// (i.e. initialized once at construction time and does not change after +// that) or like a non-final field. +// +// Note: As the underlying field is unboxed, these slots cannot be nullable. +// +// Note: Currently LoadFieldInstr::IsImmutableLengthLoad() assumes that no +// slots of non-Dart values represent length loads. +#define UNBOXED_NATIVE_NONDART_SLOTS_LIST(V) \ + V(StreamInfo, _, enabled, IntPtr, VAR) + +// List of slots that correspond to fields of non-Dart objects containing +// untagged addresses in the following format: +// +// V(class_name, _, field_name, gc_may_move, FINAL|VAR) +// +// - class_name and field_name specify the name of the host class and the name +// of the field respectively; +// - gc_may_move: whether the untagged address contained in this field is a +// pointer to memory that may be moved by the GC, which means a value loaded +// from this field is invalidated by any instruction that can cause GC; +// - the last component specifies whether field behaves like a final field +// (i.e. initialized once at construction time and does not change after +// that) or like a non-final field. +// +// Note: As the underlying field is untagged, these slots cannot be nullable. +// +// Note: All slots for fields that contain untagged addresses are given +// the kUntagged representation. +// +// Note: while Thread::isolate_ and IsolateGroup::object_store_ aren't const +// fields, they should never change during a given execution of the code +// generated for a function and the compiler only does intra-procedural +// load optimizations. +#define UNTAGGED_NATIVE_NONDART_SLOTS_LIST(V) \ + V(IsolateGroup, _, object_store, false, FINAL) \ + V(Thread, _, api_top_scope, false, VAR) \ + V(Thread, _, isolate, false, FINAL) \ + V(Thread, _, isolate_group, false, FINAL) \ + V(Thread, _, service_extension_stream, false, FINAL) + +// No untagged slot on a non-Dart object should contain a GC-movable address. +// The gc_may_move field is only there so that any code that operates on +// UNTAGGED_NATIVE_SLOTS_LIST can use that field as desired. +#define CHECK_NATIVE_NONDART_SLOT(__, ___, ____, gc_may_move, _____) \ + static_assert(!gc_may_move); +UNTAGGED_NATIVE_NONDART_SLOTS_LIST(CHECK_NATIVE_NONDART_SLOT) +#undef CHECK_NATIVE_NONDART_SLOT + +// For uses that need any native slot that contain an unboxed integer. Such uses +// can only use the following arguments for each entry: +// V(class_name, _, field_name, rep, FINAL|VAR) +#define UNBOXED_NATIVE_SLOTS_LIST(V) \ + UNBOXED_NATIVE_DART_SLOTS_LIST(V) \ + UNBOXED_NATIVE_NONDART_SLOTS_LIST(V) + +// For uses that need any native slot that contain an untagged address. Such +// uses can only use the following arguments for each entry: +// V(class_name, _, field_name, gc_may_move, FINAL|VAR) +#define UNTAGGED_NATIVE_SLOTS_LIST(V) \ + UNTAGGED_NATIVE_DART_SLOTS_LIST(V) \ + UNTAGGED_NATIVE_NONDART_SLOTS_LIST(V) + +// For uses that need any native slot that does not contain a Dart object. Such +// uses can only use the following arguments for each entry: +// V(class_name, _, field_name, _, FINAL|VAR) +#define NOT_TAGGED_NATIVE_SLOTS_LIST(V) \ + UNBOXED_NATIVE_SLOTS_LIST(V) \ + UNTAGGED_NATIVE_SLOTS_LIST(V) + +// For uses that need any native slot that is guaranteed to contain a tagged +// integer. Such uses can only use the following arguments for each entry: +// V(class_name, _, field_name, exact_type, FINAL|VAR) +#define TAGGED_INT_NATIVE_SLOTS_LIST(V) \ + NONNULLABLE_INT_TAGGED_NATIVE_DART_SLOTS_LIST(V) + +// For uses that need any native slot that contains a tagged object which is not +// guaranteed to be a integer. This includes nullable integer slots, since +// those slots may return a non-integer value (null). Such uses can +// only use the following arguments for each entry: +// V(class_name, _, field_name, exact_type, FINAL|VAR) +#define TAGGED_NONINT_NATIVE_SLOTS_LIST(V) \ + NULLABLE_TAGGED_NATIVE_DART_SLOTS_LIST(V) \ + NONNULLABLE_NONINT_TAGGED_NATIVE_DART_SLOTS_LIST(V) \ + NULLABLE_TAGGED_NATIVE_NONDART_SLOTS_LIST(V) + +// For uses that need any native slot that is not guaranteed to contain an +// integer, whether a Dart object or unboxed. Such uses can only use the +// following arguments for each entry: +// V(class_name, _, field_name, _, FINAL|VAR) +#define NOT_INT_NATIVE_SLOTS_LIST(V) \ + TAGGED_NONINT_NATIVE_SLOTS_LIST(V) \ + UNTAGGED_NATIVE_SLOTS_LIST(V) + +// For uses that need any native slot on Dart objects that contains a Dart +// object (e.g., for write barrier purposes). Such uses can use the following +// arguments for each entry: +// V(class_name, underlying_class, field_name, exact_type, FINAL|VAR) +#define TAGGED_NATIVE_DART_SLOTS_LIST(V) \ + NULLABLE_TAGGED_NATIVE_DART_SLOTS_LIST(V) \ + NONNULLABLE_INT_TAGGED_NATIVE_DART_SLOTS_LIST(V) \ + NONNULLABLE_NONINT_TAGGED_NATIVE_DART_SLOTS_LIST(V) + +// For uses that need any native slot that is not on a Dart object or does +// not contain a Dart object (e.g., for write barrier purposes). Such uses +// can only use the following arguments for each entry: +// V(class_name, _, field_name, _, FINAL|VAR) +#define NOT_TAGGED_NATIVE_DART_SLOTS_LIST(V) \ + NULLABLE_TAGGED_NATIVE_NONDART_SLOTS_LIST(V) \ + NOT_TAGGED_NATIVE_SLOTS_LIST(V) + +// For uses that need any native slot that contains a Dart object. Such uses can +// only use the following arguments for each entry: +// V(class_name, _, field_name, exact_type, FINAL|VAR) +#define TAGGED_NATIVE_SLOTS_LIST(V) \ + TAGGED_INT_NATIVE_SLOTS_LIST(V) \ + TAGGED_NONINT_NATIVE_SLOTS_LIST(V) + +// For uses that need all native slots. Such uses can only use the following +// arguments for each entry: +// V(class_name, _, field_name, _, FINAL|VAR) #define NATIVE_SLOTS_LIST(V) \ - NULLABLE_BOXED_NATIVE_SLOTS_LIST(V) \ - NONNULLABLE_BOXED_NATIVE_SLOTS_LIST(V) \ - UNBOXED_NATIVE_SLOTS_LIST(V) + TAGGED_NATIVE_SLOTS_LIST(V) \ + NOT_TAGGED_NATIVE_SLOTS_LIST(V) + +// For tagged slots, the cid should either be Dynamic or the precise cid +// of the values stored in the corresponding field. That means the cid should +// not be the cid of an abstract superclass, because then the code will assume +// the cid of retrieved values is always the given cid. +// +// Note: If we ever need native slots with CompileTypes created from an +// AbstractType instead, then a new base category should be created for those, +// possibly replacing the cid field with the name of the abstract type. +#define CHECK_TAGGED_NATIVE_SLOT(__, ___, ____, field_type, _____) \ + static_assert(k##field_type##Cid != kObjectCid); \ + static_assert(k##field_type##Cid != kInstanceCid); \ + static_assert(k##field_type##Cid != kIntegerCid); \ + static_assert(k##field_type##Cid != kStringCid); \ + static_assert(k##field_type##Cid != kAbstractTypeCid); +TAGGED_NATIVE_SLOTS_LIST(CHECK_TAGGED_NATIVE_SLOT) +#undef CHECK_NULLABLE_TAGGED_NATIVE_SLOT + +// Currently we only create slots with CompileTypes created from a precise cid, +// so integer slots listed here must only contain Smis (or Mints, but no slot +// currently does has only Mints, adjust this check if one is added). +// +// Note: If we ever add a category of native slots with AbstractType-based +// CompileTypes that always contain integers, then add additional checks that +// the AbstractTypes of those slots are subtypes of Integer. +#define CHECK_INT_NATIVE_SLOT(__, ___, ____, field_type, _____) \ + static_assert(k##field_type##Cid == kSmiCid); +TAGGED_INT_NATIVE_SLOTS_LIST(CHECK_INT_NATIVE_SLOT) +#undef CHECK_INT_NATIVE_SLOT + +// Any slot with an integer type should go into the correct category. +// +// Note: If we ever add native slots with AbstractType-based CompileTypes, then +// add appropriate checks that the AbstractType is not a subtype of Integer. +#define CHECK_NONINT_NATIVE_SLOT(__, ___, ____, field_type, _____) \ + static_assert(k##field_type##Cid != kSmiCid); \ + static_assert(k##field_type##Cid != kMintCid); +TAGGED_NONINT_NATIVE_SLOTS_LIST(CHECK_NONINT_NATIVE_SLOT) +#undef CHECK_NONINT_NATIVE_SLOT class FieldGuardState { public: @@ -313,7 +491,7 @@ class Slot : public ZoneAllocated { const ParsedFunction* parsed_function); // Convenience getters for native slots. -#define DEFINE_GETTER(ClassName, UnderlyingType, FieldName, __, ___) \ +#define DEFINE_GETTER(ClassName, __, FieldName, ___, ____) \ static const Slot& ClassName##_##FieldName() { \ return GetNativeSlot(Kind::k##ClassName##_##FieldName); \ } @@ -374,7 +552,10 @@ class Slot : public ZoneAllocated { return kind() == Kind::kCapturedVariable || kind() == Kind::kContext_parent; } - bool is_unboxed() const { return IsUnboxedBit::decode(flags_); } + bool is_tagged() const { return !IsNonTaggedBit::decode(flags_); } + bool has_untagged_instance() const { + return HasUntaggedInstanceBit::decode(flags_); + } void Write(FlowGraphSerializer* s) const; static const Slot& Read(FlowGraphDeserializer* d); @@ -404,13 +585,6 @@ class Slot : public ZoneAllocated { other.representation_, other.field_guard_state_) {} - using IsImmutableBit = BitField; - using IsGuardedBit = BitField; - using IsCompressedBit = BitField; - using IsUnboxedBit = BitField; - using MayContainInnerPointerBit = - BitField; - template const T* DataAs() const { return static_cast(data_); @@ -447,6 +621,19 @@ class Slot : public ZoneAllocated { CompileType type_; + using IsImmutableBit = BitField; + using IsGuardedBit = + BitField; + using IsCompressedBit = + BitField; + // Stores whether a field isn't tagged so that tagged is the default value + using IsNonTaggedBit = + BitField; + using MayContainInnerPointerBit = + BitField; + using HasUntaggedInstanceBit = + BitField; + friend class SlotCache; }; diff --git a/runtime/vm/compiler/frontend/base_flow_graph_builder.cc b/runtime/vm/compiler/frontend/base_flow_graph_builder.cc index 09dcb66b96c..53fbbdfdff4 100644 --- a/runtime/vm/compiler/frontend/base_flow_graph_builder.cc +++ b/runtime/vm/compiler/frontend/base_flow_graph_builder.cc @@ -472,6 +472,17 @@ Fragment BaseFlowGraphBuilder::LoadNativeField( return Fragment(load); } +Fragment BaseFlowGraphBuilder::LoadNativeField(const Slot& native_field, + bool calls_initializer) { + const InnerPointerAccess loads_inner_pointer = + native_field.representation() == kUntagged + ? (native_field.may_contain_inner_pointer() + ? InnerPointerAccess::kMayBeInnerPointer + : InnerPointerAccess::kCannotBeInnerPointer) + : InnerPointerAccess::kNotUntagged; + return LoadNativeField(native_field, loads_inner_pointer, calls_initializer); +} + Fragment BaseFlowGraphBuilder::LoadLocal(LocalVariable* variable) { ASSERT(!variable->is_captured()); LoadLocalInstr* load = new (Z) LoadLocalInstr(*variable, InstructionSource()); diff --git a/runtime/vm/compiler/frontend/base_flow_graph_builder.h b/runtime/vm/compiler/frontend/base_flow_graph_builder.h index 8b9a46ebbf8..0023ac9fbac 100644 --- a/runtime/vm/compiler/frontend/base_flow_graph_builder.h +++ b/runtime/vm/compiler/frontend/base_flow_graph_builder.h @@ -179,10 +179,7 @@ class BaseFlowGraphBuilder { InnerPointerAccess loads_inner_pointer, bool calls_initializer = false); Fragment LoadNativeField(const Slot& native_field, - bool calls_initializer = false) { - return LoadNativeField(native_field, InnerPointerAccess::kNotUntagged, - calls_initializer); - } + bool calls_initializer = false); // Pass true for index_unboxed if indexing into external typed data. Fragment LoadIndexed(classid_t class_id, intptr_t index_scale = compiler::target::kWordSize, diff --git a/runtime/vm/compiler/frontend/kernel_to_il.cc b/runtime/vm/compiler/frontend/kernel_to_il.cc index 81545a77c43..ba2bcfb4858 100644 --- a/runtime/vm/compiler/frontend/kernel_to_il.cc +++ b/runtime/vm/compiler/frontend/kernel_to_il.cc @@ -1297,8 +1297,7 @@ FlowGraph* FlowGraphBuilder::BuildGraphOfRecognizedMethod( break; case MethodRecognizer::kRecord_fieldNames: body += LoadObjectStore(); - body += RawLoadField( - compiler::target::ObjectStore::record_field_names_offset()); + body += LoadNativeField(Slot::ObjectStore_record_field_names()); body += LoadLocal(parsed_function_->RawParameterVariable(0)); body += LoadNativeField(Slot::Record_shape()); body += IntConstant(compiler::target::RecordShape::kFieldNamesIndexShift); @@ -1770,7 +1769,7 @@ FlowGraph* FlowGraphBuilder::BuildGraphOfRecognizedMethod( body += Constant(Bool::False()); #else body += LoadServiceExtensionStream(); - body += RawLoadField(compiler::target::StreamInfo::enabled_offset()); + body += LoadNativeField(Slot::StreamInfo_enabled()); // StreamInfo::enabled_ is a std::atomic. This is effectively // relaxed order access, which is acceptable for this use case. body += IntToBool(); @@ -1918,13 +1917,13 @@ FlowGraph* FlowGraphBuilder::BuildGraphOfRecognizedMethod( case MethodRecognizer::kFinalizerBase_getIsolateFinalizers: ASSERT_EQUAL(function.NumParameters(), 0); body += LoadIsolate(); - body += RawLoadField(compiler::target::Isolate::finalizers_offset()); + body += LoadNativeField(Slot::Isolate_finalizers()); break; case MethodRecognizer::kFinalizerBase_setIsolateFinalizers: ASSERT_EQUAL(function.NumParameters(), 1); body += LoadIsolate(); body += LoadLocal(parsed_function_->RawParameterVariable(0)); - body += RawStoreField(compiler::target::Isolate::finalizers_offset()); + body += StoreNativeField(Slot::Isolate_finalizers()); body += NullConstant(); break; case MethodRecognizer::kFinalizerBase_exchangeEntriesCollectedWithNull: @@ -4648,22 +4647,6 @@ Fragment FlowGraphBuilder::LoadIndexedTypedDataUnboxed( return fragment; } -Fragment FlowGraphBuilder::RawLoadField(int32_t offset) { - Fragment code; - code += UnboxedIntConstant(offset, kUnboxedIntPtr); - code += LoadIndexed(kArrayCid, /*index_scale=*/1, /*index_unboxed=*/true); - return code; -} - -Fragment FlowGraphBuilder::RawStoreField(int32_t offset) { - Fragment code; - Value* value = Pop(); - Value* base = Pop(); - auto* instr = new (Z) RawStoreFieldInstr(base, value, offset); - code <<= instr; - return code; -} - Fragment FlowGraphBuilder::UnhandledException() { const auto class_table = thread_->isolate_group()->class_table(); ASSERT(class_table->HasValidClassAt(kUnhandledExceptionCid)); @@ -4711,29 +4694,28 @@ Fragment FlowGraphBuilder::LoadThread() { Fragment FlowGraphBuilder::LoadIsolate() { Fragment body; body += LoadThread(); - body += LoadUntagged(compiler::target::Thread::isolate_offset()); + body += LoadNativeField(Slot::Thread_isolate()); return body; } Fragment FlowGraphBuilder::LoadIsolateGroup() { Fragment body; body += LoadThread(); - body += LoadUntagged(compiler::target::Thread::isolate_group_offset()); + body += LoadNativeField(Slot::Thread_isolate_group()); return body; } Fragment FlowGraphBuilder::LoadObjectStore() { Fragment body; body += LoadIsolateGroup(); - body += LoadUntagged(compiler::target::IsolateGroup::object_store_offset()); + body += LoadNativeField(Slot::IsolateGroup_object_store()); return body; } Fragment FlowGraphBuilder::LoadServiceExtensionStream() { Fragment body; body += LoadThread(); - body += - LoadUntagged(compiler::target::Thread::service_extension_stream_offset()); + body += LoadNativeField(Slot::Thread_service_extension_stream()); return body; } @@ -5276,7 +5258,7 @@ Fragment FlowGraphBuilder::FfiConvertPrimitiveToDart( } else if (marshaller.IsHandle(arg_index)) { // The top of the stack is a Dart_Handle, so retrieve the tagged pointer // out of it. - body += RawLoadField(compiler::target::LocalHandle::ptr_offset()); + body += LoadNativeField(Slot::LocalHandle_ptr()); } else if (marshaller.IsVoid(arg_index)) { // Ignore whatever value was being returned and return null. ASSERT_EQUAL(arg_index, compiler::ffi::kResultIndex); @@ -5320,7 +5302,7 @@ Fragment FlowGraphBuilder::FfiConvertPrimitiveToNative( // Get a reference to the top handle scope. body += LoadThread(); - body += LoadUntagged(compiler::target::Thread::api_top_scope_offset()); + body += LoadNativeField(Slot::Thread_api_top_scope()); arg_reps->Add(kUntagged); // Allocate a new handle in the top handle scope. @@ -5331,7 +5313,8 @@ Fragment FlowGraphBuilder::FfiConvertPrimitiveToNative( // Store the object address into the handle. body += LoadLocal(handle); body += LoadLocal(object); - body += RawStoreField(compiler::target::LocalHandle::ptr_offset()); + body += StoreNativeField(Slot::LocalHandle_ptr(), + StoreFieldInstr::Kind::kInitializing); body += DropTempsPreserveTop(1); // Drop object. } else if (marshaller.IsVoid(arg_index)) { @@ -5694,7 +5677,7 @@ FlowGraph* FlowGraphBuilder::BuildGraphOfSyncFfiCallback( body += LoadThread(); body += LoadUntagged(compiler::target::Thread::unboxed_runtime_arg_offset()); - body += RawLoadField(compiler::target::PersistentHandle::ptr_offset()); + body += LoadNativeField(Slot::PersistentHandle_ptr()); closure = MakeTemporary(); } diff --git a/runtime/vm/compiler/frontend/kernel_to_il.h b/runtime/vm/compiler/frontend/kernel_to_il.h index fa0f2116ab9..e3a6608960c 100644 --- a/runtime/vm/compiler/frontend/kernel_to_il.h +++ b/runtime/vm/compiler/frontend/kernel_to_il.h @@ -395,14 +395,6 @@ class FlowGraphBuilder : public BaseFlowGraphBuilder { ZoneGrowableArray* definitions, const GrowableArray& representations); - // Loads a tagged value from an untagged base + offset from outside the heap. - Fragment RawLoadField(int32_t offset); - - // Populates the untagged base + offset outside the heap with a tagged value. - // - // The store must be outside of the heap, does not emit a store barrier. - Fragment RawStoreField(int32_t offset); - // Wrap the current exception and stacktrace in an unhandled exception. Fragment UnhandledException(); diff --git a/runtime/vm/compiler/write_barrier_elimination.cc b/runtime/vm/compiler/write_barrier_elimination.cc index 5b8d6b8a6b3..a680e5067e8 100644 --- a/runtime/vm/compiler/write_barrier_elimination.cc +++ b/runtime/vm/compiler/write_barrier_elimination.cc @@ -363,15 +363,23 @@ bool WriteBarrierElimination::SlotEligibleForWBE(const Slot& slot) { case Slot::Kind::kRecordField: // Instance return true; -#define FOR_EACH_NATIVE_SLOT(class, underlying_type, field, __, ___) \ +#define TAGGED_NATIVE_DART_SLOT_CASE(class, underlying_type, field, __, ___) \ case Slot::Kind::k##class##_##field: \ return std::is_base_of::value || \ std::is_base_of::value || \ std::is_base_of::value; - NATIVE_SLOTS_LIST(FOR_EACH_NATIVE_SLOT) -#undef FOR_EACH_NATIVE_SLOT + TAGGED_NATIVE_DART_SLOTS_LIST(TAGGED_NATIVE_DART_SLOT_CASE) +#undef TAGGED_NATIVE_DART_SLOT_CASE + +#define OTHER_NATIVE_SLOT_CASE(class, __, field, ___, ____) \ + case Slot::Kind::k##class##_##field: + // No store barrier needed for non-tagged fields or fields of + // non-Dart objects. + NOT_TAGGED_NATIVE_DART_SLOTS_LIST(OTHER_NATIVE_SLOT_CASE) +#undef OTHER_NATIVE_SLOT_CASE + return true; default: return false; diff --git a/runtime/vm/isolate.h b/runtime/vm/isolate.h index 04b48c17c77..c320cc38d37 100644 --- a/runtime/vm/isolate.h +++ b/runtime/vm/isolate.h @@ -1553,7 +1553,7 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry { bool is_system_isolate_ = false; // End accessed from generated code. - IsolateGroup* isolate_group_; + IsolateGroup* const isolate_group_; IdleTimeHandler idle_time_handler_; std::unique_ptr isolate_object_store_; diff --git a/runtime/vm/object.cc b/runtime/vm/object.cc index 1ba730691d8..5e32f9ed29c 100644 --- a/runtime/vm/object.cc +++ b/runtime/vm/object.cc @@ -6288,11 +6288,11 @@ FunctionPtr Class::LookupFunctionReadLocked(const String& name, #if defined(DEBUG) ASSERT(thread->isolate_group()->program_lock()->IsCurrentThreadReader()); #endif + ASSERT(functions() != Array::null()); REUSABLE_ARRAY_HANDLESCOPE(thread); REUSABLE_FUNCTION_HANDLESCOPE(thread); Array& funcs = thread->ArrayHandle(); funcs = functions(); - ASSERT(!funcs.IsNull()); const intptr_t len = funcs.Length(); Function& function = thread->FunctionHandle(); if (len >= kFunctionLookupHashThreshold) { diff --git a/runtime/vm/thread.cc b/runtime/vm/thread.cc index f54c685f914..431f209457b 100644 --- a/runtime/vm/thread.cc +++ b/runtime/vm/thread.cc @@ -73,8 +73,16 @@ Thread::Thread(bool is_vm_isolate) TargetCPUFeatures::double_truncate_round_supported() ? 1 : 0), tsan_utils_(DO_IF_TSAN(new TsanUtils()) DO_IF_NOT_TSAN(nullptr)), task_kind_(kUnknownTask), +#if defined(SUPPORT_TIMELINE) + dart_stream_(ASSERT_NOTNULL(Timeline::GetDartStream())), +#else dart_stream_(nullptr), +#endif +#if !defined(PRODUCT) + service_extension_stream_(ASSERT_NOTNULL(&Service::extension_stream)), +#else service_extension_stream_(nullptr), +#endif thread_lock_(), api_reusable_scope_(nullptr), no_callback_scope_depth_(0), @@ -98,14 +106,6 @@ Thread::Thread(bool is_vm_isolate) next_(nullptr) { #endif -#if defined(SUPPORT_TIMELINE) - dart_stream_ = Timeline::GetDartStream(); - ASSERT(dart_stream_ != nullptr); -#endif -#ifndef PRODUCT - service_extension_stream_ = &Service::extension_stream; - ASSERT(service_extension_stream_ != nullptr); -#endif #define DEFAULT_INIT(type_name, member_name, init_expr, default_init_value) \ member_name = default_init_value; CACHED_CONSTANTS_LIST(DEFAULT_INIT) diff --git a/runtime/vm/thread.h b/runtime/vm/thread.h index e5c3b1aae29..1fe4d829a6d 100644 --- a/runtime/vm/thread.h +++ b/runtime/vm/thread.h @@ -1285,8 +1285,8 @@ class Thread : public ThreadState { uword true_end_ = 0; TaskKind task_kind_; - TimelineStream* dart_stream_; - StreamInfo* service_extension_stream_; + TimelineStream* const dart_stream_; + StreamInfo* const service_extension_stream_; mutable Monitor thread_lock_; ApiLocalScope* api_reusable_scope_; int32_t no_callback_scope_depth_;