From cf4a4a3c00a4e160b3c924cc3081582dd42b26f7 Mon Sep 17 00:00:00 2001 From: Tess Strickland Date: Thu, 4 Apr 2024 19:13:50 +0000 Subject: [PATCH] [vm/compiler] Add the capability for Slots on non-Dart objects. Now Slots have an extra flag that describes whether or not the instance object for that Slot is tagged (Dart objects) or untagged (non-Dart objects like Thread). Add new Slot categories for Slots with untagged instances, based on the return value of the Slot: * a tagged value (e.g., the finalizers_ field of Isolate), * an untagged value (e.g., the isolate_ field of Thread), or * an unboxed value (e.g., the enabled_ field of StreamInfo). The primary difference in the specification of slots for untagged instances vs tagged instances is that the slots for untagged instances do not include an underlying type (as only Dart objects have an Untagged for each class that specifies the actual object layout). Also do a little more refactoring of Slot categories to simplify some switch statements that now only need to be updated if the categories the switch statement deals with change. For example, methods for range analysis are only interested in Slots that are guaranteed to contain integers, so now those methods are written in a way that they only need to be updated if a new tagged integer Slot declaration is added. Adjust LoadFromSlot, StoreToSlot, and StoreToSlotNoBarrier to appropriately handle Slots on untagged instances, and change LoadField and StoreField's RequiredInputRepresentation methods to appropriately returned kTagged or kUntagged based on the slot. TEST=ci Cq-Include-Trybots: luci.dart.try:vm-aot-android-release-arm64c-try,vm-aot-android-release-arm_x64-try,vm-aot-linux-debug-x64-try,vm-aot-linux-debug-x64c-try,vm-aot-mac-release-arm64-try,vm-aot-mac-release-x64-try,vm-aot-obfuscate-linux-release-x64-try,vm-aot-optimization-level-linux-release-x64-try,vm-appjit-linux-debug-x64-try,vm-asan-linux-release-x64-try,vm-checked-mac-release-arm64-try,vm-eager-optimization-linux-release-ia32-try,vm-eager-optimization-linux-release-x64-try,vm-ffi-android-debug-arm-try,vm-ffi-android-debug-arm64c-try,vm-ffi-qemu-linux-release-arm-try,vm-ffi-qemu-linux-release-riscv64-try,vm-linux-debug-ia32-try,vm-linux-debug-x64c-try,vm-mac-debug-arm64-try,vm-mac-debug-x64-try,vm-msan-linux-release-x64-try,vm-reload-linux-debug-x64-try,vm-reload-rollback-linux-debug-x64-try,vm-ubsan-linux-release-x64-try,vm-win-debug-arm64-try,vm-win-debug-x64-try,vm-win-release-ia32-try Change-Id: Ic27de4f092c9e087486066459f68a82b4e1685b0 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/357341 Commit-Queue: Tess Strickland Reviewed-by: Daco Harkes Reviewed-by: Alexander Markov --- .../vm/compiler/assembler/assembler_base.cc | 31 +- .../vm/compiler/assembler/assembler_base.h | 3 +- .../compiler/backend/constant_propagator.cc | 4 - .../vm/compiler/backend/flow_graph_checker.cc | 7 +- runtime/vm/compiler/backend/il.cc | 59 +--- runtime/vm/compiler/backend/il.h | 56 +-- runtime/vm/compiler/backend/il_test.cc | 195 ----------- .../vm/compiler/backend/memory_copy_test.cc | 6 +- runtime/vm/compiler/backend/range_analysis.cc | 64 +--- .../backend/redundancy_elimination.cc | 4 +- runtime/vm/compiler/backend/slot.cc | 158 +++++---- runtime/vm/compiler/backend/slot.h | 327 ++++++++++++++---- .../frontend/base_flow_graph_builder.cc | 11 + .../frontend/base_flow_graph_builder.h | 5 +- runtime/vm/compiler/frontend/kernel_to_il.cc | 43 +-- runtime/vm/compiler/frontend/kernel_to_il.h | 8 - .../vm/compiler/write_barrier_elimination.cc | 14 +- runtime/vm/isolate.h | 2 +- runtime/vm/object.cc | 2 +- runtime/vm/thread.cc | 16 +- runtime/vm/thread.h | 4 +- 21 files changed, 467 insertions(+), 552 deletions(-) diff --git a/runtime/vm/compiler/assembler/assembler_base.cc b/runtime/vm/compiler/assembler/assembler_base.cc index 95c1e59070b..be9be2cccb1 100644 --- a/runtime/vm/compiler/assembler/assembler_base.cc +++ b/runtime/vm/compiler/assembler/assembler_base.cc @@ -32,7 +32,7 @@ AssemblerBase::~AssemblerBase() {} void AssemblerBase::LoadFromSlot(Register dst, Register base, const Slot& slot) { - if (slot.is_unboxed()) { + if (!slot.is_tagged()) { // The result cannot be a floating point or SIMD value. ASSERT(slot.representation() == kUntagged || RepresentationUtils::IsUnboxedInteger(slot.representation())); @@ -41,7 +41,15 @@ void AssemblerBase::LoadFromSlot(Register dst, ASSERT(RepresentationUtils::ValueSize(slot.representation()) <= compiler::target::kWordSize); auto const sz = RepresentationUtils::OperandSize(slot.representation()); - LoadFieldFromOffset(dst, base, slot.offset_in_bytes(), sz); + if (slot.has_untagged_instance()) { + LoadFromOffset(dst, base, slot.offset_in_bytes(), sz); + } else { + LoadFieldFromOffset(dst, base, slot.offset_in_bytes(), sz); + } + } else if (slot.has_untagged_instance()) { + // Non-Dart objects do not contain compressed pointers. + ASSERT(!slot.is_compressed()); + LoadFromOffset(dst, base, slot.offset_in_bytes()); } else if (!slot.is_guarded_field() && slot.type().ToCid() == kSmiCid) { if (slot.is_compressed()) { LoadCompressedSmiFieldFromOffset(dst, base, slot.offset_in_bytes()); @@ -60,10 +68,11 @@ void AssemblerBase::LoadFromSlot(Register dst, void AssemblerBase::StoreToSlot(Register src, Register base, const Slot& slot, - MemoryOrder memory_order) { + MemoryOrder memory_order, + Register scratch) { auto const can_be_smi = slot.type().CanBeSmi() ? kValueCanBeSmi : kValueIsNotSmi; - StoreToSlot(src, base, slot, can_be_smi, memory_order); + StoreToSlot(src, base, slot, can_be_smi, memory_order, scratch); } void AssemblerBase::StoreToSlot(Register src, @@ -72,7 +81,7 @@ void AssemblerBase::StoreToSlot(Register src, CanBeSmi can_be_smi, MemoryOrder memory_order, Register scratch) { - if (slot.is_unboxed()) { + if (!slot.is_tagged() || slot.has_untagged_instance()) { // Same as the no barrier case. StoreToSlotNoBarrier(src, base, slot, memory_order); } else if (slot.is_compressed()) { @@ -88,7 +97,7 @@ void AssemblerBase::StoreToSlotNoBarrier(Register src, Register base, const Slot& slot, MemoryOrder memory_order) { - if (slot.is_unboxed()) { + if (!slot.is_tagged()) { // The stored value cannot be a floating point or SIMD value. ASSERT(slot.representation() == kUntagged || RepresentationUtils::IsUnboxedInteger(slot.representation())); @@ -97,7 +106,15 @@ void AssemblerBase::StoreToSlotNoBarrier(Register src, ASSERT(RepresentationUtils::ValueSize(slot.representation()) <= compiler::target::kWordSize); auto const sz = RepresentationUtils::OperandSize(slot.representation()); - StoreFieldToOffset(src, base, slot.offset_in_bytes(), sz); + if (slot.has_untagged_instance()) { + StoreToOffset(src, base, slot.offset_in_bytes(), sz); + } else { + StoreFieldToOffset(src, base, slot.offset_in_bytes(), sz); + } + } else if (slot.has_untagged_instance()) { + // Non-Dart objects do not contain compressed pointers. + ASSERT(!slot.is_compressed()); + StoreToOffset(src, base, slot.offset_in_bytes()); } else if (slot.is_compressed()) { StoreCompressedIntoObjectOffsetNoBarrier(base, slot.offset_in_bytes(), src, memory_order); diff --git a/runtime/vm/compiler/assembler/assembler_base.h b/runtime/vm/compiler/assembler/assembler_base.h index d8ebe40bcb5..468b3017407 100644 --- a/runtime/vm/compiler/assembler/assembler_base.h +++ b/runtime/vm/compiler/assembler/assembler_base.h @@ -1063,7 +1063,8 @@ class AssemblerBase : public StackResource { void StoreToSlot(Register src, Register base, const Slot& slot, - MemoryOrder memory_order = kRelaxedNonAtomic); + MemoryOrder memory_order = kRelaxedNonAtomic, + Register scratch = TMP); // Truncates upper bits. virtual void LoadInt32FromBoxOrSmi(Register result, Register value) = 0; diff --git a/runtime/vm/compiler/backend/constant_propagator.cc b/runtime/vm/compiler/backend/constant_propagator.cc index cd0046359fd..101bd1b5265 100644 --- a/runtime/vm/compiler/backend/constant_propagator.cc +++ b/runtime/vm/compiler/backend/constant_propagator.cc @@ -780,10 +780,6 @@ void ConstantPropagator::VisitCCall(CCallInstr* instr) { SetValue(instr, non_constant_); } -void ConstantPropagator::VisitRawStoreField(RawStoreFieldInstr* instr) { - // Nothing to do. -} - void ConstantPropagator::VisitDebugStepCheck(DebugStepCheckInstr* instr) { // Nothing to do. } diff --git a/runtime/vm/compiler/backend/flow_graph_checker.cc b/runtime/vm/compiler/backend/flow_graph_checker.cc index cfe590219c4..1f39a381a8f 100644 --- a/runtime/vm/compiler/backend/flow_graph_checker.cc +++ b/runtime/vm/compiler/backend/flow_graph_checker.cc @@ -187,6 +187,8 @@ void FlowGraphChecker::VisitInstructions(BlockEntryInstr* block) { // Initial definitions are partially linked into graph. ASSERT1(def->next() == nullptr, def); ASSERT1(def->previous() == entry, def); + // No initial definition should contain an unsafe untagged pointer. + ASSERT1(!def->MayCreateUnsafeUntaggedPointer(), def); // Skip common constants as checking them could be slow. if (IsCommonConstant(def)) continue; // Visit the initial definition as instruction. @@ -460,7 +462,10 @@ void FlowGraphChecker::VisitDefUse(Definition* def, // We assume that all uses of a GC-movable untagged pointer are within the // same basic block as the definition. ASSERT2(def->GetBlock() == instruction->GetBlock(), def, instruction); - // Untagged pointers should not be returned from functions or FFI callbacks. + // Unsafe untagged pointers should not be used as inputs to Phi nodes in + // the same basic block. + ASSERT2(!instruction->IsPhi(), def, instruction); + // Unsafe untagged pointers should not be returned. ASSERT2(!instruction->IsReturnBase(), def, instruction); // Make sure no instruction between the definition and the use (including // the use) can trigger GC. diff --git a/runtime/vm/compiler/backend/il.cc b/runtime/vm/compiler/backend/il.cc index 8ff4d25c6d5..6e854311aa1 100644 --- a/runtime/vm/compiler/backend/il.cc +++ b/runtime/vm/compiler/backend/il.cc @@ -1015,11 +1015,10 @@ void AllocateTypedDataInstr::EmitNativeCode(FlowGraphCompiler* compiler) { Representation StoreFieldInstr::RequiredInputRepresentation( intptr_t index) const { - ASSERT((index == 0) || (index == 1)); if (index == 0) { - // The instance is always tagged. - return kTagged; + return slot_.has_untagged_instance() ? kUntagged : kTagged; } + ASSERT_EQUAL(index, 1); return slot().representation(); } @@ -4677,6 +4676,9 @@ LocationSummary* LoadFieldInstr::MakeLocationSummary(Zone* zone, void LoadFieldInstr::EmitNativeCode(FlowGraphCompiler* compiler) { const Register instance_reg = locs()->in(0).reg(); + ASSERT(OffsetInBytes() >= 0); // Field is finalized. + // For fields on Dart objects, the offset must point after the header. + ASSERT(OffsetInBytes() != 0 || slot().has_untagged_instance()); auto const rep = slot().representation(); if (calls_initializer()) { @@ -7181,7 +7183,7 @@ void MemoryCopyInstr::EmitUnrolledCopy(FlowGraphCompiler* compiler, #endif bool Utf8ScanInstr::IsScanFlagsUnboxed() const { - return scan_flags_field_.is_unboxed(); + return RepresentationUtils::IsUnboxed(scan_flags_field_.representation()); } InvokeMathCFunctionInstr::InvokeMathCFunctionInstr( @@ -7659,38 +7661,6 @@ void FfiCallInstr::EmitReturnMoves(FlowGraphCompiler* compiler, __ Comment("EmitReturnMovesEnd"); } -LocationSummary* RawStoreFieldInstr::MakeLocationSummary( - Zone* zone, - bool is_optimizing) const { - LocationSummary* summary = - new (zone) LocationSummary(zone, /*num_inputs=*/2, - /*num_temps=*/0, LocationSummary::kNoCall); - - summary->set_in(kBase, Location::RequiresRegister()); - summary->set_in(kValue, Location::RequiresRegister()); - - return summary; -} - -Representation RawStoreFieldInstr::RequiredInputRepresentation( - intptr_t idx) const { - switch (idx) { - case kBase: - return kUntagged; - case kValue: - return kTagged; - default: - break; - } - UNREACHABLE(); -} - -void RawStoreFieldInstr::EmitNativeCode(FlowGraphCompiler* compiler) { - const Register base_reg = locs()->in(kBase).reg(); - const Register value_reg = locs()->in(kValue).reg(); - compiler->assembler()->StoreMemoryValue(value_reg, base_reg, offset_); -} - LocationSummary* StoreFieldInstr::MakeLocationSummary(Zone* zone, bool opt) const { const intptr_t kNumInputs = 2; @@ -7756,8 +7726,9 @@ LocationSummary* StoreFieldInstr::MakeLocationSummary(Zone* zone, void StoreFieldInstr::EmitNativeCode(FlowGraphCompiler* compiler) { const Register instance_reg = locs()->in(kInstancePos).reg(); - const intptr_t offset_in_bytes = OffsetInBytes(); - ASSERT(offset_in_bytes > 0); // Field is finalized and points after header. + ASSERT(OffsetInBytes() >= 0); // Field is finalized. + // For fields on Dart objects, the offset must point after the header. + ASSERT(OffsetInBytes() != 0 || slot().has_untagged_instance()); const Representation rep = slot().representation(); if (rep == kUntagged) { @@ -7774,9 +7745,9 @@ void StoreFieldInstr::EmitNativeCode(FlowGraphCompiler* compiler) { auto const value_pair = locs()->in(kValuePos).AsPairLocation(); const Register value_lo = value_pair->At(0).reg(); const Register value_hi = value_pair->At(1).reg(); - __ StoreFieldToOffset(value_lo, instance_reg, offset_in_bytes); + __ StoreFieldToOffset(value_lo, instance_reg, OffsetInBytes()); __ StoreFieldToOffset(value_hi, instance_reg, - offset_in_bytes + compiler::target::kWordSize); + OffsetInBytes() + compiler::target::kWordSize); } } else if (RepresentationUtils::IsUnboxed(rep)) { ASSERT(slot().IsDartField()); @@ -7785,12 +7756,12 @@ void StoreFieldInstr::EmitNativeCode(FlowGraphCompiler* compiler) { switch (cid) { case kDoubleCid: __ StoreUnboxedDouble(value, instance_reg, - offset_in_bytes - kHeapObjectTag); + OffsetInBytes() - kHeapObjectTag); return; case kFloat32x4Cid: case kFloat64x2Cid: __ StoreUnboxedSimd128(value, instance_reg, - offset_in_bytes - kHeapObjectTag); + OffsetInBytes() - kHeapObjectTag); return; default: UNREACHABLE(); @@ -7804,7 +7775,7 @@ void StoreFieldInstr::EmitNativeCode(FlowGraphCompiler* compiler) { const auto& value = locs()->in(kValuePos).constant(); auto const size = slot().is_compressed() ? compiler::kObjectBytes : compiler::kWordBytes; - __ StoreObjectIntoObjectOffsetNoBarrier(instance_reg, offset_in_bytes, + __ StoreObjectIntoObjectOffsetNoBarrier(instance_reg, OffsetInBytes(), value, memory_order_, size); } else { __ StoreToSlotNoBarrier(locs()->in(kValuePos).reg(), instance_reg, slot(), @@ -7940,7 +7911,7 @@ Representation FfiCallInstr::representation() const { } if (marshaller_.IsHandle(compiler::ffi::kResultIndex)) { // The call returns a Dart_Handle, from which we need to extract the - // tagged pointer using RawLoadField. + // tagged pointer using LoadField with an appropriate slot. return kUntagged; } return marshaller_.RepInFfiCall(compiler::ffi::kResultIndex); diff --git a/runtime/vm/compiler/backend/il.h b/runtime/vm/compiler/backend/il.h index 4e6b3bfe20f..3559c308430 100644 --- a/runtime/vm/compiler/backend/il.h +++ b/runtime/vm/compiler/backend/il.h @@ -436,7 +436,6 @@ struct InstrAttrs { M(ClosureCall, _) \ M(FfiCall, _) \ M(CCall, kNoGC) \ - M(RawStoreField, kNoGC) \ M(InstanceCall, _) \ M(PolymorphicInstanceCall, _) \ M(DispatchTableCall, _) \ @@ -2809,6 +2808,13 @@ class PhiInstr : public VariadicDefinition { virtual Representation representation() const { return representation_; } + virtual bool MayCreateUnsafeUntaggedPointer() const { + // Unsafe untagged pointers should never escape the basic block in which + // they are defined, so they should never be the input to a Phi node. + // (This is checked in the FlowGraphChecker.) + return false; + } + virtual void set_representation(Representation r) { representation_ = r; } // Only Int32 phis in JIT mode are unboxed optimistically. @@ -6200,45 +6206,6 @@ class CCallInstr : public VariadicDefinition { DISALLOW_COPY_AND_ASSIGN(CCallInstr); }; -// Populates the untagged base + offset outside the heap with a tagged value. -// -// The store must be outside of the heap, does not emit a store barrier. -// For stores in the heap, use StoreIndexedInstr, which emits store barriers. -// -// Does not have a dual RawLoadFieldInstr, because for loads we do not have to -// distinguish between loading from within the heap or outside the heap. -// Use FlowGraphBuilder::RawLoadField. -class RawStoreFieldInstr : public TemplateInstruction<2, NoThrow> { - public: - RawStoreFieldInstr(Value* base, Value* value, int32_t offset) - : offset_(offset) { - SetInputAt(kBase, base); - SetInputAt(kValue, value); - } - - enum { kBase = 0, kValue = 1 }; - - DECLARE_INSTRUCTION(RawStoreField) - - virtual Representation RequiredInputRepresentation(intptr_t idx) const; - virtual bool ComputeCanDeoptimize() const { return false; } - virtual bool HasUnknownSideEffects() const { return false; } - - virtual bool CanEliminate(const BlockEntryInstr* block) const { - return false; - } - -#define FIELD_LIST(F) F(const int32_t, offset_) - - DECLARE_INSTRUCTION_SERIALIZABLE_FIELDS(RawStoreFieldInstr, - TemplateInstruction, - FIELD_LIST) -#undef FIELD_LIST - - private: - DISALLOW_COPY_AND_ASSIGN(RawStoreFieldInstr); -}; - class DebugStepCheckInstr : public TemplateInstruction<0, NoThrow> { public: DebugStepCheckInstr(const InstructionSource& source, @@ -6403,6 +6370,10 @@ class StoreFieldInstr : public TemplateInstruction<2, NoThrow> { bool is_initialization() const { return is_initialization_; } bool ShouldEmitStoreBarrier() const { + if (slot().has_untagged_instance()) { + // The instance is not a Dart object, so not traversed by the GC. + return false; + } if (slot().representation() != kTagged) { // The target field is native and unboxed, so not traversed by the GC. return false; @@ -8127,6 +8098,11 @@ class LoadFieldInstr : public TemplateLoadField<1> { loads_inner_pointer_ = value; } + virtual Representation RequiredInputRepresentation(intptr_t idx) const { + ASSERT_EQUAL(idx, 0); + return slot_.has_untagged_instance() ? kUntagged : kTagged; + } + virtual Representation representation() const; DECLARE_INSTRUCTION(LoadField) diff --git a/runtime/vm/compiler/backend/il_test.cc b/runtime/vm/compiler/backend/il_test.cc index 93b7cca75f9..742782f50b4 100644 --- a/runtime/vm/compiler/backend/il_test.cc +++ b/runtime/vm/compiler/backend/il_test.cc @@ -680,201 +680,6 @@ ISOLATE_UNIT_TEST_CASE(IRTest_DoubleEqualsSmi) { })); } -#ifdef DART_TARGET_OS_WINDOWS -const char* pointer_prefix = "0x"; -#else -const char* pointer_prefix = ""; -#endif - -ISOLATE_UNIT_TEST_CASE(IRTest_RawStoreField) { - InstancePtr ptr = Smi::New(100); - OS::Print("&ptr %p\n", &ptr); - - // clang-format off - auto kScript = Utils::CStringUniquePtr(OS::SCreate(nullptr, R"( - import 'dart:ffi'; - - void myFunction() { - final pointer = Pointer.fromAddress(%s%p); - anotherFunction(); - } - - void anotherFunction() {} - )", pointer_prefix, &ptr), std::free); - // clang-format on - - const auto& root_library = Library::Handle(LoadTestScript(kScript.get())); - Invoke(root_library, "myFunction"); - EXPECT_EQ(Smi::New(100), ptr); - - const auto& my_function = - Function::Handle(GetFunction(root_library, "myFunction")); - - TestPipeline pipeline(my_function, CompilerPass::kJIT); - FlowGraph* flow_graph = pipeline.RunPasses({ - CompilerPass::kComputeSSA, - }); - - Zone* const zone = Thread::Current()->zone(); - - StaticCallInstr* pointer = nullptr; - StaticCallInstr* another_function_call = nullptr; - { - ILMatcher cursor(flow_graph, flow_graph->graph_entry()->normal_entry()); - - EXPECT(cursor.TryMatch({ - kMoveGlob, - {kMatchAndMoveStaticCall, &pointer}, - {kMatchAndMoveStaticCall, &another_function_call}, - })); - } - auto pointer_value = Value(pointer); - auto* const load_field_instr = new (zone) LoadFieldInstr( - &pointer_value, Slot::PointerBase_data(), - InnerPointerAccess::kCannotBeInnerPointer, InstructionSource()); - flow_graph->InsertBefore(another_function_call, load_field_instr, nullptr, - FlowGraph::kValue); - auto load_field_value = Value(load_field_instr); - auto pointer_value2 = Value(pointer); - auto* const raw_store_field_instr = - new (zone) RawStoreFieldInstr(&load_field_value, &pointer_value2, 0); - flow_graph->InsertBefore(another_function_call, raw_store_field_instr, - nullptr, FlowGraph::kEffect); - another_function_call->RemoveFromGraph(); - - { - // Check we constructed the right graph. - ILMatcher cursor(flow_graph, flow_graph->graph_entry()->normal_entry()); - EXPECT(cursor.TryMatch({ - kMoveGlob, - kMatchAndMoveStaticCall, - kMatchAndMoveLoadField, - kMatchAndMoveRawStoreField, - })); - } - - pipeline.RunForcedOptimizedAfterSSAPasses(); - - { -#if !defined(PRODUCT) && !defined(USING_THREAD_SANITIZER) - SetFlagScope sfs(&FLAG_disassemble_optimized, true); -#endif - pipeline.CompileGraphAndAttachFunction(); - } - - // Ensure we can successfully invoke the function. - Invoke(root_library, "myFunction"); - - // Might be garbage if we ran a GC, but should never be a Smi. - EXPECT(!ptr.IsSmi()); -} - -// We do not have a RawLoadFieldInstr, instead we just use LoadIndexed for -// loading from outside the heap. -// -// This test constructs to instructions from FlowGraphBuilder::RawLoadField -// and exercises them to do a load from outside the heap. -ISOLATE_UNIT_TEST_CASE(IRTest_RawLoadField) { - InstancePtr ptr = Smi::New(100); - intptr_t ptr2 = 100; - OS::Print("&ptr %p &ptr2 %p\n", &ptr, &ptr2); - - // clang-format off - auto kScript = Utils::CStringUniquePtr(OS::SCreate(nullptr, R"( - import 'dart:ffi'; - - void myFunction() { - final pointer = Pointer.fromAddress(%s%p); - anotherFunction(); - final pointer2 = Pointer.fromAddress(%s%p); - pointer2.value = 3; - } - - void anotherFunction() {} - )", pointer_prefix, &ptr, pointer_prefix, &ptr2), std::free); - // clang-format on - - const auto& root_library = Library::Handle(LoadTestScript(kScript.get())); - Invoke(root_library, "myFunction"); - EXPECT_EQ(Smi::New(100), ptr); - EXPECT_EQ(3, ptr2); - - const auto& my_function = - Function::Handle(GetFunction(root_library, "myFunction")); - - TestPipeline pipeline(my_function, CompilerPass::kJIT); - FlowGraph* flow_graph = pipeline.RunPasses({ - CompilerPass::kComputeSSA, - }); - - Zone* const zone = Thread::Current()->zone(); - - StaticCallInstr* pointer = nullptr; - StaticCallInstr* another_function_call = nullptr; - StaticCallInstr* pointer2 = nullptr; - StaticCallInstr* pointer2_store = nullptr; - { - ILMatcher cursor(flow_graph, flow_graph->graph_entry()->normal_entry()); - - EXPECT(cursor.TryMatch({ - kMoveGlob, - {kMatchAndMoveStaticCall, &pointer}, - {kMatchAndMoveStaticCall, &another_function_call}, - {kMatchAndMoveStaticCall, &pointer2}, - {kMatchAndMoveStaticCall, &pointer2_store}, - })); - } - auto pointer_value = Value(pointer); - auto* const load_field_instr = new (zone) LoadFieldInstr( - &pointer_value, Slot::PointerBase_data(), - InnerPointerAccess::kCannotBeInnerPointer, InstructionSource()); - flow_graph->InsertBefore(another_function_call, load_field_instr, nullptr, - FlowGraph::kValue); - auto load_field_value = Value(load_field_instr); - auto* const constant_instr = new (zone) UnboxedConstantInstr( - Integer::ZoneHandle(zone, Integer::New(0, Heap::kOld)), kUnboxedIntPtr); - flow_graph->InsertBefore(another_function_call, constant_instr, nullptr, - FlowGraph::kValue); - auto constant_value = Value(constant_instr); - auto* const load_indexed_instr = new (zone) - LoadIndexedInstr(&load_field_value, &constant_value, - /*index_unboxed=*/true, /*index_scale=*/1, kArrayCid, - kAlignedAccess, DeoptId::kNone, InstructionSource()); - flow_graph->InsertBefore(another_function_call, load_indexed_instr, nullptr, - FlowGraph::kValue); - - another_function_call->RemoveFromGraph(); - pointer2_store->InputAt(2)->definition()->ReplaceUsesWith(load_indexed_instr); - - { - // Check we constructed the right graph. - ILMatcher cursor(flow_graph, flow_graph->graph_entry()->normal_entry()); - EXPECT(cursor.TryMatch({ - kMoveGlob, - kMatchAndMoveStaticCall, - kMatchAndMoveLoadField, - kMatchAndMoveUnboxedConstant, - kMatchAndMoveLoadIndexed, - kMatchAndMoveStaticCall, - kMatchAndMoveStaticCall, - })); - } - - pipeline.RunForcedOptimizedAfterSSAPasses(); - - { -#if !defined(PRODUCT) && !defined(USING_THREAD_SANITIZER) - SetFlagScope sfs(&FLAG_disassemble_optimized, true); -#endif - pipeline.CompileGraphAndAttachFunction(); - } - - // Ensure we can successfully invoke the function. - Invoke(root_library, "myFunction"); - EXPECT_EQ(Smi::New(100), ptr); - EXPECT_EQ(100, ptr2); -} - ISOLATE_UNIT_TEST_CASE(IRTest_LoadThread) { // clang-format off auto kScript = R"( diff --git a/runtime/vm/compiler/backend/memory_copy_test.cc b/runtime/vm/compiler/backend/memory_copy_test.cc index 7f84997c871..13f2ab788e6 100644 --- a/runtime/vm/compiler/backend/memory_copy_test.cc +++ b/runtime/vm/compiler/backend/memory_copy_test.cc @@ -14,7 +14,11 @@ namespace dart { -extern const char* pointer_prefix; +#ifdef DART_TARGET_OS_WINDOWS +const char* pointer_prefix = "0x"; +#else +const char* pointer_prefix = ""; +#endif static constexpr intptr_t kMemoryTestLength = 1024; static constexpr uint8_t kUnInitialized = 0xFE; diff --git a/runtime/vm/compiler/backend/range_analysis.cc b/runtime/vm/compiler/backend/range_analysis.cc index fccea17f747..ffe8f5b5d1b 100644 --- a/runtime/vm/compiler/backend/range_analysis.cc +++ b/runtime/vm/compiler/backend/range_analysis.cc @@ -2861,53 +2861,12 @@ void LoadFieldInstr::InferRange(RangeAnalysis* analysis, Range* range) { Definition::InferRange(analysis, range); break; - case Slot::Kind::kReceivePort_send_port: - case Slot::Kind::kReceivePort_handler: - case Slot::Kind::kLinkedHashBase_index: - case Slot::Kind::kImmutableLinkedHashBase_index: - case Slot::Kind::kLinkedHashBase_data: - case Slot::Kind::kImmutableLinkedHashBase_data: - case Slot::Kind::kGrowableObjectArray_data: - case Slot::Kind::kContext_parent: case Slot::Kind::kTypeArguments: - case Slot::Kind::kArray_type_arguments: - case Slot::Kind::kClosure_context: - case Slot::Kind::kClosure_delayed_type_arguments: - case Slot::Kind::kClosure_function: - case Slot::Kind::kClosure_function_type_arguments: - case Slot::Kind::kClosure_instantiator_type_arguments: - case Slot::Kind::kFinalizer_callback: - case Slot::Kind::kFinalizer_type_arguments: - case Slot::Kind::kFinalizerBase_all_entries: - case Slot::Kind::kFinalizerBase_detachments: - case Slot::Kind::kFinalizerBase_entries_collected: - case Slot::Kind::kFinalizerEntry_detach: - case Slot::Kind::kFinalizerEntry_finalizer: - case Slot::Kind::kFinalizerEntry_next: - case Slot::Kind::kFinalizerEntry_token: - case Slot::Kind::kFinalizerEntry_value: - case Slot::Kind::kNativeFinalizer_callback: - case Slot::Kind::kFunction_data: - case Slot::Kind::kFunction_signature: - case Slot::Kind::kFunctionType_named_parameter_names: - case Slot::Kind::kFunctionType_parameter_types: - case Slot::Kind::kFunctionType_type_parameters: - case Slot::Kind::kInstance_native_fields_array: - case Slot::Kind::kSuspendState_function_data: - case Slot::Kind::kSuspendState_then_callback: - case Slot::Kind::kSuspendState_error_callback: - case Slot::Kind::kTypedDataView_typed_data: case Slot::Kind::kTypeArgumentsIndex: - case Slot::Kind::kTypeParameters_names: - case Slot::Kind::kTypeParameters_flags: - case Slot::Kind::kTypeParameters_bounds: - case Slot::Kind::kTypeParameters_defaults: - case Slot::Kind::kUnhandledException_exception: - case Slot::Kind::kUnhandledException_stacktrace: - case Slot::Kind::kWeakProperty_key: - case Slot::Kind::kWeakProperty_value: - case Slot::Kind::kWeakReference_target: - case Slot::Kind::kWeakReference_type_arguments: +#define NATIVE_SLOT_CASE(ClassName, __, FieldName, ___, ____) \ + case Slot::Kind::k##ClassName##_##FieldName: + NOT_INT_NATIVE_SLOTS_LIST(NATIVE_SLOT_CASE) +#undef NATIVE_SLOT_CASE // Not an integer valued field. UNREACHABLE(); break; @@ -2917,22 +2876,13 @@ void LoadFieldInstr::InferRange(RangeAnalysis* analysis, Range* range) { UNREACHABLE(); break; -#define UNBOXED_NATIVE_NONADDRESS_SLOT_CASE(Class, Untagged, Field, Rep, \ - IsFinal) \ +#define UNBOXED_NATIVE_SLOT_CASE(Class, __, Field, ___, ____) \ case Slot::Kind::k##Class##_##Field: - UNBOXED_NATIVE_NONADDRESS_SLOTS_LIST(UNBOXED_NATIVE_NONADDRESS_SLOT_CASE) -#undef UNBOXED_NATIVE_NONADDRESS_SLOT_CASE + UNBOXED_NATIVE_SLOTS_LIST(UNBOXED_NATIVE_SLOT_CASE) +#undef UNBOXED_NATIVE_SLOT_CASE *range = Range::Full(slot().representation()); break; -#define UNBOXED_NATIVE_ADDRESS_SLOT_CASE(Class, Untagged, Field, MayMove, \ - IsFinal) \ - case Slot::Kind::k##Class##_##Field: - UNBOXED_NATIVE_ADDRESS_SLOTS_LIST(UNBOXED_NATIVE_ADDRESS_SLOT_CASE) -#undef UNBOXED_NATIVE_ADDRESS_SLOT_CASE - UNREACHABLE(); - break; - case Slot::Kind::kClosure_hash: case Slot::Kind::kLinkedHashBase_hash_mask: case Slot::Kind::kLinkedHashBase_used_data: diff --git a/runtime/vm/compiler/backend/redundancy_elimination.cc b/runtime/vm/compiler/backend/redundancy_elimination.cc index a409ab2d2c6..b650e4396a4 100644 --- a/runtime/vm/compiler/backend/redundancy_elimination.cc +++ b/runtime/vm/compiler/backend/redundancy_elimination.cc @@ -2217,8 +2217,8 @@ class LoadOptimizer : public ValueObject { const intptr_t pos = alloc->InputForSlot(*slot); if (pos != -1) { forward_def = alloc->InputAt(pos)->definition(); - } else if (slot->is_unboxed()) { - // Unboxed fields that are not provided as an input should not + } else if (!slot->is_tagged()) { + // Fields that do not contain tagged values should not // have a tagged null value forwarded for them, similar to // payloads of typed data arrays. continue; diff --git a/runtime/vm/compiler/backend/slot.cc b/runtime/vm/compiler/backend/slot.cc index 8f05413a71a..cd2e0eaa78a 100644 --- a/runtime/vm/compiler/backend/slot.cc +++ b/runtime/vm/compiler/backend/slot.cc @@ -70,8 +70,8 @@ Slot* SlotCache::CreateNativeSlot(Slot::Kind kind) { switch (kind) { #define FIELD_FINAL true #define FIELD_VAR false -#define DEFINE_NULLABLE_BOXED_NATIVE_FIELD(ClassName, UnderlyingType, \ - FieldName, cid, mutability) \ +#define DEFINE_NULLABLE_TAGGED_NATIVE_DART_FIELD(ClassName, UnderlyingType, \ + FieldName, cid, mutability) \ case Slot::Kind::k##ClassName##_##FieldName: \ return new (zone_) Slot( \ Slot::Kind::k##ClassName##_##FieldName, \ @@ -84,12 +84,13 @@ Slot* SlotCache::CreateNativeSlot(Slot::Kind kind) { k##cid##Cid, nullptr), \ kTagged); - NULLABLE_BOXED_NATIVE_SLOTS_LIST(DEFINE_NULLABLE_BOXED_NATIVE_FIELD) + NULLABLE_TAGGED_NATIVE_DART_SLOTS_LIST( + DEFINE_NULLABLE_TAGGED_NATIVE_DART_FIELD) -#undef DEFINE_NULLABLE_BOXED_NATIVE_FIELD +#undef DEFINE_NULLABLE_TAGGED_NATIVE_DART_FIELD -#define DEFINE_NONNULLABLE_BOXED_NATIVE_FIELD(ClassName, UnderlyingType, \ - FieldName, cid, mutability) \ +#define DEFINE_NONNULLABLE_TAGGED_NATIVE_DART_FIELD( \ + ClassName, UnderlyingType, FieldName, cid, mutability) \ case Slot::Kind::k##ClassName##_##FieldName: \ return new (zone_) Slot( \ Slot::Kind::k##ClassName##_##FieldName, \ @@ -102,40 +103,95 @@ Slot* SlotCache::CreateNativeSlot(Slot::Kind kind) { CompileType::kCannotBeSentinel, k##cid##Cid, nullptr), \ kTagged); - NONNULLABLE_BOXED_NATIVE_SLOTS_LIST(DEFINE_NONNULLABLE_BOXED_NATIVE_FIELD) + NONNULLABLE_INT_TAGGED_NATIVE_DART_SLOTS_LIST( + DEFINE_NONNULLABLE_TAGGED_NATIVE_DART_FIELD) + NONNULLABLE_NONINT_TAGGED_NATIVE_DART_SLOTS_LIST( + DEFINE_NONNULLABLE_TAGGED_NATIVE_DART_FIELD) -#undef DEFINE_NONNULLABLE_BOXED_NATIVE_FIELD +#undef DEFINE_NONNULLABLE_TAGGED_NATIVE_DART_FIELD -#define DEFINE_UNBOXED_NATIVE_NONADDRESS_FIELD( \ - ClassName, UnderlyingType, FieldName, representation, mutability) \ +#define DEFINE_UNBOXED_NATIVE_DART_FIELD(ClassName, UnderlyingType, FieldName, \ + representation, mutability) \ case Slot::Kind::k##ClassName##_##FieldName: \ return new (zone_) \ Slot(Slot::Kind::k##ClassName##_##FieldName, \ Slot::IsImmutableBit::encode(FIELD_##mutability) | \ - Slot::IsUnboxedBit::encode(true), \ + Slot::IsNonTaggedBit::encode(true), \ compiler::target::ClassName::FieldName##_offset(), \ #ClassName "." #FieldName, \ CompileType::FromUnboxedRepresentation(kUnboxed##representation), \ kUnboxed##representation); - UNBOXED_NATIVE_NONADDRESS_SLOTS_LIST(DEFINE_UNBOXED_NATIVE_NONADDRESS_FIELD) + UNBOXED_NATIVE_DART_SLOTS_LIST(DEFINE_UNBOXED_NATIVE_DART_FIELD) -#undef DEFINE_UNBOXED_NATIVE_NONADDRESS_FIELD +#undef DEFINE_UNBOXED_NATIVE_DART_FIELD -#define DEFINE_UNBOXED_NATIVE_ADDRESS_FIELD(ClassName, UnderlyingType, \ - FieldName, GcMayMove, mutability) \ +#define DEFINE_UNTAGGED_NATIVE_DART_FIELD(ClassName, UnderlyingType, \ + FieldName, GcMayMove, mutability) \ case Slot::Kind::k##ClassName##_##FieldName: \ return new (zone_) \ Slot(Slot::Kind::k##ClassName##_##FieldName, \ Slot::IsImmutableBit::encode(FIELD_##mutability) | \ Slot::MayContainInnerPointerBit::encode(GcMayMove) | \ - Slot::IsUnboxedBit::encode(true), \ + Slot::IsNonTaggedBit::encode(true), \ compiler::target::ClassName::FieldName##_offset(), \ #ClassName "." #FieldName, CompileType::Object(), kUntagged); - UNBOXED_NATIVE_ADDRESS_SLOTS_LIST(DEFINE_UNBOXED_NATIVE_ADDRESS_FIELD) + UNTAGGED_NATIVE_DART_SLOTS_LIST(DEFINE_UNTAGGED_NATIVE_DART_FIELD) + +#undef DEFINE_UNTAGGED_NATIVE_DART_FIELD + +#define DEFINE_NULLABLE_TAGGED_NATIVE_NONDART_FIELD(ClassName, __, FieldName, \ + cid, mutability) \ + case Slot::Kind::k##ClassName##_##FieldName: \ + return new (zone_) Slot( \ + Slot::Kind::k##ClassName##_##FieldName, \ + Slot::IsImmutableBit::encode(FIELD_##mutability) | \ + Slot::HasUntaggedInstanceBit::encode(true), \ + compiler::target::ClassName::FieldName##_offset(), \ + #ClassName "." #FieldName, \ + CompileType(CompileType::kCanBeNull, CompileType::kCannotBeSentinel, \ + k##cid##Cid, nullptr), \ + kTagged); + + NULLABLE_TAGGED_NATIVE_NONDART_SLOTS_LIST( + DEFINE_NULLABLE_TAGGED_NATIVE_NONDART_FIELD) + +#undef DEFINE_NULLABLE_TAGGED_NONDART_FIELD + +#define DEFINE_UNBOXED_NATIVE_NONDART_FIELD(ClassName, __, FieldName, \ + representation, mutability) \ + case Slot::Kind::k##ClassName##_##FieldName: \ + return new (zone_) \ + Slot(Slot::Kind::k##ClassName##_##FieldName, \ + Slot::IsImmutableBit::encode(FIELD_##mutability) | \ + Slot::IsNonTaggedBit::encode(true) | \ + Slot::HasUntaggedInstanceBit::encode(true), \ + compiler::target::ClassName::FieldName##_offset(), \ + #ClassName "." #FieldName, \ + CompileType::FromUnboxedRepresentation(kUnboxed##representation), \ + kUnboxed##representation); + + UNBOXED_NATIVE_NONDART_SLOTS_LIST(DEFINE_UNBOXED_NATIVE_NONDART_FIELD) + +#undef DEFINE_UNBOXED_NATIVE_NONDART_FIELD + +#define DEFINE_UNTAGGED_NATIVE_NONDART_FIELD(ClassName, __, FieldName, \ + gc_may_move, mutability) \ + case Slot::Kind::k##ClassName##_##FieldName: \ + return new (zone_) \ + Slot(Slot::Kind::k##ClassName##_##FieldName, \ + Slot::IsImmutableBit::encode(FIELD_##mutability) | \ + Slot::MayContainInnerPointerBit::encode(gc_may_move) | \ + Slot::IsNonTaggedBit::encode(true) | \ + Slot::HasUntaggedInstanceBit::encode(true), \ + compiler::target::ClassName::FieldName##_offset(), \ + #ClassName "." #FieldName, CompileType::Object(), kUntagged); + + UNTAGGED_NATIVE_NONDART_SLOTS_LIST(DEFINE_UNTAGGED_NATIVE_NONDART_FIELD) + +#undef DEFINE_UNTAGGED_NATIVE_NONDART_FIELD -#undef DEFINE_UNBOXED_NATIVE_NONADDRESS_FIELD #undef FIELD_VAR #undef FIELD_FINAL default: @@ -158,16 +214,17 @@ bool Slot::IsImmutableLengthSlot() const { return false; // Not length loads. -#define UNBOXED_NATIVE_SLOT_CASE(Class, Untagged, Field, Rep, IsFinal) \ + case Slot::Kind::kArrayElement: + case Slot::Kind::kCapturedVariable: + case Slot::Kind::kDartField: + case Slot::Kind::kRecordField: + case Slot::Kind::kTypeArguments: + case Slot::Kind::kTypeArgumentsIndex: +#define NOT_TAGGED_INT_NATIVE_SLOT_CASE(Class, __, Field, ___, ____) \ case Slot::Kind::k##Class##_##Field: - UNBOXED_NATIVE_SLOTS_LIST(UNBOXED_NATIVE_SLOT_CASE) -#undef UNBOXED_NATIVE_SLOT_CASE - case Slot::Kind::kReceivePort_send_port: - case Slot::Kind::kReceivePort_handler: - case Slot::Kind::kLinkedHashBase_index: - case Slot::Kind::kImmutableLinkedHashBase_index: - case Slot::Kind::kLinkedHashBase_data: - case Slot::Kind::kImmutableLinkedHashBase_data: + NOT_INT_NATIVE_SLOTS_LIST(NOT_TAGGED_INT_NATIVE_SLOT_CASE) + UNBOXED_NATIVE_SLOTS_LIST(NOT_TAGGED_INT_NATIVE_SLOT_CASE) +#undef NONTAGGED_NATIVE_DART_SLOT_CASE case Slot::Kind::kLinkedHashBase_hash_mask: case Slot::Kind::kLinkedHashBase_used_data: case Slot::Kind::kLinkedHashBase_deleted_keys: @@ -175,56 +232,11 @@ bool Slot::IsImmutableLengthSlot() const { case Slot::Kind::kArgumentsDescriptor_positional_count: case Slot::Kind::kArgumentsDescriptor_count: case Slot::Kind::kArgumentsDescriptor_size: - case Slot::Kind::kArrayElement: - case Slot::Kind::kInstance_native_fields_array: - case Slot::Kind::kTypeArguments: case Slot::Kind::kTypeArguments_hash: case Slot::Kind::kTypedDataView_offset_in_bytes: - case Slot::Kind::kTypedDataView_typed_data: - case Slot::Kind::kGrowableObjectArray_data: - case Slot::Kind::kArray_type_arguments: - case Slot::Kind::kContext_parent: - case Slot::Kind::kClosure_context: - case Slot::Kind::kClosure_delayed_type_arguments: - case Slot::Kind::kClosure_function: - case Slot::Kind::kClosure_function_type_arguments: - case Slot::Kind::kClosure_instantiator_type_arguments: case Slot::Kind::kClosure_hash: - case Slot::Kind::kCapturedVariable: - case Slot::Kind::kDartField: - case Slot::Kind::kFinalizer_callback: - case Slot::Kind::kFinalizer_type_arguments: - case Slot::Kind::kFinalizerBase_all_entries: - case Slot::Kind::kFinalizerBase_detachments: - case Slot::Kind::kFinalizerBase_entries_collected: - case Slot::Kind::kFinalizerEntry_detach: - case Slot::Kind::kFinalizerEntry_finalizer: - case Slot::Kind::kFinalizerEntry_next: - case Slot::Kind::kFinalizerEntry_token: - case Slot::Kind::kFinalizerEntry_value: - case Slot::Kind::kNativeFinalizer_callback: - case Slot::Kind::kFunction_data: - case Slot::Kind::kFunction_signature: - case Slot::Kind::kFunctionType_named_parameter_names: - case Slot::Kind::kFunctionType_parameter_types: - case Slot::Kind::kFunctionType_type_parameters: - case Slot::Kind::kRecordField: case Slot::Kind::kRecord_shape: - case Slot::Kind::kSuspendState_function_data: - case Slot::Kind::kSuspendState_then_callback: - case Slot::Kind::kSuspendState_error_callback: - case Slot::Kind::kTypeArgumentsIndex: case Slot::Kind::kAbstractType_hash: - case Slot::Kind::kTypeParameters_names: - case Slot::Kind::kTypeParameters_flags: - case Slot::Kind::kTypeParameters_bounds: - case Slot::Kind::kTypeParameters_defaults: - case Slot::Kind::kUnhandledException_exception: - case Slot::Kind::kUnhandledException_stacktrace: - case Slot::Kind::kWeakProperty_key: - case Slot::Kind::kWeakProperty_value: - case Slot::Kind::kWeakReference_target: - case Slot::Kind::kWeakReference_type_arguments: return false; } UNREACHABLE(); @@ -425,7 +437,7 @@ const Slot& Slot::Get(const Field& field, IsGuardedBit::encode(used_guarded_state) | IsCompressedBit::encode( compiler::target::Class::HasCompressedPointers(owner)) | - IsUnboxedBit::encode(is_unboxed), + IsNonTaggedBit::encode(is_unboxed), compiler::target::Field::OffsetOf(field), &field, type, rep, field_guard_state); diff --git a/runtime/vm/compiler/backend/slot.h b/runtime/vm/compiler/backend/slot.h index 6a0e879dfed..dba67446739 100644 --- a/runtime/vm/compiler/backend/slot.h +++ b/runtime/vm/compiler/backend/slot.h @@ -38,8 +38,8 @@ class LocalScope; class LocalVariable; class ParsedFunction; -// The list of slots that correspond to nullable boxed fields of native objects -// in the following format: +// The list of slots that correspond to nullable boxed fields of native +// Dart objects in the following format: // // V(class_name, underlying_type, field_name, exact_type, FINAL|VAR) // @@ -51,7 +51,7 @@ class ParsedFunction; // - the last component specifies whether field behaves like a final field // (i.e. initialized once at construction time and does not change after // that) or like a non-final field. -#define NULLABLE_BOXED_NATIVE_SLOTS_LIST(V) \ +#define NULLABLE_TAGGED_NATIVE_DART_SLOTS_LIST(V) \ V(Array, UntaggedArray, type_arguments, TypeArguments, FINAL) \ V(Finalizer, UntaggedFinalizer, type_arguments, TypeArguments, FINAL) \ V(FinalizerBase, UntaggedFinalizerBase, all_entries, Set, VAR) \ @@ -87,7 +87,7 @@ class ParsedFunction; V(WeakReference, UntaggedWeakReference, type_arguments, TypeArguments, FINAL) // The list of slots that correspond to non-nullable boxed fields of native -// objects in the following format: +// Dart objects that contain integers in the following format: // // V(class_name, underlying_type, field_name, exact_type, FINAL|VAR) // @@ -99,26 +99,13 @@ class ParsedFunction; // - the last component specifies whether field behaves like a final field // (i.e. initialized once at construction time and does not change after // that) or like a non-final field. -#define NONNULLABLE_BOXED_NATIVE_SLOTS_LIST(V) \ +#define NONNULLABLE_INT_TAGGED_NATIVE_DART_SLOTS_LIST(V) \ V(Array, UntaggedArray, length, Smi, FINAL) \ - V(Closure, UntaggedClosure, function, Function, FINAL) \ - V(Closure, UntaggedClosure, context, Dynamic, FINAL) \ - V(Closure, UntaggedClosure, hash, Context, VAR) \ - V(Finalizer, UntaggedFinalizer, callback, Closure, FINAL) \ - V(NativeFinalizer, UntaggedFinalizer, callback, Pointer, FINAL) \ - V(Function, UntaggedFunction, data, Dynamic, FINAL) \ - V(FunctionType, UntaggedFunctionType, named_parameter_names, Array, FINAL) \ - V(FunctionType, UntaggedFunctionType, parameter_types, Array, FINAL) \ + V(Closure, UntaggedClosure, hash, Smi, VAR) \ V(GrowableObjectArray, UntaggedGrowableObjectArray, length, Smi, VAR) \ - V(GrowableObjectArray, UntaggedGrowableObjectArray, data, Array, VAR) \ V(TypedDataBase, UntaggedTypedDataBase, length, Smi, FINAL) \ V(TypedDataView, UntaggedTypedDataView, offset_in_bytes, Smi, FINAL) \ - V(TypedDataView, UntaggedTypedDataView, typed_data, Dynamic, FINAL) \ V(String, UntaggedString, length, Smi, FINAL) \ - V(LinkedHashBase, UntaggedLinkedHashBase, index, TypedDataUint32Array, VAR) \ - V(LinkedHashBase, UntaggedLinkedHashBase, data, Array, VAR) \ - V(ImmutableLinkedHashBase, UntaggedLinkedHashBase, data, ImmutableArray, \ - FINAL) \ V(LinkedHashBase, UntaggedLinkedHashBase, hash_mask, Smi, VAR) \ V(LinkedHashBase, UntaggedLinkedHashBase, used_data, Smi, VAR) \ V(LinkedHashBase, UntaggedLinkedHashBase, deleted_keys, Smi, VAR) \ @@ -129,24 +116,41 @@ class ParsedFunction; V(Record, UntaggedRecord, shape, Smi, FINAL) \ V(TypeArguments, UntaggedTypeArguments, hash, Smi, VAR) \ V(TypeArguments, UntaggedTypeArguments, length, Smi, FINAL) \ - V(AbstractType, UntaggedTypeArguments, hash, Smi, VAR) \ + V(AbstractType, UntaggedTypeArguments, hash, Smi, VAR) + +// The list of slots that correspond to non-nullable boxed fields of native +// Dart objects that do not contain integers in the following format: +// +// V(class_name, underlying_type, field_name, exact_type, FINAL|VAR) +// +// - class_name and field_name specify the name of the host class and the name +// of the field respectively; +// - underlying_type: the Raw class which holds the field; +// - exact_type specifies exact type of the field (any load from this field +// would only yield instances of this type); +// - the last component specifies whether field behaves like a final field +// (i.e. initialized once at construction time and does not change after +// that) or like a non-final field. +#define NONNULLABLE_NONINT_TAGGED_NATIVE_DART_SLOTS_LIST(V) \ + V(Closure, UntaggedClosure, function, Function, FINAL) \ + V(Closure, UntaggedClosure, context, Dynamic, FINAL) \ + V(Finalizer, UntaggedFinalizer, callback, Closure, FINAL) \ + V(NativeFinalizer, UntaggedFinalizer, callback, Pointer, FINAL) \ + V(Function, UntaggedFunction, data, Dynamic, FINAL) \ + V(FunctionType, UntaggedFunctionType, named_parameter_names, Array, FINAL) \ + V(FunctionType, UntaggedFunctionType, parameter_types, Array, FINAL) \ + V(GrowableObjectArray, UntaggedGrowableObjectArray, data, Array, VAR) \ + V(TypedDataView, UntaggedTypedDataView, typed_data, Dynamic, FINAL) \ + V(LinkedHashBase, UntaggedLinkedHashBase, index, TypedDataUint32Array, VAR) \ + V(LinkedHashBase, UntaggedLinkedHashBase, data, Array, VAR) \ + V(ImmutableLinkedHashBase, UntaggedLinkedHashBase, data, ImmutableArray, \ + FINAL) \ V(TypeParameters, UntaggedTypeParameters, names, Array, FINAL) \ V(UnhandledException, UntaggedUnhandledException, exception, Dynamic, FINAL) \ V(UnhandledException, UntaggedUnhandledException, stacktrace, Dynamic, FINAL) -// Don't use Object or Instance, use Dynamic instead. The cid here should -// correspond to an exact type or Dynamic, not a static type. -// If we ever get a field of which the exact type is Instance (not a subtype), -// update the check below. -#define FOR_EACH_NATIVE_SLOT(_, __, ___, field_type, ____) \ - static_assert(k##field_type##Cid != kObjectCid); \ - static_assert(k##field_type##Cid != kInstanceCid); -NULLABLE_BOXED_NATIVE_SLOTS_LIST(FOR_EACH_NATIVE_SLOT) -NONNULLABLE_BOXED_NATIVE_SLOTS_LIST(FOR_EACH_NATIVE_SLOT) -#undef FOR_EACH_NATIVE_SLOT - -// List of slots that correspond to unboxed fields of native objects that -// do not contain untagged addresses in the following format: +// List of slots that correspond to fields of native objects that contain +// unboxed values in the following format: // // V(class_name, underlying_type, field_name, representation, FINAL|VAR) // @@ -163,7 +167,7 @@ NONNULLABLE_BOXED_NATIVE_SLOTS_LIST(FOR_EACH_NATIVE_SLOT) // // Note: Currently LoadFieldInstr::IsImmutableLengthLoad() assumes that no // unboxed slots represent length loads. -#define UNBOXED_NATIVE_NONADDRESS_SLOTS_LIST(V) \ +#define UNBOXED_NATIVE_DART_SLOTS_LIST(V) \ V(AbstractType, UntaggedAbstractType, flags, Uint32, FINAL) \ V(ClosureData, UntaggedClosureData, packed_fields, Uint32, FINAL) \ V(FinalizerEntry, UntaggedFinalizerEntry, external_size, IntPtr, VAR) \ @@ -174,16 +178,16 @@ NONNULLABLE_BOXED_NATIVE_SLOTS_LIST(FOR_EACH_NATIVE_SLOT) FINAL) \ V(SubtypeTestCache, UntaggedSubtypeTestCache, num_inputs, Uint32, FINAL) -// Unboxed native slots containing untagged addresses that do not exist -// in JIT mode. See UNBOXED_NATIVE_ADDRESS_SLOTS_LIST for the format. +// Native slots containing untagged addresses that do not exist in JIT mode. +// See UNTAGGED_NATIVE_DART_SLOTS_LIST for the format. #if defined(DART_PRECOMPILER) && !defined(TARGET_ARCH_IA32) -#define AOT_ONLY_UNBOXED_NATIVE_ADDRESS_SLOTS_LIST(V) \ +#define AOT_ONLY_UNTAGGED_NATIVE_DART_SLOTS_LIST(V) \ V(Closure, UntaggedClosure, entry_point, false, FINAL) #else -#define AOT_ONLY_UNBOXED_NATIVE_ADDRESS_SLOTS_LIST(V) +#define AOT_ONLY_UNTAGGED_NATIVE_DART_SLOTS_LIST(V) #endif -// List of slots that correspond to unboxed fields of native objects containing +// List of slots that correspond to fields of native Dart objects containing // untagged addresses in the following format: // // V(class_name, underlying_type, field_name, gc_may_move, FINAL|VAR) @@ -198,35 +202,209 @@ NONNULLABLE_BOXED_NATIVE_SLOTS_LIST(FOR_EACH_NATIVE_SLOT) // (i.e. initialized once at construction time and does not change after // that) or like a non-final field. // -// Note: As the underlying field is unboxed, these slots cannot be nullable. +// Note: As the underlying field is untagged, these slots cannot be nullable. // -// Note: All slots for unboxed fields that contain untagged addresses are given -// the kUntagged representation, and so a value loaded from these fields must -// be converted explicitly to an unboxed integer representation for any -// pointer arithmetic before use, and an unboxed integer must be converted -// explicitly to an untagged address before being stored to these fields. -// -// Note: Currently LoadFieldInstr::IsImmutableLengthLoad() assumes that no -// unboxed slots represent length loads. -#define UNBOXED_NATIVE_ADDRESS_SLOTS_LIST(V) \ - AOT_ONLY_UNBOXED_NATIVE_ADDRESS_SLOTS_LIST(V) \ +// Note: All slots for fields that contain untagged addresses are given +// the kUntagged representation. +#define UNTAGGED_NATIVE_DART_SLOTS_LIST(V) \ + AOT_ONLY_UNTAGGED_NATIVE_DART_SLOTS_LIST(V) \ V(Function, UntaggedFunction, entry_point, false, FINAL) \ V(FinalizerBase, UntaggedFinalizerBase, isolate, false, VAR) \ V(PointerBase, UntaggedPointerBase, data, true, VAR) -// For uses that do not need to know whether a given slot may contain an -// inner pointer to a GC-able object or not. (Generally, such users only need -// the class name, the underlying type, and/or the field name.) -#define UNBOXED_NATIVE_SLOTS_LIST(V) \ - UNBOXED_NATIVE_NONADDRESS_SLOTS_LIST(V) UNBOXED_NATIVE_ADDRESS_SLOTS_LIST(V) +// List of slots that correspond to fields of non-Dart objects containing +// tagged addresses of Dart objects in the following format: +// +// V(class_name, _, field_name, exact_type, FINAL|VAR) +// +// - class_name and field_name specify the name of the host class and the name +// of the field respectively; +// - exact_type specifies exact type of the field (any load from this field +// would only yield instances of this type); +// - the last component specifies whether field behaves like a final field +// (i.e. initialized once at construction time and does not change after +// that) or like a non-final field. +// +// Note: Currently LoadFieldInstr::IsImmutableLengthLoad() assumes that no +// slots of non-Dart values represent length loads. +#define NULLABLE_TAGGED_NATIVE_NONDART_SLOTS_LIST(V) \ + V(Isolate, _, finalizers, GrowableObjectArray, VAR) \ + V(LocalHandle, _, ptr, Dynamic, VAR) \ + V(ObjectStore, _, record_field_names, Array, VAR) \ + V(PersistentHandle, _, ptr, Dynamic, VAR) -// For uses that do not need the exact_type (boxed) or representation (unboxed) -// or whether a boxed native slot is nullable. (Generally, such users only need -// the class name, the underlying type, and/or the field name.) +// List of slots that correspond to fields of non-Dart objects containing +// unboxed values in the following format: +// +// V(class_name, _, field_name, representation, FINAL|VAR) +// +// - class_name and field_name specify the name of the host class and the name +// of the field respectively; +// - representation specifies the representation of the bits stored within +// the unboxed field (minus the kUnboxed prefix); +// - the last component specifies whether field behaves like a final field +// (i.e. initialized once at construction time and does not change after +// that) or like a non-final field. +// +// Note: As the underlying field is unboxed, these slots cannot be nullable. +// +// Note: Currently LoadFieldInstr::IsImmutableLengthLoad() assumes that no +// slots of non-Dart values represent length loads. +#define UNBOXED_NATIVE_NONDART_SLOTS_LIST(V) \ + V(StreamInfo, _, enabled, IntPtr, VAR) + +// List of slots that correspond to fields of non-Dart objects containing +// untagged addresses in the following format: +// +// V(class_name, _, field_name, gc_may_move, FINAL|VAR) +// +// - class_name and field_name specify the name of the host class and the name +// of the field respectively; +// - gc_may_move: whether the untagged address contained in this field is a +// pointer to memory that may be moved by the GC, which means a value loaded +// from this field is invalidated by any instruction that can cause GC; +// - the last component specifies whether field behaves like a final field +// (i.e. initialized once at construction time and does not change after +// that) or like a non-final field. +// +// Note: As the underlying field is untagged, these slots cannot be nullable. +// +// Note: All slots for fields that contain untagged addresses are given +// the kUntagged representation. +// +// Note: while Thread::isolate_ and IsolateGroup::object_store_ aren't const +// fields, they should never change during a given execution of the code +// generated for a function and the compiler only does intra-procedural +// load optimizations. +#define UNTAGGED_NATIVE_NONDART_SLOTS_LIST(V) \ + V(IsolateGroup, _, object_store, false, FINAL) \ + V(Thread, _, api_top_scope, false, VAR) \ + V(Thread, _, isolate, false, FINAL) \ + V(Thread, _, isolate_group, false, FINAL) \ + V(Thread, _, service_extension_stream, false, FINAL) + +// No untagged slot on a non-Dart object should contain a GC-movable address. +// The gc_may_move field is only there so that any code that operates on +// UNTAGGED_NATIVE_SLOTS_LIST can use that field as desired. +#define CHECK_NATIVE_NONDART_SLOT(__, ___, ____, gc_may_move, _____) \ + static_assert(!gc_may_move); +UNTAGGED_NATIVE_NONDART_SLOTS_LIST(CHECK_NATIVE_NONDART_SLOT) +#undef CHECK_NATIVE_NONDART_SLOT + +// For uses that need any native slot that contain an unboxed integer. Such uses +// can only use the following arguments for each entry: +// V(class_name, _, field_name, rep, FINAL|VAR) +#define UNBOXED_NATIVE_SLOTS_LIST(V) \ + UNBOXED_NATIVE_DART_SLOTS_LIST(V) \ + UNBOXED_NATIVE_NONDART_SLOTS_LIST(V) + +// For uses that need any native slot that contain an untagged address. Such +// uses can only use the following arguments for each entry: +// V(class_name, _, field_name, gc_may_move, FINAL|VAR) +#define UNTAGGED_NATIVE_SLOTS_LIST(V) \ + UNTAGGED_NATIVE_DART_SLOTS_LIST(V) \ + UNTAGGED_NATIVE_NONDART_SLOTS_LIST(V) + +// For uses that need any native slot that does not contain a Dart object. Such +// uses can only use the following arguments for each entry: +// V(class_name, _, field_name, _, FINAL|VAR) +#define NOT_TAGGED_NATIVE_SLOTS_LIST(V) \ + UNBOXED_NATIVE_SLOTS_LIST(V) \ + UNTAGGED_NATIVE_SLOTS_LIST(V) + +// For uses that need any native slot that is guaranteed to contain a tagged +// integer. Such uses can only use the following arguments for each entry: +// V(class_name, _, field_name, exact_type, FINAL|VAR) +#define TAGGED_INT_NATIVE_SLOTS_LIST(V) \ + NONNULLABLE_INT_TAGGED_NATIVE_DART_SLOTS_LIST(V) + +// For uses that need any native slot that contains a tagged object which is not +// guaranteed to be a integer. This includes nullable integer slots, since +// those slots may return a non-integer value (null). Such uses can +// only use the following arguments for each entry: +// V(class_name, _, field_name, exact_type, FINAL|VAR) +#define TAGGED_NONINT_NATIVE_SLOTS_LIST(V) \ + NULLABLE_TAGGED_NATIVE_DART_SLOTS_LIST(V) \ + NONNULLABLE_NONINT_TAGGED_NATIVE_DART_SLOTS_LIST(V) \ + NULLABLE_TAGGED_NATIVE_NONDART_SLOTS_LIST(V) + +// For uses that need any native slot that is not guaranteed to contain an +// integer, whether a Dart object or unboxed. Such uses can only use the +// following arguments for each entry: +// V(class_name, _, field_name, _, FINAL|VAR) +#define NOT_INT_NATIVE_SLOTS_LIST(V) \ + TAGGED_NONINT_NATIVE_SLOTS_LIST(V) \ + UNTAGGED_NATIVE_SLOTS_LIST(V) + +// For uses that need any native slot on Dart objects that contains a Dart +// object (e.g., for write barrier purposes). Such uses can use the following +// arguments for each entry: +// V(class_name, underlying_class, field_name, exact_type, FINAL|VAR) +#define TAGGED_NATIVE_DART_SLOTS_LIST(V) \ + NULLABLE_TAGGED_NATIVE_DART_SLOTS_LIST(V) \ + NONNULLABLE_INT_TAGGED_NATIVE_DART_SLOTS_LIST(V) \ + NONNULLABLE_NONINT_TAGGED_NATIVE_DART_SLOTS_LIST(V) + +// For uses that need any native slot that is not on a Dart object or does +// not contain a Dart object (e.g., for write barrier purposes). Such uses +// can only use the following arguments for each entry: +// V(class_name, _, field_name, _, FINAL|VAR) +#define NOT_TAGGED_NATIVE_DART_SLOTS_LIST(V) \ + NULLABLE_TAGGED_NATIVE_NONDART_SLOTS_LIST(V) \ + NOT_TAGGED_NATIVE_SLOTS_LIST(V) + +// For uses that need any native slot that contains a Dart object. Such uses can +// only use the following arguments for each entry: +// V(class_name, _, field_name, exact_type, FINAL|VAR) +#define TAGGED_NATIVE_SLOTS_LIST(V) \ + TAGGED_INT_NATIVE_SLOTS_LIST(V) \ + TAGGED_NONINT_NATIVE_SLOTS_LIST(V) + +// For uses that need all native slots. Such uses can only use the following +// arguments for each entry: +// V(class_name, _, field_name, _, FINAL|VAR) #define NATIVE_SLOTS_LIST(V) \ - NULLABLE_BOXED_NATIVE_SLOTS_LIST(V) \ - NONNULLABLE_BOXED_NATIVE_SLOTS_LIST(V) \ - UNBOXED_NATIVE_SLOTS_LIST(V) + TAGGED_NATIVE_SLOTS_LIST(V) \ + NOT_TAGGED_NATIVE_SLOTS_LIST(V) + +// For tagged slots, the cid should either be Dynamic or the precise cid +// of the values stored in the corresponding field. That means the cid should +// not be the cid of an abstract superclass, because then the code will assume +// the cid of retrieved values is always the given cid. +// +// Note: If we ever need native slots with CompileTypes created from an +// AbstractType instead, then a new base category should be created for those, +// possibly replacing the cid field with the name of the abstract type. +#define CHECK_TAGGED_NATIVE_SLOT(__, ___, ____, field_type, _____) \ + static_assert(k##field_type##Cid != kObjectCid); \ + static_assert(k##field_type##Cid != kInstanceCid); \ + static_assert(k##field_type##Cid != kIntegerCid); \ + static_assert(k##field_type##Cid != kStringCid); \ + static_assert(k##field_type##Cid != kAbstractTypeCid); +TAGGED_NATIVE_SLOTS_LIST(CHECK_TAGGED_NATIVE_SLOT) +#undef CHECK_NULLABLE_TAGGED_NATIVE_SLOT + +// Currently we only create slots with CompileTypes created from a precise cid, +// so integer slots listed here must only contain Smis (or Mints, but no slot +// currently does has only Mints, adjust this check if one is added). +// +// Note: If we ever add a category of native slots with AbstractType-based +// CompileTypes that always contain integers, then add additional checks that +// the AbstractTypes of those slots are subtypes of Integer. +#define CHECK_INT_NATIVE_SLOT(__, ___, ____, field_type, _____) \ + static_assert(k##field_type##Cid == kSmiCid); +TAGGED_INT_NATIVE_SLOTS_LIST(CHECK_INT_NATIVE_SLOT) +#undef CHECK_INT_NATIVE_SLOT + +// Any slot with an integer type should go into the correct category. +// +// Note: If we ever add native slots with AbstractType-based CompileTypes, then +// add appropriate checks that the AbstractType is not a subtype of Integer. +#define CHECK_NONINT_NATIVE_SLOT(__, ___, ____, field_type, _____) \ + static_assert(k##field_type##Cid != kSmiCid); \ + static_assert(k##field_type##Cid != kMintCid); +TAGGED_NONINT_NATIVE_SLOTS_LIST(CHECK_NONINT_NATIVE_SLOT) +#undef CHECK_NONINT_NATIVE_SLOT class FieldGuardState { public: @@ -313,7 +491,7 @@ class Slot : public ZoneAllocated { const ParsedFunction* parsed_function); // Convenience getters for native slots. -#define DEFINE_GETTER(ClassName, UnderlyingType, FieldName, __, ___) \ +#define DEFINE_GETTER(ClassName, __, FieldName, ___, ____) \ static const Slot& ClassName##_##FieldName() { \ return GetNativeSlot(Kind::k##ClassName##_##FieldName); \ } @@ -374,7 +552,10 @@ class Slot : public ZoneAllocated { return kind() == Kind::kCapturedVariable || kind() == Kind::kContext_parent; } - bool is_unboxed() const { return IsUnboxedBit::decode(flags_); } + bool is_tagged() const { return !IsNonTaggedBit::decode(flags_); } + bool has_untagged_instance() const { + return HasUntaggedInstanceBit::decode(flags_); + } void Write(FlowGraphSerializer* s) const; static const Slot& Read(FlowGraphDeserializer* d); @@ -404,13 +585,6 @@ class Slot : public ZoneAllocated { other.representation_, other.field_guard_state_) {} - using IsImmutableBit = BitField; - using IsGuardedBit = BitField; - using IsCompressedBit = BitField; - using IsUnboxedBit = BitField; - using MayContainInnerPointerBit = - BitField; - template const T* DataAs() const { return static_cast(data_); @@ -447,6 +621,19 @@ class Slot : public ZoneAllocated { CompileType type_; + using IsImmutableBit = BitField; + using IsGuardedBit = + BitField; + using IsCompressedBit = + BitField; + // Stores whether a field isn't tagged so that tagged is the default value + using IsNonTaggedBit = + BitField; + using MayContainInnerPointerBit = + BitField; + using HasUntaggedInstanceBit = + BitField; + friend class SlotCache; }; diff --git a/runtime/vm/compiler/frontend/base_flow_graph_builder.cc b/runtime/vm/compiler/frontend/base_flow_graph_builder.cc index 09dcb66b96c..53fbbdfdff4 100644 --- a/runtime/vm/compiler/frontend/base_flow_graph_builder.cc +++ b/runtime/vm/compiler/frontend/base_flow_graph_builder.cc @@ -472,6 +472,17 @@ Fragment BaseFlowGraphBuilder::LoadNativeField( return Fragment(load); } +Fragment BaseFlowGraphBuilder::LoadNativeField(const Slot& native_field, + bool calls_initializer) { + const InnerPointerAccess loads_inner_pointer = + native_field.representation() == kUntagged + ? (native_field.may_contain_inner_pointer() + ? InnerPointerAccess::kMayBeInnerPointer + : InnerPointerAccess::kCannotBeInnerPointer) + : InnerPointerAccess::kNotUntagged; + return LoadNativeField(native_field, loads_inner_pointer, calls_initializer); +} + Fragment BaseFlowGraphBuilder::LoadLocal(LocalVariable* variable) { ASSERT(!variable->is_captured()); LoadLocalInstr* load = new (Z) LoadLocalInstr(*variable, InstructionSource()); diff --git a/runtime/vm/compiler/frontend/base_flow_graph_builder.h b/runtime/vm/compiler/frontend/base_flow_graph_builder.h index 8b9a46ebbf8..0023ac9fbac 100644 --- a/runtime/vm/compiler/frontend/base_flow_graph_builder.h +++ b/runtime/vm/compiler/frontend/base_flow_graph_builder.h @@ -179,10 +179,7 @@ class BaseFlowGraphBuilder { InnerPointerAccess loads_inner_pointer, bool calls_initializer = false); Fragment LoadNativeField(const Slot& native_field, - bool calls_initializer = false) { - return LoadNativeField(native_field, InnerPointerAccess::kNotUntagged, - calls_initializer); - } + bool calls_initializer = false); // Pass true for index_unboxed if indexing into external typed data. Fragment LoadIndexed(classid_t class_id, intptr_t index_scale = compiler::target::kWordSize, diff --git a/runtime/vm/compiler/frontend/kernel_to_il.cc b/runtime/vm/compiler/frontend/kernel_to_il.cc index 81545a77c43..ba2bcfb4858 100644 --- a/runtime/vm/compiler/frontend/kernel_to_il.cc +++ b/runtime/vm/compiler/frontend/kernel_to_il.cc @@ -1297,8 +1297,7 @@ FlowGraph* FlowGraphBuilder::BuildGraphOfRecognizedMethod( break; case MethodRecognizer::kRecord_fieldNames: body += LoadObjectStore(); - body += RawLoadField( - compiler::target::ObjectStore::record_field_names_offset()); + body += LoadNativeField(Slot::ObjectStore_record_field_names()); body += LoadLocal(parsed_function_->RawParameterVariable(0)); body += LoadNativeField(Slot::Record_shape()); body += IntConstant(compiler::target::RecordShape::kFieldNamesIndexShift); @@ -1770,7 +1769,7 @@ FlowGraph* FlowGraphBuilder::BuildGraphOfRecognizedMethod( body += Constant(Bool::False()); #else body += LoadServiceExtensionStream(); - body += RawLoadField(compiler::target::StreamInfo::enabled_offset()); + body += LoadNativeField(Slot::StreamInfo_enabled()); // StreamInfo::enabled_ is a std::atomic. This is effectively // relaxed order access, which is acceptable for this use case. body += IntToBool(); @@ -1918,13 +1917,13 @@ FlowGraph* FlowGraphBuilder::BuildGraphOfRecognizedMethod( case MethodRecognizer::kFinalizerBase_getIsolateFinalizers: ASSERT_EQUAL(function.NumParameters(), 0); body += LoadIsolate(); - body += RawLoadField(compiler::target::Isolate::finalizers_offset()); + body += LoadNativeField(Slot::Isolate_finalizers()); break; case MethodRecognizer::kFinalizerBase_setIsolateFinalizers: ASSERT_EQUAL(function.NumParameters(), 1); body += LoadIsolate(); body += LoadLocal(parsed_function_->RawParameterVariable(0)); - body += RawStoreField(compiler::target::Isolate::finalizers_offset()); + body += StoreNativeField(Slot::Isolate_finalizers()); body += NullConstant(); break; case MethodRecognizer::kFinalizerBase_exchangeEntriesCollectedWithNull: @@ -4648,22 +4647,6 @@ Fragment FlowGraphBuilder::LoadIndexedTypedDataUnboxed( return fragment; } -Fragment FlowGraphBuilder::RawLoadField(int32_t offset) { - Fragment code; - code += UnboxedIntConstant(offset, kUnboxedIntPtr); - code += LoadIndexed(kArrayCid, /*index_scale=*/1, /*index_unboxed=*/true); - return code; -} - -Fragment FlowGraphBuilder::RawStoreField(int32_t offset) { - Fragment code; - Value* value = Pop(); - Value* base = Pop(); - auto* instr = new (Z) RawStoreFieldInstr(base, value, offset); - code <<= instr; - return code; -} - Fragment FlowGraphBuilder::UnhandledException() { const auto class_table = thread_->isolate_group()->class_table(); ASSERT(class_table->HasValidClassAt(kUnhandledExceptionCid)); @@ -4711,29 +4694,28 @@ Fragment FlowGraphBuilder::LoadThread() { Fragment FlowGraphBuilder::LoadIsolate() { Fragment body; body += LoadThread(); - body += LoadUntagged(compiler::target::Thread::isolate_offset()); + body += LoadNativeField(Slot::Thread_isolate()); return body; } Fragment FlowGraphBuilder::LoadIsolateGroup() { Fragment body; body += LoadThread(); - body += LoadUntagged(compiler::target::Thread::isolate_group_offset()); + body += LoadNativeField(Slot::Thread_isolate_group()); return body; } Fragment FlowGraphBuilder::LoadObjectStore() { Fragment body; body += LoadIsolateGroup(); - body += LoadUntagged(compiler::target::IsolateGroup::object_store_offset()); + body += LoadNativeField(Slot::IsolateGroup_object_store()); return body; } Fragment FlowGraphBuilder::LoadServiceExtensionStream() { Fragment body; body += LoadThread(); - body += - LoadUntagged(compiler::target::Thread::service_extension_stream_offset()); + body += LoadNativeField(Slot::Thread_service_extension_stream()); return body; } @@ -5276,7 +5258,7 @@ Fragment FlowGraphBuilder::FfiConvertPrimitiveToDart( } else if (marshaller.IsHandle(arg_index)) { // The top of the stack is a Dart_Handle, so retrieve the tagged pointer // out of it. - body += RawLoadField(compiler::target::LocalHandle::ptr_offset()); + body += LoadNativeField(Slot::LocalHandle_ptr()); } else if (marshaller.IsVoid(arg_index)) { // Ignore whatever value was being returned and return null. ASSERT_EQUAL(arg_index, compiler::ffi::kResultIndex); @@ -5320,7 +5302,7 @@ Fragment FlowGraphBuilder::FfiConvertPrimitiveToNative( // Get a reference to the top handle scope. body += LoadThread(); - body += LoadUntagged(compiler::target::Thread::api_top_scope_offset()); + body += LoadNativeField(Slot::Thread_api_top_scope()); arg_reps->Add(kUntagged); // Allocate a new handle in the top handle scope. @@ -5331,7 +5313,8 @@ Fragment FlowGraphBuilder::FfiConvertPrimitiveToNative( // Store the object address into the handle. body += LoadLocal(handle); body += LoadLocal(object); - body += RawStoreField(compiler::target::LocalHandle::ptr_offset()); + body += StoreNativeField(Slot::LocalHandle_ptr(), + StoreFieldInstr::Kind::kInitializing); body += DropTempsPreserveTop(1); // Drop object. } else if (marshaller.IsVoid(arg_index)) { @@ -5694,7 +5677,7 @@ FlowGraph* FlowGraphBuilder::BuildGraphOfSyncFfiCallback( body += LoadThread(); body += LoadUntagged(compiler::target::Thread::unboxed_runtime_arg_offset()); - body += RawLoadField(compiler::target::PersistentHandle::ptr_offset()); + body += LoadNativeField(Slot::PersistentHandle_ptr()); closure = MakeTemporary(); } diff --git a/runtime/vm/compiler/frontend/kernel_to_il.h b/runtime/vm/compiler/frontend/kernel_to_il.h index fa0f2116ab9..e3a6608960c 100644 --- a/runtime/vm/compiler/frontend/kernel_to_il.h +++ b/runtime/vm/compiler/frontend/kernel_to_il.h @@ -395,14 +395,6 @@ class FlowGraphBuilder : public BaseFlowGraphBuilder { ZoneGrowableArray* definitions, const GrowableArray& representations); - // Loads a tagged value from an untagged base + offset from outside the heap. - Fragment RawLoadField(int32_t offset); - - // Populates the untagged base + offset outside the heap with a tagged value. - // - // The store must be outside of the heap, does not emit a store barrier. - Fragment RawStoreField(int32_t offset); - // Wrap the current exception and stacktrace in an unhandled exception. Fragment UnhandledException(); diff --git a/runtime/vm/compiler/write_barrier_elimination.cc b/runtime/vm/compiler/write_barrier_elimination.cc index 5b8d6b8a6b3..a680e5067e8 100644 --- a/runtime/vm/compiler/write_barrier_elimination.cc +++ b/runtime/vm/compiler/write_barrier_elimination.cc @@ -363,15 +363,23 @@ bool WriteBarrierElimination::SlotEligibleForWBE(const Slot& slot) { case Slot::Kind::kRecordField: // Instance return true; -#define FOR_EACH_NATIVE_SLOT(class, underlying_type, field, __, ___) \ +#define TAGGED_NATIVE_DART_SLOT_CASE(class, underlying_type, field, __, ___) \ case Slot::Kind::k##class##_##field: \ return std::is_base_of::value || \ std::is_base_of::value || \ std::is_base_of::value; - NATIVE_SLOTS_LIST(FOR_EACH_NATIVE_SLOT) -#undef FOR_EACH_NATIVE_SLOT + TAGGED_NATIVE_DART_SLOTS_LIST(TAGGED_NATIVE_DART_SLOT_CASE) +#undef TAGGED_NATIVE_DART_SLOT_CASE + +#define OTHER_NATIVE_SLOT_CASE(class, __, field, ___, ____) \ + case Slot::Kind::k##class##_##field: + // No store barrier needed for non-tagged fields or fields of + // non-Dart objects. + NOT_TAGGED_NATIVE_DART_SLOTS_LIST(OTHER_NATIVE_SLOT_CASE) +#undef OTHER_NATIVE_SLOT_CASE + return true; default: return false; diff --git a/runtime/vm/isolate.h b/runtime/vm/isolate.h index 04b48c17c77..c320cc38d37 100644 --- a/runtime/vm/isolate.h +++ b/runtime/vm/isolate.h @@ -1553,7 +1553,7 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry { bool is_system_isolate_ = false; // End accessed from generated code. - IsolateGroup* isolate_group_; + IsolateGroup* const isolate_group_; IdleTimeHandler idle_time_handler_; std::unique_ptr isolate_object_store_; diff --git a/runtime/vm/object.cc b/runtime/vm/object.cc index 1ba730691d8..5e32f9ed29c 100644 --- a/runtime/vm/object.cc +++ b/runtime/vm/object.cc @@ -6288,11 +6288,11 @@ FunctionPtr Class::LookupFunctionReadLocked(const String& name, #if defined(DEBUG) ASSERT(thread->isolate_group()->program_lock()->IsCurrentThreadReader()); #endif + ASSERT(functions() != Array::null()); REUSABLE_ARRAY_HANDLESCOPE(thread); REUSABLE_FUNCTION_HANDLESCOPE(thread); Array& funcs = thread->ArrayHandle(); funcs = functions(); - ASSERT(!funcs.IsNull()); const intptr_t len = funcs.Length(); Function& function = thread->FunctionHandle(); if (len >= kFunctionLookupHashThreshold) { diff --git a/runtime/vm/thread.cc b/runtime/vm/thread.cc index f54c685f914..431f209457b 100644 --- a/runtime/vm/thread.cc +++ b/runtime/vm/thread.cc @@ -73,8 +73,16 @@ Thread::Thread(bool is_vm_isolate) TargetCPUFeatures::double_truncate_round_supported() ? 1 : 0), tsan_utils_(DO_IF_TSAN(new TsanUtils()) DO_IF_NOT_TSAN(nullptr)), task_kind_(kUnknownTask), +#if defined(SUPPORT_TIMELINE) + dart_stream_(ASSERT_NOTNULL(Timeline::GetDartStream())), +#else dart_stream_(nullptr), +#endif +#if !defined(PRODUCT) + service_extension_stream_(ASSERT_NOTNULL(&Service::extension_stream)), +#else service_extension_stream_(nullptr), +#endif thread_lock_(), api_reusable_scope_(nullptr), no_callback_scope_depth_(0), @@ -98,14 +106,6 @@ Thread::Thread(bool is_vm_isolate) next_(nullptr) { #endif -#if defined(SUPPORT_TIMELINE) - dart_stream_ = Timeline::GetDartStream(); - ASSERT(dart_stream_ != nullptr); -#endif -#ifndef PRODUCT - service_extension_stream_ = &Service::extension_stream; - ASSERT(service_extension_stream_ != nullptr); -#endif #define DEFAULT_INIT(type_name, member_name, init_expr, default_init_value) \ member_name = default_init_value; CACHED_CONSTANTS_LIST(DEFAULT_INIT) diff --git a/runtime/vm/thread.h b/runtime/vm/thread.h index e5c3b1aae29..1fe4d829a6d 100644 --- a/runtime/vm/thread.h +++ b/runtime/vm/thread.h @@ -1285,8 +1285,8 @@ class Thread : public ThreadState { uword true_end_ = 0; TaskKind task_kind_; - TimelineStream* dart_stream_; - StreamInfo* service_extension_stream_; + TimelineStream* const dart_stream_; + StreamInfo* const service_extension_stream_; mutable Monitor thread_lock_; ApiLocalScope* api_reusable_scope_; int32_t no_callback_scope_depth_;