From d34b08263c6e6e73f22cb822ee32617bff8b38d1 Mon Sep 17 00:00:00 2001 From: Tess Strickland Date: Thu, 7 Aug 2025 08:30:29 -0700 Subject: [PATCH] [vm,dyn_modules] Handle hash-based SubtypeTestCaches in the interpreter. To do this, factor out the main search loop from SubtypeTestCache::FindKeyOrUnused into a method suitable for calling from the interpreter. Then, the only work on the interpreter side is to calculate the initial index at which to start probing for entries. TEST=ci Change-Id: I620c3904458158519066074565218b6e4a77eea9 Cq-Include-Trybots: luci.dart.try:vm-linux-debug-x64-try,vm-aot-linux-debug-x64-try,vm-dyn-linux-debug-x64-try,vm-aot-dyn-linux-debug-x64-try Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/444080 Reviewed-by: Alexander Markov Commit-Queue: Tess Strickland --- runtime/vm/interpreter.cc | 119 +++++++++++++++------ runtime/vm/object.cc | 212 +++++++++++++++++++++++--------------- runtime/vm/object.h | 45 ++++++-- 3 files changed, 251 insertions(+), 125 deletions(-) diff --git a/runtime/vm/interpreter.cc b/runtime/vm/interpreter.cc index cadb125fddb..83c39646dcd 100644 --- a/runtime/vm/interpreter.cc +++ b/runtime/vm/interpreter.cc @@ -1186,9 +1186,7 @@ bool Interpreter::AssertAssignable(Thread* thread, ObjectPtr* args, SubtypeTestCachePtr cache) { ObjectPtr null_value = Object::null(); - if ((cache != null_value) && - (Smi::Value(cache->untag()->cache()->untag()->length()) <= - SubtypeTestCache::kMaxLinearCacheSize)) { + if (cache != null_value) { InstancePtr instance = Instance::RawCast(args[0]); AbstractTypePtr dst_type = AbstractType::RawCast(args[1]); TypeArgumentsPtr instantiator_type_arguments = @@ -1232,36 +1230,95 @@ bool Interpreter::AssertAssignable(Thread* thread, } ArrayPtr entries = cache->untag()->cache(); - for (intptr_t i = 0; entries->untag()->element(i) != null_value; - i += SubtypeTestCache::kTestEntryLength) { - if ((entries->untag()->element( - i + SubtypeTestCache::kInstanceCidOrSignature) == - instance_cid_or_function) && - (entries->untag()->element( - i + SubtypeTestCache::kInstanceTypeArguments) == - instance_type_arguments) && - (entries->untag()->element( - i + SubtypeTestCache::kInstantiatorTypeArguments) == - instantiator_type_arguments) && - (entries->untag()->element( - i + SubtypeTestCache::kFunctionTypeArguments) == - function_type_arguments) && - (entries->untag()->element( - i + SubtypeTestCache::kInstanceParentFunctionTypeArguments) == - parent_function_type_arguments) && - (entries->untag()->element( - i + SubtypeTestCache::kInstanceDelayedFunctionTypeArguments) == - delayed_function_type_arguments) && - (entries->untag()->element(i + SubtypeTestCache::kDestinationType) == - dst_type)) { - if (Bool::True().ptr() == - entries->untag()->element(i + SubtypeTestCache::kTestResult)) { - return true; - } else { - break; - } + const intptr_t num_inputs = cache->untag()->num_inputs_; + // The search in a linear-based STC starts at 0. + intptr_t probe = 0; + if (SubtypeTestCache::IsHash(entries)) { + // Perform the same hash as SubtypeTestCache::FindKeyOrUnused. + // + // Control flows to AssertAssignableCallRuntime if any of the individual + // hashes are 0 (which denotes the hash is not yet computed). + if (cid == kClosureCid) { + auto sig = AbstractType::RawCast(instance_cid_or_function); + probe = RawSmiValue(sig->untag()->hash()); + if (probe == 0) goto AssertAssignableCallRuntime; + } else { + probe = cid; } + switch (num_inputs) { + case 7: { + intptr_t h = RawSmiValue(dst_type->untag()->hash()); + if (h == 0) goto AssertAssignableCallRuntime; + probe = CombineHashes(probe, h); + } + FALL_THROUGH; + case 6: { + intptr_t h = TypeArguments::kAllDynamicHash; + if (delayed_function_type_arguments != null_value) { + h = RawSmiValue(delayed_function_type_arguments->untag()->hash()); + if (h == 0) goto AssertAssignableCallRuntime; + } + probe = CombineHashes(probe, h); + } + FALL_THROUGH; + case 5: { + intptr_t h = TypeArguments::kAllDynamicHash; + if (parent_function_type_arguments != null_value) { + h = RawSmiValue(parent_function_type_arguments->untag()->hash()); + if (h == 0) goto AssertAssignableCallRuntime; + } + probe = CombineHashes(probe, h); + } + FALL_THROUGH; + case 4: { + intptr_t h = TypeArguments::kAllDynamicHash; + if (function_type_arguments != null_value) { + h = RawSmiValue(function_type_arguments->untag()->hash()); + if (h == 0) goto AssertAssignableCallRuntime; + } + probe = CombineHashes(probe, h); + } + FALL_THROUGH; + case 3: { + intptr_t h = TypeArguments::kAllDynamicHash; + if (instantiator_type_arguments != null_value) { + h = RawSmiValue(instantiator_type_arguments->untag()->hash()); + if (h == 0) goto AssertAssignableCallRuntime; + } + probe = CombineHashes(probe, h); + } + FALL_THROUGH; + case 2: { + intptr_t h = TypeArguments::kAllDynamicHash; + if (instance_type_arguments != null_value) { + h = RawSmiValue(instance_type_arguments->untag()->hash()); + if (h == 0) goto AssertAssignableCallRuntime; + } + probe = CombineHashes(probe, h); + } + FALL_THROUGH; + case 1: + // Already included in the hash. + break; + default: + UNREACHABLE(); + } + probe = FinalizeHash(probe); + // The number of entries for a hash-based cache is a power of 2, + // so use it as a mask to get a valid entry index from the hash. + probe = probe & (SubtypeTestCache::NumEntries(entries) - 1); } + BoolPtr test_result = nullptr; + auto loc = SubtypeTestCache::FindKeyOrUnusedFromProbe( + entries, num_inputs, probe, instance_cid_or_function, dst_type, + instance_type_arguments, instantiator_type_arguments, + function_type_arguments, parent_function_type_arguments, + delayed_function_type_arguments, &test_result); + if (loc.present && test_result == Bool::True().ptr()) { + return true; + } + // Either there is no matching entry or the matching entry had a false test + // result, so a runtime call is needed to generate an appropriate error. } AssertAssignableCallRuntime: diff --git a/runtime/vm/object.cc b/runtime/vm/object.cc index f0a4aa9c395..8319e007670 100644 --- a/runtime/vm/object.cc +++ b/runtime/vm/object.cc @@ -19821,23 +19821,14 @@ intptr_t SubtypeTestCache::NumberOfChecks() const { return num_occupied(); } -intptr_t SubtypeTestCache::NumEntries() const { - ASSERT(!IsNull()); - return Array::LengthOf(cache()) / kTestEntryLength; +intptr_t SubtypeTestCache::NumEntries(const ArrayPtr array) { + ASSERT(array != Array::null()); + return Array::LengthOf(array) / kTestEntryLength; } -intptr_t SubtypeTestCache::NumEntries(const Array& array) { - SubtypeTestCacheTable table(array); - return table.Length(); -} - -bool SubtypeTestCache::IsHash() const { - if (IsNull()) return false; - return Array::LengthOf(cache()) > kMaxLinearCacheSize; -} - -bool SubtypeTestCache::IsHash(const Array& array) { - return array.Length() > kMaxLinearCacheSize; +bool SubtypeTestCache::IsHash(const ArrayPtr array) { + ASSERT(array != Array::null()); + return Array::LengthOf(array) > kMaxLinearCacheSize; } intptr_t SubtypeTestCache::AddCheck( @@ -19928,64 +19919,6 @@ intptr_t SubtypeTestCache::AddCheck( return loc.entry; } -static inline bool SubtypeTestCacheEntryMatches( - const SubtypeTestCacheTable::TupleView& t, - intptr_t num_inputs, - const Object& instance_class_id_or_signature, - const AbstractType& destination_type, - const TypeArguments& instance_type_arguments, - const TypeArguments& instantiator_type_arguments, - const TypeArguments& function_type_arguments, - const TypeArguments& instance_parent_function_type_arguments, - const TypeArguments& instance_delayed_type_arguments) { - switch (num_inputs) { - case 7: - if (t.Get() != - destination_type.ptr()) { - return false; - } - FALL_THROUGH; - case 6: - if (t.Get() != - instance_delayed_type_arguments.ptr()) { - return false; - } - FALL_THROUGH; - case 5: - if (t.Get() != - instance_parent_function_type_arguments.ptr()) { - return false; - } - FALL_THROUGH; - case 4: - if (t.Get() != - function_type_arguments.ptr()) { - return false; - } - FALL_THROUGH; - case 3: - if (t.Get() != - instantiator_type_arguments.ptr()) { - return false; - } - FALL_THROUGH; - case 2: - if (t.Get() != - instance_type_arguments.ptr()) { - return false; - } - FALL_THROUGH; - case 1: - // We don't need to perform load-acquire semantics when re-retrieving - // the kInstanceCidOrSignature field, as this is performed only if the - // entry is occupied, and occupied entries never change. - return t.Get() == - instance_class_id_or_signature.ptr(); - default: - UNREACHABLE(); - } -} - SubtypeTestCache::KeyLocation SubtypeTestCache::FindKeyOrUnused( const Array& array, intptr_t num_inputs, @@ -20000,14 +19933,13 @@ SubtypeTestCache::KeyLocation SubtypeTestCache::FindKeyOrUnused( if (array.ptr() == Object::empty_subtype_test_cache_array().ptr()) { return {0, false}; } - const bool is_hash = IsHash(array); + const bool is_hash = IsHash(array.ptr()); SubtypeTestCacheTable table(array); const intptr_t num_entries = table.Length(); // For a linear cache, start at the first entry and probe linearly. This can // be done because a linear cache always has at least one unoccupied entry // after all the occupied ones. intptr_t probe = 0; - intptr_t probe_distance = 1; if (is_hash) { // For a hash-based cache, instead start at an entry determined by the hash // of the keys. @@ -20045,17 +19977,126 @@ SubtypeTestCache::KeyLocation SubtypeTestCache::FindKeyOrUnused( hash = FinalizeHash(hash); probe = hash & (num_entries - 1); } + NoSafepointScope scope; + return FindKeyOrUnusedFromProbe( + array.ptr(), num_inputs, probe, instance_class_id_or_signature.ptr(), + destination_type.ptr(), instance_type_arguments.ptr(), + instantiator_type_arguments.ptr(), function_type_arguments.ptr(), + instance_parent_function_type_arguments.ptr(), + instance_delayed_type_arguments.ptr()); +} + +static inline bool CheckSubtypeTestCacheEntry( + ArrayPtr array, + intptr_t num_inputs, + intptr_t entry_start, + ObjectPtr instance_class_id_or_signature, + AbstractTypePtr destination_type, + TypeArgumentsPtr instance_type_arguments, + TypeArgumentsPtr instantiator_type_arguments, + TypeArgumentsPtr function_type_arguments, + TypeArgumentsPtr instance_parent_function_type_arguments, + TypeArgumentsPtr instance_delayed_type_arguments) { + // This function is only called when the entry slot that determines occupancy + // is non-null. This means that the elements do not need to be load-acquired + // here, as entries in STC backing arrays are never changed once added. + switch (num_inputs) { + case 7: + if (Array::ElementAt(array, + entry_start + SubtypeTestCache::kDestinationType) != + destination_type) { + return false; + } + FALL_THROUGH; + case 6: + if (Array::ElementAt( + array, + entry_start + + SubtypeTestCache::kInstanceDelayedFunctionTypeArguments) != + instance_delayed_type_arguments) { + return false; + } + FALL_THROUGH; + case 5: + if (Array::ElementAt( + array, + entry_start + + SubtypeTestCache::kInstanceParentFunctionTypeArguments) != + instance_parent_function_type_arguments) { + return false; + } + FALL_THROUGH; + case 4: + if (Array::ElementAt( + array, entry_start + SubtypeTestCache::kFunctionTypeArguments) != + function_type_arguments) { + return false; + } + FALL_THROUGH; + case 3: + if (Array::ElementAt( + array, + entry_start + SubtypeTestCache::kInstantiatorTypeArguments) != + instantiator_type_arguments) { + return false; + } + FALL_THROUGH; + case 2: + if (Array::ElementAt( + array, entry_start + SubtypeTestCache::kInstanceTypeArguments) != + instance_type_arguments) { + return false; + } + FALL_THROUGH; + case 1: + return Array::ElementAt( + array, + entry_start + SubtypeTestCache::kInstanceCidOrSignature) == + instance_class_id_or_signature; + } + UNREACHABLE(); + return false; +} + +SubtypeTestCache::KeyLocation SubtypeTestCache::FindKeyOrUnusedFromProbe( + ArrayPtr array, + intptr_t num_inputs, + intptr_t probe, + ObjectPtr instance_class_id_or_signature, + AbstractTypePtr destination_type, + TypeArgumentsPtr instance_type_arguments, + TypeArgumentsPtr instantiator_type_arguments, + TypeArgumentsPtr function_type_arguments, + TypeArgumentsPtr instance_parent_function_type_arguments, + TypeArgumentsPtr instance_delayed_type_arguments, + BoolPtr* test_result) { + // Fast case for empty STCs. + if (array == Object::empty_subtype_test_cache_array().ptr()) { + return {probe, false}; + } + const bool is_hash = IsHash(array); + const intptr_t num_entries = NumEntries(array); + intptr_t probe_distance = 1; while (true) { - const auto& tuple = table.At(probe); - if (tuple.Get() == - Object::null()) { - break; + intptr_t entry_start = probe * SubtypeTestCache::kTestEntryLength; + // First check the entry slot that determines occupancy, which requires + // load-acquire semantics. + ObjectPtr cid_or_sig = Array::ElementAt( + array, entry_start + SubtypeTestCache::kInstanceCidOrSignature); + if (cid_or_sig == Object::null()) { + // The appropriate location was found, but there's no existing entry. + return {probe, false}; } - if (SubtypeTestCacheEntryMatches( - tuple, num_inputs, instance_class_id_or_signature, destination_type, - instance_type_arguments, instantiator_type_arguments, - function_type_arguments, instance_parent_function_type_arguments, + if (CheckSubtypeTestCacheEntry( + array, num_inputs, entry_start, instance_class_id_or_signature, + destination_type, instance_type_arguments, + instantiator_type_arguments, function_type_arguments, + instance_parent_function_type_arguments, instance_delayed_type_arguments)) { + if (test_result != nullptr) { + *test_result = Bool::RawCast(Array::ElementAt( + array, entry_start + SubtypeTestCache::kTestResult)); + } return {probe, true}; } // Advance probe by the current probing distance. @@ -20068,6 +20109,7 @@ SubtypeTestCache::KeyLocation SubtypeTestCache::FindKeyOrUnused( probe_distance++; } } + UNREACHABLE(); return {probe, false}; } @@ -20078,11 +20120,11 @@ ArrayPtr SubtypeTestCache::EnsureCapacity(Zone* zone, ASSERT(new_occupied > NumberOfChecks()); ASSERT(was_grown != nullptr); // How many entries are in the current array (including unoccupied entries). - const intptr_t current_capacity = NumEntries(array); + const intptr_t current_capacity = NumEntries(array.ptr()); // Early returns for cases where no growth is needed. *was_grown = false; - const bool is_linear = IsLinear(array); + const bool is_linear = IsLinear(array.ptr()); if (is_linear) { // We need at least one unoccupied entry in addition to the occupied ones. if (current_capacity > new_occupied) return array.ptr(); diff --git a/runtime/vm/object.h b/runtime/vm/object.h index da108c88145..a5840b3307f 100644 --- a/runtime/vm/object.h +++ b/runtime/vm/object.h @@ -7922,7 +7922,10 @@ class SubtypeTestCache : public Object { intptr_t NumberOfChecks() const; // Retrieves the number of entries (occupied or unoccupied) in the cache. - intptr_t NumEntries() const; + intptr_t NumEntries() const { + ASSERT(!IsNull()); + return NumEntries(untag()->cache_); + } // Adds a check, returning the index of the new entry in the cache. intptr_t AddCheck( @@ -8011,7 +8014,10 @@ class SubtypeTestCache : public Object { bool Equals(const SubtypeTestCache& other) const; // Returns whether the cache backed by the given storage is hash-based. - bool IsHash() const; + bool IsHash() const { + if (IsNull()) return false; + return IsHash(untag()->cache_); + } // Creates a separate copy of the current STC contents. SubtypeTestCachePtr Copy(Thread* thread) const; @@ -8075,13 +8081,13 @@ class SubtypeTestCache : public Object { // Retrieves the number of entries (occupied or unoccupied) in a cache // backed by the given array. - static intptr_t NumEntries(const Array& array); + static intptr_t NumEntries(const ArrayPtr array); // Returns whether the cache backed by the given storage is linear. - static bool IsLinear(const Array& array) { return !IsHash(array); } + static bool IsLinear(const ArrayPtr array) { return !IsHash(array); } // Returns whether the cache backed by the given storage is hash-based. - static bool IsHash(const Array& array); + static bool IsHash(const ArrayPtr array); struct KeyLocation { // The entry index if [present] is true, otherwise where the entry would @@ -8109,6 +8115,22 @@ class SubtypeTestCache : public Object { const TypeArguments& instance_parent_function_type_arguments, const TypeArguments& instance_delayed_type_arguments); + // The main loop of FindKeyOrUnused() after the initial probe index has + // been calculated. Used by both FindKeyOrUnused and by the bytecode + // interpreter, so uses tagged pointers instead of handles. + static KeyLocation FindKeyOrUnusedFromProbe( + ArrayPtr array, + intptr_t num_inputs, + intptr_t probe, + ObjectPtr instance_class_id_or_signature, + AbstractTypePtr destination_type, + TypeArgumentsPtr instance_type_arguments, + TypeArgumentsPtr instantiator_type_arguments, + TypeArgumentsPtr function_type_arguments, + TypeArgumentsPtr instance_parent_function_type_arguments, + TypeArgumentsPtr instance_delayed_type_arguments, + BoolPtr* test_result = nullptr); + // If the given array can contain the requested number of entries, returns // the same array and sets [was_grown] to false. // @@ -8172,6 +8194,7 @@ class SubtypeTestCache : public Object { FINAL_HEAP_OBJECT_IMPLEMENTATION(SubtypeTestCache, Object); friend class Class; friend class FieldInvalidator; + friend class Interpreter; friend class VMSerializationRoots; friend class VMDeserializationRoots; }; @@ -11050,10 +11073,15 @@ class Array : public Instance { return array->untag()->data(); } + template + static ObjectPtr ElementAt(ArrayPtr array, intptr_t index) { + ASSERT((0 <= index) && (index < LengthOf(array))); + return array->untag()->element(index); + } + template ObjectPtr At(intptr_t index) const { - ASSERT((0 <= index) && (index < Length())); - return untag()->element(index); + return ElementAt(ptr(), index); } template void SetAt(intptr_t index, const Object& value) const { @@ -11068,8 +11096,7 @@ class Array : public Instance { // Access to the array with acquire release semantics. ObjectPtr AtAcquire(intptr_t index) const { - ASSERT((0 <= index) && (index < Length())); - return untag()->element(index); + return ElementAt(ptr(), index); } void SetAtRelease(intptr_t index, const Object& value) const { ASSERT((0 <= index) && (index < Length()));