[vm,dyn_modules] Handle hash-based SubtypeTestCaches in the interpreter.

To do this, factor out the main search loop from
SubtypeTestCache::FindKeyOrUnused into a method suitable for calling
from the interpreter. Then, the only work on the interpreter side is to
calculate the initial index at which to start probing for entries.

TEST=ci

Change-Id: I620c3904458158519066074565218b6e4a77eea9
Cq-Include-Trybots: luci.dart.try:vm-linux-debug-x64-try,vm-aot-linux-debug-x64-try,vm-dyn-linux-debug-x64-try,vm-aot-dyn-linux-debug-x64-try
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/444080
Reviewed-by: Alexander Markov <alexmarkov@google.com>
Commit-Queue: Tess Strickland <sstrickl@google.com>

runtime/vm/interpreter.cc

@@ -1186,9 +1186,7 @@ bool Interpreter::AssertAssignable(Thread* thread,
                                    ObjectPtr* args,
                                    SubtypeTestCachePtr cache) {
   ObjectPtr null_value = Object::null();
-  if ((cache != null_value) &&
-      (Smi::Value(cache->untag()->cache()->untag()->length()) <=
-       SubtypeTestCache::kMaxLinearCacheSize)) {
+  if (cache != null_value) {
     InstancePtr instance = Instance::RawCast(args[0]);
     AbstractTypePtr dst_type = AbstractType::RawCast(args[1]);
     TypeArgumentsPtr instantiator_type_arguments =
@@ -1232,36 +1230,95 @@ bool Interpreter::AssertAssignable(Thread* thread,
     }
 
     ArrayPtr entries = cache->untag()->cache();
-    for (intptr_t i = 0; entries->untag()->element(i) != null_value;
-         i += SubtypeTestCache::kTestEntryLength) {
-      if ((entries->untag()->element(
-               i + SubtypeTestCache::kInstanceCidOrSignature) ==
-           instance_cid_or_function) &&
-          (entries->untag()->element(
-               i + SubtypeTestCache::kInstanceTypeArguments) ==
-           instance_type_arguments) &&
-          (entries->untag()->element(
-               i + SubtypeTestCache::kInstantiatorTypeArguments) ==
-           instantiator_type_arguments) &&
-          (entries->untag()->element(
-               i + SubtypeTestCache::kFunctionTypeArguments) ==
-           function_type_arguments) &&
-          (entries->untag()->element(
-               i + SubtypeTestCache::kInstanceParentFunctionTypeArguments) ==
-           parent_function_type_arguments) &&
-          (entries->untag()->element(
-               i + SubtypeTestCache::kInstanceDelayedFunctionTypeArguments) ==
-           delayed_function_type_arguments) &&
-          (entries->untag()->element(i + SubtypeTestCache::kDestinationType) ==
-           dst_type)) {
-        if (Bool::True().ptr() ==
-            entries->untag()->element(i + SubtypeTestCache::kTestResult)) {
-          return true;
-        } else {
-          break;
-        }
+    const intptr_t num_inputs = cache->untag()->num_inputs_;
+    // The search in a linear-based STC starts at 0.
+    intptr_t probe = 0;
+    if (SubtypeTestCache::IsHash(entries)) {
+      // Perform the same hash as SubtypeTestCache::FindKeyOrUnused.
+      //
+      // Control flows to AssertAssignableCallRuntime if any of the individual
+      // hashes are 0 (which denotes the hash is not yet computed).
+      if (cid == kClosureCid) {
+        auto sig = AbstractType::RawCast(instance_cid_or_function);
+        probe = RawSmiValue(sig->untag()->hash());
+        if (probe == 0) goto AssertAssignableCallRuntime;
+      } else {
+        probe = cid;
       }
+      switch (num_inputs) {
+        case 7: {
+          intptr_t h = RawSmiValue(dst_type->untag()->hash());
+          if (h == 0) goto AssertAssignableCallRuntime;
+          probe = CombineHashes(probe, h);
+        }
+          FALL_THROUGH;
+        case 6: {
+          intptr_t h = TypeArguments::kAllDynamicHash;
+          if (delayed_function_type_arguments != null_value) {
+            h = RawSmiValue(delayed_function_type_arguments->untag()->hash());
+            if (h == 0) goto AssertAssignableCallRuntime;
+          }
+          probe = CombineHashes(probe, h);
+        }
+          FALL_THROUGH;
+        case 5: {
+          intptr_t h = TypeArguments::kAllDynamicHash;
+          if (parent_function_type_arguments != null_value) {
+            h = RawSmiValue(parent_function_type_arguments->untag()->hash());
+            if (h == 0) goto AssertAssignableCallRuntime;
+          }
+          probe = CombineHashes(probe, h);
+        }
+          FALL_THROUGH;
+        case 4: {
+          intptr_t h = TypeArguments::kAllDynamicHash;
+          if (function_type_arguments != null_value) {
+            h = RawSmiValue(function_type_arguments->untag()->hash());
+            if (h == 0) goto AssertAssignableCallRuntime;
+          }
+          probe = CombineHashes(probe, h);
+        }
+          FALL_THROUGH;
+        case 3: {
+          intptr_t h = TypeArguments::kAllDynamicHash;
+          if (instantiator_type_arguments != null_value) {
+            h = RawSmiValue(instantiator_type_arguments->untag()->hash());
+            if (h == 0) goto AssertAssignableCallRuntime;
+          }
+          probe = CombineHashes(probe, h);
+        }
+          FALL_THROUGH;
+        case 2: {
+          intptr_t h = TypeArguments::kAllDynamicHash;
+          if (instance_type_arguments != null_value) {
+            h = RawSmiValue(instance_type_arguments->untag()->hash());
+            if (h == 0) goto AssertAssignableCallRuntime;
+          }
+          probe = CombineHashes(probe, h);
+        }
+          FALL_THROUGH;
+        case 1:
+          // Already included in the hash.
+          break;
+        default:
+          UNREACHABLE();
+      }
+      probe = FinalizeHash(probe);
+      // The number of entries for a hash-based cache is a power of 2,
+      // so use it as a mask to get a valid entry index from the hash.
+      probe = probe & (SubtypeTestCache::NumEntries(entries) - 1);
     }
+    BoolPtr test_result = nullptr;
+    auto loc = SubtypeTestCache::FindKeyOrUnusedFromProbe(
+        entries, num_inputs, probe, instance_cid_or_function, dst_type,
+        instance_type_arguments, instantiator_type_arguments,
+        function_type_arguments, parent_function_type_arguments,
+        delayed_function_type_arguments, &test_result);
+    if (loc.present && test_result == Bool::True().ptr()) {
+      return true;
+    }
+    // Either there is no matching entry or the matching entry had a false test
+    // result, so a runtime call is needed to generate an appropriate error.
   }
 
 AssertAssignableCallRuntime:

runtime/vm/object.cc

@@ -19821,23 +19821,14 @@ intptr_t SubtypeTestCache::NumberOfChecks() const {
   return num_occupied();
 }
 
-intptr_t SubtypeTestCache::NumEntries() const {
-  ASSERT(!IsNull());
-  return Array::LengthOf(cache()) / kTestEntryLength;
+intptr_t SubtypeTestCache::NumEntries(const ArrayPtr array) {
+  ASSERT(array != Array::null());
+  return Array::LengthOf(array) / kTestEntryLength;
 }
 
-intptr_t SubtypeTestCache::NumEntries(const Array& array) {
-  SubtypeTestCacheTable table(array);
-  return table.Length();
-}
-
-bool SubtypeTestCache::IsHash() const {
-  if (IsNull()) return false;
-  return Array::LengthOf(cache()) > kMaxLinearCacheSize;
-}
-
-bool SubtypeTestCache::IsHash(const Array& array) {
-  return array.Length() > kMaxLinearCacheSize;
+bool SubtypeTestCache::IsHash(const ArrayPtr array) {
+  ASSERT(array != Array::null());
+  return Array::LengthOf(array) > kMaxLinearCacheSize;
 }
 
 intptr_t SubtypeTestCache::AddCheck(
@@ -19928,64 +19919,6 @@ intptr_t SubtypeTestCache::AddCheck(
   return loc.entry;
 }
 
-static inline bool SubtypeTestCacheEntryMatches(
-    const SubtypeTestCacheTable::TupleView& t,
-    intptr_t num_inputs,
-    const Object& instance_class_id_or_signature,
-    const AbstractType& destination_type,
-    const TypeArguments& instance_type_arguments,
-    const TypeArguments& instantiator_type_arguments,
-    const TypeArguments& function_type_arguments,
-    const TypeArguments& instance_parent_function_type_arguments,
-    const TypeArguments& instance_delayed_type_arguments) {
-  switch (num_inputs) {
-    case 7:
-      if (t.Get<SubtypeTestCache::kDestinationType>() !=
-          destination_type.ptr()) {
-        return false;
-      }
-      FALL_THROUGH;
-    case 6:
-      if (t.Get<SubtypeTestCache::kInstanceDelayedFunctionTypeArguments>() !=
-          instance_delayed_type_arguments.ptr()) {
-        return false;
-      }
-      FALL_THROUGH;
-    case 5:
-      if (t.Get<SubtypeTestCache::kInstanceParentFunctionTypeArguments>() !=
-          instance_parent_function_type_arguments.ptr()) {
-        return false;
-      }
-      FALL_THROUGH;
-    case 4:
-      if (t.Get<SubtypeTestCache::kFunctionTypeArguments>() !=
-          function_type_arguments.ptr()) {
-        return false;
-      }
-      FALL_THROUGH;
-    case 3:
-      if (t.Get<SubtypeTestCache::kInstantiatorTypeArguments>() !=
-          instantiator_type_arguments.ptr()) {
-        return false;
-      }
-      FALL_THROUGH;
-    case 2:
-      if (t.Get<SubtypeTestCache::kInstanceTypeArguments>() !=
-          instance_type_arguments.ptr()) {
-        return false;
-      }
-      FALL_THROUGH;
-    case 1:
-      // We don't need to perform load-acquire semantics when re-retrieving
-      // the kInstanceCidOrSignature field, as this is performed only if the
-      // entry is occupied, and occupied entries never change.
-      return t.Get<SubtypeTestCache::kInstanceCidOrSignature>() ==
-             instance_class_id_or_signature.ptr();
-    default:
-      UNREACHABLE();
-  }
-}
-
 SubtypeTestCache::KeyLocation SubtypeTestCache::FindKeyOrUnused(
     const Array& array,
     intptr_t num_inputs,
@@ -20000,14 +19933,13 @@ SubtypeTestCache::KeyLocation SubtypeTestCache::FindKeyOrUnused(
   if (array.ptr() == Object::empty_subtype_test_cache_array().ptr()) {
     return {0, false};
   }
-  const bool is_hash = IsHash(array);
+  const bool is_hash = IsHash(array.ptr());
   SubtypeTestCacheTable table(array);
   const intptr_t num_entries = table.Length();
   // For a linear cache, start at the first entry and probe linearly. This can
   // be done because a linear cache always has at least one unoccupied entry
   // after all the occupied ones.
   intptr_t probe = 0;
-  intptr_t probe_distance = 1;
   if (is_hash) {
     // For a hash-based cache, instead start at an entry determined by the hash
     // of the keys.
@@ -20045,17 +19977,126 @@ SubtypeTestCache::KeyLocation SubtypeTestCache::FindKeyOrUnused(
     hash = FinalizeHash(hash);
     probe = hash & (num_entries - 1);
   }
+  NoSafepointScope scope;
+  return FindKeyOrUnusedFromProbe(
+      array.ptr(), num_inputs, probe, instance_class_id_or_signature.ptr(),
+      destination_type.ptr(), instance_type_arguments.ptr(),
+      instantiator_type_arguments.ptr(), function_type_arguments.ptr(),
+      instance_parent_function_type_arguments.ptr(),
+      instance_delayed_type_arguments.ptr());
+}
+
+static inline bool CheckSubtypeTestCacheEntry(
+    ArrayPtr array,
+    intptr_t num_inputs,
+    intptr_t entry_start,
+    ObjectPtr instance_class_id_or_signature,
+    AbstractTypePtr destination_type,
+    TypeArgumentsPtr instance_type_arguments,
+    TypeArgumentsPtr instantiator_type_arguments,
+    TypeArgumentsPtr function_type_arguments,
+    TypeArgumentsPtr instance_parent_function_type_arguments,
+    TypeArgumentsPtr instance_delayed_type_arguments) {
+  // This function is only called when the entry slot that determines occupancy
+  // is non-null. This means that the elements do not need to be load-acquired
+  // here, as entries in STC backing arrays are never changed once added.
+  switch (num_inputs) {
+    case 7:
+      if (Array::ElementAt(array,
+                           entry_start + SubtypeTestCache::kDestinationType) !=
+          destination_type) {
+        return false;
+      }
+      FALL_THROUGH;
+    case 6:
+      if (Array::ElementAt(
+              array,
+              entry_start +
+                  SubtypeTestCache::kInstanceDelayedFunctionTypeArguments) !=
+          instance_delayed_type_arguments) {
+        return false;
+      }
+      FALL_THROUGH;
+    case 5:
+      if (Array::ElementAt(
+              array,
+              entry_start +
+                  SubtypeTestCache::kInstanceParentFunctionTypeArguments) !=
+          instance_parent_function_type_arguments) {
+        return false;
+      }
+      FALL_THROUGH;
+    case 4:
+      if (Array::ElementAt(
+              array, entry_start + SubtypeTestCache::kFunctionTypeArguments) !=
+          function_type_arguments) {
+        return false;
+      }
+      FALL_THROUGH;
+    case 3:
+      if (Array::ElementAt(
+              array,
+              entry_start + SubtypeTestCache::kInstantiatorTypeArguments) !=
+          instantiator_type_arguments) {
+        return false;
+      }
+      FALL_THROUGH;
+    case 2:
+      if (Array::ElementAt(
+              array, entry_start + SubtypeTestCache::kInstanceTypeArguments) !=
+          instance_type_arguments) {
+        return false;
+      }
+      FALL_THROUGH;
+    case 1:
+      return Array::ElementAt(
+                 array,
+                 entry_start + SubtypeTestCache::kInstanceCidOrSignature) ==
+             instance_class_id_or_signature;
+  }
+  UNREACHABLE();
+  return false;
+}
+
+SubtypeTestCache::KeyLocation SubtypeTestCache::FindKeyOrUnusedFromProbe(
+    ArrayPtr array,
+    intptr_t num_inputs,
+    intptr_t probe,
+    ObjectPtr instance_class_id_or_signature,
+    AbstractTypePtr destination_type,
+    TypeArgumentsPtr instance_type_arguments,
+    TypeArgumentsPtr instantiator_type_arguments,
+    TypeArgumentsPtr function_type_arguments,
+    TypeArgumentsPtr instance_parent_function_type_arguments,
+    TypeArgumentsPtr instance_delayed_type_arguments,
+    BoolPtr* test_result) {
+  // Fast case for empty STCs.
+  if (array == Object::empty_subtype_test_cache_array().ptr()) {
+    return {probe, false};
+  }
+  const bool is_hash = IsHash(array);
+  const intptr_t num_entries = NumEntries(array);
+  intptr_t probe_distance = 1;
   while (true) {
-    const auto& tuple = table.At(probe);
-    if (tuple.Get<kInstanceCidOrSignature, std::memory_order_acquire>() ==
-        Object::null()) {
-      break;
+    intptr_t entry_start = probe * SubtypeTestCache::kTestEntryLength;
+    // First check the entry slot that determines occupancy, which requires
+    // load-acquire semantics.
+    ObjectPtr cid_or_sig = Array::ElementAt<std::memory_order_acquire>(
+        array, entry_start + SubtypeTestCache::kInstanceCidOrSignature);
+    if (cid_or_sig == Object::null()) {
+      // The appropriate location was found, but there's no existing entry.
+      return {probe, false};
     }
-    if (SubtypeTestCacheEntryMatches(
-            tuple, num_inputs, instance_class_id_or_signature, destination_type,
-            instance_type_arguments, instantiator_type_arguments,
-            function_type_arguments, instance_parent_function_type_arguments,
+    if (CheckSubtypeTestCacheEntry(
+            array, num_inputs, entry_start, instance_class_id_or_signature,
+            destination_type, instance_type_arguments,
+            instantiator_type_arguments, function_type_arguments,
+            instance_parent_function_type_arguments,
             instance_delayed_type_arguments)) {
+      if (test_result != nullptr) {
+        *test_result = Bool::RawCast(Array::ElementAt(
+            array, entry_start + SubtypeTestCache::kTestResult));
+      }
       return {probe, true};
     }
     // Advance probe by the current probing distance.
@@ -20068,6 +20109,7 @@ SubtypeTestCache::KeyLocation SubtypeTestCache::FindKeyOrUnused(
       probe_distance++;
     }
   }
+  UNREACHABLE();
   return {probe, false};
 }
 
@@ -20078,11 +20120,11 @@ ArrayPtr SubtypeTestCache::EnsureCapacity(Zone* zone,
   ASSERT(new_occupied > NumberOfChecks());
   ASSERT(was_grown != nullptr);
   // How many entries are in the current array (including unoccupied entries).
-  const intptr_t current_capacity = NumEntries(array);
+  const intptr_t current_capacity = NumEntries(array.ptr());
 
   // Early returns for cases where no growth is needed.
   *was_grown = false;
-  const bool is_linear = IsLinear(array);
+  const bool is_linear = IsLinear(array.ptr());
   if (is_linear) {
     // We need at least one unoccupied entry in addition to the occupied ones.
     if (current_capacity > new_occupied) return array.ptr();

runtime/vm/object.h

@@ -7922,7 +7922,10 @@ class SubtypeTestCache : public Object {
   intptr_t NumberOfChecks() const;
 
   // Retrieves the number of entries (occupied or unoccupied) in the cache.
-  intptr_t NumEntries() const;
+  intptr_t NumEntries() const {
+    ASSERT(!IsNull());
+    return NumEntries(untag()->cache_);
+  }
 
   // Adds a check, returning the index of the new entry in the cache.
   intptr_t AddCheck(
@@ -8011,7 +8014,10 @@ class SubtypeTestCache : public Object {
   bool Equals(const SubtypeTestCache& other) const;
 
   // Returns whether the cache backed by the given storage is hash-based.
-  bool IsHash() const;
+  bool IsHash() const {
+    if (IsNull()) return false;
+    return IsHash(untag()->cache_);
+  }
 
   // Creates a separate copy of the current STC contents.
   SubtypeTestCachePtr Copy(Thread* thread) const;
@@ -8075,13 +8081,13 @@ class SubtypeTestCache : public Object {
 
   // Retrieves the number of entries (occupied or unoccupied) in a cache
   // backed by the given array.
-  static intptr_t NumEntries(const Array& array);
+  static intptr_t NumEntries(const ArrayPtr array);
 
   // Returns whether the cache backed by the given storage is linear.
-  static bool IsLinear(const Array& array) { return !IsHash(array); }
+  static bool IsLinear(const ArrayPtr array) { return !IsHash(array); }
 
   // Returns whether the cache backed by the given storage is hash-based.
-  static bool IsHash(const Array& array);
+  static bool IsHash(const ArrayPtr array);
 
   struct KeyLocation {
     // The entry index if [present] is true, otherwise where the entry would
@@ -8109,6 +8115,22 @@ class SubtypeTestCache : public Object {
       const TypeArguments& instance_parent_function_type_arguments,
       const TypeArguments& instance_delayed_type_arguments);
 
+  // The main loop of FindKeyOrUnused() after the initial probe index has
+  // been calculated. Used by both FindKeyOrUnused and by the bytecode
+  // interpreter, so uses tagged pointers instead of handles.
+  static KeyLocation FindKeyOrUnusedFromProbe(
+      ArrayPtr array,
+      intptr_t num_inputs,
+      intptr_t probe,
+      ObjectPtr instance_class_id_or_signature,
+      AbstractTypePtr destination_type,
+      TypeArgumentsPtr instance_type_arguments,
+      TypeArgumentsPtr instantiator_type_arguments,
+      TypeArgumentsPtr function_type_arguments,
+      TypeArgumentsPtr instance_parent_function_type_arguments,
+      TypeArgumentsPtr instance_delayed_type_arguments,
+      BoolPtr* test_result = nullptr);
+
   // If the given array can contain the requested number of entries, returns
   // the same array and sets [was_grown] to false.
   //
@@ -8172,6 +8194,7 @@ class SubtypeTestCache : public Object {
   FINAL_HEAP_OBJECT_IMPLEMENTATION(SubtypeTestCache, Object);
   friend class Class;
   friend class FieldInvalidator;
+  friend class Interpreter;
   friend class VMSerializationRoots;
   friend class VMDeserializationRoots;
 };
@@ -11050,10 +11073,15 @@ class Array : public Instance {
     return array->untag()->data();
   }
 
+  template <std::memory_order order = std::memory_order_relaxed>
+  static ObjectPtr ElementAt(ArrayPtr array, intptr_t index) {
+    ASSERT((0 <= index) && (index < LengthOf(array)));
+    return array->untag()->element<order>(index);
+  }
+
   template <std::memory_order order = std::memory_order_relaxed>
   ObjectPtr At(intptr_t index) const {
-    ASSERT((0 <= index) && (index < Length()));
-    return untag()->element<order>(index);
+    return ElementAt<order>(ptr(), index);
   }
   template <std::memory_order order = std::memory_order_relaxed>
   void SetAt(intptr_t index, const Object& value) const {
@@ -11068,8 +11096,7 @@ class Array : public Instance {
 
   // Access to the array with acquire release semantics.
   ObjectPtr AtAcquire(intptr_t index) const {
-    ASSERT((0 <= index) && (index < Length()));
-    return untag()->element<std::memory_order_acquire>(index);
+    return ElementAt<std::memory_order_acquire>(ptr(), index);
   }
   void SetAtRelease(intptr_t index, const Object& value) const {
     ASSERT((0 <= index) && (index < Length()));