dart-lang/sdk
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[vm] Add support for hash-based caches in SubtypeNTestCache stubs.

Browse Source 
This also lowers the threshold for converting from a linear cache
to a hash-based cache from 100 to 30 on non-IA32 architectures.
(IA32 still goes to runtime if there's a hash-based cache.)

For SubtypeTestCache benchmarks above this threshold, we see an
improvement varying from ~50-100% to ~700-800% on all architectures,
from lowest number of checks (50) to highest (1000).

For SubtypeTestCache benchmarks below this threshold, no major
changes are seen: generally <5% improvement or <5% regression per
check at most.

TEST=vm/cc/TTS

Change-Id: I83aa7c085ab5a411e944ec660d6b8eba7a788ee0
Cq-Include-Trybots: luci.dart.try:vm-aot-linux-debug-simriscv64-try,vm-aot-linux-debug-x64-try,vm-aot-linux-release-x64-try,vm-aot-linux-product-x64-try,vm-aot-linux-release-simarm64-try,vm-aot-linux-release-simarm_x64-try,vm-aot-linux-debug-x64c-try,vm-aot-tsan-linux-release-x64-try,vm-aot-mac-release-arm64-try,vm-kernel-precomp-linux-release-x64-try,vm-kernel-precomp-linux-debug-x64-try,vm-aot-dwarf-linux-product-x64-try,vm-linux-release-ia32-try,vm-linux-debug-x64c-try,vm-linux-debug-x64-try,vm-linux-debug-simriscv64-try,vm-linux-release-simarm64-try,vm-linux-release-simarm-try,vm-mac-release-arm64-try,vm-mac-release-x64-try,vm-tsan-linux-release-x64-try,vm-reload-rollback-linux-release-x64-try,vm-reload-linux-release-x64-try,vm-ffi-qemu-linux-release-arm-try
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/308941
Reviewed-by: Martin Kustermann <kustermann@google.com>
Commit-Queue: Tess Strickland <sstrickl@google.com>
This commit is contained in:
Tess Strickland
2023-06-24 12:56:12 +00:00
committed by Commit Queue
parent a92a1e5064
commit f7e26c508d
17 changed files with 700 additions and 242 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/vm/compiler/assembler/assembler_riscv.cc
+3 -3
View File
@@ -3241,13 +3241,13 @@ void Assembler::LoadDFromOffset(FRegister dest, Register base, int32_t offset) {
}
void Assembler::LoadFromStack(Register dst, intptr_t depth) {
UNIMPLEMENTED();
LoadFromOffset(dst, SPREG, target::kWordSize * depth);
}
void Assembler::StoreToStack(Register src, intptr_t depth) {
UNIMPLEMENTED();
StoreToOffset(src, SPREG, target::kWordSize * depth);
}
void Assembler::CompareToStack(Register src, intptr_t depth) {
UNIMPLEMENTED();
CompareWithMemoryValue(src, Address(SPREG, target::kWordSize * depth));
}
void Assembler::StoreToOffset(Register src,
runtime/vm/compiler/assembler/assembler_x64.cc
+2 -1
View File
@@ -1130,7 +1130,8 @@ void Assembler::StoreToStack(Register src, intptr_t depth) {
}
void Assembler::CompareToStack(Register src, intptr_t depth) {
cmpq(Address(SPREG, depth * target::kWordSize), src);
ASSERT(depth >= 0);
cmpq(src, Address(SPREG, depth * target::kWordSize));
}
void Assembler::ExtendValue(Register to, Register from, OperandSize sz) {
runtime/vm/compiler/backend/range_analysis.cc
+1
View File
@@ -2766,6 +2766,7 @@ void LoadFieldInstr::InferRange(RangeAnalysis* analysis, Range* range) {
*range = Range(RangeBoundary::FromConstant(0), RangeBoundary::MaxSmi());
break;
case Slot::Kind::kAbstractType_hash:
case Slot::Kind::kTypeArguments_hash:
*range = Range(RangeBoundary::MinSmi(), RangeBoundary::MaxSmi());
break;
runtime/vm/compiler/backend/slot.cc
+1
View File
@@ -197,6 +197,7 @@ bool Slot::IsImmutableLengthSlot() const {
case Slot::Kind::kSuspendState_then_callback:
case Slot::Kind::kSuspendState_error_callback:
case Slot::Kind::kTypeArgumentsIndex:
case Slot::Kind::kAbstractType_hash:
case Slot::Kind::kTypeParameters_names:
case Slot::Kind::kTypeParameters_flags:
case Slot::Kind::kTypeParameters_bounds:
runtime/vm/compiler/backend/slot.h
+1
View File
@@ -129,6 +129,7 @@ class ParsedFunction;
V(Record, UntaggedRecord, shape, Smi, FINAL) \
V(TypeArguments, UntaggedTypeArguments, hash, Smi, VAR) \
V(TypeArguments, UntaggedTypeArguments, length, Smi, FINAL) \
V(AbstractType, UntaggedTypeArguments, hash, Smi, VAR) \
V(TypeParameters, UntaggedTypeParameters, names, Array, FINAL) \
V(UnhandledException, UntaggedUnhandledException, exception, Dynamic, FINAL) \
V(UnhandledException, UntaggedUnhandledException, stacktrace, Dynamic, FINAL)
runtime/vm/compiler/stub_code_compiler.cc
+419 -44
View File
@@ -2571,17 +2571,17 @@ void StubCodeCompiler::InsertBSSRelocation(BSS::Relocation reloc) {
}
#if !defined(TARGET_ARCH_IA32)
static void GenerateSubtypeTestCacheLoop(Assembler* assembler,
int n,
Register null_reg,
Register cache_entry_reg,
Register instance_cid_or_sig_reg,
Register instance_type_args_reg,
Register parent_fun_type_args_reg,
Register delayed_type_args_reg,
Label* found,
Label* not_found,
Label* next_iteration) {
static void GenerateSubtypeTestCacheLoopBody(Assembler* assembler,
int n,
Register null_reg,
Register cache_entry_reg,
Register instance_cid_or_sig_reg,
Register instance_type_args_reg,
Register parent_fun_type_args_reg,
Register delayed_type_args_reg,
Label* found,
Label* not_found,
Label* next_iteration) {
__ Comment("Loop");
// LoadAcquireCompressed assumes the loaded value is a heap object and
// extends it with the heap bits if compressed. However, the entry may be
@@ -2672,6 +2672,369 @@ static void GenerateSubtypeTestCacheLoop(Assembler* assembler,
__ BranchIf(EQUAL, found, Assembler::kNearJump);
}
// An object that uses RAII to load from and store to the stack when
// appropriate, allowing the code within that scope to act as if the given
// register is always provided. Either the Register value stored at [reg] must
// be a valid register (not kNoRegister) or [depth] must be a valid stack depth
// (not StackRegisterScope::kNoDepth).
//
// When the Register value stored at [reg] is a valid register, this scope
// generates no assembly and does not change the value stored at [reg].
//
// When [depth] is a valid stack depth, this scope object performs the
// following actions:
//
// On construction:
// * Generates assembly to load the value on the stack at [depth] into [alt].
// * Sets the Register value pointed to by [reg] to [alt].
//
// On destruction:
// * Generates assembly to store the value of [alt] into the stack at [depth].
// * Resets the Register value pointed to by [reg] to kNoRegister.
class StackRegisterScope : ValueObject {
public:
StackRegisterScope(Assembler* assembler,
Register* reg,
intptr_t depth,
Register alt = TMP)
: assembler(assembler), reg_(reg), depth_(depth), alt_(alt) {
if (depth_ != kNoDepth) {
ASSERT(depth_ >= 0);
ASSERT(*reg_ == kNoRegister);
ASSERT(alt_ != kNoRegister);
__ LoadFromStack(alt_, depth_);
*reg_ = alt_;
} else {
ASSERT(*reg_ != kNoRegister);
}
}
~StackRegisterScope() {
if (depth_ != kNoDepth) {
__ StoreToStack(alt_, depth_);
*reg_ = kNoRegister;
}
}
static constexpr intptr_t kNoDepth = kIntptrMin;
private:
Assembler* const assembler;
Register* const reg_;
const intptr_t depth_;
const Register alt_;
};
// Same inputs as StubCodeCompiler::GenerateSubtypeTestCacheSearch with
// the following additional requirements:
// - catch_entry_reg: the address of the backing array for the cache.
// - TypeTestABI::kScratchReg: the Smi value of the length field for the
// backing array in cache_entry_reg
//
// Also expects that all the STC entry input registers have been filled.
static void GenerateSubtypeTestCacheHashSearch(
Assembler* assembler,
int n,
Register null_reg,
Register cache_entry_reg,
Register instance_cid_or_sig_reg,
Register instance_type_args_reg,
Register parent_fun_type_args_reg,
Register delayed_type_args_reg,
Register cache_entry_end_reg,
Register cache_contents_size_reg,
Register probe_distance_reg,
const StubCodeCompiler::STCSearchExitGenerator& gen_found,
const StubCodeCompiler::STCSearchExitGenerator& gen_not_found) {
// Since the test entry size is a power of 2, we can use shr to divide.
const intptr_t kTestEntryLengthLog2 =
Utils::ShiftForPowerOfTwo(target::SubtypeTestCache::kTestEntryLength);
// Before we finish calculating the initial probe entry, we'll need the
// starting cache entry and the number of entries. We'll store these in
// [cache_contents_size_reg] and [probe_distance_reg] (or their equivalent
// stack slots), respectively.
__ Comment("Hash cache traversal");
__ Comment("Calculating number of entries");
// The array length is a Smi so it needs to be untagged.
__ SmiUntag(TypeTestABI::kScratchReg);
__ LsrImmediate(TypeTestABI::kScratchReg, kTestEntryLengthLog2);
if (probe_distance_reg != kNoRegister) {
__ MoveRegister(probe_distance_reg, TypeTestABI::kScratchReg);
} else {
__ PushRegister(TypeTestABI::kScratchReg);
}
__ Comment("Calculating starting entry address");
__ AddImmediate(cache_entry_reg,
target::Array::data_offset() - kHeapObjectTag);
if (cache_contents_size_reg != kNoRegister) {
__ MoveRegister(cache_contents_size_reg, cache_entry_reg);
} else {
__ PushRegister(cache_entry_reg);
}
__ Comment("Calculating end of entries address");
__ LslImmediate(TypeTestABI::kScratchReg,
kTestEntryLengthLog2 + target::kCompressedWordSizeLog2);
__ AddRegisters(TypeTestABI::kScratchReg, cache_entry_reg);
if (cache_entry_end_reg != kNoRegister) {
__ MoveRegister(cache_entry_end_reg, TypeTestABI::kScratchReg);
} else {
__ PushRegister(TypeTestABI::kScratchReg);
}
// At this point, the stack is in the following order, if the corresponding
// value doesn't have a register assignment:
// <number of total entries in cache array>
// <cache array entries start>
// <cache array entries end>
// --------- top of stack
//
// and after calculating the initial entry, we'll replace them as follows:
// <probe distance>
// <-cache array contents size> (note this is _negative_)
// <cache array entries end>
// ---------- top of stack
//
// So name them according to their later use.
intptr_t kProbeDistanceDepth = StackRegisterScope::kNoDepth;
intptr_t kHashStackElements = 0;
if (probe_distance_reg == kNoRegister) {
kProbeDistanceDepth = 0;
kHashStackElements++;
}
intptr_t kCacheContentsSizeDepth = StackRegisterScope::kNoDepth;
if (cache_contents_size_reg == kNoRegister) {
kProbeDistanceDepth++;
kHashStackElements++;
kCacheContentsSizeDepth = 0;
}
intptr_t kCacheArrayEndDepth = StackRegisterScope::kNoDepth;
if (cache_entry_end_reg == kNoRegister) {
kProbeDistanceDepth++;
kCacheContentsSizeDepth++;
kHashStackElements++;
kCacheArrayEndDepth = 0;
}
// After this point, any exits should go through one of these two labels,
// which will pop the extra stack elements pushed above.
Label found, not_found;
// When retrieving hashes from objects below, note that a hash of 0 means
// the hash hasn't been computed yet and we need to go to runtime.
auto get_abstract_type_hash = [&](Register dst, Register src,
const char* name) {
ASSERT(dst != kNoRegister);
ASSERT(src != kNoRegister);
__ Comment("Loading %s type hash", name);
#if defined(DEBUG)
// Verify the object in the given register is not null and break if not.
// Can't use EnsureHasClassIdInDEBUG here, as it could be an instance of
// any concrete subclass of AbstractType.
Label is_not_null, is_abstract_type;
__ CompareRegisters(src, null_reg);
__ BranchIf(NOT_EQUAL, &is_not_null, Assembler::kNearJump);
__ Comment("Stop: Expected non-null");
__ Breakpoint();
__ Bind(&is_not_null);
__ LoadClassIdMayBeSmi(TMP, src);
__ AddImmediate(TMP, -kTypeCid);
__ CompareImmediate(TMP, kTypeParameterCid - kTypeCid);
__ BranchIf(UNSIGNED_LESS_EQUAL, &is_abstract_type, Assembler::kNearJump);
__ Comment("Stop: Expected AbstractType");
__ Breakpoint();
__ Bind(&is_abstract_type);
#endif
__ LoadFromSlot(dst, src, Slot::AbstractType_hash());
__ SmiUntag(dst);
__ CompareImmediate(dst, 0);
__ BranchIf(EQUAL, &not_found);
};
auto get_type_arguments_hash = [&](Register dst, Register src,
const char* name) {
ASSERT(dst != kNoRegister);
ASSERT(src != kNoRegister);
Label done;
__ Comment("Loading %s type arguments hash", name);
// Preload the hash value for TypeArguments::null() so control can jump
// to done if null.
__ LoadImmediate(dst, TypeArguments::kAllDynamicHash);
__ CompareRegisters(src, null_reg);
__ BranchIf(EQUAL, &done, Assembler::kNearJump);
__ EnsureHasClassIdInDEBUG(kTypeArgumentsCid, src, TMP,
/*can_be_null=*/false);
__ LoadFromSlot(dst, src, Slot::TypeArguments_hash());
__ SmiUntag(dst);
__ CompareImmediate(dst, 0);
__ BranchIf(EQUAL, &not_found);
__ Bind(&done);
};
__ Comment("Hash the entry inputs");
{
Label done;
// Assume a Smi tagged instance cid to avoid a branch in the common case.
__ MoveRegister(cache_entry_reg, instance_cid_or_sig_reg);
__ SmiUntag(cache_entry_reg);
__ BranchIfSmi(instance_cid_or_sig_reg, &done, Assembler::kNearJump);
get_abstract_type_hash(cache_entry_reg, instance_cid_or_sig_reg,
"closure signature");
__ Bind(&done);
}
if (n >= 7) {
get_abstract_type_hash(TypeTestABI::kScratchReg, TypeTestABI::kDstTypeReg,
"destination");
__ CombineHashes(cache_entry_reg, TypeTestABI::kScratchReg);
}
if (n >= 6) {
get_type_arguments_hash(TypeTestABI::kScratchReg, delayed_type_args_reg,
"delayed");
__ CombineHashes(cache_entry_reg, TypeTestABI::kScratchReg);
}
if (n >= 5) {
get_type_arguments_hash(TypeTestABI::kScratchReg, parent_fun_type_args_reg,
"parent function");
__ CombineHashes(cache_entry_reg, TypeTestABI::kScratchReg);
}
if (n >= 4) {
get_type_arguments_hash(TypeTestABI::kScratchReg,
TypeTestABI::kFunctionTypeArgumentsReg, "function");
__ CombineHashes(cache_entry_reg, TypeTestABI::kScratchReg);
}
if (n >= 3) {
get_type_arguments_hash(TypeTestABI::kScratchReg,
TypeTestABI::kInstantiatorTypeArgumentsReg,
"instantiator");
__ CombineHashes(cache_entry_reg, TypeTestABI::kScratchReg);
}
if (n >= 2) {
get_type_arguments_hash(TypeTestABI::kScratchReg, instance_type_args_reg,
"instance");
__ CombineHashes(cache_entry_reg, TypeTestABI::kScratchReg);
}
__ FinalizeHash(cache_entry_reg);
// This requires the number of entries in a hash cache to be a power of 2.
__ Comment("Converting hash to probe entry index");
{
StackRegisterScope scope(assembler, &probe_distance_reg,
kProbeDistanceDepth, TypeTestABI::kScratchReg);
// The entry count is not needed after this point; create the mask in place.
__ AddImmediate(probe_distance_reg, -1);
__ AndRegisters(cache_entry_reg, probe_distance_reg);
// Now set the register to the initial probe distance in words.
__ Comment("Set initial probe distance");
__ LoadImmediate(probe_distance_reg,
target::kCompressedWordSize *
target::SubtypeTestCache::kTestEntryLength);
}
// Now cache_entry_reg is the starting probe entry index.
__ Comment("Converting probe entry index to probe entry address");
{
StackRegisterScope scope(assembler, &cache_contents_size_reg,
kCacheContentsSizeDepth, TypeTestABI::kScratchReg);
__ LslImmediate(cache_entry_reg,
kTestEntryLengthLog2 + target::kCompressedWordSizeLog2);
__ AddRegisters(cache_entry_reg, cache_contents_size_reg);
// Now set the register to the negated size of the cache contents in words.
__ Comment("Set negated cache contents size");
if (cache_entry_end_reg != kNoRegister) {
__ SubRegisters(cache_contents_size_reg, cache_entry_end_reg);
} else {
__ LoadFromStack(TMP, kCacheArrayEndDepth);
__ SubRegisters(cache_contents_size_reg, TMP);
}
}
Label loop, next_iteration;
__ Bind(&loop);
GenerateSubtypeTestCacheLoopBody(
assembler, n, null_reg, cache_entry_reg, instance_cid_or_sig_reg,
instance_type_args_reg, parent_fun_type_args_reg, delayed_type_args_reg,
&found, &not_found, &next_iteration);
__ Bind(&next_iteration);
__ Comment("Move to next entry");
{
StackRegisterScope scope(assembler, &probe_distance_reg,
kProbeDistanceDepth, TypeTestABI::kScratchReg);
__ AddRegisters(cache_entry_reg, probe_distance_reg);
__ Comment("Adjust probe distance");
__ AddImmediate(probe_distance_reg,
target::kCompressedWordSize *
target::SubtypeTestCache::kTestEntryLength);
}
__ Comment("Check for leaving array");
// Make sure we haven't run off the array.
if (cache_entry_end_reg != kNoRegister) {
__ CompareRegisters(cache_entry_reg, cache_entry_end_reg);
} else {
__ CompareToStack(cache_entry_reg, kCacheArrayEndDepth);
}
__ BranchIf(LESS, &loop, Assembler::kNearJump);
__ Comment("Wrap around to start of entries");
// Add the negated size of the cache contents.
if (cache_contents_size_reg != kNoRegister) {
__ AddRegisters(cache_entry_reg, cache_contents_size_reg);
} else {
__ LoadFromStack(TypeTestABI::kScratchReg, kCacheContentsSizeDepth);
__ AddRegisters(cache_entry_reg, TypeTestABI::kScratchReg);
}
__ Jump(&loop, Assembler::kNearJump);
__ Bind(&found);
__ Comment("Hash found");
__ Drop(kHashStackElements);
gen_found(assembler, n);
__ Bind(&not_found);
__ Comment("Hash not found");
__ Drop(kHashStackElements);
gen_not_found(assembler, n);
}
// Same inputs as StubCodeCompiler::GenerateSubtypeTestCacheSearch with
// the following additional requirement:
// - catch_entry_reg: the address of the backing array for the cache.
//
// Also expects that all the STC entry input registers have been filled.
static void GenerateSubtypeTestCacheLinearSearch(
Assembler* assembler,
int n,
Register null_reg,
Register cache_entry_reg,
Register instance_cid_or_sig_reg,
Register instance_type_args_reg,
Register parent_fun_type_args_reg,
Register delayed_type_args_reg,
const StubCodeCompiler::STCSearchExitGenerator& gen_found,
const StubCodeCompiler::STCSearchExitGenerator& gen_not_found) {
__ Comment("Linear cache traversal");
__ AddImmediate(cache_entry_reg,
target::Array::data_offset() - kHeapObjectTag);
Label found, not_found, loop, next_iteration;
__ Bind(&loop);
GenerateSubtypeTestCacheLoopBody(
assembler, n, null_reg, cache_entry_reg, instance_cid_or_sig_reg,
instance_type_args_reg, parent_fun_type_args_reg, delayed_type_args_reg,
&found, &not_found, &next_iteration);
__ Bind(&next_iteration);
__ Comment("Next iteration");
__ AddImmediate(
cache_entry_reg,
target::kCompressedWordSize * target::SubtypeTestCache::kTestEntryLength);
__ Jump(&loop, Assembler::kNearJump);
__ Bind(&found);
__ Comment("Linear found");
gen_found(assembler, n);
__ Bind(&not_found);
__ Comment("Linear not found");
gen_not_found(assembler, n);
}
void StubCodeCompiler::GenerateSubtypeTestCacheSearch(
Assembler* assembler,
int n,
@@ -2681,7 +3044,11 @@ void StubCodeCompiler::GenerateSubtypeTestCacheSearch(
Register instance_type_args_reg,
Register parent_fun_type_args_reg,
Register delayed_type_args_reg,
Label* not_found) {
Register cache_entry_end_reg,
Register cache_contents_size_reg,
Register probe_distance_reg,
const StubCodeCompiler::STCSearchExitGenerator& gen_found,
const StubCodeCompiler::STCSearchExitGenerator& gen_not_found) {
#if defined(DEBUG)
RegisterSet input_regs;
ASSERT(null_reg != kNoRegister);
@@ -2702,13 +3069,23 @@ void StubCodeCompiler::GenerateSubtypeTestCacheSearch(
ASSERT(!input_regs.ContainsRegister(parent_fun_type_args_reg));
input_regs.AddRegister(parent_fun_type_args_reg);
}
ASSERT(!input_regs.ContainsRegister(TypeTestABI::kInstanceReg));
if (n >= 6) {
ASSERT(!input_regs.ContainsRegister(TypeTestABI::kInstanceReg));
ASSERT(delayed_type_args_reg != kNoRegister);
ASSERT(!input_regs.ContainsRegister(delayed_type_args_reg));
input_regs.AddRegister(delayed_type_args_reg);
} else {
ASSERT(!input_regs.ContainsRegister(TypeTestABI::kInstanceReg));
}
if (cache_entry_end_reg != kNoRegister) {
ASSERT(!input_regs.ContainsRegister(cache_entry_end_reg));
input_regs.AddRegister(cache_entry_end_reg);
}
if (cache_contents_size_reg != kNoRegister) {
ASSERT(!input_regs.ContainsRegister(cache_contents_size_reg));
input_regs.AddRegister(cache_contents_size_reg);
}
if (probe_distance_reg != kNoRegister) {
ASSERT(!input_regs.ContainsRegister(probe_distance_reg));
input_regs.AddRegister(probe_distance_reg);
}
// We can allow the use of the registers below only if we're not expecting
// them as an inspected input.
@@ -2726,26 +3103,13 @@ void StubCodeCompiler::GenerateSubtypeTestCacheSearch(
// We use this as a scratch, so it has to be distinct from the others.
ASSERT(!input_regs.ContainsRegister(TypeTestABI::kScratchReg));
#endif
Label loop;
__ LoadAcquireCompressed(
cache_entry_reg, TypeTestABI::kSubtypeTestCacheReg,
target::SubtypeTestCache::cache_offset() - kHeapObjectTag);
// There is a maximum size for linear caches that is smaller than the size
// of any hash-based cache, so we check the size of the backing array to
// determine if this is a linear or hash-based cache.
__ LoadFromSlot(TypeTestABI::kScratchReg, cache_entry_reg,
Slot::Array_length());
__ CompareImmediate(TypeTestABI::kScratchReg,
target::ToRawSmi(SubtypeTestCache::kMaxLinearCacheSize));
// TODO(sstrickl): Handle hash-based tables in the stub and load the first
// entry to check here instead of generating a false negative.
__ BranchIf(GREATER, not_found);
__ AddImmediate(cache_entry_reg,
target::Array::data_offset() - kHeapObjectTag);
Label not_closure;
// Fill in all the STC input registers.
Label initialized, not_closure;
if (n >= 4) {
__ LoadClassIdMayBeSmi(instance_cid_or_sig_reg, TypeTestABI::kInstanceReg);
} else {
@@ -2793,7 +3157,7 @@ void StubCodeCompiler::GenerateSubtypeTestCacheSearch(
target::Closure::delayed_type_arguments_offset()));
}
__ Jump(&loop, Assembler::kNearJump);
__ Jump(&initialized, Assembler::kNearJump);
}
// Non-Closure handling.
@@ -2826,20 +3190,31 @@ void StubCodeCompiler::GenerateSubtypeTestCacheSearch(
}
}
Label next_iteration, found;
__ Bind(&loop);
GenerateSubtypeTestCacheLoop(assembler, n, null_reg, cache_entry_reg,
instance_cid_or_sig_reg, instance_type_args_reg,
parent_fun_type_args_reg, delayed_type_args_reg,
&found, not_found, &next_iteration);
__ Bind(&next_iteration);
__ Comment("Next iteration");
__ AddImmediate(
cache_entry_reg,
target::kCompressedWordSize * target::SubtypeTestCache::kTestEntryLength);
__ Jump(&loop, Assembler::kNearJump);
__ Bind(&initialized);
// There is a maximum size for linear caches that is smaller than the size
// of any hash-based cache, so we check the size of the backing array to
// determine if this is a linear or hash-based cache.
//
// We load it into TypeTestABI::kScratchReg as the hash search code expects
// it there.
Label is_hash;
__ LoadFromSlot(TypeTestABI::kScratchReg, cache_entry_reg,
Slot::Array_length());
__ CompareImmediate(TypeTestABI::kScratchReg,
target::ToRawSmi(SubtypeTestCache::kMaxLinearCacheSize));
__ BranchIf(GREATER, &is_hash);
__ Bind(&found);
GenerateSubtypeTestCacheLinearSearch(
assembler, n, null_reg, cache_entry_reg, instance_cid_or_sig_reg,
instance_type_args_reg, parent_fun_type_args_reg, delayed_type_args_reg,
gen_found, gen_not_found);
__ Bind(&is_hash);
GenerateSubtypeTestCacheHashSearch(
assembler, n, null_reg, cache_entry_reg, instance_cid_or_sig_reg,
instance_type_args_reg, parent_fun_type_args_reg, delayed_type_args_reg,
cache_entry_end_reg, cache_contents_size_reg, probe_distance_reg,
gen_found, gen_not_found);
}
#endif
runtime/vm/compiler/stub_code_compiler.h
+42 -15
View File
@@ -119,13 +119,23 @@ class StubCodeCompiler {
// `StubCode::*<stub-name>Shared{With,Without}FpuRegsStub()`
static intptr_t WordOffsetFromFpToCpuRegister(Register cpu_register);
#if !defined(TARGET_ARCH_IA32)
// Used for passing functions that generate exit branches for a
// SubtypeTestCache search stub. Must generate a return instruction.
using STCSearchExitGenerator = std::function<void(Assembler*, int)>;
#endif
private:
#if !defined(TARGET_ARCH_IA32)
// Generates the code for searching a subtype test cache for an entry that
// matches the contents of the TypeTestABI registers. If no matching
// entry is found, then the loop jumps to [not_found], otherwise execution
// continues immediately after the loop and [cache_entry_reg] points to
// the start of the matching cache entry.
// entry is found, then the code generated by [not_found] is executed.
// Otherwise, the code generated by [found] is executed, which can assume
// that [cache_entry_reg] points to the start of the matching cache entry.
// Both generators should return from the stub and not fall through.
//
// Inputs in addition to those in TypeTestABI:
// - null_reg: a register containing the address Object::null().
//
// The following registers from TypeTestABI are inputs under the following
// conditions:
@@ -140,27 +150,44 @@ class StubCodeCompiler {
// - cache_entry_reg: contains the ArrayPtr for the backing array of the STC
//
// The following registers must be provided for the given conditions and
// are clobbered:
// are clobbered, and can be kNoRegister otherwise:
// - instance_cid_or_sig_reg: always
// - instance_type_args_reg: [n] >= 2
// - parent_fun_type_args_reg: [n] >= 6
// - delayed_type_args_reg: [n] >= 7
//
// The following registers must be distinct from other inputs when provided,
// but can be kNoRegister:
// - cache_entry_end_reg: used in the hash-based cache iteration loop
// on each iteration
// - cache_entry_start_reg, used in the hash-based cache iteration loop
// to reset if iteration hits the end of the entries
// - cache_entry_count_reg, used to calculate the starting index to probe
// Note that if any of these are kNoRegister, then a stack slot is used to
// store and retrieve the corresponding value.
//
// Note that all input registers must be distinct, except for the case
// of kInstanceReg and [delayed_type_args_reg], which are allowed to overlap.
// of kInstanceReg, which can be used for one of [delayed_type_args_reg],
// [cache_entry_end_reg], [cache_entry_start_reg], or [cache_entry_count_reg],
// which are all set after the last use of kInstanceReg.
//
// Also note that if any non-TypeTestABI registers overlap with any
// non-scratch TypeTestABI registers, the original value of the TypeTestABI
// register must be restored afterwards.
static void GenerateSubtypeTestCacheSearch(Assembler* assembler,
int n,
Register null_reg,
Register cache_entry_reg,
Register instance_cid_or_sig_reg,
Register instance_type_args_reg,
Register parent_fun_type_args_reg,
Register delayed_type_args_reg,
Label* not_found);
// register must be stored before the generated code and restored afterwards.
static void GenerateSubtypeTestCacheSearch(
Assembler* assembler,
int n,
Register null_reg,
Register cache_entry_reg,
Register instance_cid_or_sig_reg,
Register instance_type_args_reg,
Register parent_fun_type_args_reg,
Register delayed_type_args_reg,
Register cache_entry_end_reg,
Register cache_entry_start_reg,
Register cache_entry_count_reg,
const STCSearchExitGenerator& gen_found,
const STCSearchExitGenerator& gen_not_found);
#endif
// Common function for generating the different SubtypeTestCache search
runtime/vm/compiler/stub_code_compiler_arm.cc
+54 -26
View File
@@ -2706,10 +2706,11 @@ void StubCodeCompiler::GenerateSubtypeNTestCacheStub(Assembler* assembler,
const Register kCacheArrayReg = TypeTestABI::kSubtypeTestCacheReg;
saved_registers.AddRegister(kCacheArrayReg);
// NOTFP must be preserved for bare payloads, otherwise CODE_REG.
const bool use_bare_payloads = FLAG_precompiled_mode;
// For this, we choose the register that need not be preserved of the pair.
const Register kNullReg = use_bare_payloads ? CODE_REG : NOTFP;
// CODE_REG is used only in JIT mode, and the dispatch table only exists in
// AOT mode, so we can use the corresponding register for the mode we're not
// in without having to preserve it.
const Register kNullReg =
FLAG_precompiled_mode ? CODE_REG : DISPATCH_TABLE_REG;
__ LoadObject(kNullReg, NullObject());
// Free up additional registers needed for checks in the loop. Initially
@@ -2721,9 +2722,10 @@ void StubCodeCompiler::GenerateSubtypeNTestCacheStub(Assembler* assembler,
}
Register kInstanceParentFunctionTypeArgumentsReg = kNoRegister;
if (n >= 5) {
// For this, we choose the register that must be preserved of the pair.
// For this, we look at the pair of Registers we considered for kNullReg
// and use the one that must be preserved instead.
kInstanceParentFunctionTypeArgumentsReg =
use_bare_payloads ? NOTFP : CODE_REG;
FLAG_precompiled_mode ? DISPATCH_TABLE_REG : CODE_REG;
saved_registers.AddRegister(kInstanceParentFunctionTypeArgumentsReg);
}
Register kInstanceDelayedFunctionTypeArgumentsReg = kNoRegister;
@@ -2736,29 +2738,55 @@ void StubCodeCompiler::GenerateSubtypeNTestCacheStub(Assembler* assembler,
kInstanceDelayedFunctionTypeArgumentsReg = TypeTestABI::kInstanceReg;
saved_registers.AddRegister(kInstanceDelayedFunctionTypeArgumentsReg);
}
// We'll replace these with actual registers if possible, but fall back to
// the stack if register pressure is too great. The last two values are
// used in every loop iteration, and so are more important to put in
// registers if possible, whereas the first is used only when we go off
// the end of the backing array (usually at most once per check).
Register kCacheContentsSizeReg = kNoRegister;
if (n < 5) {
// Use the register we would have used for the parent function type args.
kCacheContentsSizeReg =
FLAG_precompiled_mode ? DISPATCH_TABLE_REG : CODE_REG;
saved_registers.AddRegister(kCacheContentsSizeReg);
}
Register kProbeDistanceReg = kNoRegister;
if (n < 6) {
// Use the register we would have used for the delayed type args.
kProbeDistanceReg = TypeTestABI::kInstanceReg;
saved_registers.AddRegister(kProbeDistanceReg);
}
Register kCacheEntryEndReg = kNoRegister;
if (n < 7) {
// Use the destination type, as that is the last input that might be unused.
kCacheEntryEndReg = TypeTestABI::kDstTypeReg;
saved_registers.AddRegister(TypeTestABI::kDstTypeReg);
}
__ PushRegisters(saved_registers);
Label not_found;
GenerateSubtypeTestCacheSearch(assembler, n, kNullReg, kCacheArrayReg,
STCInternalRegs::kInstanceCidOrSignatureReg,
kInstanceInstantiatorTypeArgumentsReg,
kInstanceParentFunctionTypeArgumentsReg,
kInstanceDelayedFunctionTypeArgumentsReg,
&not_found);
__ Comment("Found");
__ LoadCompressed(
TypeTestABI::kSubtypeTestCacheResultReg,
Address(kCacheArrayReg, target::kCompressedWordSize *
target::SubtypeTestCache::kTestResult));
__ PopRegisters(saved_registers);
__ Ret();
__ Bind(&not_found);
__ Comment("Not found");
__ MoveRegister(TypeTestABI::kSubtypeTestCacheResultReg, kNullReg);
__ PopRegisters(saved_registers);
__ Ret();
GenerateSubtypeTestCacheSearch(
assembler, n, kNullReg, kCacheArrayReg,
STCInternalRegs::kInstanceCidOrSignatureReg,
kInstanceInstantiatorTypeArgumentsReg,
kInstanceParentFunctionTypeArgumentsReg,
kInstanceDelayedFunctionTypeArgumentsReg, kCacheEntryEndReg,
kCacheContentsSizeReg, kProbeDistanceReg,
[&](Assembler* assembler, int n) {
__ LoadCompressed(
TypeTestABI::kSubtypeTestCacheResultReg,
Address(kCacheArrayReg, target::kCompressedWordSize *
target::SubtypeTestCache::kTestResult));
__ PopRegisters(saved_registers);
__ Ret();
},
[&](Assembler* assembler, int n) {
__ MoveRegister(TypeTestABI::kSubtypeTestCacheResultReg, kNullReg);
__ PopRegisters(saved_registers);
__ Ret();
});
}
// Return the current stack pointer address, used to do stack alignment checks.
runtime/vm/compiler/stub_code_compiler_arm64.cc
+15 -13
View File
@@ -3036,19 +3036,21 @@ void StubCodeCompiler::GenerateSubtypeNTestCacheStub(Assembler* assembler,
STCInternalRegs::kInstanceCidOrSignatureReg,
STCInternalRegs::kInstanceInstantiatorTypeArgumentsReg,
STCInternalRegs::kInstanceParentFunctionTypeArgumentsReg,
STCInternalRegs::kInstanceDelayedFunctionTypeArgumentsReg, &not_found);
__ Comment("Found");
__ LoadCompressed(
TypeTestABI::kSubtypeTestCacheResultReg,
Address(kCacheArrayReg, target::kCompressedWordSize *
target::SubtypeTestCache::kTestResult));
__ Ret();
__ Bind(&not_found);
__ Comment("Not found");
__ MoveRegister(TypeTestABI::kSubtypeTestCacheResultReg, NULL_REG);
__ Ret();
STCInternalRegs::kInstanceDelayedFunctionTypeArgumentsReg,
STCInternalRegs::kCacheEntriesEndReg,
STCInternalRegs::kCacheContentsSizeReg,
STCInternalRegs::kProbeDistanceReg,
[](Assembler* assembler, int n) {
__ LoadCompressed(
TypeTestABI::kSubtypeTestCacheResultReg,
Address(kCacheArrayReg, target::kCompressedWordSize *
target::SubtypeTestCache::kTestResult));
__ Ret();
},
[](Assembler* assembler, int n) {
__ MoveRegister(TypeTestABI::kSubtypeTestCacheResultReg, NULL_REG);
__ Ret();
});
}
void StubCodeCompiler::GenerateGetCStackPointerStub() {
runtime/vm/compiler/stub_code_compiler_riscv.cc
+15 -13
View File
@@ -2831,19 +2831,21 @@ void StubCodeCompiler::GenerateSubtypeNTestCacheStub(Assembler* assembler,
STCInternalRegs::kInstanceCidOrSignatureReg,
STCInternalRegs::kInstanceInstantiatorTypeArgumentsReg,
STCInternalRegs::kInstanceParentFunctionTypeArgumentsReg,
STCInternalRegs::kInstanceDelayedFunctionTypeArgumentsReg, &not_found);
__ Comment("Found");
__ LoadCompressed(
TypeTestABI::kSubtypeTestCacheResultReg,
Address(kCacheArrayReg, target::kCompressedWordSize *
target::SubtypeTestCache::kTestResult));
__ Ret();
__ Bind(&not_found);
__ Comment("Not found");
__ MoveRegister(TypeTestABI::kSubtypeTestCacheResultReg, NULL_REG);
__ Ret();
STCInternalRegs::kInstanceDelayedFunctionTypeArgumentsReg,
STCInternalRegs::kCacheEntriesEndReg,
STCInternalRegs::kCacheContentsSizeReg,
STCInternalRegs::kProbeDistanceReg,
[](Assembler* assembler, int n) {
__ LoadCompressed(
TypeTestABI::kSubtypeTestCacheResultReg,
Address(kCacheArrayReg, target::kCompressedWordSize *
target::SubtypeTestCache::kTestResult));
__ Ret();
},
[](Assembler* assembler, int n) {
__ MoveRegister(TypeTestABI::kSubtypeTestCacheResultReg, NULL_REG);
__ Ret();
});
}
void StubCodeCompiler::GenerateGetCStackPointerStub() {
runtime/vm/compiler/stub_code_compiler_x64.cc
+44 -15
View File
@@ -2995,6 +2995,34 @@ void StubCodeCompiler::GenerateSubtypeNTestCacheStub(Assembler* assembler,
kInstanceDelayedFunctionTypeArgumentsReg = CODE_REG;
saved_registers.AddRegister(kInstanceDelayedFunctionTypeArgumentsReg);
}
// We'll replace these with actual registers if possible, but fall back to
// the stack if register pressure is too great. The last two values are
// used in every loop iteration, and so are more important to put in
// registers if possible, whereas the first is used only when we go off
// the end of the backing array (usually at most once per check).
Register kCacheContentsSizeReg = kNoRegister;
if (n < 5) {
// Use the register we would have used for the parent function type args.
kCacheContentsSizeReg = PP;
saved_registers.AddRegister(kCacheContentsSizeReg);
}
Register kProbeDistanceReg = kNoRegister;
if (n < 6) {
// Use the register we would have used for the delayed type args.
kProbeDistanceReg = CODE_REG;
saved_registers.AddRegister(kProbeDistanceReg);
}
Register kCacheEntryEndReg = kNoRegister;
if (n < 2) {
// This register isn't in use and doesn't require saving/restoring.
kCacheEntryEndReg = STCInternalRegs::kInstanceInstantiatorTypeArgumentsReg;
} else if (n < 7) {
// Use the destination type, as that is the last input that might be unused.
kCacheEntryEndReg = TypeTestABI::kDstTypeReg;
saved_registers.AddRegister(TypeTestABI::kDstTypeReg);
}
__ PushRegisters(saved_registers);
Label done;
@@ -3003,21 +3031,22 @@ void StubCodeCompiler::GenerateSubtypeNTestCacheStub(Assembler* assembler,
STCInternalRegs::kInstanceCidOrSignatureReg,
STCInternalRegs::kInstanceInstantiatorTypeArgumentsReg,
kInstanceParentFunctionTypeArgumentsReg,
kInstanceDelayedFunctionTypeArgumentsReg, &done);
__ Comment("Found");
__ LoadCompressed(TypeTestABI::kSubtypeTestCacheResultReg,
Address(STCInternalRegs::kCacheEntryReg,
target::kCompressedWordSize *
target::SubtypeTestCache::kTestResult));
__ Bind(&done);
__ Comment("Done");
// We initialize kSubtypeTestCacheResultReg to null so it can be used for
// null checks, so the result value is already correct in the not found case
// and so popping and exiting can be shared between both branches.
__ PopRegisters(saved_registers);
__ Ret();
kInstanceDelayedFunctionTypeArgumentsReg, kCacheEntryEndReg,
kCacheContentsSizeReg, kProbeDistanceReg,
[&](Assembler* assembler, int n) {
__ LoadCompressed(TypeTestABI::kSubtypeTestCacheResultReg,
Address(STCInternalRegs::kCacheEntryReg,
target::kCompressedWordSize *
target::SubtypeTestCache::kTestResult));
__ PopRegisters(saved_registers);
__ Ret();
},
[&](Assembler* assembler, int n) {
// We initialize kSubtypeTestCacheResultReg to null so it can be used
// for null checks, so the result value is already set.
__ PopRegisters(saved_registers);
__ Ret();
});
}
// Return the current stack pointer address, used to stack alignment
runtime/vm/constants_arm64.h
+6 -1
View File
@@ -220,12 +220,17 @@ struct STCInternalRegs {
static constexpr Register kInstanceInstantiatorTypeArgumentsReg = R5;
static constexpr Register kInstanceParentFunctionTypeArgumentsReg = R9;
static constexpr Register kInstanceDelayedFunctionTypeArgumentsReg = R10;
static constexpr Register kCacheEntriesEndReg = R11;
static constexpr Register kCacheContentsSizeReg = R12;
static constexpr Register kProbeDistanceReg = R13;
static constexpr intptr_t kInternalRegisters =
(1 << kInstanceCidOrSignatureReg) |
(1 << kInstanceInstantiatorTypeArgumentsReg) |
(1 << kInstanceParentFunctionTypeArgumentsReg) |
(1 << kInstanceDelayedFunctionTypeArgumentsReg);
(1 << kInstanceDelayedFunctionTypeArgumentsReg) |
(1 << kCacheEntriesEndReg) | (1 << kCacheContentsSizeReg) |
(1 << kProbeDistanceReg);
};
// Calling convention when calling TypeTestingStub and SubtypeTestCacheStub.
runtime/vm/constants_riscv.h
+6 -1
View File
@@ -229,12 +229,17 @@ struct STCInternalRegs {
static constexpr Register kInstanceInstantiatorTypeArgumentsReg = S3;
static constexpr Register kInstanceParentFunctionTypeArgumentsReg = S4;
static constexpr Register kInstanceDelayedFunctionTypeArgumentsReg = S5;
static constexpr Register kCacheEntriesEndReg = S6;
static constexpr Register kCacheContentsSizeReg = A6;
static constexpr Register kProbeDistanceReg = A7;
static constexpr intptr_t kInternalRegisters =
(1 << kInstanceCidOrSignatureReg) |
(1 << kInstanceInstantiatorTypeArgumentsReg) |
(1 << kInstanceParentFunctionTypeArgumentsReg) |
(1 << kInstanceDelayedFunctionTypeArgumentsReg);
(1 << kInstanceDelayedFunctionTypeArgumentsReg) |
(1 << kCacheEntriesEndReg) | (1 << kCacheContentsSizeReg) |
(1 << kProbeDistanceReg);
};
// Calling convention when calling TypeTestingStub and SubtypeTestCacheStub.
runtime/vm/object.cc
+9 -1
View File
@@ -19220,7 +19220,8 @@ intptr_t SubtypeTestCache::NumberOfChecks() const {
}
intptr_t SubtypeTestCache::NumEntries() const {
return NumEntries(Array::Handle(cache()));
ASSERT(!IsNull());
return Array::LengthOf(cache()) / kTestEntryLength;
}
intptr_t SubtypeTestCache::NumEntries(const Array& array) {
@@ -19526,6 +19527,12 @@ ArrayPtr SubtypeTestCache::EnsureCapacity(Zone* zone,
// find an empty slot). This is described in Knuth's The Art of Computer
// Programming Volume 2, Chapter 6.4, exercise 20 (solution in the
// appendix, 2nd edition).
//
// This is also important because when we do hash probing, we take the
// calculated hash from the inputs and then calculate (hash % capacity) to get
// the initial probe index. To ensure this is a fast calculation in the stubs,
// we ensure the capacity is a power of 2, which allows (hash % capacity) to
// be calculated as (hash & (capacity - 1)).
ASSERT(Utils::IsPowerOfTwo(new_capacity));
ASSERT(LoadFactor(new_occupied, new_capacity) < kMaxLoadFactor);
const intptr_t new_size = new_capacity * kTestEntryLength;
@@ -19911,6 +19918,7 @@ SubtypeTestCachePtr SubtypeTestCache::Copy(Thread* thread) const {
bool SubtypeTestCache::IsOccupied(intptr_t index) const {
ASSERT(!IsNull());
ASSERT(index < NumEntries());
const intptr_t cache_index =
index * kTestEntryLength + kInstanceCidOrSignature;
NoSafepointScope no_safepoint;
runtime/vm/object.h
+1 -3
View File
@@ -7574,9 +7574,7 @@ class SubtypeTestCache : public Object {
// force runtime checks.
static constexpr intptr_t kMaxLinearCacheEntries = 100;
#else
// TODO(sstrickl): Currently we don't generate hash cache probing in the
// other architectures, so use 100 like IA32. Update this to 10 once we do.
static constexpr intptr_t kMaxLinearCacheEntries = 100;
static constexpr intptr_t kMaxLinearCacheEntries = 30;
#endif
// Whether the entry at the given index in the cache is occupied. Exposed
runtime/vm/runtime_entry.cc
+18 -44
View File
@@ -840,9 +840,8 @@ static void PrintTypeCheck(const char* message,
}
}
// Checks for false negatives in the SubtypeNTestCache stubs and returns the
// result found if any, otherwise null.
static BoolPtr ResultForExistingTypeTestCacheEntry(
#if defined(TARGET_ARCH_IA32)
static BoolPtr CheckHashBasedSubtypeTestCache(
Zone* zone,
Thread* thread,
const Instance& instance,
@@ -850,10 +849,7 @@ static BoolPtr ResultForExistingTypeTestCacheEntry(
const TypeArguments& instantiator_type_arguments,
const TypeArguments& function_type_arguments,
const SubtypeTestCache& cache) {
ASSERT(destination_type.IsCanonical());
ASSERT(instantiator_type_arguments.IsCanonical());
ASSERT(function_type_arguments.IsCanonical());
if (cache.IsNull()) return Bool::null();
ASSERT(cache.IsHash());
// Record instances are not added to the cache as they don't have a valid
// key (type of a record depends on types of all its fields).
if (instance.IsRecord()) return Bool::null();
@@ -874,19 +870,13 @@ static BoolPtr ResultForExistingTypeTestCacheEntry(
const auto& closure = Closure::Cast(instance);
const auto& function = Function::Handle(zone, closure.function());
instance_class_id_or_signature = function.signature();
ASSERT(instance_class_id_or_signature.IsFunctionType());
instance_type_arguments = closure.instantiator_type_arguments();
instance_parent_function_type_arguments = closure.function_type_arguments();
instance_delayed_type_arguments = closure.delayed_type_arguments();
ASSERT(instance_class_id_or_signature.IsCanonical());
ASSERT(instance_type_arguments.IsCanonical());
ASSERT(instance_parent_function_type_arguments.IsCanonical());
ASSERT(instance_delayed_type_arguments.IsCanonical());
} else {
instance_class_id_or_signature = Smi::New(instance_class.id());
if (instance_class.NumTypeArguments() > 0) {
instance_type_arguments = instance.GetTypeArguments();
ASSERT(instance_type_arguments.IsCanonical());
}
}
@@ -902,6 +892,7 @@ static BoolPtr ResultForExistingTypeTestCacheEntry(
return Bool::null();
}
#endif // defined(TARGET_ARCH_IA32)
// This updates the type test cache, an array containing 8 elements:
// - instance class (or function if the instance is a closure)
@@ -1068,22 +1059,20 @@ DEFINE_RUNTIME_ENTRY(Instanceof, 5) {
ASSERT(type.IsFinalized());
ASSERT(!type.IsDynamicType()); // No need to check assignment.
ASSERT(!cache.IsNull());
// Handle cases where currently the SubtypeNTestCache stubs return a false
// negative and the information is already in the cache.
//
// TODO(sstrickl): Remove this check and the associated helper function when
// the SubtypeNTestCache stubs have been updated to handle hash-based caches.
#if defined(TARGET_ARCH_IA32)
// Hash-based caches are still not handled by the stubs on IA32.
if (cache.IsHash()) {
const auto& result = Bool::Handle(
zone, ResultForExistingTypeTestCacheEntry(
zone, thread, instance, type, instantiator_type_arguments,
function_type_arguments, cache));
zone, CheckHashBasedSubtypeTestCache(zone, thread, instance, type,
instantiator_type_arguments,
function_type_arguments, cache));
if (!result.IsNull()) {
// Early exit because an entry already exists in the cache.
arguments.SetReturn(result);
return;
}
}
#endif // defined(TARGET_ARCH_IA32)
const Bool& result = Bool::Get(instance.IsInstanceOf(
type, instantiator_type_arguments, function_type_arguments));
if (FLAG_trace_type_checks) {
@@ -1099,12 +1088,6 @@ DEFINE_RUNTIME_ENTRY(Instanceof, 5) {
// Used only in type_testing_stubs_test.cc. If DRT_TypeCheck is entered, then
// this flag is set to true.
bool TESTING_runtime_entered_on_TTS_invocation = false;
// Used only in type_testing_stubs_test.cc. Set to true if we got a hit in
// the hash-based STC.
//
// TODO(sstrickl): Remove this when the SubtypeNTestCache stubs handle
// hash-based caches.
bool TESTING_found_hash_STC_entry = false;
#endif
// Check that the type of the given instance is a subtype of the given type and
@@ -1142,30 +1125,21 @@ DEFINE_RUNTIME_ENTRY(TypeCheck, 7) {
TESTING_runtime_entered_on_TTS_invocation = true;
#endif
// Handle cases where currently the SubtypeNTestCache stubs return a false
// negative and the information is already in the cache.
//
// TODO(sstrickl): Remove this check and the associated helper function when
// the SubtypeNTestCache stubs have been updated to handle hash-based caches.
#if defined(TARGET_ARCH_IA32)
ASSERT(mode == kTypeCheckFromInline);
// Hash-based caches are still not handled by the stubs on IA32.
if (cache.IsHash()) {
const auto& result = Bool::Handle(
zone, ResultForExistingTypeTestCacheEntry(
zone, CheckHashBasedSubtypeTestCache(
zone, thread, src_instance, dst_type,
instantiator_type_arguments, function_type_arguments, cache));
if (!result.IsNull() && result.value()) {
// Early exit because a positive entry already exists in the cache.
// (Negative entries should fall through to generating an exception.)
arguments.SetReturn(src_instance);
#if defined(TESTING)
TESTING_found_hash_STC_entry = true;
#endif
if (!result.IsNull()) {
// Early exit because an entry already exists in the cache.
arguments.SetReturn(result);
return;
}
}
#if defined(TARGET_ARCH_IA32)
ASSERT(mode == kTypeCheckFromInline);
#endif
#endif // defined(TARGET_ARCH_IA32)
// These are guaranteed on the calling side.
ASSERT(!dst_type.IsDynamicType());
runtime/vm/type_testing_stubs_test.cc
+63 -62
View File
@@ -215,12 +215,6 @@ enum TTSTestResult {
// The TTS invocation should enter the runtime and add a new entry to the
// STC, creating a new STC if necessary.
kNewSTCEntry,
// The TTS invocation should enter the runtime but return early after
// checking the STC. The STC prior to the invocation must be hash-based.
//
// TODO(sstrickl): Remove this possibility when the SubtypeNTestCache stubs
// have been updated to handle hash-based caches.
kRuntimeHash,
// The TTS invocation should enter the runtime and return a successful check
// but without adding an entry to the STC. This should only happen when the
// STC has hit the limit described by FLAG_max_subtype_cache_entries prior
@@ -241,6 +235,8 @@ enum TTSTestResult {
//
// Used only when calling TTSTestState::InvokeExistingStub.
kRespecialize,
// Used for static assert below only.
kNumTestResults,
};
static const char* kTestResultStrings[] = {
@@ -248,12 +244,19 @@ static const char* kTestResultStrings[] = {
"passes in TTS",
"passes in STC stub",
"passes in runtime, adding new STC entry",
"passes in runtime, checking hash-based STC entry",
"passes in runtime, no changes to max size STC",
"passes in runtime, initial TTS stub specialization",
"passes in runtime, TTS stub respecialized",
};
// Just to make sure the above are kept in sync.
static_assert(sizeof(kTestResultStrings) >=
kNumTestResults * sizeof(*kTestResultStrings),
"kTestResultStrings has too few entries");
static_assert(sizeof(kTestResultStrings) <=
kNumTestResults * sizeof(*kTestResultStrings),
"kTestResultStrings has extra entries");
struct TTSTestCase {
const Object& instance;
const TypeArguments& instantiator_tav;
@@ -276,7 +279,6 @@ struct TTSTestCase {
return false;
case kFail:
case kNewSTCEntry:
case kRuntimeHash:
case kRuntimeCheck:
case kSpecialize:
case kRespecialize:
@@ -377,16 +379,6 @@ static TTSTestCase FalseNegative(const TTSTestCase& original) {
original.function_tav, kNewSTCEntry);
}
// Takes an existing test case and creates a test case that should go to the
// runtime but find the entry in the hash-based STC.
//
// TODO(sstrickl): Remove this and its uses when the SubtypeNTestCache stubs can
// find entries in hash-based caches.
static TTSTestCase HashCheck(const TTSTestCase& original) {
return TTSTestCase(original.instance, original.instantiator_tav,
original.function_tav, kRuntimeHash);
}
// Takes an existing test case and creates a test case that should go to the
// runtime and pass but not modify the STC, as the STC has grown too large.
static TTSTestCase RuntimeCheck(const TTSTestCase& original) {
@@ -592,9 +584,8 @@ class TTSTestState : public ValueObject {
previous_stc_ = previous_stc_.Copy(thread_);
}
#if defined(TESTING)
// Clear the runtime entered and hash cache hit flags prior to invocation.
// Clear the runtime entered flag prior to invocation.
TESTING_runtime_entered_on_TTS_invocation = false;
TESTING_found_hash_STC_entry = false;
#endif
{
TraceStubInvocationScope scope;
@@ -604,8 +595,6 @@ class TTSTestState : public ValueObject {
#if defined(TESTING)
EXPECT_EQ(test_case.ShouldEnterRuntime(),
TESTING_runtime_entered_on_TTS_invocation);
EXPECT_EQ(test_case.expected_result == kRuntimeHash,
TESTING_found_hash_STC_entry);
#endif
new_tts_stub_ = last_tested_type_.type_test_stub();
last_stc_ = current_stc();
@@ -742,8 +731,7 @@ class TTSTestState : public ValueObject {
// We also expect an entry if we expect an STC hit in the STC stub or in
// the runtime.
const bool expects_stc_entry =
should_update_cache || test_case.expected_result == kExistingSTCEntry ||
test_case.expected_result == kRuntimeHash;
should_update_cache || test_case.expected_result == kExistingSTCEntry;
if ((!expects_stc_entry && has_stc_entry) ||
(expects_stc_entry && !has_stc_entry)) {
ZoneTextBuffer buffer(zone());
@@ -799,7 +787,6 @@ static void RunTTSTest(const AbstractType& dst_type,
EXPECT_NE(kRespecialize, test_case.expected_result);
// We're creating a new STC so it _can't_ use an existing entry.
EXPECT_NE(kExistingSTCEntry, test_case.expected_result);
EXPECT_NE(kRuntimeHash, test_case.expected_result);
bool null_should_fail = !Instance::NullIsAssignableTo(
dst_type, test_case.instantiator_tav, test_case.function_tav);
@@ -2618,7 +2605,13 @@ ISOLATE_UNIT_TEST_CASE(TTS_Regress_CidRangeChecks) {
state.InvokeEagerlySpecializedStub(Failure({obj_i, tav_null, tav_null}));
}
static void SubtypeTestCacheHashTest(Thread* thread, intptr_t num_classes) {
struct STCTestResults {
bool became_hash_cache = false;
bool cache_capped = false;
};
static STCTestResults SubtypeTestCacheTest(Thread* thread,
intptr_t num_classes) {
TextBuffer buffer(MB);
buffer.AddString("class D<S> {}\n");
buffer.AddString("D<int> Function() createClosureD() => () => D<int>();\n");
@@ -2654,14 +2647,18 @@ static void SubtypeTestCacheHashTest(Thread* thread, intptr_t num_classes) {
ASSERT(!object_d.IsNull());
auto& type_closure_d_int =
AbstractType::Handle(zone, object_d.GetType(Heap::kNew));
const bool can_be_null = Instance::NullIsAssignableTo(type_closure_d_int);
const auto& tav_null = Object::null_type_arguments();
TTSTestState state(thread, type_closure_d_int);
// Prime the stub before the loop with the null object.
state.InvokeEagerlySpecializedStub(
{Object::null_object(), tav_null, tav_null, can_be_null ? kTTS : kFail});
auto& class_c = Class::Handle(zone);
auto& object_c = Object::Handle(zone);
bool became_hash_cache = false;
STCTestResults results;
for (intptr_t i = 0; i < num_classes; ++i) {
auto const class_name = OS::SCreate(zone, "C%" Pd "", i);
class_c = GetClass(root_lib, class_name);
@@ -2682,55 +2679,59 @@ static void SubtypeTestCacheHashTest(Thread* thread, intptr_t num_classes) {
state.InvokeExistingStub(RuntimeCheck(base_case));
// Rerunning the test doesn't change the fact we can't add to the STC.
state.InvokeExistingStub(RuntimeCheck(base_case));
results.cache_capped = true;
} else {
const bool was_hash = state.last_stc().IsHash();
// All the rest of the tests should create or modify an STC.
state.InvokeExistingStub(FalseNegative(base_case));
if (i == 0) {
state.InvokeEagerlySpecializedStub(FalseNegative(base_case));
// We should get a linear cache the first time.
EXPECT(!state.last_stc().IsHash());
} else {
const bool was_hash = state.last_stc().IsHash();
state.InvokeExistingStub(FalseNegative(base_case));
if (was_hash) {
// We should never change from hash back to linear.
EXPECT(state.last_stc().IsHash());
} else if (state.last_stc().IsHash()) {
became_hash_cache = true;
}
}
if (became_hash_cache) {
// TODO(sstrickl): Remove this special case when the STC stubs are
// updated.
state.InvokeExistingStub(HashCheck(base_case));
} else {
state.InvokeExistingStub(STCCheck(base_case));
} else if (was_hash) {
// We should never change from hash back to linear.
EXPECT(state.last_stc().IsHash());
} else if (state.last_stc().IsHash()) {
results.became_hash_cache = true;
}
state.InvokeExistingStub(STCCheck(base_case));
}
}
// Ensure we're actually testing hash caches at some point.
EXPECT(became_hash_cache);
return results;
}
// A smaller version of the following test case, just to ensure some coverage
// on slower builds.
TEST_CASE(TTS_STC_SomeAsserts) {
SubtypeTestCacheHashTest(thread,
2 * SubtypeTestCache::kMaxLinearCacheEntries);
// The smallest test that just checks linear caches.
TEST_CASE(TTS_STC_LinearOnly) {
const intptr_t num_classes =
Utils::Minimum(static_cast<intptr_t>(FLAG_max_subtype_cache_entries),
SubtypeTestCache::kMaxLinearCacheEntries);
EXPECT(num_classes > 0);
const auto& results = SubtypeTestCacheTest(thread, num_classes);
EXPECT(!results.became_hash_cache);
EXPECT(!results.cache_capped);
}
// Too slow in debug mode. Also avoid the sanitizers and simulators for similar
// reasons. Any core issues will likely be found by TTS_STC_SomeAsserts.
#if !defined(DEBUG) && !defined(USING_MEMORY_SANITIZER) && \
!defined(USING_THREAD_SANITIZER) && !defined(USING_LEAK_SANITIZER) && \
!defined(USING_UNDEFINED_BEHAVIOR_SANITIZER) && !defined(USING_SIMULATOR)
TEST_CASE(TTS_STC_ManyAsserts) {
const intptr_t kNumClasses = 5000;
static_assert(kNumClasses > SubtypeTestCache::kMaxLinearCacheEntries,
"too few classes to trigger change to a hash-based cache");
SubtypeTestCacheHashTest(thread, kNumClasses);
// A larger test that ensures we convert to a hash table at some point.
TEST_CASE(TTS_STC_Hash) {
const intptr_t num_classes =
Utils::Minimum(static_cast<intptr_t>(FLAG_max_subtype_cache_entries),
2 * SubtypeTestCache::kMaxLinearCacheEntries);
EXPECT(num_classes > SubtypeTestCache::kMaxLinearCacheEntries);
const auto& results = SubtypeTestCacheTest(thread, num_classes);
EXPECT(results.became_hash_cache);
EXPECT(!results.cache_capped);
}
// A larger test that ensures that we use enough entries to hit the max STC
// size and test what happens when we go above it.
TEST_CASE(TTS_STC_Capped) {
const intptr_t num_classes = 1.1 * FLAG_max_subtype_cache_entries;
EXPECT(num_classes > 0);
const auto& results = SubtypeTestCacheTest(thread, num_classes);
EXPECT_EQ(SubtypeTestCache::kMaxLinearCacheEntries < num_classes,
results.became_hash_cache);
EXPECT(results.cache_capped);
}
#endif
} // namespace dart