diff --git a/runtime/vm/aot_optimizer.cc b/runtime/vm/aot_optimizer.cc index 41d43e9b46f..a30e36b328a 100644 --- a/runtime/vm/aot_optimizer.cc +++ b/runtime/vm/aot_optimizer.cc @@ -17,6 +17,7 @@ #include "vm/flow_graph_range_analysis.h" #include "vm/hash_map.h" #include "vm/il_printer.h" +#include "vm/jit_optimizer.h" #include "vm/intermediate_language.h" #include "vm/object.h" #include "vm/object_store.h" @@ -1841,8 +1842,9 @@ void AotOptimizer::VisitInstanceCall(InstanceCallInstr* instr) { RawFunction::Kind function_kind = Function::Handle(Z, unary_checks.GetTargetAt(0)).kind(); if (!flow_graph()->InstanceCallNeedsClassCheck(instr, function_kind)) { + CallTargets* targets = CallTargets::Create(Z, unary_checks); PolymorphicInstanceCallInstr* call = - new (Z) PolymorphicInstanceCallInstr(instr, unary_checks, + new (Z) PolymorphicInstanceCallInstr(instr, *targets, /* with_checks = */ false, /* complete = */ true); instr->ReplaceWith(call, current_iterator()); @@ -1903,17 +1905,16 @@ void AotOptimizer::VisitInstanceCall(InstanceCallInstr* instr) { Array::Handle(Z, ArgumentsDescriptor::New(instr->ArgumentCount(), instr->argument_names())); ArgumentsDescriptor args_desc(args_desc_array); - const Function& function = Function::Handle( + Function& function = Function::Handle( Z, Resolver::ResolveDynamicForReceiverClass( receiver_class, instr->function_name(), args_desc)); if (!function.IsNull()) { - const ICData& ic_data = ICData::Handle( - ICData::New(flow_graph_->function(), instr->function_name(), - args_desc_array, Thread::kNoDeoptId, - /* args_tested = */ 1, false)); - ic_data.AddReceiverCheck(receiver_class.id(), function); + CallTargets* targets = new (Z) CallTargets(); + Function& target = Function::ZoneHandle(Z, function.raw()); + targets->Add(CidRangeTarget(receiver_class.id(), receiver_class.id(), + &target, /*count = */ 1)); PolymorphicInstanceCallInstr* call = - new (Z) PolymorphicInstanceCallInstr(instr, ic_data, + new (Z) PolymorphicInstanceCallInstr(instr, *targets, /* with_checks = */ false, /* complete = */ true); instr->ReplaceWith(call, current_iterator()); @@ -2036,8 +2037,9 @@ void AotOptimizer::VisitInstanceCall(InstanceCallInstr* instr) { return; } else if ((ic_data.raw() != ICData::null()) && !ic_data.NumberOfChecksIs(0)) { + CallTargets* targets = CallTargets::Create(Z, ic_data); PolymorphicInstanceCallInstr* call = - new (Z) PolymorphicInstanceCallInstr(instr, ic_data, + new (Z) PolymorphicInstanceCallInstr(instr, *targets, /* with_checks = */ true, /* complete = */ true); instr->ReplaceWith(call, current_iterator()); @@ -2052,8 +2054,9 @@ void AotOptimizer::VisitInstanceCall(InstanceCallInstr* instr) { ASSERT(!FLAG_polymorphic_with_deopt); // OK to use checks with PolymorphicInstanceCallInstr since no // deoptimization is allowed. + CallTargets* targets = CallTargets::Create(Z, *instr->ic_data()); PolymorphicInstanceCallInstr* call = - new (Z) PolymorphicInstanceCallInstr(instr, unary_checks, + new (Z) PolymorphicInstanceCallInstr(instr, *targets, /* with_checks = */ true, /* complete = */ false); instr->ReplaceWith(call, current_iterator()); diff --git a/runtime/vm/assembler_arm.h b/runtime/vm/assembler_arm.h index 049d76951b9..bd0a2044a77 100644 --- a/runtime/vm/assembler_arm.h +++ b/runtime/vm/assembler_arm.h @@ -980,6 +980,11 @@ class Assembler : public ValueObject { b(label, NE); } + void BranchIfSmi(Register reg, Label* label) { + tst(reg, Operand(kSmiTagMask)); + b(label, EQ); + } + void CheckCodePointer(); // Function frame setup and tear down. diff --git a/runtime/vm/flow_graph_compiler.cc b/runtime/vm/flow_graph_compiler.cc index 866cab4de25..3fbeb86ece5 100644 --- a/runtime/vm/flow_graph_compiler.cc +++ b/runtime/vm/flow_graph_compiler.cc @@ -23,6 +23,7 @@ #include "vm/object_store.h" #include "vm/parser.h" #include "vm/raw_object.h" +#include "vm/resolver.h" #include "vm/stack_frame.h" #include "vm/stub_code.h" #include "vm/symbols.h" @@ -1169,8 +1170,12 @@ void FlowGraphCompiler::GenerateInstanceCall(intptr_t deopt_id, } if (is_optimizing()) { - EmitMegamorphicInstanceCall(ic_data_in, argument_count, deopt_id, token_pos, - locs, CatchClauseNode::kInvalidTryIndex); + String& name = String::Handle(ic_data_in.target_name()); + Array& arguments_descriptor = + Array::Handle(ic_data_in.arguments_descriptor()); + EmitMegamorphicInstanceCall(name, arguments_descriptor, argument_count, + deopt_id, token_pos, locs, + CatchClauseNode::kInvalidTryIndex); return; } @@ -1787,84 +1792,70 @@ void FlowGraphCompiler::EndCodeSourceRange(TokenPosition token_pos) { } -const ICData& FlowGraphCompiler::TrySpecializeICDataByReceiverCid( - const ICData& ic_data, - intptr_t cid) { +const CallTargets* FlowGraphCompiler::ResolveCallTargetsForReceiverCid( + intptr_t cid, + const String& selector, + const Array& args_desc_array) { Zone* zone = Thread::Current()->zone(); - if (ic_data.NumArgsTested() != 1) return ic_data; - if ((ic_data.NumberOfUsedChecks() == 1) && ic_data.HasReceiverClassId(cid)) { - return ic_data; // Nothing to do - } + ArgumentsDescriptor args_desc(args_desc_array); - intptr_t count = 1; - const Function& function = - Function::Handle(zone, ic_data.GetTargetForReceiverClassId(cid, &count)); - // TODO(fschneider): Try looking up the function on the class if it is - // not found in the ICData. - if (!function.IsNull()) { - const ICData& new_ic_data = ICData::ZoneHandle( - zone, ICData::New(Function::Handle(zone, ic_data.Owner()), - String::Handle(zone, ic_data.target_name()), - Object::empty_array(), // Dummy argument descriptor. - ic_data.deopt_id(), ic_data.NumArgsTested(), false)); - new_ic_data.SetDeoptReasons(ic_data.DeoptReasons()); - new_ic_data.AddReceiverCheck(cid, function, count); - return new_ic_data; - } + Function& fn = Function::ZoneHandle(zone); + if (!LookupMethodFor(cid, selector, args_desc, &fn)) return NULL; - return ic_data; + CallTargets* targets = new (zone) CallTargets(); + targets->Add(CidRangeTarget(cid, cid, &fn, /* count = */ 1)); + + return targets; } -intptr_t FlowGraphCompiler::ComputeGoodBiasForCidComparison( - const GrowableArray& sorted, - intptr_t max_immediate) { - // Sometimes a bias can be useful so we can emit more compact compare - // instructions. - intptr_t min_cid = 1000000; - intptr_t max_cid = -1; +bool FlowGraphCompiler::LookupMethodFor(int class_id, + const String& name, + const ArgumentsDescriptor& args_desc, + Function* fn_return) { + Thread* thread = Thread::Current(); + Isolate* isolate = thread->isolate(); + Zone* zone = thread->zone(); + if (class_id < 0) return false; + if (class_id >= isolate->class_table()->NumCids()) return false; - const intptr_t sorted_len = sorted.length(); + RawClass* raw_class = isolate->class_table()->At(class_id); + if (raw_class == NULL) return false; + Class& cls = Class::Handle(zone, raw_class); + if (cls.IsNull()) return false; + if (!cls.is_finalized()) return false; + if (Array::Handle(cls.functions()).IsNull()) return false; - for (intptr_t i = 0; i < sorted_len + 1; i++) { - bool done = (i == sorted_len); - intptr_t start = done ? 0 : sorted[i].cid_start; - intptr_t end = done ? 0 : sorted[i].cid_end; - bool is_range = start != end; - bool spread_too_big = start - min_cid > max_immediate; - if (done || is_range || spread_too_big) { - if (i >= 2 && max_cid - min_cid <= max_immediate && - max_cid > max_immediate) { - return min_cid; - } else { - return 0; - } - } - min_cid = Utils::Minimum(min_cid, start); - max_cid = Utils::Maximum(max_cid, end); - } - UNREACHABLE(); - return 0; + const bool allow_add = false; + Function& target_function = + Function::Handle(zone, Resolver::ResolveDynamicForReceiverClass( + cls, name, args_desc, allow_add)); + if (target_function.IsNull()) return false; + *fn_return ^= target_function.raw(); + return true; } #if !defined(TARGET_ARCH_DBC) // DBC emits calls very differently from other architectures due to its // interpreted nature. -void FlowGraphCompiler::EmitPolymorphicInstanceCall(const ICData& ic_data, - intptr_t argument_count, - const Array& argument_names, - intptr_t deopt_id, - TokenPosition token_pos, - LocationSummary* locs, - bool complete, - intptr_t total_ic_calls) { +void FlowGraphCompiler::EmitPolymorphicInstanceCall( + const CallTargets& targets, + const InstanceCallInstr& original_call, + intptr_t argument_count, + const Array& argument_names, + intptr_t deopt_id, + TokenPosition token_pos, + LocationSummary* locs, + bool complete, + intptr_t total_ic_calls) { if (FLAG_polymorphic_with_deopt) { Label* deopt = AddDeoptStub(deopt_id, ICData::kDeoptPolymorphicInstanceCallTestFail); Label ok; - EmitTestAndCall(ic_data, argument_count, argument_names, + EmitTestAndCall(targets, original_call.function_name(), argument_count, + argument_names, deopt, // No cid match. &ok, // Found cid. deopt_id, token_pos, locs, complete, total_ic_calls); @@ -1872,17 +1863,141 @@ void FlowGraphCompiler::EmitPolymorphicInstanceCall(const ICData& ic_data, } else { if (complete) { Label ok; - EmitTestAndCall(ic_data, argument_count, argument_names, + EmitTestAndCall(targets, original_call.function_name(), argument_count, + argument_names, NULL, // No cid match. &ok, // Found cid. deopt_id, token_pos, locs, true, total_ic_calls); assembler()->Bind(&ok); } else { - EmitSwitchableInstanceCall(ic_data, argument_count, deopt_id, token_pos, - locs); + const ICData& unary_checks = ICData::ZoneHandle( + zone(), original_call.ic_data()->AsUnaryClassChecks()); + EmitSwitchableInstanceCall(unary_checks, argument_count, deopt_id, + token_pos, locs); } } } + + +#define __ assembler()-> +void FlowGraphCompiler::EmitTestAndCall(const CallTargets& targets, + const String& function_name, + intptr_t argument_count, + const Array& argument_names, + Label* failed, + Label* match_found, + intptr_t deopt_id, + TokenPosition token_index, + LocationSummary* locs, + bool complete, + intptr_t total_ic_calls) { + ASSERT(is_optimizing()); + + const Array& arguments_descriptor = Array::ZoneHandle( + zone(), ArgumentsDescriptor::New(argument_count, argument_names)); + + EmitTestAndCallLoadReceiver(argument_count, arguments_descriptor); + + static const int kNoCase = -1; + int smi_case = kNoCase; + int which_case_to_skip = kNoCase; + + const int length = targets.length(); + ASSERT(length > 0); + int non_smi_length = length; + + // Find out if one of the classes in one of the cases is the Smi class. We + // will be handling that specially. + for (int i = 0; i < length; i++) { + const intptr_t start = targets[i].cid_start; + if (start > kSmiCid) continue; + const intptr_t end = targets[i].cid_end; + if (end >= kSmiCid) { + smi_case = i; + if (start == kSmiCid && end == kSmiCid) { + // If this case has only the Smi class then we won't need to emit it at + // all later. + which_case_to_skip = i; + non_smi_length--; + } + break; + } + } + + if (smi_case != kNoCase) { + Label after_smi_test; + EmitTestAndCallSmiBranch(non_smi_length == 0 ? failed : &after_smi_test, + /* jump_if_smi= */ false); + + // Do not use the code from the function, but let the code be patched so + // that we can record the outgoing edges to other code. + const Function& function = *targets[smi_case].target; + GenerateStaticDartCall(deopt_id, token_index, + *StubCode::CallStaticFunction_entry(), + RawPcDescriptors::kOther, locs, function); + __ Drop(argument_count); + if (match_found != NULL) { + __ Jump(match_found); + } + __ Bind(&after_smi_test); + } else { + if (!complete) { + // Smi is not a valid class. + EmitTestAndCallSmiBranch(failed, /* jump_if_smi = */ true); + } + } + + if (non_smi_length == 0) { + // If non_smi_length is 0 then only a Smi check was needed; the Smi check + // above will fail if there was only one check and receiver is not Smi. + return; + } + + bool add_megamorphic_call = false; + int bias = 0; + + // Value is not Smi. + EmitTestAndCallLoadCid(); + + int last_check = which_case_to_skip == length - 1 ? length - 2 : length - 1; + + for (intptr_t i = 0; i < length; i++) { + if (i == which_case_to_skip) continue; + const bool is_last_check = (i == last_check); + const int count = targets[i].count; + if (!is_last_check && !complete && count < (total_ic_calls >> 5)) { + // This case is hit too rarely to be worth writing class-id checks inline + // for. Note that we can't do this for calls with only one target because + // the type propagator may have made use of that and expects a deopt if + // a new class is seen at this calls site. See IsMonomorphic. + add_megamorphic_call = true; + break; + } + Label next_test; + if (!complete || !is_last_check) { + bias = EmitTestAndCallCheckCid(is_last_check ? failed : &next_test, + targets[i], bias); + } + // Do not use the code from the function, but let the code be patched so + // that we can record the outgoing edges to other code. + const Function& function = *targets[i].target; + GenerateStaticDartCall(deopt_id, token_index, + *StubCode::CallStaticFunction_entry(), + RawPcDescriptors::kOther, locs, function); + __ Drop(argument_count); + if (!is_last_check || add_megamorphic_call) { + __ Jump(match_found); + } + __ Bind(&next_test); + } + if (add_megamorphic_call) { + int try_index = CatchClauseNode::kInvalidTryIndex; + EmitMegamorphicInstanceCall(function_name, arguments_descriptor, + argument_count, deopt_id, token_index, locs, + try_index); + } +} +#undef __ #endif #if defined(DEBUG) && !defined(TARGET_ARCH_DBC) diff --git a/runtime/vm/flow_graph_compiler.h b/runtime/vm/flow_graph_compiler.h index 3288d963a5f..7aaafe43a99 100644 --- a/runtime/vm/flow_graph_compiler.h +++ b/runtime/vm/flow_graph_compiler.h @@ -233,22 +233,6 @@ class SlowPathCode : public ZoneAllocated { }; -struct CidRangeTarget { - intptr_t cid_start; - intptr_t cid_end; - Function* target; - intptr_t count; - CidRangeTarget(intptr_t cid_start_arg, - intptr_t cid_end_arg, - Function* target_arg, - intptr_t count_arg) - : cid_start(cid_start_arg), - cid_end(cid_end_arg), - target(target_arg), - count(count_arg) {} -}; - - class FlowGraphCompiler : public ValueObject { private: class BlockInfo : public ZoneAllocated { @@ -434,17 +418,20 @@ class FlowGraphCompiler : public ValueObject { TokenPosition token_pos, LocationSummary* locs); - void EmitPolymorphicInstanceCall(const ICData& ic_data, - intptr_t argument_count, - const Array& argument_names, - intptr_t deopt_id, - TokenPosition token_pos, - LocationSummary* locs, - bool complete, - intptr_t total_call_count); + void EmitPolymorphicInstanceCall( + const CallTargets& targets, + const InstanceCallInstr& original_instruction, + intptr_t argument_count, + const Array& argument_names, + intptr_t deopt_id, + TokenPosition token_pos, + LocationSummary* locs, + bool complete, + intptr_t total_call_count); // Pass a value for try-index where block is not available (e.g. slow path). - void EmitMegamorphicInstanceCall(const ICData& ic_data, + void EmitMegamorphicInstanceCall(const String& function_name, + const Array& arguments_descriptor, intptr_t argument_count, intptr_t deopt_id, TokenPosition token_pos, @@ -458,7 +445,8 @@ class FlowGraphCompiler : public ValueObject { TokenPosition token_pos, LocationSummary* locs); - void EmitTestAndCall(const ICData& ic_data, + void EmitTestAndCall(const CallTargets& targets, + const String& function_name, intptr_t arg_count, const Array& arg_names, Label* failed, @@ -601,8 +589,10 @@ class FlowGraphCompiler : public ValueObject { const Array& arguments_descriptor, intptr_t num_args_tested); - static const ICData& TrySpecializeICDataByReceiverCid(const ICData& ic_data, - intptr_t cid); + static const CallTargets* ResolveCallTargetsForReceiverCid( + intptr_t cid, + const String& selector, + const Array& args_desc_array); const ZoneGrowableArray& deopt_id_to_ic_data() const { return *deopt_id_to_ic_data_; @@ -621,6 +611,11 @@ class FlowGraphCompiler : public ValueObject { void BeginCodeSourceRange(); void EndCodeSourceRange(TokenPosition token_pos); + static bool LookupMethodFor(int class_id, + const String& name, + const ArgumentsDescriptor& args_desc, + Function* fn_return); + #if defined(TARGET_ARCH_DBC) enum CallResult { kHasResult, @@ -666,9 +661,21 @@ class FlowGraphCompiler : public ValueObject { // Helper for TestAndCall that calculates a good bias that // allows more compact instructions to be emitted. - intptr_t ComputeGoodBiasForCidComparison( - const GrowableArray& sorted, - intptr_t max_immediate); + intptr_t ComputeGoodBiasForCidComparison(const CallTargets& sorted, + intptr_t max_immediate); + + // More helpers for EmitTestAndCall. + void EmitTestAndCallLoadReceiver(intptr_t argument_count, + const Array& arguments_descriptor); + + void EmitTestAndCallSmiBranch(Label* label, bool jump_if_smi); + + void EmitTestAndCallLoadCid(); + + // Returns new class-id bias. + int EmitTestAndCallCheckCid(Label* next_label, + const CidRangeTarget& target, + int bias); // DBC handles type tests differently from all other architectures due // to its interpreted nature. diff --git a/runtime/vm/flow_graph_compiler_arm.cc b/runtime/vm/flow_graph_compiler_arm.cc index 047c40b6278..b0b936d6fc1 100644 --- a/runtime/vm/flow_graph_compiler_arm.cc +++ b/runtime/vm/flow_graph_compiler_arm.cc @@ -1239,16 +1239,14 @@ void FlowGraphCompiler::EmitInstanceCall(const StubEntry& stub_entry, void FlowGraphCompiler::EmitMegamorphicInstanceCall( - const ICData& ic_data, + const String& name, + const Array& arguments_descriptor, intptr_t argument_count, intptr_t deopt_id, TokenPosition token_pos, LocationSummary* locs, intptr_t try_index, intptr_t slow_path_argument_count) { - const String& name = String::Handle(zone(), ic_data.target_name()); - const Array& arguments_descriptor = - Array::ZoneHandle(zone(), ic_data.arguments_descriptor()); ASSERT(!arguments_descriptor.IsNull() && (arguments_descriptor.Length() > 0)); const MegamorphicCache& cache = MegamorphicCache::ZoneHandle( zone(), @@ -1502,120 +1500,43 @@ void FlowGraphCompiler::ClobberDeadTempRegisters(LocationSummary* locs) { #endif -void FlowGraphCompiler::EmitTestAndCall(const ICData& ic_data, - intptr_t argument_count, - const Array& argument_names, - Label* failed, - Label* match_found, - intptr_t deopt_id, - TokenPosition token_index, - LocationSummary* locs, - bool complete, - intptr_t total_ic_calls) { - ASSERT(is_optimizing()); +void FlowGraphCompiler::EmitTestAndCallLoadReceiver( + intptr_t argument_count, + const Array& arguments_descriptor) { __ Comment("EmitTestAndCall"); - const Array& arguments_descriptor = Array::ZoneHandle( - zone(), ArgumentsDescriptor::New(argument_count, argument_names)); - // Load receiver into R0. __ LoadFromOffset(kWord, R0, SP, (argument_count - 1) * kWordSize); __ LoadObject(R4, arguments_descriptor); +} - const bool kFirstCheckIsSmi = ic_data.GetReceiverClassIdAt(0) == kSmiCid; - const intptr_t num_checks = ic_data.NumberOfChecks(); - ASSERT(!ic_data.IsNull() && (num_checks > 0)); +void FlowGraphCompiler::EmitTestAndCallSmiBranch(Label* label, bool if_smi) { + __ tst(R0, Operand(kSmiTagMask)); + // Jump if receiver is not Smi. + __ b(label, if_smi ? EQ : NE); +} - Label after_smi_test; - if (kFirstCheckIsSmi) { - __ tst(R0, Operand(kSmiTagMask)); - // Jump if receiver is not Smi. - if (num_checks == 1) { - __ b(failed, NE); - } else { - __ b(&after_smi_test, NE); - } - // Do not use the code from the function, but let the code be patched so - // that we can record the outgoing edges to other code. - const Function& function = - Function::ZoneHandle(zone(), ic_data.GetTargetAt(0)); - GenerateStaticDartCall(deopt_id, token_index, - *StubCode::CallStaticFunction_entry(), - RawPcDescriptors::kOther, locs, function); - __ Drop(argument_count); - if (num_checks > 1) { - __ b(match_found); - } - } else { - // Receiver is Smi, but Smi is not a valid class therefore fail. - // (Smi class must be first in the list). - if (!complete) { - __ tst(R0, Operand(kSmiTagMask)); - __ b(failed, EQ); - } - } - __ Bind(&after_smi_test); - ASSERT(!ic_data.IsNull() && (num_checks > 0)); - GrowableArray sorted(num_checks); - SortICDataByCount(ic_data, &sorted, /* drop_smi = */ true); - - const intptr_t sorted_len = sorted.length(); - // If sorted_len is 0 then only a Smi check was needed; the Smi check above - // will fail if there was only one check and receiver is not Smi. - if (sorted_len == 0) return; - - // Value is not Smi, +void FlowGraphCompiler::EmitTestAndCallLoadCid() { __ LoadClassId(R2, R0); +} - bool add_megamorphic_call = false; - const int kMaxImmediateInInstruction = 256; - int bias = - ComputeGoodBiasForCidComparison(sorted, kMaxImmediateInInstruction); - if (bias != 0) __ AddImmediate(R2, R2, -bias); - for (intptr_t i = 0; i < sorted_len; i++) { - const bool is_last_check = (i == (sorted_len - 1)); - int cid_start = sorted[i].cid_start; - int cid_end = sorted[i].cid_end; - int count = sorted[i].count; - if (!is_last_check && !complete && count < (total_ic_calls >> 5)) { - // This case is hit too rarely to be worth writing class-id checks inline - // for. - add_megamorphic_call = true; - break; - } - ASSERT(cid_start > kSmiCid || cid_end < kSmiCid); - Label next_test; - if (!complete || !is_last_check) { - Label* next_label = is_last_check ? failed : &next_test; - if (cid_start == cid_end) { - __ CompareImmediate(R2, cid_start - bias); - __ b(next_label, NE); - } else { - __ AddImmediate(R2, R2, bias - cid_start); - bias = cid_start; - __ CompareImmediate(R2, cid_end - cid_start); - __ b(next_label, HI); // Unsigned higher. - } - } - // Do not use the code from the function, but let the code be patched so - // that we can record the outgoing edges to other code. - const Function& function = *sorted[i].target; - GenerateStaticDartCall(deopt_id, token_index, - *StubCode::CallStaticFunction_entry(), - RawPcDescriptors::kOther, locs, function); - __ Drop(argument_count); - if (!is_last_check) { - __ b(match_found); - } - __ Bind(&next_test); - } - if (add_megamorphic_call) { - int try_index = CatchClauseNode::kInvalidTryIndex; - EmitMegamorphicInstanceCall(ic_data, argument_count, deopt_id, token_index, - locs, try_index, argument_count); +int FlowGraphCompiler::EmitTestAndCallCheckCid(Label* next_label, + const CidRangeTarget& target, + int bias) { + intptr_t cid_start = target.cid_start; + intptr_t cid_end = target.cid_end; + if (cid_start == cid_end) { + __ CompareImmediate(R2, cid_start - bias); + __ b(next_label, NE); + } else { + __ AddImmediate(R2, R2, bias - cid_start); + bias = cid_start; + __ CompareImmediate(R2, cid_end - cid_start); + __ b(next_label, HI); // Unsigned higher. } + return bias; } diff --git a/runtime/vm/flow_graph_compiler_arm64.cc b/runtime/vm/flow_graph_compiler_arm64.cc index 3a8310a8977..e51c0ab389f 100644 --- a/runtime/vm/flow_graph_compiler_arm64.cc +++ b/runtime/vm/flow_graph_compiler_arm64.cc @@ -1224,16 +1224,14 @@ void FlowGraphCompiler::EmitInstanceCall(const StubEntry& stub_entry, void FlowGraphCompiler::EmitMegamorphicInstanceCall( - const ICData& ic_data, + const String& name, + const Array& arguments_descriptor, intptr_t argument_count, intptr_t deopt_id, TokenPosition token_pos, LocationSummary* locs, intptr_t try_index, intptr_t slow_path_argument_count) { - const String& name = String::Handle(zone(), ic_data.target_name()); - const Array& arguments_descriptor = - Array::ZoneHandle(zone(), ic_data.arguments_descriptor()); ASSERT(!arguments_descriptor.IsNull() && (arguments_descriptor.Length() > 0)); const MegamorphicCache& cache = MegamorphicCache::ZoneHandle( zone(), @@ -1466,120 +1464,43 @@ void FlowGraphCompiler::ClobberDeadTempRegisters(LocationSummary* locs) { #endif -void FlowGraphCompiler::EmitTestAndCall(const ICData& ic_data, - intptr_t argument_count, - const Array& argument_names, - Label* failed, - Label* match_found, - intptr_t deopt_id, - TokenPosition token_index, - LocationSummary* locs, - bool complete, - intptr_t total_ic_calls) { - ASSERT(is_optimizing()); +void FlowGraphCompiler::EmitTestAndCallLoadReceiver( + intptr_t argument_count, + const Array& arguments_descriptor) { __ Comment("EmitTestAndCall"); - const Array& arguments_descriptor = Array::ZoneHandle( - zone(), ArgumentsDescriptor::New(argument_count, argument_names)); - // Load receiver into R0. __ LoadFromOffset(R0, SP, (argument_count - 1) * kWordSize); __ LoadObject(R4, arguments_descriptor); +} - const bool kFirstCheckIsSmi = ic_data.GetReceiverClassIdAt(0) == kSmiCid; - const intptr_t num_checks = ic_data.NumberOfChecks(); - ASSERT(!ic_data.IsNull() && (num_checks > 0)); +void FlowGraphCompiler::EmitTestAndCallSmiBranch(Label* label, bool if_smi) { + __ tsti(R0, Immediate(kSmiTagMask)); + // Jump if receiver is not Smi. + __ b(label, if_smi ? EQ : NE); +} - Label after_smi_test; - if (kFirstCheckIsSmi) { - __ tsti(R0, Immediate(kSmiTagMask)); - // Jump if receiver is not Smi. - if (num_checks == 1) { - __ b(failed, NE); - } else { - __ b(&after_smi_test, NE); - } - // Do not use the code from the function, but let the code be patched so - // that we can record the outgoing edges to other code. - const Function& function = - Function::ZoneHandle(zone(), ic_data.GetTargetAt(0)); - GenerateStaticDartCall(deopt_id, token_index, - *StubCode::CallStaticFunction_entry(), - RawPcDescriptors::kOther, locs, function); - __ Drop(argument_count); - if (num_checks > 1) { - __ b(match_found); - } - } else { - // Receiver is Smi, but Smi is not a valid class therefore fail. - // (Smi class must be first in the list). - if (!complete) { - __ tsti(R0, Immediate(kSmiTagMask)); - __ b(failed, EQ); - } - } - __ Bind(&after_smi_test); - ASSERT(!ic_data.IsNull() && (num_checks > 0)); - GrowableArray sorted(num_checks); - SortICDataByCount(ic_data, &sorted, /* drop_smi = */ true); - - const intptr_t sorted_len = sorted.length(); - // If sorted_len is 0 then only a Smi check was needed; the Smi check above - // will fail if there was only one check and receiver is not Smi. - if (sorted_len == 0) return; - - // Value is not Smi, +void FlowGraphCompiler::EmitTestAndCallLoadCid() { __ LoadClassId(R2, R0); +} - bool add_megamorphic_call = false; - const int kMaxImmediateInInstruction = 256; - int bias = - ComputeGoodBiasForCidComparison(sorted, kMaxImmediateInInstruction); - if (bias != 0) __ AddImmediate(R2, R2, -bias); - for (intptr_t i = 0; i < sorted_len; i++) { - const bool is_last_check = (i == (sorted_len - 1)); - int cid_start = sorted[i].cid_start; - int cid_end = sorted[i].cid_end; - int count = sorted[i].count; - if (!is_last_check && !complete && count < (total_ic_calls >> 5)) { - // This case is hit too rarely to be worth writing class-id checks inline - // for. - add_megamorphic_call = true; - break; - } - ASSERT(cid_start > kSmiCid || cid_end < kSmiCid); - Label next_test; - if (!complete || !is_last_check) { - Label* next_label = is_last_check ? failed : &next_test; - if (cid_start == cid_end) { - __ CompareImmediate(R2, cid_start - bias); - __ b(next_label, NE); - } else { - __ AddImmediate(R2, R2, bias - cid_start); - bias = cid_start; - __ CompareImmediate(R2, cid_end - cid_start); - __ b(next_label, HI); // Unsigned higher. - } - } - // Do not use the code from the function, but let the code be patched so - // that we can record the outgoing edges to other code. - const Function& function = *sorted[i].target; - GenerateStaticDartCall(deopt_id, token_index, - *StubCode::CallStaticFunction_entry(), - RawPcDescriptors::kOther, locs, function); - __ Drop(argument_count); - if (!is_last_check) { - __ b(match_found); - } - __ Bind(&next_test); - } - if (add_megamorphic_call) { - int try_index = CatchClauseNode::kInvalidTryIndex; - EmitMegamorphicInstanceCall(ic_data, argument_count, deopt_id, token_index, - locs, try_index, argument_count); +int FlowGraphCompiler::EmitTestAndCallCheckCid(Label* next_label, + const CidRangeTarget& target, + int bias) { + intptr_t cid_start = target.cid_start; + intptr_t cid_end = target.cid_end; + if (cid_start == cid_end) { + __ CompareImmediate(R2, cid_start - bias); + __ b(next_label, NE); + } else { + __ AddImmediate(R2, R2, bias - cid_start); + bias = cid_start; + __ CompareImmediate(R2, cid_end - cid_start); + __ b(next_label, HI); // Unsigned higher. } + return bias; } diff --git a/runtime/vm/flow_graph_compiler_ia32.cc b/runtime/vm/flow_graph_compiler_ia32.cc index 86c709da9be..34d3452ec62 100644 --- a/runtime/vm/flow_graph_compiler_ia32.cc +++ b/runtime/vm/flow_graph_compiler_ia32.cc @@ -1131,6 +1131,17 @@ void FlowGraphCompiler::GenerateDartCall(intptr_t deopt_id, } +void FlowGraphCompiler::GenerateStaticDartCall(intptr_t deopt_id, + TokenPosition token_pos, + const StubEntry& stub_entry, + RawPcDescriptors::Kind kind, + LocationSummary* locs, + const Function& target) { + GenerateDartCall(deopt_id, token_pos, stub_entry, kind, locs); + AddStaticCallTarget(target); +} + + void FlowGraphCompiler::GenerateRuntimeCall(TokenPosition token_pos, intptr_t deopt_id, const RuntimeEntry& entry, @@ -1216,16 +1227,14 @@ void FlowGraphCompiler::EmitInstanceCall(const StubEntry& stub_entry, void FlowGraphCompiler::EmitMegamorphicInstanceCall( - const ICData& ic_data, + const String& name, + const Array& arguments_descriptor, intptr_t argument_count, intptr_t deopt_id, TokenPosition token_pos, LocationSummary* locs, intptr_t try_index, intptr_t slow_path_argument_count) { - const String& name = String::Handle(zone(), ic_data.target_name()); - const Array& arguments_descriptor = - Array::ZoneHandle(zone(), ic_data.arguments_descriptor()); ASSERT(!arguments_descriptor.IsNull() && (arguments_descriptor.Length() > 0)); const MegamorphicCache& cache = MegamorphicCache::ZoneHandle( zone(), @@ -1425,119 +1434,43 @@ void FlowGraphCompiler::ClobberDeadTempRegisters(LocationSummary* locs) { #endif -void FlowGraphCompiler::EmitTestAndCall(const ICData& ic_data, - intptr_t argument_count, - const Array& argument_names, - Label* failed, - Label* match_found, - intptr_t deopt_id, - TokenPosition token_index, - LocationSummary* locs, - bool complete, - intptr_t total_ic_calls) { - ASSERT(is_optimizing()); - ASSERT(!complete); +void FlowGraphCompiler::EmitTestAndCallLoadReceiver( + intptr_t argument_count, + const Array& arguments_descriptor) { __ Comment("EmitTestAndCall"); - const Array& arguments_descriptor = Array::ZoneHandle( - zone(), ArgumentsDescriptor::New(argument_count, argument_names)); // Load receiver into EAX. __ movl(EAX, Address(ESP, (argument_count - 1) * kWordSize)); __ LoadObject(EDX, arguments_descriptor); +} - const bool kFirstCheckIsSmi = ic_data.GetReceiverClassIdAt(0) == kSmiCid; - const intptr_t num_checks = ic_data.NumberOfChecks(); - ASSERT(!ic_data.IsNull() && (num_checks > 0)); - - Label after_smi_test; +void FlowGraphCompiler::EmitTestAndCallSmiBranch(Label* label, bool if_smi) { __ testl(EAX, Immediate(kSmiTagMask)); - if (kFirstCheckIsSmi) { - // Jump if receiver is not Smi. - if (num_checks == 1) { - __ j(NOT_ZERO, failed); - } else { - __ j(NOT_ZERO, &after_smi_test); - } - // Do not use the code from the function, but let the code be patched so - // that we can record the outgoing edges to other code. - GenerateDartCall(deopt_id, token_index, - *StubCode::CallStaticFunction_entry(), - RawPcDescriptors::kOther, locs); - const Function& function = - Function::ZoneHandle(zone(), ic_data.GetTargetAt(0)); - AddStaticCallTarget(function); - __ Drop(argument_count); - if (num_checks > 1) { - __ jmp(match_found); - } - } else { - // Receiver is Smi, but Smi is not a valid class therefore fail. - // (Smi class must be first in the list). - __ j(ZERO, failed); - } - __ Bind(&after_smi_test); + // Jump if receiver is (not) Smi. + __ j(if_smi ? ZERO : NOT_ZERO, label); +} - ASSERT(!ic_data.IsNull() && (num_checks > 0)); - GrowableArray sorted(num_checks); - SortICDataByCount(ic_data, &sorted, /* drop_smi = */ true); - const intptr_t sorted_len = sorted.length(); - // If sorted_len is 0 then only a Smi check was needed; the Smi check above - // will fail if there was only one check and receiver is not Smi. - if (sorted_len == 0) return; - - // Value is not Smi, +void FlowGraphCompiler::EmitTestAndCallLoadCid() { __ LoadClassId(EDI, EAX); +} - bool add_megamorphic_call = false; - const int kMaxImmediateInInstruction = 127; - int bias = - ComputeGoodBiasForCidComparison(sorted, kMaxImmediateInInstruction); - if (bias != 0) __ addl(EDI, Immediate(-bias)); - for (intptr_t i = 0; i < sorted_len; i++) { - const bool is_last_check = (i == (sorted_len - 1)); - int cid_start = sorted[i].cid_start; - int cid_end = sorted[i].cid_end; - int count = sorted[i].count; - if (!is_last_check && !complete && count < (total_ic_calls >> 5)) { - // This case is hit too rarely to be worth writing class-id checks inline - // for. - add_megamorphic_call = true; - break; - } - ASSERT(cid_start > kSmiCid || cid_end < kSmiCid); - Label next_test; - if (!complete || !is_last_check) { - Label* next_label = is_last_check ? failed : &next_test; - if (cid_start == cid_end) { - __ cmpl(EDI, Immediate(cid_start - bias)); - __ j(NOT_EQUAL, next_label); - } else { - __ addl(EDI, Immediate(bias - cid_start)); - bias = cid_start; - __ cmpl(EDI, Immediate(cid_end - cid_start)); - __ j(ABOVE, next_label); // Unsigned higher. - } - } - // Do not use the code from the function, but let the code be patched so - // that we can record the outgoing edges to other code. - const Function& function = *sorted[i].target; - GenerateDartCall(deopt_id, token_index, - *StubCode::CallStaticFunction_entry(), - RawPcDescriptors::kOther, locs); - AddStaticCallTarget(function); - __ Drop(argument_count); - if (!is_last_check) { - __ jmp(match_found); - } - __ Bind(&next_test); - } - if (add_megamorphic_call) { - int try_index = CatchClauseNode::kInvalidTryIndex; - EmitMegamorphicInstanceCall(ic_data, argument_count, deopt_id, token_index, - locs, try_index, argument_count); +int FlowGraphCompiler::EmitTestAndCallCheckCid(Label* next_label, + const CidRangeTarget& target, + int bias) { + intptr_t cid_start = target.cid_start; + intptr_t cid_end = target.cid_end; + if (cid_start == cid_end) { + __ cmpl(EDI, Immediate(cid_start - bias)); + __ j(NOT_EQUAL, next_label); + } else { + __ addl(EDI, Immediate(bias - cid_start)); + bias = cid_start; + __ cmpl(EDI, Immediate(cid_end - cid_start)); + __ j(ABOVE, next_label); // Unsigned higher. } + return bias; } diff --git a/runtime/vm/flow_graph_compiler_mips.cc b/runtime/vm/flow_graph_compiler_mips.cc index 96be50c3d17..5f7a62f6b3a 100644 --- a/runtime/vm/flow_graph_compiler_mips.cc +++ b/runtime/vm/flow_graph_compiler_mips.cc @@ -1249,16 +1249,14 @@ void FlowGraphCompiler::EmitInstanceCall(const StubEntry& stub_entry, void FlowGraphCompiler::EmitMegamorphicInstanceCall( - const ICData& ic_data, + const String& name, + const Array& arguments_descriptor, intptr_t argument_count, intptr_t deopt_id, TokenPosition token_pos, LocationSummary* locs, intptr_t try_index, intptr_t slow_path_argument_count) { - const String& name = String::Handle(zone(), ic_data.target_name()); - const Array& arguments_descriptor = - Array::ZoneHandle(zone(), ic_data.arguments_descriptor()); ASSERT(!arguments_descriptor.IsNull() && (arguments_descriptor.Length() > 0)); const MegamorphicCache& cache = MegamorphicCache::ZoneHandle( zone(), @@ -1532,123 +1530,52 @@ void FlowGraphCompiler::ClobberDeadTempRegisters(LocationSummary* locs) { #endif -void FlowGraphCompiler::EmitTestAndCall(const ICData& ic_data, - intptr_t argument_count, - const Array& argument_names, - Label* failed, - Label* match_found, - intptr_t deopt_id, - TokenPosition token_index, - LocationSummary* locs, - bool complete, - intptr_t total_ic_calls) { - ASSERT(is_optimizing()); +void FlowGraphCompiler::EmitTestAndCallLoadReceiver( + intptr_t argument_count, + const Array& arguments_descriptor) { __ Comment("EmitTestAndCall"); - const Array& arguments_descriptor = Array::ZoneHandle( - zone(), ArgumentsDescriptor::New(argument_count, argument_names)); - // Load receiver into T0. __ LoadFromOffset(T0, SP, (argument_count - 1) * kWordSize); __ LoadObject(S4, arguments_descriptor); +} - const bool kFirstCheckIsSmi = ic_data.GetReceiverClassIdAt(0) == kSmiCid; - const intptr_t num_checks = ic_data.NumberOfChecks(); - ASSERT(!ic_data.IsNull() && (num_checks > 0)); - - Label after_smi_test; - if (kFirstCheckIsSmi) { - __ andi(CMPRES1, T0, Immediate(kSmiTagMask)); - // Jump if receiver is not Smi. - if (num_checks == 1) { - __ bne(CMPRES1, ZR, failed); - } else { - __ bne(CMPRES1, ZR, &after_smi_test); - } - // Do not use the code from the function, but let the code be patched so - // that we can record the outgoing edges to other code. - const Function& function = - Function::ZoneHandle(zone(), ic_data.GetTargetAt(0)); - GenerateStaticDartCall(deopt_id, token_index, - *StubCode::CallStaticFunction_entry(), - RawPcDescriptors::kOther, locs, function); - __ Drop(argument_count); - if (num_checks > 1) { - __ b(match_found); - } +void FlowGraphCompiler::EmitTestAndCallSmiBranch(Label* label, bool if_smi) { + __ andi(CMPRES1, T0, Immediate(kSmiTagMask)); + if (if_smi) { + // Jump if receiver is Smi. + __ beq(CMPRES1, ZR, label); } else { - // Receiver is Smi, but Smi is not a valid class therefore fail. - // (Smi class must be first in the list). - if (!complete) { - __ andi(CMPRES1, T0, Immediate(kSmiTagMask)); - __ beq(CMPRES1, ZR, failed); - } + // Jump if receiver is not Smi. + __ bne(CMPRES1, ZR, label); } +} - __ Bind(&after_smi_test); - ASSERT(!ic_data.IsNull() && (num_checks > 0)); - GrowableArray sorted(num_checks); - SortICDataByCount(ic_data, &sorted, /* drop_smi = */ true); - - const intptr_t sorted_len = sorted.length(); - // If sorted_len is 0 then only a Smi check was needed; the Smi check above - // will fail if there was only one check and receiver is not Smi. - if (sorted_len == 0) return; - - // Value is not Smi, +void FlowGraphCompiler::EmitTestAndCallLoadCid() { __ LoadClassId(T2, T0); +} - bool add_megamorphic_call = false; - int bias = 0; - for (intptr_t i = 0; i < sorted_len; i++) { - const bool is_last_check = (i == (sorted_len - 1)); - int cid_start = sorted[i].cid_start; - int cid_end = sorted[i].cid_end; - int count = sorted[i].count; - if (!is_last_check && !complete && count < (total_ic_calls >> 5)) { - // This case is hit too rarely to be worth writing class-id checks inline - // for. - add_megamorphic_call = true; - break; - } - ASSERT(cid_start > kSmiCid || cid_end < kSmiCid); - Label next_test; - Condition no_match; - if (!complete || !is_last_check) { - Label* next_label = is_last_check ? failed : &next_test; - if (cid_start == cid_end) { - __ BranchNotEqual(T2, Immediate(cid_start - bias), next_label); - } else { - __ AddImmediate(T2, T2, bias - cid_start); - bias = cid_start; - // TODO(erikcorry): We should use sltiu instead of the temporary TMP if - // the range is small enough. - __ LoadImmediate(TMP, cid_end - cid_end); - // Reverse comparison so we get 1 if biased cid > tmp ie cid is out of - // range. - __ sltu(TMP, TMP, T2); - __ bne(TMP, ZR, next_label); - } - } - // Do not use the code from the function, but let the code be patched so - // that we can record the outgoing edges to other code. - const Function& function = *sorted[i].target; - GenerateStaticDartCall(deopt_id, token_index, - *StubCode::CallStaticFunction_entry(), - RawPcDescriptors::kOther, locs, function); - __ Drop(argument_count); - if (!is_last_check) { - __ b(match_found); - } - __ Bind(&next_test); - } - if (add_megamorphic_call) { - int try_index = CatchClauseNode::kInvalidTryIndex; - EmitMegamorphicInstanceCall(ic_data, argument_count, deopt_id, token_index, - locs, try_index, argument_count); +int FlowGraphCompiler::EmitTestAndCallCheckCid(Label* next_label, + const CidRangeTarget& target, + int bias) { + intptr_t cid_start = target.cid_start; + intptr_t cid_end = target.cid_end; + if (cid_start == cid_end) { + __ BranchNotEqual(T2, Immediate(cid_start - bias), next_label); + } else { + __ AddImmediate(T2, T2, bias - cid_start); + bias = cid_start; + // TODO(erikcorry): We should use sltiu instead of the temporary TMP if + // the range is small enough. + __ LoadImmediate(TMP, cid_end - cid_end); + // Reverse comparison so we get 1 if biased cid > tmp ie cid is out of + // range. + __ sltu(TMP, TMP, T2); + __ bne(TMP, ZR, next_label); } + return bias; } diff --git a/runtime/vm/flow_graph_compiler_x64.cc b/runtime/vm/flow_graph_compiler_x64.cc index 7e92a96a2fe..13af4ded995 100644 --- a/runtime/vm/flow_graph_compiler_x64.cc +++ b/runtime/vm/flow_graph_compiler_x64.cc @@ -1250,16 +1250,14 @@ void FlowGraphCompiler::EmitInstanceCall(const StubEntry& stub_entry, void FlowGraphCompiler::EmitMegamorphicInstanceCall( - const ICData& ic_data, + const String& name, + const Array& arguments_descriptor, intptr_t argument_count, intptr_t deopt_id, TokenPosition token_pos, LocationSummary* locs, intptr_t try_index, intptr_t slow_path_argument_count) { - const String& name = String::Handle(zone(), ic_data.target_name()); - const Array& arguments_descriptor = - Array::ZoneHandle(zone(), ic_data.arguments_descriptor()); ASSERT(!arguments_descriptor.IsNull() && (arguments_descriptor.Length() > 0)); const MegamorphicCache& cache = MegamorphicCache::ZoneHandle( zone(), @@ -1446,121 +1444,43 @@ void FlowGraphCompiler::ClobberDeadTempRegisters(LocationSummary* locs) { #endif -void FlowGraphCompiler::EmitTestAndCall(const ICData& ic_data, - intptr_t argument_count, - const Array& argument_names, - Label* failed, - Label* match_found, - intptr_t deopt_id, - TokenPosition token_index, - LocationSummary* locs, - bool complete, - intptr_t total_ic_calls) { - ASSERT(is_optimizing()); - +void FlowGraphCompiler::EmitTestAndCallLoadReceiver( + intptr_t argument_count, + const Array& arguments_descriptor) { __ Comment("EmitTestAndCall"); - const Array& arguments_descriptor = Array::ZoneHandle( - zone(), ArgumentsDescriptor::New(argument_count, argument_names)); // Load receiver into RAX. __ movq(RAX, Address(RSP, (argument_count - 1) * kWordSize)); __ LoadObject(R10, arguments_descriptor); +} - const bool kFirstCheckIsSmi = ic_data.GetReceiverClassIdAt(0) == kSmiCid; - const intptr_t num_checks = ic_data.NumberOfChecks(); - ASSERT(!ic_data.IsNull() && (num_checks > 0)); +void FlowGraphCompiler::EmitTestAndCallSmiBranch(Label* label, bool if_smi) { + __ testq(RAX, Immediate(kSmiTagMask)); + // Jump if receiver is (not) Smi. + __ j(if_smi ? ZERO : NOT_ZERO, label); +} - Label after_smi_test; - if (kFirstCheckIsSmi) { - __ testq(RAX, Immediate(kSmiTagMask)); - // Jump if receiver is not Smi. - if (num_checks == 1) { - __ j(NOT_ZERO, failed); - } else { - __ j(NOT_ZERO, &after_smi_test); - } - // Do not use the code from the function, but let the code be patched so - // that we can record the outgoing edges to other code. - const Function& function = - Function::ZoneHandle(zone(), ic_data.GetTargetAt(0)); - GenerateStaticDartCall(deopt_id, token_index, - *StubCode::CallStaticFunction_entry(), - RawPcDescriptors::kOther, locs, function); - __ Drop(argument_count, RCX); - if (num_checks > 1) { - __ jmp(match_found); - } - } else { - // Receiver is Smi, but Smi is not a valid class therefore fail. - // (Smi class must be first in the list). - if (!complete) { - __ testq(RAX, Immediate(kSmiTagMask)); - __ j(ZERO, failed); - } - } - __ Bind(&after_smi_test); - ASSERT(!ic_data.IsNull() && (num_checks > 0)); - GrowableArray sorted(num_checks); - SortICDataByCount(ic_data, &sorted, /* drop_smi = */ true); - - const intptr_t sorted_len = sorted.length(); - // If sorted_len is 0 then only a Smi check was needed; the Smi check above - // will fail if there was only one check and receiver is not Smi. - if (sorted_len == 0) return; - - // Value is not Smi, +void FlowGraphCompiler::EmitTestAndCallLoadCid() { __ LoadClassId(RDI, RAX); +} - bool add_megamorphic_call = false; - const int kMaxImmediateInInstruction = 127; - int bias = - ComputeGoodBiasForCidComparison(sorted, kMaxImmediateInInstruction); - if (bias != 0) __ addl(RDI, Immediate(-bias)); - for (intptr_t i = 0; i < sorted_len; i++) { - const bool is_last_check = (i == (sorted_len - 1)); - int cid_start = sorted[i].cid_start; - int cid_end = sorted[i].cid_end; - int count = sorted[i].count; - if (!is_last_check && !complete && count < (total_ic_calls >> 5)) { - // This case is hit too rarely to be worth writing class-id checks inline - // for. - add_megamorphic_call = true; - break; - } - ASSERT(cid_start > kSmiCid || cid_end < kSmiCid); - Label next_test; - if (!complete || !is_last_check) { - Label* next_label = is_last_check ? failed : &next_test; - bool near = is_last_check ? Assembler::kFarJump : Assembler::kNearJump; - if (cid_start == cid_end) { - __ cmpl(RDI, Immediate(cid_start - bias)); - __ j(NOT_EQUAL, next_label, near); - } else { - __ addl(RDI, Immediate(bias - cid_start)); - bias = cid_start; - __ cmpl(RDI, Immediate(cid_end - cid_start)); - __ j(ABOVE, next_label, near); // Unsigned higher. - } - } - // Do not use the code from the function, but let the code be patched so - // that we can record the outgoing edges to other code. - const Function& function = *sorted[i].target; - GenerateStaticDartCall(deopt_id, token_index, - *StubCode::CallStaticFunction_entry(), - RawPcDescriptors::kOther, locs, function); - __ Drop(argument_count, RCX); - if (!is_last_check) { - __ jmp(match_found); - } - __ Bind(&next_test); - } - if (add_megamorphic_call) { - int try_index = CatchClauseNode::kInvalidTryIndex; - EmitMegamorphicInstanceCall(ic_data, argument_count, deopt_id, token_index, - locs, try_index, argument_count); +int FlowGraphCompiler::EmitTestAndCallCheckCid(Label* next_label, + const CidRangeTarget& target, + int bias) { + intptr_t cid_start = target.cid_start; + intptr_t cid_end = target.cid_end; + if (cid_start == cid_end) { + __ cmpl(RDI, Immediate(cid_start - bias)); + __ j(NOT_EQUAL, next_label); + } else { + __ addl(RDI, Immediate(bias - cid_start)); + bias = cid_start; + __ cmpl(RDI, Immediate(cid_end - cid_start)); + __ j(ABOVE, next_label); // Unsigned higher. } + return bias; } diff --git a/runtime/vm/flow_graph_inliner.cc b/runtime/vm/flow_graph_inliner.cc index 094a557e337..95e634b1a18 100644 --- a/runtime/vm/flow_graph_inliner.cc +++ b/runtime/vm/flow_graph_inliner.cc @@ -177,7 +177,7 @@ class GraphInfoCollector : public ValueObject { // parameters was fixed. // TODO(fschneider): Determine new heuristic parameters that avoid // these checks entirely. - if (!call->HasSingleRecognizedTarget() && + if (!call->IsSureToCallSingleRecognizedTarget() && (call->instance_call()->token_kind() != Token::kEQ)) { ++call_site_count_; } @@ -290,9 +290,7 @@ class CallSites : public ValueObject { GrowableArray instance_call_counts(num_instance_calls); for (intptr_t i = 0; i < num_instance_calls; ++i) { const intptr_t aggregate_count = - instance_calls_[i + instance_call_start_ix] - .call->ic_data() - .AggregateCount(); + instance_calls_[i + instance_call_start_ix].call->CallCount(); instance_call_counts.Add(aggregate_count); if (aggregate_count > max_count) max_count = aggregate_count; } @@ -343,11 +341,11 @@ class CallSites : public ValueObject { if (current->IsPolymorphicInstanceCall()) { PolymorphicInstanceCallInstr* instance_call = current->AsPolymorphicInstanceCall(); - target = instance_call->ic_data().GetTargetAt(0); + target ^= instance_call->targets().FirstTarget().raw(); call = instance_call; } else if (current->IsStaticCall()) { StaticCallInstr* static_call = current->AsStaticCall(); - target = static_call->function().raw(); + target ^= static_call->function().raw(); call = static_call; } else if (current->IsClosureCall()) { // TODO(srdjan): Add data for closure calls. @@ -388,17 +386,15 @@ class CallSites : public ValueObject { PolymorphicInstanceCallInstr* instance_call = current->AsPolymorphicInstanceCall(); if (!inline_only_recognized_methods || - instance_call->HasSingleRecognizedTarget() || - instance_call->ic_data() - .HasOnlyDispatcherOrImplicitAccessorTargets()) { + instance_call->IsSureToCallSingleRecognizedTarget() || + instance_call->HasOnlyDispatcherOrImplicitAccessorTargets()) { instance_calls_.Add(InstanceCallInfo(instance_call, graph)); } else { // Method not inlined because inlining too deep and method // not recognized. if (FLAG_print_inlining_tree) { const Function* caller = &graph->function(); - const Function* target = &Function::ZoneHandle( - instance_call->ic_data().GetTargetAt(0)); + const Function* target = &instance_call->targets().FirstTarget(); inlined_info->Add(InlinedInfo(caller, target, depth + 1, instance_call, "Too deep")); } @@ -491,10 +487,12 @@ class PolymorphicInliner : public ValueObject { CallSiteInliner* const owner_; PolymorphicInstanceCallInstr* const call_; const intptr_t num_variants_; - GrowableArray variants_; + const CallTargets& variants_; - GrowableArray inlined_variants_; - GrowableArray non_inlined_variants_; + CallTargets inlined_variants_; + // The non_inlined_variants_ can be used in a long-lived instruction object, + // so they are not embedded into the shorter-lived PolymorphicInliner object. + CallTargets* non_inlined_variants_; GrowableArray inlined_entries_; InlineExitCollector* exit_collector_; @@ -1305,8 +1303,7 @@ class CallSiteInliner : public ValueObject { continue; } - const ICData& ic_data = call->ic_data(); - const Function& target = Function::ZoneHandle(ic_data.GetTargetAt(0)); + const Function& target = call->targets().MostPopularTarget(); if (!inliner_->AlwaysInline(target) && (call_info[call_idx].ratio * 100) < FLAG_inlining_hotness) { if (trace_inlining()) { @@ -1447,10 +1444,10 @@ PolymorphicInliner::PolymorphicInliner(CallSiteInliner* owner, intptr_t caller_inlining_id) : owner_(owner), call_(call), - num_variants_(call->ic_data().NumberOfChecks()), - variants_(num_variants_), - inlined_variants_(num_variants_), - non_inlined_variants_(num_variants_), + num_variants_(call->NumberOfChecks()), + variants_(call->targets_), + inlined_variants_(), + non_inlined_variants_(new (zone()) CallTargets()), inlined_entries_(num_variants_), exit_collector_(new (Z) InlineExitCollector(owner->caller_graph(), call)), caller_function_(caller_function), @@ -1534,8 +1531,8 @@ bool PolymorphicInliner::CheckInlinedDuplicate(const Function& target) { bool PolymorphicInliner::CheckNonInlinedDuplicate(const Function& target) { - for (intptr_t i = 0; i < non_inlined_variants_.length(); ++i) { - if (target.raw() == non_inlined_variants_[i].target->raw()) { + for (intptr_t i = 0; i < non_inlined_variants_->length(); ++i) { + if (target.raw() == non_inlined_variants_->At(i).target->raw()) { return true; } } @@ -1559,8 +1556,8 @@ bool PolymorphicInliner::TryInliningPoly(const CidRangeTarget& range) { } InlinedCallData call_data(call_, &arguments, caller_function_, caller_inlining_id_); - if (!owner_->TryInlining(*range.target, - call_->instance_call()->argument_names(), + Function& target = Function::ZoneHandle(zone(), range.target->raw()); + if (!owner_->TryInlining(target, call_->instance_call()->argument_names(), &call_data)) { return false; } @@ -1708,7 +1705,7 @@ TargetEntryInstr* PolymorphicInliner::BuildDecisionGraph() { // arbitrary CidRangeTarget. Currently we don't go into this branch if the // last test is a range test - instead we set the follow_with_deopt flag. if (is_last_test && (!test_is_range || call_->complete()) && - non_inlined_variants_.is_empty()) { + non_inlined_variants_->is_empty()) { // If it is the last variant use a check class id instruction which can // deoptimize, followed unconditionally by the body. Omit the check if // we know that we have covered all possible classes. @@ -1884,7 +1881,7 @@ TargetEntryInstr* PolymorphicInliner::BuildDecisionGraph() { } // Handle any non-inlined variants. - if (!non_inlined_variants_.is_empty()) { + if (!non_inlined_variants_->is_empty()) { // Move push arguments of the call. for (intptr_t i = 0; i < call_->ArgumentCount(); ++i) { PushArgumentInstr* push = call_->PushArgumentAt(i); @@ -1893,29 +1890,10 @@ TargetEntryInstr* PolymorphicInliner::BuildDecisionGraph() { cursor->LinkTo(push); cursor = push; } - const ICData& old_checks = call_->ic_data(); - const ICData& new_checks = ICData::ZoneHandle(ICData::New( - Function::Handle(old_checks.Owner()), - String::Handle(old_checks.target_name()), - Array::Handle(old_checks.arguments_descriptor()), old_checks.deopt_id(), - 1, // Number of args tested. - false)); // is_static_call - for (intptr_t i = 0; i < non_inlined_variants_.length(); ++i) { - // We are adding all the cids in each range. They will be joined - // together again by the PolymorphicInstanceCall instruction, which is a - // bit messy. - intptr_t count = non_inlined_variants_[i].count; - - for (intptr_t j = non_inlined_variants_[i].cid_start; - j <= non_inlined_variants_[i].cid_end; j++) { - new_checks.AddReceiverCheck(j, *non_inlined_variants_[i].target, count); - count = 0; - } - } PolymorphicInstanceCallInstr* fallback_call = - new PolymorphicInstanceCallInstr(call_->instance_call(), new_checks, - /* with_checks = */ true, - call_->complete()); + new PolymorphicInstanceCallInstr( + call_->instance_call(), *non_inlined_variants_, + /* with_checks = */ true, call_->complete()); fallback_call->set_ssa_temp_index( owner_->caller_graph()->alloc_ssa_temp_index()); fallback_call->InheritDeoptTarget(zone(), call_); @@ -1965,13 +1943,12 @@ bool PolymorphicInliner::trace_inlining() const { void PolymorphicInliner::Inline() { - // Consider the polymorphic variants in order by frequency. - FlowGraphCompiler::SortICDataByCount(call_->ic_data(), &variants_, - /* drop_smi = */ false); + ASSERT(&variants_ == &call_->targets_); + intptr_t total = call_->total_call_count(); for (intptr_t var_idx = 0; var_idx < variants_.length(); ++var_idx) { if (variants_.length() > FLAG_max_polymorphic_checks) { - non_inlined_variants_.Add(variants_[var_idx]); + non_inlined_variants_->Add(variants_[var_idx]); continue; } @@ -1979,7 +1956,7 @@ void PolymorphicInliner::Inline() { // the last two, because it's a big win if we inline all of them (compiler // can see all side effects). const bool try_harder = (var_idx >= variants_.length() - 2) && - non_inlined_variants_.length() == 0; + non_inlined_variants_->length() == 0; const Function& target = *variants_[var_idx].target; const intptr_t count = variants_[var_idx].count; @@ -1992,7 +1969,7 @@ void PolymorphicInliner::Inline() { if (!try_harder && count < (total >> 5)) { TRACE_INLINING( TracePolyInlining(variants_[var_idx], total, "way too infrequent")); - non_inlined_variants_.Add(variants_[var_idx]); + non_inlined_variants_->Add(variants_[var_idx]); continue; } @@ -2010,7 +1987,7 @@ void PolymorphicInliner::Inline() { if (!try_harder && count < (total >> (small ? 4 : 3))) { TRACE_INLINING( TracePolyInlining(variants_[var_idx], total, "too infrequent")); - non_inlined_variants_.Add(variants_[var_idx]); + non_inlined_variants_->Add(variants_[var_idx]); continue; } @@ -2020,7 +1997,7 @@ void PolymorphicInliner::Inline() { if (CheckNonInlinedDuplicate(target)) { TRACE_INLINING( TracePolyInlining(variants_[var_idx], total, "already not inlined")); - non_inlined_variants_.Add(variants_[var_idx]); + non_inlined_variants_->Add(variants_[var_idx]); continue; } @@ -2031,7 +2008,7 @@ void PolymorphicInliner::Inline() { } else { TRACE_INLINING( TracePolyInlining(variants_[var_idx], total, "not inlined")); - non_inlined_variants_.Add(variants_[var_idx]); + non_inlined_variants_->Add(variants_[var_idx]); } } diff --git a/runtime/vm/flow_graph_type_propagator.cc b/runtime/vm/flow_graph_type_propagator.cc index 92a4d78ae68..9617c03b348 100644 --- a/runtime/vm/flow_graph_type_propagator.cc +++ b/runtime/vm/flow_graph_type_propagator.cc @@ -297,7 +297,7 @@ void FlowGraphTypePropagator::VisitInstanceCall(InstanceCallInstr* instr) { void FlowGraphTypePropagator::VisitPolymorphicInstanceCall( PolymorphicInstanceCallInstr* instr) { if (instr->instance_call()->has_unique_selector()) { - SetCid(instr->ArgumentAt(0), instr->ic_data().GetReceiverClassIdAt(0)); + SetCid(instr->ArgumentAt(0), instr->targets().MonomorphicReceiverCid()); return; } CheckNonNullSelector(instr, instr->ArgumentAt(0), @@ -1024,8 +1024,8 @@ CompileType AllocateUninitializedContextInstr::ComputeType() const { CompileType PolymorphicInstanceCallInstr::ComputeType() const { - if (!HasSingleRecognizedTarget()) return CompileType::Dynamic(); - const Function& target = Function::Handle(ic_data().GetTargetAt(0)); + if (!IsSureToCallSingleRecognizedTarget()) return CompileType::Dynamic(); + const Function& target = *targets_[0].target; return (target.recognized_kind() != MethodRecognizer::kUnknown) ? CompileType::FromCid(MethodRecognizer::ResultCid(target)) : CompileType::Dynamic(); diff --git a/runtime/vm/il_printer.cc b/runtime/vm/il_printer.cc index dcfc067a11b..fde3ffd2629 100644 --- a/runtime/vm/il_printer.cc +++ b/runtime/vm/il_printer.cc @@ -195,6 +195,42 @@ const char* CompileType::ToCString() const { } +static void PrintTargetsHelper(BufferFormatter* f, + const CallTargets& targets, + intptr_t num_checks_to_print) { + f->Print(" IC["); + f->Print("%" Pd ": ", targets.length()); + Function& target = Function::Handle(); + if ((num_checks_to_print == FlowGraphPrinter::kPrintAll) || + (num_checks_to_print > targets.length())) { + num_checks_to_print = targets.length(); + } + for (intptr_t i = 0; i < num_checks_to_print; i++) { + const CidRangeTarget& range = targets[i]; + const intptr_t count = range.count; + target ^= range.target->raw(); + if (i > 0) { + f->Print(" | "); + } + if (range.cid_start == range.cid_end) { + const Class& cls = + Class::Handle(Isolate::Current()->class_table()->At(range.cid_start)); + f->Print("%s", String::Handle(cls.Name()).ToCString()); + f->Print(" cnt:%" Pd " trgt:'%s'", count, target.ToQualifiedCString()); + } else { + const Class& cls = Class::Handle(range.target->Owner()); + f->Print("cid %" Pd "-%" Pd " %s", range.cid_start, range.cid_end, + String::Handle(cls.Name()).ToCString()); + f->Print(" cnt:%" Pd " trgt:'%s'", count, target.ToQualifiedCString()); + } + } + if (num_checks_to_print < targets.length()) { + f->Print("..."); + } + f->Print("]"); +} + + static void PrintICDataHelper(BufferFormatter* f, const ICData& ic_data, intptr_t num_checks_to_print) { @@ -256,6 +292,16 @@ void FlowGraphPrinter::PrintICData(const ICData& ic_data, } +void FlowGraphPrinter::PrintCidRangeData(const CallTargets& targets, + intptr_t num_checks_to_print) { + char buffer[1024]; + BufferFormatter f(buffer, sizeof(buffer)); + PrintTargetsHelper(&f, targets, num_checks_to_print); + THR_Print("%s ", buffer); + // TODO(erikcorry): Print args descriptor. +} + + static void PrintUse(BufferFormatter* f, const Definition& definition) { if (definition.HasSSATemp()) { if (definition.HasPairRepresentation()) { @@ -467,11 +513,7 @@ void PolymorphicInstanceCallInstr::PrintOperandsTo(BufferFormatter* f) const { f->Print(", "); PushArgumentAt(i)->value()->PrintTo(f); } - if (FLAG_display_sorted_ic_data) { - PrintICDataSortedHelper(f, ic_data()); - } else { - PrintICDataHelper(f, ic_data(), FlowGraphPrinter::kPrintAll); - } + PrintTargetsHelper(f, targets_, FlowGraphPrinter::kPrintAll); if (with_checks()) { f->Print(" WITH-CHECKS"); } diff --git a/runtime/vm/il_printer.h b/runtime/vm/il_printer.h index 7b27a6174fa..b3553e4462a 100644 --- a/runtime/vm/il_printer.h +++ b/runtime/vm/il_printer.h @@ -62,6 +62,11 @@ class FlowGraphPrinter : public ValueObject { static void PrintICData(const ICData& ic_data, intptr_t num_checks_to_print = kPrintAll); + // Debugging helper function. If 'num_checks_to_print' is not specified + // all checks will be printed. + static void PrintCidRangeData(const CallTargets& ic_data, + intptr_t num_checks_to_print = kPrintAll); + static bool ShouldPrint(const Function& function); static bool PassesFilter(const char* filter, const Function& function); diff --git a/runtime/vm/intermediate_language.cc b/runtime/vm/intermediate_language.cc index beb86543b04..45c3129456b 100644 --- a/runtime/vm/intermediate_language.cc +++ b/runtime/vm/intermediate_language.cc @@ -2741,6 +2741,104 @@ bool UnboxInstr::CanConvertSmi() const { } +static int OrderById(const CidRangeTarget* a, const CidRangeTarget* b) { + // Negative if 'a' should sort before 'b'. + ASSERT(a->cid_start == a->cid_end); + ASSERT(b->cid_start == b->cid_end); + return a->cid_start - b->cid_start; +} + + +static int OrderByFrequency(const CidRangeTarget* a, const CidRangeTarget* b) { + // Negative if 'a' should sort before 'b'. + return b->count - a->count; +} + + +CallTargets* CallTargets::Create(Zone* zone, const ICData& ic_data) { + CallTargets* targets = new (zone) CallTargets(); + if (ic_data.NumberOfChecks() == 0) return targets; + + Function& dummy = Function::Handle(zone); + + bool check_one_arg = ic_data.NumArgsTested() == 1; + + int checks = ic_data.NumberOfChecks(); + for (int i = 0; i < checks; i++) { + intptr_t id = 0; + if (check_one_arg) { + ic_data.GetOneClassCheckAt(i, &id, &dummy); + } else { + // The API works for multi dispatch ICs that check more than one + // argument, but we know we will only check one arg here, so only the 0th + // element of id will be used. + GrowableArray arg_ids; + ic_data.GetCheckAt(i, &arg_ids, &dummy); + id = arg_ids[0]; + } + Function& function = Function::ZoneHandle(zone, ic_data.GetTargetAt(i)); + targets->Add(CidRangeTarget(id, id, &function, ic_data.GetCountAt(i))); + } + + targets->Sort(OrderById); + + Array& args_desc_array = Array::Handle(zone, ic_data.arguments_descriptor()); + ArgumentsDescriptor args_desc(args_desc_array); + String& name = String::Handle(zone, ic_data.target_name()); + + Function& fn = Function::Handle(zone); + + intptr_t length = targets->length(); + + // Spread class-ids to preceding classes where a lookup yields the same + // method. + for (int idx = 0; idx < length; idx++) { + int lower_limit_cid = (idx == 0) ? -1 : targets->At(idx - 1).cid_end; + const Function& target = *targets->At(idx).target; + for (int i = targets->At(idx).cid_start - 1; i > lower_limit_cid; i--) { + if (FlowGraphCompiler::LookupMethodFor(i, name, args_desc, &fn) && + fn.raw() == target.raw()) { + CidRangeTarget t = targets->At(idx); + t.cid_start = i; + (*targets)[idx] = t; + } else { + break; + } + } + } + // Spread class-ids to following classes where a lookup yields the same + // method. + for (int idx = 0; idx < length; idx++) { + int upper_limit_cid = + (idx == length - 1) ? 1000000000 : targets->At(idx + 1).cid_start; + const Function& target = *targets->At(idx).target; + for (int i = targets->At(idx).cid_end + 1; i < upper_limit_cid; i++) { + if (FlowGraphCompiler::LookupMethodFor(i, name, args_desc, &fn) && + fn.raw() == target.raw()) { + (*targets)[idx].cid_end = i; + } else { + break; + } + } + } + // Merge adjacent class id ranges. + int dest = 0; + for (int src = 1; src < length; src++) { + if (targets->At(dest).cid_end + 1 == targets->At(src).cid_start && + targets->At(dest).target->raw() == targets->At(src).target->raw()) { + (*targets)[dest].cid_end = targets->At(src).cid_end; + (*targets)[dest].count += targets->At(src).count; + } else { + dest++; + if (src != dest) (*targets)[dest] = targets->At(src); + } + } + targets->SetLength(dest + 1); + targets->Sort(OrderByFrequency); + return targets; +} + + // Shared code generation methods (EmitNativeCode and // MakeLocationSummary). Only assembly code that can be shared across all // architectures can be used. Machine specific register allocation and code @@ -3169,12 +3267,77 @@ bool InstanceCallInstr::MatchesCoreName(const String& name) { } -bool PolymorphicInstanceCallInstr::HasSingleRecognizedTarget() const { - if (FLAG_precompiled_mode && with_checks()) return false; +bool CallTargets::HasSingleRecognizedTarget() const { + if (!HasSingleTarget()) return false; + return MethodRecognizer::RecognizeKind(FirstTarget()) != + MethodRecognizer::kUnknown; +} - return ic_data().HasOneTarget() && - (MethodRecognizer::RecognizeKind(Function::Handle( - ic_data().GetTargetAt(0))) != MethodRecognizer::kUnknown); + +bool CallTargets::HasSingleTarget() const { + ASSERT(length() != 0); + for (int i = 0; i < length(); i++) { + if (cid_ranges_[i].target->raw() != cid_ranges_[0].target->raw()) + return false; + } + return true; +} + + +bool CallTargets::IsMonomorphic() const { + if (length() != 1) return false; + return cid_ranges_[0].cid_start == cid_ranges_[0].cid_end; +} + + +intptr_t CallTargets::MonomorphicReceiverCid() const { + ASSERT(IsMonomorphic()); + return cid_ranges_[0].cid_start; +} + + +Function& CallTargets::FirstTarget() const { + ASSERT(length() != 0); + ASSERT(cid_ranges_[0].target->IsZoneHandle()); + return *cid_ranges_[0].target; +} + + +Function& CallTargets::MostPopularTarget() const { + ASSERT(length() != 0); + ASSERT(cid_ranges_[0].target->IsZoneHandle()); + for (int i = 1; i < length(); i++) { + ASSERT(cid_ranges_[i].count <= cid_ranges_[0].count); + } + return *cid_ranges_[0].target; +} + + +intptr_t CallTargets::AggregateCallCount() const { + intptr_t sum = 0; + for (int i = 0; i < length(); i++) { + sum += cid_ranges_[i].count; + } + return sum; +} + + +bool PolymorphicInstanceCallInstr::HasOnlyDispatcherOrImplicitAccessorTargets() + const { + const intptr_t len = targets_.length(); + Function& target = Function::Handle(); + for (intptr_t i = 0; i < len; i++) { + target ^= targets_[i].target->raw(); + if (!target.IsDispatcherOrImplicitAccessor()) { + return false; + } + } + return true; +} + + +intptr_t PolymorphicInstanceCallInstr::CallCount() const { + return targets().AggregateCallCount(); } @@ -3182,10 +3345,9 @@ bool PolymorphicInstanceCallInstr::HasSingleRecognizedTarget() const { // PolymorphicInstanceCallInstr. #if !defined(TARGET_ARCH_DBC) void PolymorphicInstanceCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { - ASSERT(ic_data().NumArgsTested() == 1); if (!with_checks()) { - ASSERT(ic_data().HasOneTarget()); - const Function& target = Function::ZoneHandle(ic_data().GetTargetAt(0)); + ASSERT(targets().HasSingleTarget()); + const Function& target = targets().FirstTarget(); compiler->GenerateStaticCall(deopt_id(), instance_call()->token_pos(), target, instance_call()->ArgumentCount(), instance_call()->argument_names(), locs(), @@ -3194,7 +3356,7 @@ void PolymorphicInstanceCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { } compiler->EmitPolymorphicInstanceCall( - ic_data(), instance_call()->ArgumentCount(), + targets_, *instance_call(), instance_call()->ArgumentCount(), instance_call()->argument_names(), deopt_id(), instance_call()->token_pos(), locs(), complete(), total_call_count()); } @@ -3202,22 +3364,29 @@ void PolymorphicInstanceCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { RawType* PolymorphicInstanceCallInstr::ComputeRuntimeType( - const ICData& ic_data) { + const CallTargets& targets) { bool is_string = true; bool is_integer = true; bool is_double = true; - const intptr_t num_checks = ic_data.NumberOfChecks(); + const intptr_t num_checks = targets.length(); for (intptr_t i = 0; i < num_checks; i++) { - const intptr_t cid = ic_data.GetReceiverClassIdAt(i); - is_string = is_string && RawObject::IsStringClassId(cid); - is_integer = is_integer && RawObject::IsIntegerClassId(cid); - is_double = is_double && (cid == kDoubleCid); + ASSERT(targets[i].target->raw() == targets[0].target->raw()); + const intptr_t start = targets[i].cid_start; + const intptr_t end = targets[i].cid_end; + for (intptr_t cid = start; cid <= end; cid++) { + is_string = is_string && RawObject::IsStringClassId(cid); + is_integer = is_integer && RawObject::IsIntegerClassId(cid); + is_double = is_double && (cid == kDoubleCid); + } } if (is_string) { + ASSERT(!is_integer); + ASSERT(!is_double); return Type::StringType(); } else if (is_integer) { + ASSERT(!is_double); return Type::IntType(); } else if (is_double) { return Type::Double(); @@ -3230,12 +3399,18 @@ RawType* PolymorphicInstanceCallInstr::ComputeRuntimeType( Definition* InstanceCallInstr::Canonicalize(FlowGraph* flow_graph) { const intptr_t receiver_cid = PushArgumentAt(0)->value()->Type()->ToCid(); - if (!HasICData()) return this; + // TODO(erikcorry): Even for cold call sites we could still try to look up + // methods when we know the receiver cid. We don't currently do this because + // it turns the InstanceCall into a PolymorphicInstanceCall which doesn't get + // recognized or inlined when it is cold. + if (ic_data()->NumberOfUsedChecks() == 0) return this; - const ICData& new_ic_data = - FlowGraphCompiler::TrySpecializeICDataByReceiverCid(*ic_data(), - receiver_cid); - if (new_ic_data.raw() == ic_data()->raw()) { + const CallTargets* new_target = + FlowGraphCompiler::ResolveCallTargetsForReceiverCid( + receiver_cid, + String::Handle(flow_graph->zone(), ic_data()->target_name()), + Array::Handle(flow_graph->zone(), ic_data()->arguments_descriptor())); + if (new_target == NULL) { // No specialization. return this; } @@ -3243,21 +3418,21 @@ Definition* InstanceCallInstr::Canonicalize(FlowGraph* flow_graph) { const bool with_checks = false; const bool complete = false; PolymorphicInstanceCallInstr* specialized = new PolymorphicInstanceCallInstr( - this, new_ic_data, with_checks, complete); + this, *new_target, with_checks, complete); flow_graph->InsertBefore(this, specialized, env(), FlowGraph::kValue); return specialized; } Definition* PolymorphicInstanceCallInstr::Canonicalize(FlowGraph* flow_graph) { - if (!HasSingleRecognizedTarget() || with_checks()) { + if (!IsSureToCallSingleRecognizedTarget()) { return this; } - const Function& target = Function::Handle(ic_data().GetTargetAt(0)); + const Function& target = targets().FirstTarget(); if (target.recognized_kind() == MethodRecognizer::kObjectRuntimeType) { const AbstractType& type = - AbstractType::Handle(ComputeRuntimeType(ic_data())); + AbstractType::Handle(ComputeRuntimeType(targets_)); if (!type.IsNull()) { return flow_graph->GetConstant(type); } @@ -3267,6 +3442,12 @@ Definition* PolymorphicInstanceCallInstr::Canonicalize(FlowGraph* flow_graph) { } +bool PolymorphicInstanceCallInstr::IsSureToCallSingleRecognizedTarget() const { + if (FLAG_precompiled_mode && with_checks()) return false; + return targets_.HasSingleRecognizedTarget(); +} + + Definition* StaticCallInstr::Canonicalize(FlowGraph* flow_graph) { if (!FLAG_precompiled_mode) { return this; diff --git a/runtime/vm/intermediate_language.h b/runtime/vm/intermediate_language.h index 715b5369ecd..3521d11a98e 100644 --- a/runtime/vm/intermediate_language.h +++ b/runtime/vm/intermediate_language.h @@ -548,6 +548,68 @@ FOR_EACH_ABSTRACT_INSTRUCTION(FORWARD_DECLARATION) #define PRINT_OPERANDS_TO_SUPPORT #endif // !PRODUCT + +// Represents a mapping from a range of class-ids to a method for a given +// selector (method name). Also can contain an indication of how frequently a +// given method has been called at a call site. This information can be +// harvested from the inline caches (ICs). +struct CidRangeTarget { + intptr_t cid_start; + intptr_t cid_end; + Function* target; + intptr_t count; + CidRangeTarget(intptr_t cid_start_arg, + intptr_t cid_end_arg, + Function* target_arg, + intptr_t count_arg) + : cid_start(cid_start_arg), + cid_end(cid_end_arg), + target(target_arg), + count(count_arg) { + ASSERT(target->IsZoneHandle()); + } +}; + + +class CallTargets : public ZoneAllocated { + public: + // Creates the off-heap CallTargets object that reflects the contents + // of the on-VM-heap IC data. Also expands the class-ids to neighbouring + // classes that inherit the same method. + static CallTargets* Create(Zone* zone, const ICData& ic_data); + + void Add(const CidRangeTarget& target) { cid_ranges_.Add(target); } + + CidRangeTarget& operator[](intptr_t index) const { + return cid_ranges_[index]; + } + + CidRangeTarget At(int index) { return cid_ranges_.At(index); } + + intptr_t length() const { return cid_ranges_.length(); } + + void SetLength(intptr_t len) { cid_ranges_.SetLength(len); } + + bool is_empty() const { return cid_ranges_.is_empty(); } + + void Sort(int compare(const CidRangeTarget* a, const CidRangeTarget* b)) { + cid_ranges_.Sort(compare); + } + + intptr_t AggregateCallCount() const; + + bool HasSingleTarget() const; + bool HasSingleRecognizedTarget() const; + Function& FirstTarget() const; + Function& MostPopularTarget() const; + bool IsMonomorphic() const; + intptr_t MonomorphicReceiverCid() const; + + private: + GrowableArray cid_ranges_; +}; + + class Instruction : public ZoneAllocated { public: #define DECLARE_TAG(type) k##type, @@ -2787,16 +2849,16 @@ class InstanceCallInstr : public TemplateDefinition<0, Throws> { class PolymorphicInstanceCallInstr : public TemplateDefinition<0, Throws> { public: PolymorphicInstanceCallInstr(InstanceCallInstr* instance_call, - const ICData& ic_data, + const CallTargets& targets, bool with_checks, bool complete) : TemplateDefinition(instance_call->deopt_id()), instance_call_(instance_call), - ic_data_(ic_data), + targets_(targets), with_checks_(with_checks), complete_(complete) { ASSERT(instance_call_ != NULL); - ASSERT(!ic_data.NumberOfChecksIs(0)); + ASSERT(targets.length() != 0); total_call_count_ = CallCount(); } @@ -2817,9 +2879,14 @@ class PolymorphicInstanceCallInstr : public TemplateDefinition<0, Throws> { return instance_call()->PushArgumentAt(index); } - bool HasSingleRecognizedTarget() const; + bool HasOnlyDispatcherOrImplicitAccessorTargets() const; - virtual intptr_t CallCount() const { return ic_data().AggregateCount(); } + const CallTargets& targets() const { return targets_; } + intptr_t NumberOfChecks() const { return targets_.length(); } + + bool IsSureToCallSingleRecognizedTarget() const; + + virtual intptr_t CallCount() const; // If this polymophic call site was created to cover the remaining cids after // inlinng then we need to keep track of the total number of calls including @@ -2834,25 +2901,25 @@ class PolymorphicInstanceCallInstr : public TemplateDefinition<0, Throws> { DECLARE_INSTRUCTION(PolymorphicInstanceCall) - const ICData& ic_data() const { return ic_data_; } - virtual bool ComputeCanDeoptimize() const { return true; } virtual EffectSet Effects() const { return EffectSet::All(); } virtual Definition* Canonicalize(FlowGraph* graph); - static RawType* ComputeRuntimeType(const ICData& ic_data); + static RawType* ComputeRuntimeType(const CallTargets& targets); PRINT_OPERANDS_TO_SUPPORT private: InstanceCallInstr* instance_call_; - const ICData& ic_data_; + const CallTargets& targets_; bool with_checks_; const bool complete_; intptr_t total_call_count_; + friend class PolymorphicInliner; + DISALLOW_COPY_AND_ASSIGN(PolymorphicInstanceCallInstr); }; diff --git a/runtime/vm/intermediate_language_arm.cc b/runtime/vm/intermediate_language_arm.cc index 238cfc11bb4..0fd2ada46bb 100644 --- a/runtime/vm/intermediate_language_arm.cc +++ b/runtime/vm/intermediate_language_arm.cc @@ -3158,8 +3158,12 @@ class CheckedSmiSlowPath : public SlowPathCode { } __ Push(locs->in(0).reg()); __ Push(locs->in(1).reg()); + const String& selector = + String::Handle(instruction_->call()->ic_data()->target_name()); + const Array& argument_names = + Array::Handle(instruction_->call()->ic_data()->arguments_descriptor()); compiler->EmitMegamorphicInstanceCall( - *instruction_->call()->ic_data(), instruction_->call()->ArgumentCount(), + selector, argument_names, instruction_->call()->ArgumentCount(), instruction_->call()->deopt_id(), instruction_->call()->token_pos(), locs, try_index_, /* slow_path_argument_count = */ 2); @@ -3297,8 +3301,12 @@ class CheckedSmiComparisonSlowPath : public SlowPathCode { } __ Push(locs->in(0).reg()); __ Push(locs->in(1).reg()); + String& selector = + String::Handle(instruction_->call()->ic_data()->target_name()); + Array& argument_names = + Array::Handle(instruction_->call()->ic_data()->arguments_descriptor()); compiler->EmitMegamorphicInstanceCall( - *instruction_->call()->ic_data(), instruction_->call()->ArgumentCount(), + selector, argument_names, instruction_->call()->ArgumentCount(), instruction_->call()->deopt_id(), instruction_->call()->token_pos(), locs, try_index_, /* slow_path_argument_count = */ 2); diff --git a/runtime/vm/intermediate_language_arm64.cc b/runtime/vm/intermediate_language_arm64.cc index 640e7e527c8..0091d4e4bf3 100644 --- a/runtime/vm/intermediate_language_arm64.cc +++ b/runtime/vm/intermediate_language_arm64.cc @@ -2888,8 +2888,12 @@ class CheckedSmiSlowPath : public SlowPathCode { } __ Push(locs->in(0).reg()); __ Push(locs->in(1).reg()); + const String& selector = + String::Handle(instruction_->call()->ic_data()->target_name()); + const Array& argument_names = + Array::Handle(instruction_->call()->ic_data()->arguments_descriptor()); compiler->EmitMegamorphicInstanceCall( - *instruction_->call()->ic_data(), instruction_->call()->ArgumentCount(), + selector, argument_names, instruction_->call()->ArgumentCount(), instruction_->call()->deopt_id(), instruction_->call()->token_pos(), locs, try_index_, /* slow_path_argument_count = */ 2); @@ -3030,8 +3034,12 @@ class CheckedSmiComparisonSlowPath : public SlowPathCode { } __ Push(locs->in(0).reg()); __ Push(locs->in(1).reg()); + String& selector = + String::Handle(instruction_->call()->ic_data()->target_name()); + Array& argument_names = + Array::Handle(instruction_->call()->ic_data()->arguments_descriptor()); compiler->EmitMegamorphicInstanceCall( - *instruction_->call()->ic_data(), instruction_->call()->ArgumentCount(), + selector, argument_names, instruction_->call()->ArgumentCount(), instruction_->call()->deopt_id(), instruction_->call()->token_pos(), locs, try_index_, /* slow_path_argument_count = */ 2); diff --git a/runtime/vm/intermediate_language_dbc.cc b/runtime/vm/intermediate_language_dbc.cc index f4a252f130c..3838de2cbf2 100644 --- a/runtime/vm/intermediate_language_dbc.cc +++ b/runtime/vm/intermediate_language_dbc.cc @@ -239,26 +239,21 @@ EMIT_NATIVE_CODE(PolymorphicInstanceCall, 0, Location::RegisterLocation(0), LocationSummary::kCall) { - ASSERT(ic_data().NumArgsTested() == 1); const Array& arguments_descriptor = Array::Handle(ArgumentsDescriptor::New( instance_call()->ArgumentCount(), instance_call()->argument_names())); const intptr_t argdesc_kidx = __ AddConstant(arguments_descriptor); + const CallTargets& ic_data = targets(); // Push the target onto the stack. if (with_checks()) { - const intptr_t may_be_smi = - (ic_data().GetReceiverClassIdAt(0) == kSmiCid) ? 1 : 0; - GrowableArray sorted_ic_data; - FlowGraphCompiler::SortICDataByCount(ic_data(), &sorted_ic_data, - /* drop_smi = */ true); - const intptr_t sorted_length = sorted_ic_data.length(); - if (!Utils::IsUint(8, sorted_length)) { + const intptr_t length = ic_data.length(); + if (!Utils::IsUint(8, length)) { Unsupported(compiler); UNREACHABLE(); } bool using_ranges = false; - for (intptr_t i = 0; i < sorted_length; i++) { - if (sorted_ic_data[i].cid_start != sorted_ic_data[i].cid_end) { + for (intptr_t i = 0; i < length; i++) { + if (ic_data[i].cid_start != ic_data[i].cid_end) { using_ranges = true; break; } @@ -266,24 +261,14 @@ EMIT_NATIVE_CODE(PolymorphicInstanceCall, if (using_ranges) { __ PushPolymorphicInstanceCallByRange(instance_call()->ArgumentCount(), - sorted_length + may_be_smi); + length); } else { - __ PushPolymorphicInstanceCall(instance_call()->ArgumentCount(), - sorted_length + may_be_smi); + __ PushPolymorphicInstanceCall(instance_call()->ArgumentCount(), length); } - if (may_be_smi == 1) { - const Function& target = - Function::ZoneHandle(compiler->zone(), ic_data().GetTargetAt(0)); - __ Nop(compiler->ToEmbeddableCid(kSmiCid, this)); - if (using_ranges) { - __ Nop(compiler->ToEmbeddableCid(1, this)); - } - __ Nop(__ AddConstant(target)); - } - for (intptr_t i = 0; i < sorted_length; i++) { - const Function& target = *sorted_ic_data[i].target; - intptr_t cid_start = sorted_ic_data[i].cid_start; - intptr_t cid_end = sorted_ic_data[i].cid_end; + for (intptr_t i = 0; i < length; i++) { + const Function& target = *ic_data[i].target; + intptr_t cid_start = ic_data[i].cid_start; + intptr_t cid_end = ic_data[i].cid_end; __ Nop(compiler->ToEmbeddableCid(cid_start, this)); if (using_ranges) { @@ -294,8 +279,8 @@ EMIT_NATIVE_CODE(PolymorphicInstanceCall, compiler->EmitDeopt(deopt_id(), ICData::kDeoptPolymorphicInstanceCallTestFail, 0); } else { - ASSERT(ic_data().HasOneTarget()); - const Function& target = Function::ZoneHandle(ic_data().GetTargetAt(0)); + ASSERT(targets().HasSingleTarget()); + const Function& target = targets().FirstTarget(); __ PushConstant(target); } diff --git a/runtime/vm/intermediate_language_mips.cc b/runtime/vm/intermediate_language_mips.cc index 66c4f3698fa..6446dddda33 100644 --- a/runtime/vm/intermediate_language_mips.cc +++ b/runtime/vm/intermediate_language_mips.cc @@ -3024,8 +3024,12 @@ class CheckedSmiSlowPath : public SlowPathCode { } __ Push(locs->in(0).reg()); __ Push(locs->in(1).reg()); + const String& selector = + String::Handle(instruction_->call()->ic_data()->target_name()); + const Array& argument_names = + Array::Handle(instruction_->call()->ic_data()->arguments_descriptor()); compiler->EmitMegamorphicInstanceCall( - *instruction_->call()->ic_data(), instruction_->call()->ArgumentCount(), + selector, argument_names, instruction_->call()->ArgumentCount(), instruction_->call()->deopt_id(), instruction_->call()->token_pos(), locs, try_index_, /* slow_path_argument_count = */ 2); @@ -3160,8 +3164,12 @@ class CheckedSmiComparisonSlowPath : public SlowPathCode { } __ Push(locs->in(0).reg()); __ Push(locs->in(1).reg()); + String& selector = + String::Handle(instruction_->call()->ic_data()->target_name()); + Array& argument_names = + Array::Handle(instruction_->call()->ic_data()->arguments_descriptor()); compiler->EmitMegamorphicInstanceCall( - *instruction_->call()->ic_data(), instruction_->call()->ArgumentCount(), + selector, argument_names, instruction_->call()->ArgumentCount(), instruction_->call()->deopt_id(), instruction_->call()->token_pos(), locs, try_index_, /* slow_path_argument_count = */ 2); diff --git a/runtime/vm/intermediate_language_x64.cc b/runtime/vm/intermediate_language_x64.cc index 03a3741b984..644e8b2e9d1 100644 --- a/runtime/vm/intermediate_language_x64.cc +++ b/runtime/vm/intermediate_language_x64.cc @@ -2856,8 +2856,12 @@ class CheckedSmiSlowPath : public SlowPathCode { } __ pushq(locs->in(0).reg()); __ pushq(locs->in(1).reg()); + const String& selector = + String::Handle(instruction_->call()->ic_data()->target_name()); + const Array& argument_names = + Array::Handle(instruction_->call()->ic_data()->arguments_descriptor()); compiler->EmitMegamorphicInstanceCall( - *instruction_->call()->ic_data(), instruction_->call()->ArgumentCount(), + selector, argument_names, instruction_->call()->ArgumentCount(), instruction_->call()->deopt_id(), instruction_->call()->token_pos(), locs, try_index_, /* slow_path_argument_count = */ 2); @@ -3021,8 +3025,12 @@ class CheckedSmiComparisonSlowPath : public SlowPathCode { } __ pushq(locs->in(0).reg()); __ pushq(locs->in(1).reg()); + String& selector = + String::Handle(instruction_->call()->ic_data()->target_name()); + Array& argument_names = + Array::Handle(instruction_->call()->ic_data()->arguments_descriptor()); compiler->EmitMegamorphicInstanceCall( - *instruction_->call()->ic_data(), instruction_->call()->ArgumentCount(), + selector, argument_names, instruction_->call()->ArgumentCount(), instruction_->call()->deopt_id(), instruction_->call()->token_pos(), locs, try_index_, /* slow_path_argument_count = */ 2); diff --git a/runtime/vm/jit_optimizer.cc b/runtime/vm/jit_optimizer.cc index bdfcddcf09c..fb6ee1d43be 100644 --- a/runtime/vm/jit_optimizer.cc +++ b/runtime/vm/jit_optimizer.cc @@ -209,9 +209,13 @@ void JitOptimizer::SpecializePolymorphicInstanceCall( return; // No information about receiver was infered. } - const ICData& ic_data = FlowGraphCompiler::TrySpecializeICDataByReceiverCid( - call->ic_data(), receiver_cid); - if (ic_data.raw() == call->ic_data().raw()) { + const ICData& ic_data = *call->instance_call()->ic_data(); + + const CallTargets* targets = + FlowGraphCompiler::ResolveCallTargetsForReceiverCid( + receiver_cid, String::Handle(zone(), ic_data.target_name()), + Array::Handle(zone(), ic_data.arguments_descriptor())); + if (targets == NULL) { // No specialization. return; } @@ -219,7 +223,7 @@ void JitOptimizer::SpecializePolymorphicInstanceCall( const bool with_checks = false; const bool complete = false; PolymorphicInstanceCallInstr* specialized = - new (Z) PolymorphicInstanceCallInstr(call->instance_call(), ic_data, + new (Z) PolymorphicInstanceCallInstr(call->instance_call(), *targets, with_checks, complete); call->ReplaceWith(specialized, current_iterator()); } @@ -1448,85 +1452,6 @@ void JitOptimizer::ReplaceWithTypeCast(InstanceCallInstr* call) { } -bool JitOptimizer::LookupMethodFor(int class_id, - const ArgumentsDescriptor& args_desc, - const String& name, - Function* fn_return) { - if (class_id < 0) return false; - if (class_id >= I->class_table()->NumCids()) return false; - - RawClass* raw_class = I->class_table()->At(class_id); - if (raw_class == NULL) return false; - Class& cls = Class::Handle(Z, raw_class); - if (cls.IsNull()) return false; - if (!cls.is_finalized()) return false; - if (Array::Handle(cls.functions()).IsNull()) return false; - - bool allow_add = false; - Function& target_function = - Function::Handle(Z, Resolver::ResolveDynamicForReceiverClass( - cls, name, args_desc, allow_add)); - if (target_function.IsNull()) return false; - *fn_return ^= target_function.raw(); - return true; -} - - -static int OrderById(const intptr_t* a, const intptr_t* b) { - // Negative if 'a' should sort before 'b'. - return *a - *b; -} - - -void JitOptimizer::TryExpandClassesInICData(const ICData& ic_data) { - if (ic_data.NumberOfChecks() == 0) return; - - Function& dummy = Function::Handle(Z); - - GrowableArray ids; - for (int i = 0; i < ic_data.NumberOfChecks(); i++) { - // The API works for multi dispatch ICs that check more than one argument, - // but we know we only check one arg here, so only the 0th element of id - // will be used. - GrowableArray id; - ic_data.GetCheckAt(i, &id, &dummy); - ids.Add(id[0]); - } - ids.Sort(OrderById); - - Array& args_desc_array = Array::Handle(Z, ic_data.arguments_descriptor()); - ArgumentsDescriptor args_desc(args_desc_array); - String& name = String::Handle(Z, ic_data.target_name()); - - Function& fn = Function::Handle(Z); - Function& fn_high = Function::Handle(Z); - Function& possible_match = Function::Handle(Z); - - for (int cid_index = 0; cid_index < ids.length() - 1; cid_index++) { - int low_cid = ids[cid_index]; - int high_cid = ids[cid_index + 1]; - if (low_cid + 1 == high_cid) continue; - if (LookupMethodFor(low_cid, args_desc, name, &fn) && - LookupMethodFor(high_cid, args_desc, name, &fn_high) && - fn.raw() == fn_high.raw()) { - // Try to fill in the IC table by going downwards from a known class-id. - bool can_fill_in = true; - for (int i = low_cid + 1; i < high_cid; i++) { - if (!LookupMethodFor(i, args_desc, name, &possible_match) || - possible_match.raw() != fn.raw()) { - can_fill_in = false; - break; - } - } - if (can_fill_in) { - for (int i = low_cid + 1; i < high_cid; i++) { - ic_data.AddReceiverCheck(i, fn, 0); - } - } - } - } -} - // Tries to optimize instance call by replacing it with a faster instruction // (e.g, binary op, field load, ..). void JitOptimizer::VisitInstanceCall(InstanceCallInstr* instr) { @@ -1584,11 +1509,9 @@ void JitOptimizer::VisitInstanceCall(InstanceCallInstr* instr) { return; } - // Now we are done trying the inlining options that benefit from only having - // 1 entry in the IC table. - TryExpandClassesInICData(unary_checks); + CallTargets* targets = CallTargets::Create(Z, unary_checks); - bool has_one_target = unary_checks.HasOneTarget(); + bool has_one_target = targets->HasSingleTarget(); if (has_one_target) { // Check if the single target is a polymorphic target, if it is, @@ -1596,7 +1519,7 @@ void JitOptimizer::VisitInstanceCall(InstanceCallInstr* instr) { const Function& target = Function::Handle(Z, unary_checks.GetTargetAt(0)); if (target.recognized_kind() == MethodRecognizer::kObjectRuntimeType) { has_one_target = PolymorphicInstanceCallInstr::ComputeRuntimeType( - unary_checks) != Type::null(); + *targets) != Type::null(); } else { const bool polymorphic_target = MethodRecognizer::PolymorphicTarget(target); @@ -1609,7 +1532,7 @@ void JitOptimizer::VisitInstanceCall(InstanceCallInstr* instr) { const RawFunction::Kind function_kind = target.kind(); if (!flow_graph()->InstanceCallNeedsClassCheck(instr, function_kind)) { PolymorphicInstanceCallInstr* call = - new (Z) PolymorphicInstanceCallInstr(instr, unary_checks, + new (Z) PolymorphicInstanceCallInstr(instr, *targets, /* call_with_checks = */ false, /* complete = */ false); instr->ReplaceWith(call, current_iterator()); @@ -1632,15 +1555,17 @@ void JitOptimizer::VisitInstanceCall(InstanceCallInstr* instr) { (!instr->ic_data()->HasDeoptReason(ICData::kDeoptCheckClass) || unary_checks.NumberOfChecks() <= FLAG_max_polymorphic_checks)) { // Type propagation has not run yet, we cannot eliminate the check. + // TODO(erikcorry): The receiver check should use the off-heap targets + // array, not the IC array. AddReceiverCheck(instr); // Call can still deoptimize, do not detach environment from instr. call_with_checks = false; } else { call_with_checks = true; } - PolymorphicInstanceCallInstr* call = new (Z) - PolymorphicInstanceCallInstr(instr, unary_checks, call_with_checks, - /* complete = */ false); + PolymorphicInstanceCallInstr* call = + new (Z) PolymorphicInstanceCallInstr(instr, *targets, call_with_checks, + /* complete = */ false); instr->ReplaceWith(call, current_iterator()); } diff --git a/runtime/vm/jit_optimizer.h b/runtime/vm/jit_optimizer.h index 23dd278db3d..6fe4562056b 100644 --- a/runtime/vm/jit_optimizer.h +++ b/runtime/vm/jit_optimizer.h @@ -52,12 +52,6 @@ class JitOptimizer : public FlowGraphVisitor { bool TryReplaceWithIndexedOp(InstanceCallInstr* call); - void TryExpandClassesInICData(const ICData& ic_data); - bool LookupMethodFor(int class_id, - const ArgumentsDescriptor& args_desc, - const String& name, - Function* fn_return); - bool TryReplaceWithBinaryOp(InstanceCallInstr* call, Token::Kind op_kind); bool TryReplaceWithUnaryOp(InstanceCallInstr* call, Token::Kind op_kind); diff --git a/runtime/vm/object.cc b/runtime/vm/object.cc index 08821a27c54..ff790974353 100644 --- a/runtime/vm/object.cc +++ b/runtime/vm/object.cc @@ -13882,19 +13882,6 @@ bool ICData::HasOneTarget() const { } -bool ICData::HasOnlyDispatcherOrImplicitAccessorTargets() const { - const intptr_t len = NumberOfChecks(); - Function& target = Function::Handle(); - for (intptr_t i = 0; i < len; i++) { - target = GetTargetAt(i); - if (!target.IsDispatcherOrImplicitAccessor()) { - return false; - } - } - return true; -} - - void ICData::GetUsedCidsForTwoArgs(GrowableArray* first, GrowableArray* second) const { ASSERT(NumArgsTested() == 2); diff --git a/runtime/vm/object.h b/runtime/vm/object.h index 4faf54604b7..3fee9886a10 100644 --- a/runtime/vm/object.h +++ b/runtime/vm/object.h @@ -2074,7 +2074,6 @@ class ICData : public Object { bool AllTargetsHaveSameOwner(intptr_t owner_cid) const; bool AllReceiversAreNumbers() const; bool HasOneTarget() const; - bool HasOnlyDispatcherOrImplicitAccessorTargets() const; bool HasReceiverClassId(intptr_t class_id) const; static RawICData* New(const Function& owner,