diff --git a/runtime/vm/code_generator.cc b/runtime/vm/code_generator.cc index bc33bca5998..9b3eb295c83 100644 --- a/runtime/vm/code_generator.cc +++ b/runtime/vm/code_generator.cc @@ -61,6 +61,7 @@ DEFINE_FLAG(bool, trace_type_checks, false, "Trace runtime type checks."); DECLARE_FLAG(int, deoptimization_counter_threshold); DECLARE_FLAG(bool, trace_compiler); DECLARE_FLAG(bool, warn_on_javascript_compatibility); +DECLARE_FLAG(int, max_polymorphic_checks); DEFINE_FLAG(bool, use_osr, true, "Use on-stack replacement."); DEFINE_FLAG(bool, trace_osr, false, "Trace attempts at on-stack replacement."); @@ -779,11 +780,9 @@ static bool ResolveCallThroughGetter(const Instance& receiver, // Handle other invocations (implicit closures, noSuchMethod). RawFunction* InlineCacheMissHelper( const Instance& receiver, - const ICData& ic_data) { - const Array& args_descriptor = Array::Handle(ic_data.arguments_descriptor()); - + const Array& args_descriptor, + const String& target_name) { const Class& receiver_class = Class::Handle(receiver.clazz()); - const String& target_name = String::Handle(ic_data.target_name()); Function& result = Function::Handle(); if (!ResolveCallThroughGetter(receiver, @@ -828,7 +827,12 @@ static RawFunction* InlineCacheMissHandler( String::Handle(ic_data.target_name()).ToCString(), receiver.ToCString()); } - target_function = InlineCacheMissHelper(receiver, ic_data); + const Array& args_descriptor = + Array::Handle(ic_data.arguments_descriptor()); + const String& target_name = String::Handle(ic_data.target_name()); + target_function = InlineCacheMissHelper(receiver, + args_descriptor, + target_name); } if (target_function.IsNull()) { ASSERT(!FLAG_lazy_dispatchers); @@ -1008,18 +1012,21 @@ DEFINE_RUNTIME_ENTRY(StaticCallMissHandlerTwoArgs, 3) { // Handle a miss of a megamorphic cache. // Arg0: Receiver. -// Arg1: ICData object. +// Arg1: ICData or MegamorphicCache. // Arg2: Arguments descriptor array. - // Returns: target function to call. DEFINE_RUNTIME_ENTRY(MegamorphicCacheMissHandler, 3) { - const Instance& receiver = Instance::CheckedHandle(arguments.ArgAt(0)); - const ICData& ic_data = ICData::CheckedHandle(arguments.ArgAt(1)); - const Array& descriptor = Array::CheckedHandle(arguments.ArgAt(2)); - const String& name = String::Handle(ic_data.target_name()); - const MegamorphicCache& cache = MegamorphicCache::Handle( - MegamorphicCacheTable::Lookup(isolate, name, descriptor)); - Class& cls = Class::Handle(receiver.clazz()); + const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(0)); + const Object& ic_data_or_cache = Object::Handle(zone, arguments.ArgAt(1)); + const Array& descriptor = Array::CheckedHandle(zone, arguments.ArgAt(2)); + String& name = String::Handle(zone); + if (ic_data_or_cache.IsICData()) { + name = ICData::Cast(ic_data_or_cache).target_name(); + } else { + ASSERT(ic_data_or_cache.IsMegamorphicCache()); + name = MegamorphicCache::Cast(ic_data_or_cache).target_name(); + } + Class& cls = Class::Handle(zone, receiver.clazz()); ASSERT(!cls.IsNull()); if (FLAG_trace_ic || FLAG_trace_ic_miss_in_optimized) { OS::PrintErr("Megamorphic IC miss, class=%s, function=%s\n", @@ -1027,41 +1034,70 @@ DEFINE_RUNTIME_ENTRY(MegamorphicCacheMissHandler, 3) { } ArgumentsDescriptor args_desc(descriptor); - Function& target_function = Function::Handle( + Function& target_function = Function::Handle(zone, Resolver::ResolveDynamicForReceiverClass(cls, name, args_desc)); if (target_function.IsNull()) { - target_function = InlineCacheMissHelper(receiver, ic_data); + target_function = InlineCacheMissHelper(receiver, descriptor, name); } if (target_function.IsNull()) { ASSERT(!FLAG_lazy_dispatchers); arguments.SetReturn(target_function); return; } - // Insert function found into cache and return it. - cache.EnsureCapacity(); - const Smi& class_id = Smi::Handle(Smi::New(cls.id())); - cache.Insert(class_id, target_function); + + if (ic_data_or_cache.IsICData()) { + const ICData& ic_data = ICData::Cast(ic_data_or_cache); + ic_data.AddReceiverCheck(receiver.GetClassId(), target_function); + if (ic_data.NumberOfChecks() > FLAG_max_polymorphic_checks) { + // Switch to megamorphic call. + const MegamorphicCache& cache = MegamorphicCache::Handle(zone, + MegamorphicCacheTable::Lookup(isolate, name, descriptor)); + DartFrameIterator iterator; + StackFrame* miss_function_frame = iterator.NextFrame(); + ASSERT(miss_function_frame->IsDartFrame()); + StackFrame* caller_frame = iterator.NextFrame(); + ASSERT(caller_frame->IsDartFrame()); + const Code& code = Code::Handle(zone, caller_frame->LookupDartCode()); + const Code& stub = + Code::Handle(zone, StubCode::MegamorphicLookup_entry()->code()); + CodePatcher::PatchSwitchableCallAt(caller_frame->pc(), + code, ic_data, cache, stub); + } + } else { + const MegamorphicCache& cache = MegamorphicCache::Cast(ic_data_or_cache); + // Insert function found into cache and return it. + cache.EnsureCapacity(); + const Smi& class_id = Smi::Handle(zone, Smi::New(cls.id())); + cache.Insert(class_id, target_function); + } arguments.SetReturn(target_function); } // Invoke appropriate noSuchMethod or closure from getter. // Arg0: receiver -// Arg1: IC data +// Arg1: ICData or MegamorphicCache // Arg2: arguments descriptor array // Arg3: arguments array DEFINE_RUNTIME_ENTRY(InvokeNoSuchMethodDispatcher, 4) { ASSERT(!FLAG_lazy_dispatchers); - const Instance& receiver = Instance::CheckedHandle(arguments.ArgAt(0)); - const ICData& ic_data = ICData::CheckedHandle(arguments.ArgAt(1)); - const Array& orig_arguments_desc = Array::CheckedHandle(arguments.ArgAt(2)); - const Array& orig_arguments = Array::CheckedHandle(arguments.ArgAt(3)); - const String& target_name = String::Handle(ic_data.target_name()); + const Instance& receiver = Instance::CheckedHandle(zone, arguments.ArgAt(0)); + const Object& ic_data_or_cache = Object::Handle(zone, arguments.ArgAt(1)); + const Array& orig_arguments_desc = + Array::CheckedHandle(zone, arguments.ArgAt(2)); + const Array& orig_arguments = Array::CheckedHandle(zone, arguments.ArgAt(3)); + String& target_name = String::Handle(zone); + if (ic_data_or_cache.IsICData()) { + target_name = ICData::Cast(ic_data_or_cache).target_name(); + } else { + ASSERT(ic_data_or_cache.IsMegamorphicCache()); + target_name = MegamorphicCache::Cast(ic_data_or_cache).target_name(); + } - Class& cls = Class::Handle(receiver.clazz()); - Function& function = Function::Handle(); + Class& cls = Class::Handle(zone, receiver.clazz()); + Function& function = Function::Handle(zone); // Dart distinguishes getters and regular methods and allows their calls // to mix with conversions, and its selectors are independent of arity. So do @@ -1069,7 +1105,7 @@ DEFINE_RUNTIME_ENTRY(InvokeNoSuchMethodDispatcher, 4) { // need for conversion, or there really is no such method. #define NO_SUCH_METHOD() \ - const Object& result = Object::Handle( \ + const Object& result = Object::Handle(zone, \ DartEntry::InvokeNoSuchMethod(receiver, \ target_name, \ orig_arguments, \ @@ -1079,9 +1115,9 @@ DEFINE_RUNTIME_ENTRY(InvokeNoSuchMethodDispatcher, 4) { #define CLOSURIZE(some_function) \ const Function& closure_function = \ - Function::Handle(some_function.ImplicitClosureFunction()); \ + Function::Handle(zone, some_function.ImplicitClosureFunction()); \ const Object& result = \ - Object::Handle(closure_function.ImplicitInstanceClosure(receiver)); \ + Object::Handle(zone, closure_function.ImplicitInstanceClosure(receiver));\ arguments.SetReturn(result); \ const bool is_getter = Field::IsGetterName(target_name); @@ -1091,7 +1127,7 @@ DEFINE_RUNTIME_ENTRY(InvokeNoSuchMethodDispatcher, 4) { // encountered first on the inheritance chain. Or, // o#foo= (o.get:#set:foo) failed, closurize o.foo= if it exists. String& field_name = - String::Handle(Field::NameFromGetter(target_name)); + String::Handle(zone, Field::NameFromGetter(target_name)); const bool is_extractor = field_name.CharAt(0) == '#'; if (is_extractor) { @@ -1143,14 +1179,15 @@ DEFINE_RUNTIME_ENTRY(InvokeNoSuchMethodDispatcher, 4) { // call method and with lazy dispatchers the field-invocation-dispatcher // would perform the closure call. const Object& result = - Object::Handle(DartEntry::InvokeClosure(orig_arguments, - orig_arguments_desc)); + Object::Handle(zone, DartEntry::InvokeClosure(orig_arguments, + orig_arguments_desc)); CheckResultError(result); arguments.SetReturn(result); return; } - const String& getter_name = String::Handle(Field::GetterName(target_name)); + const String& getter_name = + String::Handle(zone, Field::GetterName(target_name)); while (!cls.IsNull()) { function ^= cls.LookupDynamicFunction(target_name); if (!function.IsNull()) { @@ -1163,15 +1200,15 @@ DEFINE_RUNTIME_ENTRY(InvokeNoSuchMethodDispatcher, 4) { const Array& getter_arguments = Array::Handle(Array::New(1)); getter_arguments.SetAt(0, receiver); const Object& getter_result = - Object::Handle(DartEntry::InvokeFunction(function, - getter_arguments)); + Object::Handle(zone, DartEntry::InvokeFunction(function, + getter_arguments)); CheckResultError(getter_result); ASSERT(getter_result.IsNull() || getter_result.IsInstance()); orig_arguments.SetAt(0, getter_result); const Object& call_result = - Object::Handle(DartEntry::InvokeClosure(orig_arguments, - orig_arguments_desc)); + Object::Handle(zone, DartEntry::InvokeClosure(orig_arguments, + orig_arguments_desc)); CheckResultError(call_result); arguments.SetReturn(call_result); return; diff --git a/runtime/vm/code_patcher.h b/runtime/vm/code_patcher.h index dc2716856a9..ebcdeaaed66 100644 --- a/runtime/vm/code_patcher.h +++ b/runtime/vm/code_patcher.h @@ -74,6 +74,12 @@ class CodePatcher : public AllStatic { const Code& code, const Code& new_target); + static void PatchSwitchableCallAt(uword return_address, + const Code& code, + const ICData& ic_data, + const MegamorphicCache& new_cache, + const Code& lookup_stub); + static RawCode* GetNativeCallAt(uword return_address, const Code& code, NativeFunction* target); diff --git a/runtime/vm/code_patcher_arm.cc b/runtime/vm/code_patcher_arm.cc index a9d5a102280..7e0e8eaddbb 100644 --- a/runtime/vm/code_patcher_arm.cc +++ b/runtime/vm/code_patcher_arm.cc @@ -69,6 +69,19 @@ RawFunction* CodePatcher::GetUnoptimizedStaticCallAt( } +void CodePatcher::PatchSwitchableCallAt(uword return_address, + const Code& code, + const ICData& ic_data, + const MegamorphicCache& cache, + const Code& lookup_stub) { + ASSERT(code.ContainsInstructionAt(return_address)); + SwitchableCallPattern call(return_address, code); + ASSERT(call.cache() == ic_data.raw()); + call.SetLookupStub(lookup_stub); + call.SetCache(cache); +} + + void CodePatcher::PatchNativeCallAt(uword return_address, const Code& code, NativeFunction target, diff --git a/runtime/vm/code_patcher_arm64.cc b/runtime/vm/code_patcher_arm64.cc index 4e0bbee3319..53f26edd22f 100644 --- a/runtime/vm/code_patcher_arm64.cc +++ b/runtime/vm/code_patcher_arm64.cc @@ -109,6 +109,19 @@ RawFunction* CodePatcher::GetUnoptimizedStaticCallAt( } +void CodePatcher::PatchSwitchableCallAt(uword return_address, + const Code& code, + const ICData& ic_data, + const MegamorphicCache& cache, + const Code& lookup_stub) { + ASSERT(code.ContainsInstructionAt(return_address)); + SwitchableCallPattern call(return_address, code); + ASSERT(call.cache() == ic_data.raw()); + call.SetLookupStub(lookup_stub); + call.SetCache(cache); +} + + void CodePatcher::PatchNativeCallAt(uword return_address, const Code& code, NativeFunction target, diff --git a/runtime/vm/code_patcher_ia32.cc b/runtime/vm/code_patcher_ia32.cc index d0a4f07c921..90f187659b5 100644 --- a/runtime/vm/code_patcher_ia32.cc +++ b/runtime/vm/code_patcher_ia32.cc @@ -211,6 +211,16 @@ RawFunction* CodePatcher::GetUnoptimizedStaticCallAt( } +void CodePatcher::PatchSwitchableCallAt(uword return_address, + const Code& code, + const ICData& ic_data, + const MegamorphicCache& cache, + const Code& lookup_stub) { + // Switchable instance calls only generated for precompilation. + UNREACHABLE(); +} + + void CodePatcher::PatchNativeCallAt(uword return_address, const Code& code, NativeFunction target, diff --git a/runtime/vm/code_patcher_mips.cc b/runtime/vm/code_patcher_mips.cc index 09b06a2f249..4c8ab992faf 100644 --- a/runtime/vm/code_patcher_mips.cc +++ b/runtime/vm/code_patcher_mips.cc @@ -68,6 +68,19 @@ RawFunction* CodePatcher::GetUnoptimizedStaticCallAt( } +void CodePatcher::PatchSwitchableCallAt(uword return_address, + const Code& code, + const ICData& ic_data, + const MegamorphicCache& cache, + const Code& lookup_stub) { + ASSERT(code.ContainsInstructionAt(return_address)); + SwitchableCallPattern call(return_address, code); + ASSERT(call.cache() == ic_data.raw()); + call.SetLookupStub(lookup_stub); + call.SetCache(cache); +} + + void CodePatcher::PatchNativeCallAt(uword return_address, const Code& code, NativeFunction target, diff --git a/runtime/vm/code_patcher_x64.cc b/runtime/vm/code_patcher_x64.cc index d0ac739b386..2e6ec0257cd 100644 --- a/runtime/vm/code_patcher_x64.cc +++ b/runtime/vm/code_patcher_x64.cc @@ -184,6 +184,64 @@ class PoolPointerCall : public ValueObject { }; +// Instance call that can switch from an IC call to a megamorphic call +// load ICData load MegamorphicCache +// call ICLookup stub -> call MegamorphicLookup stub +// call target call target +class SwitchableCall : public ValueObject { + public: + SwitchableCall(uword return_address, const Code& code) + : start_(return_address - kCallPatternSize), + object_pool_(ObjectPool::Handle(code.GetObjectPool())) { + ASSERT(IsValid()); + } + + static const int kCallPatternSize = 24; + + bool IsValid() const { + static int16_t pattern[kCallPatternSize] = { + 0x49, 0x8b, 0x9f, -1, -1, -1, -1, // movq rbx, [PP + cache_offs] + 0x4d, 0x8b, 0xa7, -1, -1, -1, -1, // movq r12, [PP + code_offs] + 0x4d, 0x8b, 0x5c, 0x24, 0x07, // movq r11, [r12 + entrypoint_off] + 0x41, 0xff, 0xd3, // call r11 + 0xff, 0xd1, // call rcx + }; + return MatchesPattern(start_, pattern, kCallPatternSize); + } + + intptr_t cache_index() const { + return IndexFromPPLoad(start_ + 3); + } + intptr_t lookup_stub_index() const { + return IndexFromPPLoad(start_ + 10); + } + + RawObject* cache() const { + return object_pool_.ObjectAt(cache_index()); + } + + void SetCache(const MegamorphicCache& cache) const { + ASSERT(Object::Handle(object_pool_.ObjectAt(cache_index())).IsICData()); + object_pool_.SetObjectAt(cache_index(), cache); + // No need to flush the instruction cache, since the code is not modified. + } + + void SetLookupStub(const Code& lookup_stub) const { + ASSERT(Object::Handle(object_pool_.ObjectAt(lookup_stub_index())).IsCode()); + object_pool_.SetObjectAt(lookup_stub_index(), lookup_stub); + // No need to flush the instruction cache, since the code is not modified. + } + + protected: + uword start_; + const ObjectPool& object_pool_; + + private: + DISALLOW_IMPLICIT_CONSTRUCTORS(SwitchableCall); +}; + + + RawCode* CodePatcher::GetStaticCallTargetAt(uword return_address, const Code& code) { ASSERT(code.ContainsInstructionAt(return_address)); @@ -248,6 +306,19 @@ RawFunction* CodePatcher::GetUnoptimizedStaticCallAt( } +void CodePatcher::PatchSwitchableCallAt(uword return_address, + const Code& code, + const ICData& ic_data, + const MegamorphicCache& cache, + const Code& lookup_stub) { + ASSERT(code.ContainsInstructionAt(return_address)); + SwitchableCall call(return_address, code); + ASSERT(call.cache() == ic_data.raw()); + call.SetLookupStub(lookup_stub); + call.SetCache(cache); +} + + void CodePatcher::PatchNativeCallAt(uword return_address, const Code& code, NativeFunction target, diff --git a/runtime/vm/dart.cc b/runtime/vm/dart.cc index 9d9090a005e..4b63f6f454b 100644 --- a/runtime/vm/dart.cc +++ b/runtime/vm/dart.cc @@ -350,6 +350,10 @@ RawError* Dart::InitializeIsolate(const uint8_t* snapshot_buffer, void* data) { if (!Dart::IsRunningPrecompiledCode()) { MegamorphicCacheTable::InitMissHandler(I); } + const Code& miss_code = + Code::Handle(I->object_store()->megamorphic_miss_code()); + I->set_ic_miss_code(miss_code); + if (snapshot_buffer == NULL) { if (!I->object_store()->PreallocateObjects()) { return I->object_store()->sticky_error(); diff --git a/runtime/vm/flow_graph_compiler.cc b/runtime/vm/flow_graph_compiler.cc index 4c6b4b7f32c..d43c6847cbf 100644 --- a/runtime/vm/flow_graph_compiler.cc +++ b/runtime/vm/flow_graph_compiler.cc @@ -1089,6 +1089,11 @@ void FlowGraphCompiler::GenerateInstanceCall( intptr_t argument_count, LocationSummary* locs, const ICData& ic_data) { + if (Compiler::always_optimize()) { + EmitSwitchableInstanceCall(ic_data, argument_count, + deopt_id, token_pos, locs); + return; + } if (FLAG_always_megamorphic_calls) { EmitMegamorphicInstanceCall(ic_data, argument_count, deopt_id, token_pos, locs); diff --git a/runtime/vm/flow_graph_compiler.h b/runtime/vm/flow_graph_compiler.h index 47e45af22f5..2bd474e4af6 100644 --- a/runtime/vm/flow_graph_compiler.h +++ b/runtime/vm/flow_graph_compiler.h @@ -417,6 +417,12 @@ class FlowGraphCompiler : public ValueObject { intptr_t token_pos, LocationSummary* locs); + void EmitSwitchableInstanceCall(const ICData& ic_data, + intptr_t argument_count, + intptr_t deopt_id, + intptr_t token_pos, + LocationSummary* locs); + void EmitTestAndCall(const ICData& ic_data, intptr_t arg_count, const Array& arg_names, diff --git a/runtime/vm/flow_graph_compiler_arm.cc b/runtime/vm/flow_graph_compiler_arm.cc index fb944158a28..a11916db9e6 100644 --- a/runtime/vm/flow_graph_compiler_arm.cc +++ b/runtime/vm/flow_graph_compiler_arm.cc @@ -1265,20 +1265,57 @@ void FlowGraphCompiler::EmitMegamorphicInstanceCall( ASSERT(!arguments_descriptor.IsNull() && (arguments_descriptor.Length() > 0)); const MegamorphicCache& cache = MegamorphicCache::ZoneHandle(zone(), MegamorphicCacheTable::Lookup(isolate(), name, arguments_descriptor)); - const Register receiverR = R0; - const Register cacheR = R1; - const Register targetR = R1; - __ LoadFromOffset(kWord, receiverR, SP, (argument_count - 1) * kWordSize); - __ LoadObject(cacheR, cache); + __ Comment("MegamorphicCall"); + __ LoadFromOffset(kWord, R0, SP, (argument_count - 1) * kWordSize); + __ LoadObject(R9, cache); if (FLAG_use_megamorphic_stub) { __ BranchLink(*StubCode::MegamorphicLookup_entry()); - } else { - StubCode::EmitMegamorphicLookup(assembler(), receiverR, cacheR, targetR); + } else { + StubCode::EmitMegamorphicLookup(assembler()); } - __ LoadObject(R9, ic_data); - __ LoadObject(R4, arguments_descriptor); - __ blx(targetR); + __ blx(R1); + + AddCurrentDescriptor(RawPcDescriptors::kOther, Thread::kNoDeoptId, token_pos); + RecordSafepoint(locs); + const intptr_t deopt_id_after = Thread::ToDeoptAfter(deopt_id); + if (is_optimizing()) { + AddDeoptIndexAtCall(deopt_id_after, token_pos); + } else { + // Add deoptimization continuation point after the call and before the + // arguments are removed. + AddCurrentDescriptor(RawPcDescriptors::kDeopt, + deopt_id_after, token_pos); + } + __ Drop(argument_count); +} + + +void FlowGraphCompiler::EmitSwitchableInstanceCall( + const ICData& ic_data, + intptr_t argument_count, + intptr_t deopt_id, + intptr_t token_pos, + LocationSummary* locs) { + __ Comment("SwitchableCall"); + __ LoadFromOffset(kWord, R0, SP, (argument_count - 1) * kWordSize); + if (ic_data.NumArgsTested() == 1) { + __ LoadUniqueObject(R9, ic_data); + __ BranchLinkPatchable(*StubCode::ICLookup_entry()); + } else { + const String& name = String::Handle(zone(), ic_data.target_name()); + const Array& arguments_descriptor = + Array::ZoneHandle(zone(), ic_data.arguments_descriptor()); + ASSERT(!arguments_descriptor.IsNull() && + (arguments_descriptor.Length() > 0)); + const MegamorphicCache& cache = MegamorphicCache::ZoneHandle(zone(), + MegamorphicCacheTable::Lookup(isolate(), name, arguments_descriptor)); + + __ LoadUniqueObject(R9, cache); + __ BranchLinkPatchable(*StubCode::MegamorphicLookup_entry()); + } + __ blx(R1); + AddCurrentDescriptor(RawPcDescriptors::kOther, Thread::kNoDeoptId, token_pos); RecordSafepoint(locs); const intptr_t deopt_id_after = Thread::ToDeoptAfter(deopt_id); diff --git a/runtime/vm/flow_graph_compiler_arm64.cc b/runtime/vm/flow_graph_compiler_arm64.cc index 6592a906675..53429c8bda2 100644 --- a/runtime/vm/flow_graph_compiler_arm64.cc +++ b/runtime/vm/flow_graph_compiler_arm64.cc @@ -1255,20 +1255,57 @@ void FlowGraphCompiler::EmitMegamorphicInstanceCall( ASSERT(!arguments_descriptor.IsNull() && (arguments_descriptor.Length() > 0)); const MegamorphicCache& cache = MegamorphicCache::ZoneHandle(zone(), MegamorphicCacheTable::Lookup(isolate(), name, arguments_descriptor)); - const Register receiverR = R0; - const Register cacheR = R1; - const Register targetR = R1; - __ LoadFromOffset(receiverR, SP, (argument_count - 1) * kWordSize); - __ LoadObject(cacheR, cache); + __ Comment("MegamorphicCall"); + __ LoadFromOffset(R0, SP, (argument_count - 1) * kWordSize); + __ LoadObject(R5, cache); if (FLAG_use_megamorphic_stub) { __ BranchLink(*StubCode::MegamorphicLookup_entry()); } else { - StubCode::EmitMegamorphicLookup(assembler(), receiverR, cacheR, targetR); + StubCode::EmitMegamorphicLookup(assembler()); } - __ LoadObject(R5, ic_data); - __ LoadObject(R4, arguments_descriptor); - __ blr(targetR); + __ blr(R1); + + AddCurrentDescriptor(RawPcDescriptors::kOther, + Thread::kNoDeoptId, token_pos); + RecordSafepoint(locs); + const intptr_t deopt_id_after = Thread::ToDeoptAfter(deopt_id); + if (is_optimizing()) { + AddDeoptIndexAtCall(deopt_id_after, token_pos); + } else { + // Add deoptimization continuation point after the call and before the + // arguments are removed. + AddCurrentDescriptor(RawPcDescriptors::kDeopt, deopt_id_after, token_pos); + } + __ Drop(argument_count); +} + + +void FlowGraphCompiler::EmitSwitchableInstanceCall( + const ICData& ic_data, + intptr_t argument_count, + intptr_t deopt_id, + intptr_t token_pos, + LocationSummary* locs) { + __ Comment("SwitchableCall"); + __ LoadFromOffset(R0, SP, (argument_count - 1) * kWordSize); + if (ic_data.NumArgsTested() == 1) { + __ LoadUniqueObject(R5, ic_data); + __ BranchLinkPatchable(*StubCode::ICLookup_entry()); + } else { + const String& name = String::Handle(zone(), ic_data.target_name()); + const Array& arguments_descriptor = + Array::ZoneHandle(zone(), ic_data.arguments_descriptor()); + ASSERT(!arguments_descriptor.IsNull() && + (arguments_descriptor.Length() > 0)); + const MegamorphicCache& cache = MegamorphicCache::ZoneHandle(zone(), + MegamorphicCacheTable::Lookup(isolate(), name, arguments_descriptor)); + + __ LoadUniqueObject(R5, cache); + __ BranchLinkPatchable(*StubCode::MegamorphicLookup_entry()); + } + __ blr(R1); + AddCurrentDescriptor(RawPcDescriptors::kOther, Thread::kNoDeoptId, token_pos); RecordSafepoint(locs); diff --git a/runtime/vm/flow_graph_compiler_ia32.cc b/runtime/vm/flow_graph_compiler_ia32.cc index 72dda0a3e17..c6094fe3d72 100644 --- a/runtime/vm/flow_graph_compiler_ia32.cc +++ b/runtime/vm/flow_graph_compiler_ia32.cc @@ -1285,21 +1285,17 @@ void FlowGraphCompiler::EmitMegamorphicInstanceCall( ASSERT(!arguments_descriptor.IsNull() && (arguments_descriptor.Length() > 0)); const MegamorphicCache& cache = MegamorphicCache::ZoneHandle(zone(), MegamorphicCacheTable::Lookup(isolate(), name, arguments_descriptor)); - const Register receiverR = ECX; - const Register cacheR = EBX; - const Register targetR = EBX; - __ movl(receiverR, Address(ESP, (argument_count - 1) * kWordSize)); - __ LoadObject(cacheR, cache); + __ Comment("MegamorphicCall"); + __ movl(EBX, Address(ESP, (argument_count - 1) * kWordSize)); + __ LoadObject(ECX, cache); if (FLAG_use_megamorphic_stub) { __ Call(*StubCode::MegamorphicLookup_entry()); } else { - StubCode::EmitMegamorphicLookup(assembler(), receiverR, cacheR, targetR); + StubCode::EmitMegamorphicLookup(assembler()); } + __ call(EBX); - __ LoadObject(ECX, ic_data); - __ LoadObject(EDX, arguments_descriptor); - __ call(targetR); AddCurrentDescriptor(RawPcDescriptors::kOther, Thread::kNoDeoptId, token_pos); RecordSafepoint(locs); @@ -1315,6 +1311,17 @@ void FlowGraphCompiler::EmitMegamorphicInstanceCall( } +void FlowGraphCompiler::EmitSwitchableInstanceCall( + const ICData& ic_data, + intptr_t argument_count, + intptr_t deopt_id, + intptr_t token_pos, + LocationSummary* locs) { + // Only generated with precompilation. + UNREACHABLE(); +} + + void FlowGraphCompiler::EmitOptimizedStaticCall( const Function& function, const Array& arguments_descriptor, diff --git a/runtime/vm/flow_graph_compiler_mips.cc b/runtime/vm/flow_graph_compiler_mips.cc index 14e99afcf94..448b70a8f7d 100644 --- a/runtime/vm/flow_graph_compiler_mips.cc +++ b/runtime/vm/flow_graph_compiler_mips.cc @@ -1276,21 +1276,57 @@ void FlowGraphCompiler::EmitMegamorphicInstanceCall( ASSERT(!arguments_descriptor.IsNull() && (arguments_descriptor.Length() > 0)); const MegamorphicCache& cache = MegamorphicCache::ZoneHandle(zone(), MegamorphicCacheTable::Lookup(isolate(), name, arguments_descriptor)); - __ Comment("MegamorphicInstanceCall"); - const Register receiverR = T0; - const Register cacheR = T1; - const Register targetR = T1; - __ lw(receiverR, Address(SP, (argument_count - 1) * kWordSize)); - __ LoadObject(cacheR, cache); + __ Comment("MegamorphicCall"); + __ lw(T0, Address(SP, (argument_count - 1) * kWordSize)); + __ LoadObject(S5, cache); if (FLAG_use_megamorphic_stub) { __ BranchLink(*StubCode::MegamorphicLookup_entry()); } else { - StubCode::EmitMegamorphicLookup(assembler(), receiverR, cacheR, targetR); + StubCode::EmitMegamorphicLookup(assembler()); } - __ LoadObject(S5, ic_data); - __ LoadObject(S4, arguments_descriptor); - __ jalr(targetR); + __ jalr(T1); + + AddCurrentDescriptor(RawPcDescriptors::kOther, + Thread::kNoDeoptId, token_pos); + RecordSafepoint(locs); + const intptr_t deopt_id_after = Thread::ToDeoptAfter(deopt_id); + if (is_optimizing()) { + AddDeoptIndexAtCall(deopt_id_after, token_pos); + } else { + // Add deoptimization continuation point after the call and before the + // arguments are removed. + AddCurrentDescriptor(RawPcDescriptors::kDeopt, deopt_id_after, token_pos); + } + __ Drop(argument_count); +} + + +void FlowGraphCompiler::EmitSwitchableInstanceCall( + const ICData& ic_data, + intptr_t argument_count, + intptr_t deopt_id, + intptr_t token_pos, + LocationSummary* locs) { + __ Comment("SwitchableCall"); + __ lw(T0, Address(SP, (argument_count - 1) * kWordSize)); + if (ic_data.NumArgsTested() == 1) { + __ LoadUniqueObject(S5, ic_data); + __ BranchLink(*StubCode::ICLookup_entry()); + } else { + const String& name = String::Handle(zone(), ic_data.target_name()); + const Array& arguments_descriptor = + Array::ZoneHandle(zone(), ic_data.arguments_descriptor()); + ASSERT(!arguments_descriptor.IsNull() && + (arguments_descriptor.Length() > 0)); + const MegamorphicCache& cache = MegamorphicCache::ZoneHandle(zone(), + MegamorphicCacheTable::Lookup(isolate(), name, arguments_descriptor)); + + __ LoadUniqueObject(S5, cache); + __ BranchLink(*StubCode::MegamorphicLookup_entry()); + } + __ jalr(T1); + AddCurrentDescriptor(RawPcDescriptors::kOther, Thread::kNoDeoptId, token_pos); RecordSafepoint(locs); diff --git a/runtime/vm/flow_graph_compiler_x64.cc b/runtime/vm/flow_graph_compiler_x64.cc index 2a80633d890..42b3677bc82 100644 --- a/runtime/vm/flow_graph_compiler_x64.cc +++ b/runtime/vm/flow_graph_compiler_x64.cc @@ -1286,20 +1286,17 @@ void FlowGraphCompiler::EmitMegamorphicInstanceCall( ASSERT(!arguments_descriptor.IsNull() && (arguments_descriptor.Length() > 0)); const MegamorphicCache& cache = MegamorphicCache::ZoneHandle(zone(), MegamorphicCacheTable::Lookup(isolate(), name, arguments_descriptor)); - const Register receiverR = RDI; - const Register cacheR = RBX; - const Register targetR = RCX; - __ movq(receiverR, Address(RSP, (argument_count - 1) * kWordSize)); - __ LoadObject(cacheR, cache); + __ Comment("MegamorphicCall"); + __ movq(RDI, Address(RSP, (argument_count - 1) * kWordSize)); + __ LoadObject(RBX, cache); if (FLAG_use_megamorphic_stub) { __ Call(*StubCode::MegamorphicLookup_entry()); } else { - StubCode::EmitMegamorphicLookup(assembler(), receiverR, cacheR, targetR); + StubCode::EmitMegamorphicLookup(assembler()); } - __ LoadObject(RBX, ic_data); - __ LoadObject(R10, arguments_descriptor); - __ call(targetR); + __ call(RCX); + AddCurrentDescriptor(RawPcDescriptors::kOther, Thread::kNoDeoptId, token_pos); RecordSafepoint(locs); @@ -1315,6 +1312,46 @@ void FlowGraphCompiler::EmitMegamorphicInstanceCall( } +void FlowGraphCompiler::EmitSwitchableInstanceCall( + const ICData& ic_data, + intptr_t argument_count, + intptr_t deopt_id, + intptr_t token_pos, + LocationSummary* locs) { + __ Comment("SwitchableCall"); + __ movq(RDI, Address(RSP, (argument_count - 1) * kWordSize)); + if (ic_data.NumArgsTested() == 1) { + __ LoadUniqueObject(RBX, ic_data); + __ CallPatchable(*StubCode::ICLookup_entry()); + } else { + const String& name = String::Handle(zone(), ic_data.target_name()); + const Array& arguments_descriptor = + Array::ZoneHandle(zone(), ic_data.arguments_descriptor()); + ASSERT(!arguments_descriptor.IsNull() && + (arguments_descriptor.Length() > 0)); + const MegamorphicCache& cache = MegamorphicCache::ZoneHandle(zone(), + MegamorphicCacheTable::Lookup(isolate(), name, arguments_descriptor)); + + __ LoadUniqueObject(RBX, cache); + __ CallPatchable(*StubCode::MegamorphicLookup_entry()); + } + __ call(RCX); + + AddCurrentDescriptor(RawPcDescriptors::kOther, + Thread::kNoDeoptId, token_pos); + RecordSafepoint(locs); + const intptr_t deopt_id_after = Thread::ToDeoptAfter(deopt_id); + if (is_optimizing()) { + AddDeoptIndexAtCall(deopt_id_after, token_pos); + } else { + // Add deoptimization continuation point after the call and before the + // arguments are removed. + AddCurrentDescriptor(RawPcDescriptors::kDeopt, deopt_id_after, token_pos); + } + __ Drop(argument_count, RCX); +} + + void FlowGraphCompiler::EmitOptimizedStaticCall( const Function& function, const Array& arguments_descriptor, diff --git a/runtime/vm/instructions_arm.cc b/runtime/vm/instructions_arm.cc index dc513344a31..de74bd5ed29 100644 --- a/runtime/vm/instructions_arm.cc +++ b/runtime/vm/instructions_arm.cc @@ -273,6 +273,45 @@ void CallPattern::InsertDeoptCallAt(uword pc, uword target_address) { } +SwitchableCallPattern::SwitchableCallPattern(uword pc, const Code& code) + : object_pool_(ObjectPool::Handle(code.GetObjectPool())), + cache_pool_index_(-1), + stub_pool_index_(-1) { + ASSERT(code.ContainsInstructionAt(pc)); + // Last instruction: blx r1. + ASSERT(*(reinterpret_cast(pc) - 1) == 0xe12fff31); + + Register reg; + uword stub_load_end = + InstructionPattern::DecodeLoadWordFromPool(pc - 3 * Instr::kInstrSize, + ®, + &stub_pool_index_); + ASSERT(reg == CODE_REG); + InstructionPattern::DecodeLoadWordFromPool(stub_load_end, + ®, + &cache_pool_index_); + ASSERT(reg == R9); +} + + +RawObject* SwitchableCallPattern::cache() const { + return reinterpret_cast( + object_pool_.ObjectAt(cache_pool_index_)); +} + + +void SwitchableCallPattern::SetCache(const MegamorphicCache& cache) const { + ASSERT(Object::Handle(object_pool_.ObjectAt(cache_pool_index_)).IsICData()); + object_pool_.SetObjectAt(cache_pool_index_, cache); +} + + +void SwitchableCallPattern::SetLookupStub(const Code& lookup_stub) const { + ASSERT(Object::Handle(object_pool_.ObjectAt(stub_pool_index_)).IsCode()); + object_pool_.SetObjectAt(stub_pool_index_, lookup_stub); +} + + ReturnPattern::ReturnPattern(uword pc) : pc_(pc) { } diff --git a/runtime/vm/instructions_arm.h b/runtime/vm/instructions_arm.h index ad82da1daba..57d48244d55 100644 --- a/runtime/vm/instructions_arm.h +++ b/runtime/vm/instructions_arm.h @@ -98,6 +98,27 @@ class NativeCallPattern : public ValueObject { }; +// Instance call that can switch from an IC call to a megamorphic call +// load ICData load MegamorphicCache +// call ICLookup stub -> call MegamorphicLookup stub +// call target call target +class SwitchableCallPattern : public ValueObject { + public: + SwitchableCallPattern(uword pc, const Code& code); + + RawObject* cache() const; + void SetCache(const MegamorphicCache& cache) const; + void SetLookupStub(const Code& stub) const; + + private: + const ObjectPool& object_pool_; + intptr_t cache_pool_index_; + intptr_t stub_pool_index_; + + DISALLOW_COPY_AND_ASSIGN(SwitchableCallPattern); +}; + + class ReturnPattern : public ValueObject { public: explicit ReturnPattern(uword pc); diff --git a/runtime/vm/instructions_arm64.cc b/runtime/vm/instructions_arm64.cc index 62b3ff224b9..a7e58538488 100644 --- a/runtime/vm/instructions_arm64.cc +++ b/runtime/vm/instructions_arm64.cc @@ -329,6 +329,45 @@ void CallPattern::InsertDeoptCallAt(uword pc, uword target_address) { } +SwitchableCallPattern::SwitchableCallPattern(uword pc, const Code& code) + : object_pool_(ObjectPool::Handle(code.GetObjectPool())), + cache_pool_index_(-1), + stub_pool_index_(-1) { + ASSERT(code.ContainsInstructionAt(pc)); + // Last instruction: blr r1. + ASSERT(*(reinterpret_cast(pc) - 1) == 0xd63f0020); + + Register reg; + uword stub_load_end = + InstructionPattern::DecodeLoadWordFromPool(pc - 3 * Instr::kInstrSize, + ®, + &stub_pool_index_); + ASSERT(reg == CODE_REG); + InstructionPattern::DecodeLoadWordFromPool(stub_load_end, + ®, + &cache_pool_index_); + ASSERT(reg == R5); +} + + +RawObject* SwitchableCallPattern::cache() const { + return reinterpret_cast( + object_pool_.ObjectAt(cache_pool_index_)); +} + + +void SwitchableCallPattern::SetCache(const MegamorphicCache& cache) const { + ASSERT(Object::Handle(object_pool_.ObjectAt(cache_pool_index_)).IsICData()); + object_pool_.SetObjectAt(cache_pool_index_, cache); +} + + +void SwitchableCallPattern::SetLookupStub(const Code& lookup_stub) const { + ASSERT(Object::Handle(object_pool_.ObjectAt(stub_pool_index_)).IsCode()); + object_pool_.SetObjectAt(stub_pool_index_, lookup_stub); +} + + ReturnPattern::ReturnPattern(uword pc) : pc_(pc) { } diff --git a/runtime/vm/instructions_arm64.h b/runtime/vm/instructions_arm64.h index 0d2c96d70ce..6195d52b594 100644 --- a/runtime/vm/instructions_arm64.h +++ b/runtime/vm/instructions_arm64.h @@ -106,6 +106,27 @@ class NativeCallPattern : public ValueObject { }; +// Instance call that can switch from an IC call to a megamorphic call +// load ICData load MegamorphicCache +// call ICLookup stub -> call MegamorphicLookup stub +// call target call target +class SwitchableCallPattern : public ValueObject { + public: + SwitchableCallPattern(uword pc, const Code& code); + + RawObject* cache() const; + void SetCache(const MegamorphicCache& cache) const; + void SetLookupStub(const Code& stub) const; + + private: + const ObjectPool& object_pool_; + intptr_t cache_pool_index_; + intptr_t stub_pool_index_; + + DISALLOW_COPY_AND_ASSIGN(SwitchableCallPattern); +}; + + class ReturnPattern : public ValueObject { public: explicit ReturnPattern(uword pc); diff --git a/runtime/vm/instructions_mips.cc b/runtime/vm/instructions_mips.cc index dcca54fd949..1c243efd252 100644 --- a/runtime/vm/instructions_mips.cc +++ b/runtime/vm/instructions_mips.cc @@ -210,6 +210,46 @@ void CallPattern::InsertDeoptCallAt(uword pc, uword target_address) { } +SwitchableCallPattern::SwitchableCallPattern(uword pc, const Code& code) + : object_pool_(ObjectPool::Handle(code.GetObjectPool())), + cache_pool_index_(-1), + stub_pool_index_(-1) { + ASSERT(code.ContainsInstructionAt(pc)); + // Last instruction: jalr t1. + ASSERT(*(reinterpret_cast(pc) - 1) == 0); // Delay slot. + ASSERT(*(reinterpret_cast(pc) - 2) == 0x0120f809); + + Register reg; + uword stub_load_end = + InstructionPattern::DecodeLoadWordFromPool(pc - 5 * Instr::kInstrSize, + ®, + &stub_pool_index_); + ASSERT(reg == CODE_REG); + InstructionPattern::DecodeLoadWordFromPool(stub_load_end, + ®, + &cache_pool_index_); + ASSERT(reg == S5); +} + + +RawObject* SwitchableCallPattern::cache() const { + return reinterpret_cast( + object_pool_.ObjectAt(cache_pool_index_)); +} + + +void SwitchableCallPattern::SetCache(const MegamorphicCache& cache) const { + ASSERT(Object::Handle(object_pool_.ObjectAt(cache_pool_index_)).IsICData()); + object_pool_.SetObjectAt(cache_pool_index_, cache); +} + + +void SwitchableCallPattern::SetLookupStub(const Code& lookup_stub) const { + ASSERT(Object::Handle(object_pool_.ObjectAt(stub_pool_index_)).IsCode()); + object_pool_.SetObjectAt(stub_pool_index_, lookup_stub); +} + + ReturnPattern::ReturnPattern(uword pc) : pc_(pc) { } diff --git a/runtime/vm/instructions_mips.h b/runtime/vm/instructions_mips.h index fb2e22ba673..607e835e524 100644 --- a/runtime/vm/instructions_mips.h +++ b/runtime/vm/instructions_mips.h @@ -97,6 +97,27 @@ class NativeCallPattern : public ValueObject { }; +// Instance call that can switch from an IC call to a megamorphic call +// load ICData load MegamorphicCache +// call ICLookup stub -> call MegamorphicLookup stub +// call target call target +class SwitchableCallPattern : public ValueObject { + public: + SwitchableCallPattern(uword pc, const Code& code); + + RawObject* cache() const; + void SetCache(const MegamorphicCache& cache) const; + void SetLookupStub(const Code& stub) const; + + private: + const ObjectPool& object_pool_; + intptr_t cache_pool_index_; + intptr_t stub_pool_index_; + + DISALLOW_COPY_AND_ASSIGN(SwitchableCallPattern); +}; + + class ReturnPattern : public ValueObject { public: explicit ReturnPattern(uword pc); diff --git a/runtime/vm/isolate.cc b/runtime/vm/isolate.cc index 79398c5a579..491cac220f3 100644 --- a/runtime/vm/isolate.cc +++ b/runtime/vm/isolate.cc @@ -1926,6 +1926,10 @@ void Isolate::set_default_tag(const UserTag& tag) { default_tag_ = tag.raw(); } +void Isolate::set_ic_miss_code(const Code& code) { + ic_miss_code_ = code.raw(); +} + void Isolate::set_deoptimized_code_array(const GrowableObjectArray& value) { ASSERT(Thread::Current()->IsMutatorThread()); diff --git a/runtime/vm/isolate.h b/runtime/vm/isolate.h index f7beafac3b3..ac192ff14b7 100644 --- a/runtime/vm/isolate.h +++ b/runtime/vm/isolate.h @@ -138,6 +138,10 @@ class Isolate : public BaseIsolate { return OFFSET_OF(Isolate, class_table_); } + static intptr_t ic_miss_code_offset() { + return OFFSET_OF(Isolate, ic_miss_code_); + } + Dart_MessageNotifyCallback message_notify_callback() const { return message_notify_callback_; } @@ -593,6 +597,8 @@ class Isolate : public BaseIsolate { RawUserTag* default_tag() const { return default_tag_; } void set_default_tag(const UserTag& tag); + void set_ic_miss_code(const Code& code); + Metric* metrics_list_head() { return metrics_list_head_; } @@ -723,6 +729,7 @@ class Isolate : public BaseIsolate { uword user_tag_; RawUserTag* current_tag_; RawUserTag* default_tag_; + RawCode* ic_miss_code_; ClassTable class_table_; bool single_step_; diff --git a/runtime/vm/object_store.h b/runtime/vm/object_store.h index c4294789080..09c24d736d9 100644 --- a/runtime/vm/object_store.h +++ b/runtime/vm/object_store.h @@ -454,6 +454,9 @@ class ObjectStore { void set_megamorphic_cache_table(const GrowableObjectArray& value) { megamorphic_cache_table_ = value.raw(); } + RawCode* megamorphic_miss_code() const { + return megamorphic_miss_code_; + } RawFunction* megamorphic_miss_function() const { return megamorphic_miss_function_; } diff --git a/runtime/vm/stub_code.h b/runtime/vm/stub_code.h index bad4a860916..f451638f9fc 100644 --- a/runtime/vm/stub_code.h +++ b/runtime/vm/stub_code.h @@ -35,6 +35,7 @@ class SnapshotWriter; V(OptimizeFunction) \ V(InvokeDartCode) \ V(DebugStepCheck) \ + V(ICLookup) \ V(MegamorphicLookup) \ V(FixAllocationStubTarget) \ V(Deoptimize) \ @@ -60,7 +61,7 @@ class SnapshotWriter; V(Subtype1TestCache) \ V(Subtype2TestCache) \ V(Subtype3TestCache) \ - V(CallClosureNoSuchMethod) + V(CallClosureNoSuchMethod) \ // Is it permitted for the stubs above to refer to Object::null(), which is // allocated in the VM isolate and shared across all isolates. @@ -139,8 +140,7 @@ class StubCode : public AllStatic { static const intptr_t kNoInstantiator = 0; - static void EmitMegamorphicLookup( - Assembler*, Register recv, Register cache, Register target); + static void EmitMegamorphicLookup(Assembler* assembler); private: friend class MegamorphicCacheTable; diff --git a/runtime/vm/stub_code_arm.cc b/runtime/vm/stub_code_arm.cc index fcf6843ec12..94cf00b33d0 100644 --- a/runtime/vm/stub_code_arm.cc +++ b/runtime/vm/stub_code_arm.cc @@ -539,9 +539,9 @@ static void GenerateDispatcherCode(Assembler* assembler, __ add(IP, FP, Operand(R2, LSL, 1)); // R2 is Smi. __ ldr(R8, Address(IP, kParamEndSlotFromFp * kWordSize)); __ PushObject(Object::null_object()); - __ Push(R8); - __ Push(R9); - __ Push(R4); + __ Push(R8); // Receiver. + __ Push(R9); // ICData/MegamorphicCache. + __ Push(R4); // Arguments descriptor. // R2: Smi-tagged arguments array length. PushArgumentsArray(assembler); const intptr_t kNumArgs = 4; @@ -2044,18 +2044,18 @@ void StubCode::GenerateOptimizedIdenticalWithNumberCheckStub( } -void StubCode::EmitMegamorphicLookup( - Assembler* assembler, Register receiver, Register cache, Register target) { - ASSERT((cache != R0) && (cache != R2)); - __ LoadTaggedClassIdMayBeSmi(R0, receiver); - // R0: class ID of the receiver (smi). - __ ldr(R2, FieldAddress(cache, MegamorphicCache::buckets_offset())); - __ ldr(R1, FieldAddress(R1, MegamorphicCache::mask_offset())); +void StubCode::EmitMegamorphicLookup(Assembler* assembler) { + __ LoadTaggedClassIdMayBeSmi(R0, R0); + // R0: receiver cid as Smi. + __ ldr(R4, FieldAddress(R9, MegamorphicCache::arguments_descriptor_offset())); + __ ldr(R2, FieldAddress(R9, MegamorphicCache::buckets_offset())); + __ ldr(R1, FieldAddress(R9, MegamorphicCache::mask_offset())); // R2: cache buckets array. // R1: mask. __ mov(R3, Operand(R0)); + // R3: probe. - Label loop, update, call_target_function; + Label loop, update, load_target_function; __ b(&loop); __ Bind(&update); @@ -2065,33 +2065,77 @@ void StubCode::EmitMegamorphicLookup( const intptr_t base = Array::data_offset(); // R3 is smi tagged, but table entries are two words, so LSL 2. __ add(IP, R2, Operand(R3, LSL, 2)); - __ ldr(R4, FieldAddress(IP, base)); + __ ldr(R6, FieldAddress(IP, base)); ASSERT(kIllegalCid == 0); - __ tst(R4, Operand(R4)); - __ b(&call_target_function, EQ); - __ cmp(R4, Operand(R0)); + __ tst(R6, Operand(R6)); + __ b(&load_target_function, EQ); + __ cmp(R6, Operand(R0)); __ b(&update, NE); - __ Bind(&call_target_function); + __ Bind(&load_target_function); // Call the target found in the cache. For a class id match, this is a // proper target for the given name and arguments descriptor. If the // illegal class id was found, the target is a cache miss handler that can // be invoked as a normal Dart function. __ add(IP, R2, Operand(R3, LSL, 2)); __ ldr(R0, FieldAddress(IP, base + kWordSize)); + __ ldr(R1, FieldAddress(R0, Function::entry_point_offset())); __ ldr(CODE_REG, FieldAddress(R0, Function::code_offset())); - __ ldr(target, FieldAddress(R0, Function::entry_point_offset())); } // Called from megamorphic calls. -// R0: receiver. -// R1: lookup cache. +// R0: receiver +// R9: MegamorphicCache (preserved) // Result: -// R1: entry point. +// R1: target entry point +// CODE_REG: target Code +// R4: arguments descriptor void StubCode::GenerateMegamorphicLookupStub(Assembler* assembler) { - EmitMegamorphicLookup(assembler, R0, R1, R1); + EmitMegamorphicLookup(assembler); + __ Ret(); +} + + +// Called from switchable IC calls. +// R0: receiver +// R9: ICData (preserved) +// Result: +// R1: target entry point +// CODE_REG: target Code object +// R4: arguments descriptor +void StubCode::GenerateICLookupStub(Assembler* assembler) { + Label loop, found, miss; + __ ldr(R4, FieldAddress(R9, ICData::arguments_descriptor_offset())); + __ ldr(R8, FieldAddress(R9, ICData::ic_data_offset())); + __ AddImmediate(R8, R8, Array::data_offset() - kHeapObjectTag); + // R8: first IC entry + __ LoadTaggedClassIdMayBeSmi(R1, R0); + // R1: receiver cid as Smi + + __ Bind(&loop); + __ ldr(R2, Address(R8, 0)); + __ cmp(R1, Operand(R2)); + __ b(&found, EQ); + __ CompareImmediate(R2, Smi::RawValue(kIllegalCid)); + __ b(&miss, EQ); + + const intptr_t entry_length = ICData::TestEntryLengthFor(1) * kWordSize; + __ AddImmediate(R8, entry_length); // Next entry. + __ b(&loop); + + __ Bind(&found); + const intptr_t target_offset = ICData::TargetIndexFor(1) * kWordSize; + __ LoadFromOffset(kWord, R0, R8, target_offset); + __ ldr(R1, FieldAddress(R0, Function::entry_point_offset())); + __ ldr(CODE_REG, FieldAddress(R0, Function::code_offset())); + __ Ret(); + + __ Bind(&miss); + __ LoadIsolate(R2); + __ ldr(CODE_REG, Address(R2, Isolate::ic_miss_code_offset())); + __ ldr(R1, FieldAddress(CODE_REG, Code::entry_point_offset())); __ Ret(); } diff --git a/runtime/vm/stub_code_arm64.cc b/runtime/vm/stub_code_arm64.cc index 7d6f022fe51..8c197ae9633 100644 --- a/runtime/vm/stub_code_arm64.cc +++ b/runtime/vm/stub_code_arm64.cc @@ -567,9 +567,9 @@ static void GenerateDispatcherCode(Assembler* assembler, __ add(TMP, FP, Operand(R2, LSL, 2)); // R2 is Smi. __ LoadFromOffset(R6, TMP, kParamEndSlotFromFp * kWordSize); __ PushObject(Object::null_object()); - __ Push(R6); - __ Push(R5); - __ Push(R4); + __ Push(R6); // Receiver. + __ Push(R5); // ICData/MegamorphicCache. + __ Push(R4); // Arguments descriptor. // R2: Smi-tagged arguments array length. PushArgumentsArray(assembler); const intptr_t kNumArgs = 4; @@ -2098,18 +2098,18 @@ void StubCode::GenerateOptimizedIdenticalWithNumberCheckStub( } -void StubCode::EmitMegamorphicLookup( - Assembler* assembler, Register receiver, Register cache, Register target) { - ASSERT((cache != R0) && (cache != R2)); - __ LoadTaggedClassIdMayBeSmi(R0, receiver); +void StubCode::EmitMegamorphicLookup(Assembler* assembler) { + __ LoadTaggedClassIdMayBeSmi(R0, R0); // R0: class ID of the receiver (smi). - __ LoadFieldFromOffset(R2, cache, MegamorphicCache::buckets_offset()); - __ LoadFieldFromOffset(R1, cache, MegamorphicCache::mask_offset()); + __ ldr(R4, FieldAddress(R5, MegamorphicCache::arguments_descriptor_offset())); + __ ldr(R2, FieldAddress(R5, MegamorphicCache::buckets_offset())); + __ ldr(R1, FieldAddress(R5, MegamorphicCache::mask_offset())); // R2: cache buckets array. // R1: mask. __ mov(R3, R0); + // R3: probe. - Label loop, update, call_target_function; + Label loop, update, load_target_function; __ b(&loop); __ Bind(&update); @@ -2119,33 +2119,77 @@ void StubCode::EmitMegamorphicLookup( const intptr_t base = Array::data_offset(); // R3 is smi tagged, but table entries are 16 bytes, so LSL 3. __ add(TMP, R2, Operand(R3, LSL, 3)); - __ LoadFieldFromOffset(R4, TMP, base); + __ ldr(R6, FieldAddress(TMP, base)); ASSERT(kIllegalCid == 0); - __ tst(R4, Operand(R4)); - __ b(&call_target_function, EQ); - __ CompareRegisters(R4, R0); + __ tst(R6, Operand(R6)); + __ b(&load_target_function, EQ); + __ CompareRegisters(R6, R0); __ b(&update, NE); - __ Bind(&call_target_function); + __ Bind(&load_target_function); // Call the target found in the cache. For a class id match, this is a // proper target for the given name and arguments descriptor. If the // illegal class id was found, the target is a cache miss handler that can // be invoked as a normal Dart function. __ add(TMP, R2, Operand(R3, LSL, 3)); - __ LoadFieldFromOffset(R0, TMP, base + kWordSize); - __ LoadFieldFromOffset(CODE_REG, R0, Function::code_offset()); - __ LoadFieldFromOffset(R1, R0, Function::entry_point_offset()); + __ ldr(R0, FieldAddress(TMP, base + kWordSize)); + __ ldr(R1, FieldAddress(R0, Function::entry_point_offset())); + __ ldr(CODE_REG, FieldAddress(R0, Function::code_offset())); } // Called from megamorphic calls. -// R0: receiver. -// R1: lookup cache. +// R0: receiver +// R5: MegamorphicCache (preserved) // Result: -// R1: entry point. +// R1: target entry point +// CODE_REG: target Code +// R4: arguments descriptor void StubCode::GenerateMegamorphicLookupStub(Assembler* assembler) { - EmitMegamorphicLookup(assembler, R0, R1, R1); + EmitMegamorphicLookup(assembler); + __ ret(); +} + + +// Called from switchable IC calls. +// R0: receiver +// R5: ICData (preserved) +// Result: +// R1: target entry point +// CODE_REG: target Code object +// R4: arguments descriptor +void StubCode::GenerateICLookupStub(Assembler* assembler) { + Label loop, found, miss; + __ ldr(R4, FieldAddress(R5, ICData::arguments_descriptor_offset())); + __ ldr(R8, FieldAddress(R5, ICData::ic_data_offset())); + __ AddImmediate(R8, R8, Array::data_offset() - kHeapObjectTag); + // R8: first IC entry + __ LoadTaggedClassIdMayBeSmi(R1, R0); + // R1: receiver cid as Smi + + __ Bind(&loop); + __ ldr(R2, Address(R8, 0)); + __ cmp(R1, Operand(R2)); + __ b(&found, EQ); + __ CompareImmediate(R2, Smi::RawValue(kIllegalCid)); + __ b(&miss, EQ); + + const intptr_t entry_length = ICData::TestEntryLengthFor(1) * kWordSize; + __ AddImmediate(R8, R8, entry_length); // Next entry. + __ b(&loop); + + __ Bind(&found); + const intptr_t target_offset = ICData::TargetIndexFor(1) * kWordSize; + __ ldr(R0, Address(R8, target_offset)); + __ ldr(R1, FieldAddress(R0, Function::entry_point_offset())); + __ ldr(CODE_REG, FieldAddress(R0, Function::code_offset())); + __ ret(); + + __ Bind(&miss); + __ LoadIsolate(R2); + __ ldr(CODE_REG, Address(R2, Isolate::ic_miss_code_offset())); + __ ldr(R1, FieldAddress(CODE_REG, Code::entry_point_offset())); __ ret(); } diff --git a/runtime/vm/stub_code_ia32.cc b/runtime/vm/stub_code_ia32.cc index 8ca11ef882f..b6302d333ff 100644 --- a/runtime/vm/stub_code_ia32.cc +++ b/runtime/vm/stub_code_ia32.cc @@ -481,7 +481,7 @@ static void GenerateDispatcherCode(Assembler* assembler, EBP, EDI, TIMES_HALF_WORD_SIZE, kParamEndSlotFromFp * kWordSize)); __ pushl(raw_null); // Setup space on stack for result. __ pushl(EAX); // Receiver. - __ pushl(ECX); + __ pushl(ECX); // ICData/MegamorphicCache. __ pushl(EDX); // Arguments descriptor array. __ movl(EDX, EDI); // EDX: Smi-tagged arguments array length. @@ -2022,19 +2022,18 @@ void StubCode::GenerateOptimizedIdenticalWithNumberCheckStub( } -void StubCode::EmitMegamorphicLookup( - Assembler* assembler, Register receiver, Register cache, Register target) { - ASSERT((cache != EAX) && (cache != EDI)); - __ LoadTaggedClassIdMayBeSmi(EAX, receiver); - +void StubCode::EmitMegamorphicLookup(Assembler* assembler) { + __ LoadTaggedClassIdMayBeSmi(EAX, EBX); // EAX: class ID of the receiver (smi). - __ movl(EDI, FieldAddress(cache, MegamorphicCache::buckets_offset())); - __ movl(EBX, FieldAddress(cache, MegamorphicCache::mask_offset())); + __ movl(EDI, FieldAddress(ECX, MegamorphicCache::buckets_offset())); + __ movl(EBX, FieldAddress(ECX, MegamorphicCache::mask_offset())); // EDI: cache buckets array. // EBX: mask. + __ pushl(ECX); // Spill MegamorphicCache. __ movl(ECX, EAX); + // ECX: probe. - Label loop, update, call_target_function; + Label loop, update, load_target_function; __ jmp(&loop); __ Bind(&update); @@ -2047,31 +2046,45 @@ void StubCode::EmitMegamorphicLookup( ASSERT(kIllegalCid == 0); __ testl(EDX, EDX); - __ j(ZERO, &call_target_function, Assembler::kNearJump); + __ j(ZERO, &load_target_function, Assembler::kNearJump); __ cmpl(EDX, EAX); __ j(NOT_EQUAL, &update, Assembler::kNearJump); - __ Bind(&call_target_function); + __ Bind(&load_target_function); // Call the target found in the cache. For a class id match, this is a // proper target for the given name and arguments descriptor. If the // illegal class id was found, the target is a cache miss handler that can // be invoked as a normal Dart function. __ movl(EAX, FieldAddress(EDI, ECX, TIMES_4, base + kWordSize)); - __ movl(target, FieldAddress(EAX, Function::entry_point_offset())); + __ popl(ECX); // Restore MegamorphicCache. + __ movl(EDX, + FieldAddress(ECX, MegamorphicCache::arguments_descriptor_offset())); + __ movl(EBX, FieldAddress(EAX, Function::entry_point_offset())); } // Called from megamorphic calls. -// ECX: receiver. -// EBX: lookup cache. +// EBX: receiver +// ECX: MegamorphicCache (preserved) // Result: -// EBX: entry point. +// EBX: target entry point +// EDX: argument descriptor void StubCode::GenerateMegamorphicLookupStub(Assembler* assembler) { - EmitMegamorphicLookup(assembler, ECX, EBX, EBX); + EmitMegamorphicLookup(assembler); __ ret(); } +// Called from switchable IC calls. +// EBX: receiver +// ECX: ICData (preserved) +// Result: +// EBX: target entry point +// EDX: arguments descriptor +void StubCode::GenerateICLookupStub(Assembler* assembler) { + __ int3(); +} + } // namespace dart #endif // defined TARGET_ARCH_IA32 diff --git a/runtime/vm/stub_code_mips.cc b/runtime/vm/stub_code_mips.cc index 30a7cdf6622..58af41870b0 100644 --- a/runtime/vm/stub_code_mips.cc +++ b/runtime/vm/stub_code_mips.cc @@ -563,7 +563,7 @@ static void GenerateDispatcherCode(Assembler* assembler, // Push space for the return value. // Push the receiver. - // Push IC data object. + // Push ICData/MegamorphicCache object. // Push arguments descriptor array. // Push original arguments array. __ addiu(SP, SP, Immediate(-4 * kWordSize)); @@ -2211,16 +2211,16 @@ void StubCode::GenerateOptimizedIdenticalWithNumberCheckStub( } -void StubCode::EmitMegamorphicLookup( - Assembler* assembler, Register receiver, Register cache, Register target) { - ASSERT((cache != T0) && (cache != T2)); - __ LoadTaggedClassIdMayBeSmi(T0, receiver); +void StubCode::EmitMegamorphicLookup(Assembler* assembler) { + __ LoadTaggedClassIdMayBeSmi(T0, T0); // T0: class ID of the receiver (smi). - __ lw(T2, FieldAddress(cache, MegamorphicCache::buckets_offset())); - __ lw(T1, FieldAddress(cache, MegamorphicCache::mask_offset())); + __ lw(S4, FieldAddress(S5, MegamorphicCache::arguments_descriptor_offset())); + __ lw(T2, FieldAddress(S5, MegamorphicCache::buckets_offset())); + __ lw(T1, FieldAddress(S5, MegamorphicCache::mask_offset())); // T2: cache buckets array. // T1: mask. __ mov(T3, T0); + // T3: probe. Label loop, update, call_target_function; __ b(&loop); @@ -2248,18 +2248,61 @@ void StubCode::EmitMegamorphicLookup( __ addu(T1, T2, T1); __ lw(T0, FieldAddress(T1, base + kWordSize)); + __ lw(T1, FieldAddress(T0, Function::entry_point_offset())); __ lw(CODE_REG, FieldAddress(T0, Function::code_offset())); - __ lw(target, FieldAddress(T0, Function::entry_point_offset())); } // Called from megamorphic calls. -// T0: receiver. -// T1: lookup cache. +// T0: receiver +// S5: MegamorphicCache (preserved) // Result: -// T1: entry point. +// T1: target entry point +// CODE_REG: target Code +// S4: arguments descriptor void StubCode::GenerateMegamorphicLookupStub(Assembler* assembler) { - EmitMegamorphicLookup(assembler, T0, T1, T1); + EmitMegamorphicLookup(assembler); + __ Ret(); +} + + +// Called from switchable IC calls. +// T0: receiver +// S5: ICData (preserved) +// Result: +// T1: target entry point +// CODE_REG: target Code object +// S4: arguments descriptor +void StubCode::GenerateICLookupStub(Assembler* assembler) { + Label loop, found, miss; + __ lw(T6, FieldAddress(S5, ICData::ic_data_offset())); + __ lw(S4, FieldAddress(S5, ICData::arguments_descriptor_offset())); + __ AddImmediate(T6, T6, Array::data_offset() - kHeapObjectTag); + // T6: first IC entry. + __ LoadTaggedClassIdMayBeSmi(T1, T0); + // T1: receiver cid as Smi + + __ Bind(&loop); + __ lw(T2, Address(T6, 0)); + __ beq(T1, T2, &found); + ASSERT(Smi::RawValue(kIllegalCid) == 0); + __ beq(T2, ZR, &miss); + + const intptr_t entry_length = ICData::TestEntryLengthFor(1) * kWordSize; + __ AddImmediate(T6, entry_length); // Next entry. + __ b(&loop); + + __ Bind(&found); + const intptr_t target_offset = ICData::TargetIndexFor(1) * kWordSize; + __ lw(T0, Address(T6, target_offset)); + __ lw(T1, FieldAddress(T0, Function::entry_point_offset())); + __ lw(CODE_REG, FieldAddress(T0, Function::code_offset())); + __ Ret(); + + __ Bind(&miss); + __ LoadIsolate(T2); + __ lw(CODE_REG, Address(T2, Isolate::ic_miss_code_offset())); + __ lw(T1, FieldAddress(CODE_REG, Code::entry_point_offset())); __ Ret(); } diff --git a/runtime/vm/stub_code_x64.cc b/runtime/vm/stub_code_x64.cc index 44d5f9632a6..ea58adb223a 100644 --- a/runtime/vm/stub_code_x64.cc +++ b/runtime/vm/stub_code_x64.cc @@ -510,7 +510,7 @@ static void GenerateDispatcherCode(Assembler* assembler, RBP, RDI, TIMES_HALF_WORD_SIZE, kParamEndSlotFromFp * kWordSize)); __ PushObject(Object::null_object()); // Setup space on stack for result. __ pushq(RAX); // Receiver. - __ pushq(RBX); + __ pushq(RBX); // ICData/MegamorphicCache. __ pushq(R10); // Arguments descriptor array. __ movq(R10, RDI); // EDX: Smi-tagged arguments array length. @@ -2088,52 +2088,99 @@ void StubCode::GenerateOptimizedIdenticalWithNumberCheckStub( } -void StubCode::EmitMegamorphicLookup( - Assembler* assembler, Register receiver, Register cache, Register target) { - ASSERT((cache != RAX) && (cache != RDI)); - __ LoadTaggedClassIdMayBeSmi(RAX, receiver); +void StubCode::EmitMegamorphicLookup(Assembler* assembler) { + __ LoadTaggedClassIdMayBeSmi(RAX, RDI); // RAX: class ID of the receiver (smi). - __ movq(RDI, FieldAddress(cache, MegamorphicCache::buckets_offset())); - __ movq(RBX, FieldAddress(cache, MegamorphicCache::mask_offset())); + __ movq(R10, + FieldAddress(RBX, MegamorphicCache::arguments_descriptor_offset())); + __ movq(RDI, FieldAddress(RBX, MegamorphicCache::buckets_offset())); + __ movq(R9, FieldAddress(RBX, MegamorphicCache::mask_offset())); // RDI: cache buckets array. // RBX: mask. __ movq(RCX, RAX); - Label loop, update, call_target_function; + Label loop, update, load_target_function; __ jmp(&loop); __ Bind(&update); __ AddImmediate(RCX, Immediate(Smi::RawValue(1))); __ Bind(&loop); - __ andq(RCX, RBX); + __ andq(RCX, R9); const intptr_t base = Array::data_offset(); // RCX is smi tagged, but table entries are two words, so TIMES_8. __ movq(RDX, FieldAddress(RDI, RCX, TIMES_8, base)); ASSERT(kIllegalCid == 0); __ testq(RDX, RDX); - __ j(ZERO, &call_target_function, Assembler::kNearJump); + __ j(ZERO, &load_target_function, Assembler::kNearJump); __ cmpq(RDX, RAX); __ j(NOT_EQUAL, &update, Assembler::kNearJump); - __ Bind(&call_target_function); + __ Bind(&load_target_function); // Call the target found in the cache. For a class id match, this is a // proper target for the given name and arguments descriptor. If the // illegal class id was found, the target is a cache miss handler that can // be invoked as a normal Dart function. __ movq(RAX, FieldAddress(RDI, RCX, TIMES_8, base + kWordSize)); + __ movq(RCX, FieldAddress(RAX, Function::entry_point_offset())); __ movq(CODE_REG, FieldAddress(RAX, Function::code_offset())); - __ movq(target, FieldAddress(RAX, Function::entry_point_offset())); } // Called from megamorphic calls. -// RDI: receiver. -// RBX: lookup cache. +// RDI: receiver +// RBX: MegamorphicCache (preserved) // Result: -// RCX: entry point. +// RCX: target entry point +// CODE_REG: target Code +// R10: arguments descriptor void StubCode::GenerateMegamorphicLookupStub(Assembler* assembler) { - EmitMegamorphicLookup(assembler, RDI, RBX, RCX); + EmitMegamorphicLookup(assembler); + __ ret(); +} + + +// Called from switchable IC calls. +// RDI: receiver +// RBX: ICData (preserved) +// Result: +// RCX: target entry point +// CODE_REG: target Code object +// R10: arguments descriptor +void StubCode::GenerateICLookupStub(Assembler* assembler) { + Label loop, found, miss; + + __ movq(R13, FieldAddress(RBX, ICData::ic_data_offset())); + __ movq(R10, FieldAddress(RBX, ICData::arguments_descriptor_offset())); + __ leaq(R13, FieldAddress(R13, Array::data_offset())); + // R13: first IC entry + __ LoadTaggedClassIdMayBeSmi(RAX, RDI); + // RAX: receiver cid as Smi + + __ Bind(&loop); + __ movq(R9, Address(R13, 0)); + __ cmpq(RAX, R9); + __ j(EQUAL, &found, Assembler::kNearJump); + + ASSERT(Smi::RawValue(kIllegalCid) == 0); + __ cmpq(R9, R9); + __ j(EQUAL, &miss, Assembler::kNearJump); + + const intptr_t entry_length = ICData::TestEntryLengthFor(1) * kWordSize; + __ addq(R13, Immediate(entry_length)); // Next entry. + __ jmp(&loop); + + __ Bind(&found); + const intptr_t target_offset = ICData::TargetIndexFor(1) * kWordSize; + __ movq(RAX, Address(R13, target_offset)); + __ movq(RCX, FieldAddress(RAX, Function::entry_point_offset())); + __ movq(CODE_REG, FieldAddress(RAX, Function::code_offset())); + __ ret(); + + __ Bind(&miss); + __ LoadIsolate(RAX); + __ movq(CODE_REG, Address(RAX, Isolate::ic_miss_code_offset())); + __ movq(RCX, FieldAddress(CODE_REG, Code::entry_point_offset())); __ ret(); }