diff --git a/runtime/vm/code_patcher.h b/runtime/vm/code_patcher.h index 30828a1c137..9e1e3cb9ff4 100644 --- a/runtime/vm/code_patcher.h +++ b/runtime/vm/code_patcher.h @@ -86,8 +86,8 @@ class CodePatcher : public AllStatic { const Code& target); static ObjectPtr GetSwitchableCallDataAt(uword return_address, const Code& caller_code); - static CodePtr GetSwitchableCallTargetAt(uword return_address, - const Code& caller_code); + static uword GetSwitchableCallTargetEntryAt(uword return_address, + const Code& caller_code); static CodePtr GetNativeCallAt(uword return_address, const Code& caller_code, diff --git a/runtime/vm/code_patcher_arm.cc b/runtime/vm/code_patcher_arm.cc index 89de68a2136..0947b58f6c6 100644 --- a/runtime/vm/code_patcher_arm.cc +++ b/runtime/vm/code_patcher_arm.cc @@ -108,15 +108,15 @@ void CodePatcher::PatchSwitchableCallAtWithMutatorsStopped( } } -CodePtr CodePatcher::GetSwitchableCallTargetAt(uword return_address, - const Code& caller_code) { +uword CodePatcher::GetSwitchableCallTargetEntryAt(uword return_address, + const Code& caller_code) { ASSERT(caller_code.ContainsInstructionAt(return_address)); if (FLAG_precompiled_mode && FLAG_use_bare_instructions) { BareSwitchableCallPattern call(return_address, caller_code); - return call.target(); + return call.target_entry(); } else { SwitchableCallPattern call(return_address, caller_code); - return call.target(); + return call.target_entry(); } } diff --git a/runtime/vm/code_patcher_arm64.cc b/runtime/vm/code_patcher_arm64.cc index a1d4c4dc629..b7590dbfae8 100644 --- a/runtime/vm/code_patcher_arm64.cc +++ b/runtime/vm/code_patcher_arm64.cc @@ -144,15 +144,15 @@ void CodePatcher::PatchSwitchableCallAtWithMutatorsStopped( } } -CodePtr CodePatcher::GetSwitchableCallTargetAt(uword return_address, - const Code& caller_code) { +uword CodePatcher::GetSwitchableCallTargetEntryAt(uword return_address, + const Code& caller_code) { ASSERT(caller_code.ContainsInstructionAt(return_address)); if (FLAG_precompiled_mode && FLAG_use_bare_instructions) { BareSwitchableCallPattern call(return_address, caller_code); - return call.target(); + return call.target_entry(); } else { SwitchableCallPattern call(return_address, caller_code); - return call.target(); + return call.target_entry(); } } diff --git a/runtime/vm/code_patcher_ia32.cc b/runtime/vm/code_patcher_ia32.cc index 3104115bc0c..08ce568369e 100644 --- a/runtime/vm/code_patcher_ia32.cc +++ b/runtime/vm/code_patcher_ia32.cc @@ -261,11 +261,11 @@ void CodePatcher::PatchSwitchableCallAtWithMutatorsStopped( UNREACHABLE(); } -CodePtr CodePatcher::GetSwitchableCallTargetAt(uword return_address, - const Code& caller_code) { +uword CodePatcher::GetSwitchableCallTargetEntryAt(uword return_address, + const Code& caller_code) { // Switchable instance calls only generated for precompilation. UNREACHABLE(); - return Code::null(); + return 0; } ObjectPtr CodePatcher::GetSwitchableCallDataAt(uword return_address, diff --git a/runtime/vm/code_patcher_x64.cc b/runtime/vm/code_patcher_x64.cc index c9167899f82..1ba005f311e 100644 --- a/runtime/vm/code_patcher_x64.cc +++ b/runtime/vm/code_patcher_x64.cc @@ -321,8 +321,9 @@ class SwitchableCall : public SwitchableCallBase { // No need to flush the instruction cache, since the code is not modified. } - CodePtr target() const { - return static_cast(object_pool_.ObjectAt(target_index())); + uword target_entry() const { + return Code::Handle(Code::RawCast(object_pool_.ObjectAt(target_index()))) + .MonomorphicEntryPoint(); } }; @@ -395,18 +396,7 @@ class BareSwitchableCall : public SwitchableCallBase { object_pool_.SetRawValueAt(target_index(), target.MonomorphicEntryPoint()); } - CodePtr target() const { - const uword pc = object_pool_.RawValueAt(target_index()); - CodePtr result = ReversePc::Lookup(IsolateGroup::Current(), pc); - if (result != Code::null()) { - return result; - } - result = ReversePc::Lookup(Dart::vm_isolate_group(), pc); - if (result != Code::null()) { - return result; - } - UNREACHABLE(); - } + uword target_entry() const { return object_pool_.RawValueAt(target_index()); } }; CodePtr CodePatcher::GetStaticCallTargetAt(uword return_address, @@ -511,15 +501,15 @@ void CodePatcher::PatchSwitchableCallAtWithMutatorsStopped( } } -CodePtr CodePatcher::GetSwitchableCallTargetAt(uword return_address, - const Code& caller_code) { +uword CodePatcher::GetSwitchableCallTargetEntryAt(uword return_address, + const Code& caller_code) { ASSERT(caller_code.ContainsInstructionAt(return_address)); if (FLAG_precompiled_mode && FLAG_use_bare_instructions) { BareSwitchableCall call(return_address, caller_code); - return call.target(); + return call.target_entry(); } else { SwitchableCall call(return_address, caller_code); - return call.target(); + return call.target_entry(); } } diff --git a/runtime/vm/instructions_arm.cc b/runtime/vm/instructions_arm.cc index 16bb992750e..c2b70142fc7 100644 --- a/runtime/vm/instructions_arm.cc +++ b/runtime/vm/instructions_arm.cc @@ -292,9 +292,11 @@ SwitchableCallPattern::SwitchableCallPattern(uword pc, const Code& code) ASSERT(reg == CODE_REG); } -CodePtr SwitchableCallPattern::target() const { - return static_cast(object_pool_.ObjectAt(target_pool_index_)); +uword SwitchableCallPattern::target_entry() const { + return Code::Handle(Code::RawCast(object_pool_.ObjectAt(target_pool_index_))) + .MonomorphicEntryPoint(); } + void SwitchableCallPattern::SetTarget(const Code& target) const { ASSERT(Object::Handle(object_pool_.ObjectAt(target_pool_index_)).IsCode()); object_pool_.SetObjectAt(target_pool_index_, target); @@ -316,17 +318,8 @@ BareSwitchableCallPattern::BareSwitchableCallPattern(uword pc, const Code& code) ASSERT(reg == LINK_REGISTER); } -CodePtr BareSwitchableCallPattern::target() const { - const uword pc = object_pool_.RawValueAt(target_pool_index_); - CodePtr result = ReversePc::Lookup(IsolateGroup::Current(), pc); - if (result != Code::null()) { - return result; - } - result = ReversePc::Lookup(Dart::vm_isolate_group(), pc); - if (result != Code::null()) { - return result; - } - UNREACHABLE(); +uword BareSwitchableCallPattern::target_entry() const { + return object_pool_.RawValueAt(target_pool_index_); } void BareSwitchableCallPattern::SetTarget(const Code& target) const { diff --git a/runtime/vm/instructions_arm.h b/runtime/vm/instructions_arm.h index 5fe9ab375b9..7b7b43bde3d 100644 --- a/runtime/vm/instructions_arm.h +++ b/runtime/vm/instructions_arm.h @@ -153,7 +153,7 @@ class SwitchableCallPattern : public SwitchableCallPatternBase { public: SwitchableCallPattern(uword pc, const Code& code); - CodePtr target() const; + uword target_entry() const; void SetTarget(const Code& target) const; private: @@ -168,7 +168,7 @@ class BareSwitchableCallPattern : public SwitchableCallPatternBase { public: BareSwitchableCallPattern(uword pc, const Code& code); - CodePtr target() const; + uword target_entry() const; void SetTarget(const Code& target) const; private: diff --git a/runtime/vm/instructions_arm64.cc b/runtime/vm/instructions_arm64.cc index e7a0d24153b..a38a07ea9c6 100644 --- a/runtime/vm/instructions_arm64.cc +++ b/runtime/vm/instructions_arm64.cc @@ -428,8 +428,9 @@ SwitchableCallPattern::SwitchableCallPattern(uword pc, const Code& code) target_pool_index_ = pool_index + 1; } -CodePtr SwitchableCallPattern::target() const { - return static_cast(object_pool_.ObjectAt(target_pool_index_)); +uword SwitchableCallPattern::target_entry() const { + return Code::Handle(Code::RawCast(object_pool_.ObjectAt(target_pool_index_))) + .MonomorphicEntryPoint(); } void SwitchableCallPattern::SetTarget(const Code& target) const { @@ -454,17 +455,8 @@ BareSwitchableCallPattern::BareSwitchableCallPattern(uword pc, const Code& code) target_pool_index_ = pool_index + 1; } -CodePtr BareSwitchableCallPattern::target() const { - const uword pc = object_pool_.RawValueAt(target_pool_index_); - CodePtr result = ReversePc::Lookup(IsolateGroup::Current(), pc); - if (result != Code::null()) { - return result; - } - result = ReversePc::Lookup(Dart::vm_isolate_group(), pc); - if (result != Code::null()) { - return result; - } - UNREACHABLE(); +uword BareSwitchableCallPattern::target_entry() const { + return object_pool_.RawValueAt(target_pool_index_); } void BareSwitchableCallPattern::SetTarget(const Code& target) const { diff --git a/runtime/vm/instructions_arm64.h b/runtime/vm/instructions_arm64.h index 4a8f7e2aada..ae755397827 100644 --- a/runtime/vm/instructions_arm64.h +++ b/runtime/vm/instructions_arm64.h @@ -163,7 +163,7 @@ class SwitchableCallPattern : public SwitchableCallPatternBase { public: SwitchableCallPattern(uword pc, const Code& code); - CodePtr target() const; + uword target_entry() const; void SetTarget(const Code& target) const; private: @@ -178,7 +178,7 @@ class BareSwitchableCallPattern : public SwitchableCallPatternBase { public: BareSwitchableCallPattern(uword pc, const Code& code); - CodePtr target() const; + uword target_entry() const; void SetTarget(const Code& target) const; private: diff --git a/runtime/vm/runtime_entry.cc b/runtime/vm/runtime_entry.cc index 2a7cbd532d2..2c31114944e 100644 --- a/runtime/vm/runtime_entry.cc +++ b/runtime/vm/runtime_entry.cc @@ -1520,11 +1520,12 @@ class PatchableCallHandler { private: FunctionPtr ResolveTargetFunction(const Object& data); - void HandleMiss(const Object& old_data, - const Code& old_target, - const Function& target_function); #if defined(DART_PRECOMPILED_RUNTIME) + void HandleMissAOT(const Object& old_data, + uword old_entry, + const Function& target_function); + void DoUnlinkedCallAOT(const UnlinkedCall& unlinked, const Function& target_function); void DoMonomorphicMissAOT(const Object& data, @@ -1538,6 +1539,10 @@ class PatchableCallHandler { intptr_t* lower, intptr_t* upper); #else + void HandleMissJIT(const Object& old_data, + const Code& old_target, + const Function& target_function); + void DoMonomorphicMissJIT(const Object& data, const Function& target_function); void DoICDataMissJIT(const ICData& data, @@ -2086,7 +2091,6 @@ void PatchableCallHandler::ResolveSwitchAndReturn(const Object& old_data) { const auto& target_function = Function::Handle(zone_, ResolveTargetFunction(old_data)); - auto& code = Code::Handle(zone_); auto& data = Object::Handle(zone_); // We ensure any transition in a patchable calls are done in an atomic @@ -2100,9 +2104,12 @@ void PatchableCallHandler::ResolveSwitchAndReturn(const Object& old_data) { #if defined(DART_PRECOMPILED_RUNTIME) data = CodePatcher::GetSwitchableCallDataAt(caller_frame_->pc(), caller_code_); - DEBUG_ONLY(code = CodePatcher::GetSwitchableCallTargetAt(caller_frame_->pc(), - caller_code_)); + uword target_entry = 0; + DEBUG_ONLY(target_entry = CodePatcher::GetSwitchableCallTargetEntryAt( + caller_frame_->pc(), caller_code_)); + HandleMissAOT(data, target_entry, target_function); #else + auto& code = Code::Handle(zone_); if (should_consider_patching()) { code ^= CodePatcher::GetInstanceCallAt(caller_frame_->pc(), caller_code_, &data); @@ -2110,34 +2117,52 @@ void PatchableCallHandler::ResolveSwitchAndReturn(const Object& old_data) { ASSERT(old_data.IsICData() || old_data.IsMegamorphicCache()); data = old_data.ptr(); } + HandleMissJIT(data, code, target_function); #endif - HandleMiss(data, code, target_function); } -void PatchableCallHandler::HandleMiss(const Object& old_data, - const Code& old_code, - const Function& target_function) { - switch (old_data.GetClassId()) { #if defined(DART_PRECOMPILED_RUNTIME) + +void PatchableCallHandler::HandleMissAOT(const Object& old_data, + uword old_entry, + const Function& target_function) { + switch (old_data.GetClassId()) { case kUnlinkedCallCid: - ASSERT(old_code.ptr() == StubCode::SwitchableCallMiss().ptr()); + ASSERT(old_entry == + StubCode::SwitchableCallMiss().MonomorphicEntryPoint()); DoUnlinkedCallAOT(UnlinkedCall::Cast(old_data), target_function); break; case kMonomorphicSmiableCallCid: - ASSERT(old_code.ptr() == StubCode::MonomorphicSmiableCheck().ptr()); + ASSERT(old_entry == + StubCode::MonomorphicSmiableCheck().MonomorphicEntryPoint()); FALL_THROUGH; case kSmiCid: DoMonomorphicMissAOT(old_data, target_function); break; case kSingleTargetCacheCid: - ASSERT(old_code.ptr() == StubCode::SingleTargetCall().ptr()); + ASSERT(old_entry == StubCode::SingleTargetCall().MonomorphicEntryPoint()); DoSingleTargetMissAOT(SingleTargetCache::Cast(old_data), target_function); break; case kICDataCid: - ASSERT(old_code.ptr() == StubCode::ICCallThroughCode().ptr()); + ASSERT(old_entry == + StubCode::ICCallThroughCode().MonomorphicEntryPoint()); DoICDataMissAOT(ICData::Cast(old_data), target_function); break; + case kMegamorphicCacheCid: + ASSERT(old_entry == StubCode::MegamorphicCall().MonomorphicEntryPoint()); + DoMegamorphicMiss(MegamorphicCache::Cast(old_data), target_function); + break; + default: + UNREACHABLE(); + } +} + #else + +void PatchableCallHandler::HandleMissJIT(const Object& old_data, + const Code& old_code, + const Function& target_function) { + switch (old_data.GetClassId()) { case kArrayCid: // ICData three-element array: Smi(receiver CID), Smi(count), // Function(target). It is the Array from ICData::entries_. @@ -2146,7 +2171,6 @@ void PatchableCallHandler::HandleMiss(const Object& old_data, case kICDataCid: DoICDataMissJIT(ICData::Cast(old_data), old_code, target_function); break; -#endif // defined(DART_PRECOMPILED_RUNTIME) case kMegamorphicCacheCid: ASSERT(old_code.ptr() == StubCode::MegamorphicCall().ptr() || (old_code.IsNull() && !should_consider_patching())); @@ -2156,6 +2180,7 @@ void PatchableCallHandler::HandleMiss(const Object& old_data, UNREACHABLE(); } } +#endif // defined(DART_PRECOMPILED_RUNTIME) static void InlineCacheMissHandler(Thread* thread, Zone* zone,