diff --git a/runtime/vm/code_generator.cc b/runtime/vm/code_generator.cc index ef1ff7ad10e..7cf44df8580 100644 --- a/runtime/vm/code_generator.cc +++ b/runtime/vm/code_generator.cc @@ -729,8 +729,8 @@ DEFINE_RUNTIME_ENTRY(ReThrow, 2) { } -// Patches static call with the target's entry point. Compiles target if -// necessary. +// Patches static call in optimized code with the target's entry point. +// Compiles target if necessary. DEFINE_RUNTIME_ENTRY(PatchStaticCall, 0) { ASSERT(arguments.ArgCount() == kPatchStaticCallRuntimeEntry.argument_count()); DartFrameIterator iterator; @@ -738,6 +738,7 @@ DEFINE_RUNTIME_ENTRY(PatchStaticCall, 0) { ASSERT(caller_frame != NULL); const Code& caller_code = Code::Handle(caller_frame->LookupDartCode()); ASSERT(!caller_code.IsNull()); + ASSERT(caller_code.is_optimized()); const Function& target_function = Function::Handle( caller_code.GetStaticCallTargetFunctionAt(caller_frame->pc())); if (!target_function.HasCode()) { @@ -835,8 +836,10 @@ DEFINE_RUNTIME_ENTRY(BreakpointStaticHandler, 0) { StackFrame* caller_frame = iterator.NextFrame(); ASSERT(caller_frame != NULL); const Code& code = Code::Handle(caller_frame->LookupDartCode()); + ASSERT(!code.is_optimized()); const Function& function = - Function::Handle(code.GetStaticCallTargetFunctionAt(caller_frame->pc())); + Function::Handle(CodePatcher::GetUnoptimizedStaticCallTargetAt( + caller_frame->pc(), code)); if (!function.HasCode()) { const Error& error = Error::Handle(Compiler::CompileFunction(function)); @@ -1403,6 +1406,7 @@ DEFINE_RUNTIME_ENTRY(FixCallersTarget, 0) { } ASSERT(frame->IsDartFrame()); const Code& caller_code = Code::Handle(frame->LookupDartCode()); + ASSERT(caller_code.is_optimized()); const Function& target_function = Function::Handle( caller_code.GetStaticCallTargetFunctionAt(frame->pc())); const Code& target_code = Code::Handle(target_function.CurrentCode()); diff --git a/runtime/vm/code_patcher.h b/runtime/vm/code_patcher.h index 179993e6f5d..e912b446f37 100644 --- a/runtime/vm/code_patcher.h +++ b/runtime/vm/code_patcher.h @@ -17,6 +17,7 @@ class ExternalLabel; class Function; class ICData; class RawArray; +class RawFunction; class RawICData; class String; @@ -55,6 +56,10 @@ class CodePatcher : public AllStatic { const Code& code, ICData* ic_data); + // Return target of an unoptimized static call (calls target via a stub). + static RawFunction* GetUnoptimizedStaticCallTargetAt(uword return_address, + const Code& code); + // Return the arguments descriptor array of the closure call // before the given return address. static RawArray* GetClosureArgDescAt(uword return_address, diff --git a/runtime/vm/code_patcher_arm.cc b/runtime/vm/code_patcher_arm.cc index 1d644108738..3f1a35b8843 100644 --- a/runtime/vm/code_patcher_arm.cc +++ b/runtime/vm/code_patcher_arm.cc @@ -71,6 +71,16 @@ intptr_t CodePatcher::InstanceCallSizeInBytes() { return 0; } + +RawFunction* CodePatcher::GetUnoptimizedStaticCallTargetAt( + uword return_address, const Code& code) { + ASSERT(code.ContainsInstructionAt(return_address)); + CallPattern static_call(return_address, code); + ICData& ic_data = ICData::Handle(); + ic_data ^= static_call.IcData(); + return ic_data.GetTargetAt(0); +} + } // namespace dart #endif // defined TARGET_ARCH_ARM diff --git a/runtime/vm/code_patcher_arm_test.cc b/runtime/vm/code_patcher_arm_test.cc index 152006a331b..a9bfc71b8f3 100644 --- a/runtime/vm/code_patcher_arm_test.cc +++ b/runtime/vm/code_patcher_arm_test.cc @@ -18,30 +18,6 @@ namespace dart { -CODEGEN_TEST_GENERATE(NativePatchStaticCall, test) { - SequenceNode* node_seq = test->node_sequence(); - const String& native_name = - String::ZoneHandle(Symbols::New("TestStaticCallPatching")); - NativeFunction native_function = - reinterpret_cast(TestStaticCallPatching); - test->function().set_is_native(true); - node_seq->Add(new ReturnNode(Scanner::kDummyTokenIndex, - new NativeBodyNode(Scanner::kDummyTokenIndex, - test->function(), - native_name, - native_function))); -} - -CODEGEN_TEST2_GENERATE(PatchStaticCall, function, test) { - SequenceNode* node_seq = test->node_sequence(); - ArgumentListNode* arguments = new ArgumentListNode(Scanner::kDummyTokenIndex); - node_seq->Add(new ReturnNode(Scanner::kDummyTokenIndex, - new StaticCallNode(Scanner::kDummyTokenIndex, - function, arguments))); -} - -CODEGEN_TEST2_RUN(PatchStaticCall, NativePatchStaticCall, Instance::null()); - #define __ assembler-> ASSEMBLER_TEST_GENERATE(IcDataAccess, assembler) { diff --git a/runtime/vm/code_patcher_ia32.cc b/runtime/vm/code_patcher_ia32.cc index 45f8e47afc6..b5dcbf9130a 100644 --- a/runtime/vm/code_patcher_ia32.cc +++ b/runtime/vm/code_patcher_ia32.cc @@ -15,13 +15,13 @@ namespace dart { -// The expected pattern of a dart instance call: +// The expected pattern of a Dart unoptimized call (static and instance): // mov ECX, ic-data -// call target_address +// call target_address (stub) // <- return address -class InstanceCall : public ValueObject { +class UnoptimizedCall : public ValueObject { public: - explicit InstanceCall(uword return_address) + explicit UnoptimizedCall(uword return_address) : start_(return_address - (kNumInstructions * kInstructionSize)) { ASSERT(IsValid(return_address)); ASSERT(kInstructionSize == Assembler::kCallExternalLabelSize); @@ -64,10 +64,42 @@ class InstanceCall : public ValueObject { } uword start_; + DISALLOW_IMPLICIT_CONSTRUCTORS(UnoptimizedCall); +}; + + +class InstanceCall : public UnoptimizedCall { + public: + explicit InstanceCall(uword return_address) + : UnoptimizedCall(return_address) { +#if defined(DEBUG) + ICData& test_ic_data = ICData::Handle(); + test_ic_data ^= ic_data(); + ASSERT(test_ic_data.num_args_tested() > 0); +#endif // DEBUG + } + + private: DISALLOW_IMPLICIT_CONSTRUCTORS(InstanceCall); }; +class UnoptimizedStaticCall : public UnoptimizedCall { + public: + explicit UnoptimizedStaticCall(uword return_address) + : UnoptimizedCall(return_address) { +#if defined(DEBUG) + ICData& test_ic_data = ICData::Handle(); + test_ic_data ^= ic_data(); + ASSERT(test_ic_data.num_args_tested() == 0); +#endif // DEBUG + } + + private: + DISALLOW_IMPLICIT_CONSTRUCTORS(UnoptimizedStaticCall); +}; + + // The expected pattern of a dart static call: // mov EDX, arguments_descriptor_array (optional in polymorphic calls) // call target_address @@ -211,6 +243,16 @@ uword CodePatcher::GetInstanceCallAt(uword return_address, } +RawFunction* CodePatcher::GetUnoptimizedStaticCallTargetAt( + uword return_address, const Code& code) { + ASSERT(code.ContainsInstructionAt(return_address)); + UnoptimizedStaticCall static_call(return_address); + ICData& ic_data = ICData::Handle(); + ic_data ^= static_call.ic_data(); + return ic_data.GetTargetAt(0); +} + + intptr_t CodePatcher::InstanceCallSizeInBytes() { return InstanceCall::kNumInstructions * InstanceCall::kInstructionSize; } diff --git a/runtime/vm/code_patcher_ia32_test.cc b/runtime/vm/code_patcher_ia32_test.cc index f88f7ea5786..bb893d46426 100644 --- a/runtime/vm/code_patcher_ia32_test.cc +++ b/runtime/vm/code_patcher_ia32_test.cc @@ -18,30 +18,6 @@ namespace dart { -CODEGEN_TEST_GENERATE(NativePatchStaticCall, test) { - SequenceNode* node_seq = test->node_sequence(); - const String& native_name = - String::ZoneHandle(Symbols::New("TestStaticCallPatching")); - NativeFunction native_function = - reinterpret_cast(TestStaticCallPatching); - test->function().set_is_native(true); - node_seq->Add(new ReturnNode(Scanner::kDummyTokenIndex, - new NativeBodyNode(Scanner::kDummyTokenIndex, - test->function(), - native_name, - native_function))); -} - -CODEGEN_TEST2_GENERATE(PatchStaticCall, function, test) { - SequenceNode* node_seq = test->node_sequence(); - ArgumentListNode* arguments = new ArgumentListNode(Scanner::kDummyTokenIndex); - node_seq->Add(new ReturnNode(Scanner::kDummyTokenIndex, - new StaticCallNode(Scanner::kDummyTokenIndex, - function, arguments))); -} - -CODEGEN_TEST2_RUN(PatchStaticCall, NativePatchStaticCall, Instance::null()); - #define __ assembler-> ASSEMBLER_TEST_GENERATE(IcDataAccess, assembler) { diff --git a/runtime/vm/code_patcher_mips.cc b/runtime/vm/code_patcher_mips.cc index 67456eefc83..450c0a3d9da 100644 --- a/runtime/vm/code_patcher_mips.cc +++ b/runtime/vm/code_patcher_mips.cc @@ -71,6 +71,16 @@ intptr_t CodePatcher::InstanceCallSizeInBytes() { return 0; } + +RawFunction* CodePatcher::GetUnoptimizedStaticCallTargetAt( + uword return_address, const Code& code) { + ASSERT(code.ContainsInstructionAt(return_address)); + CallPattern static_call(return_address, code); + ICData& ic_data = ICData::Handle(); + ic_data ^= static_call.IcData(); + return ic_data.GetTargetAt(0); +} + } // namespace dart #endif // defined TARGET_ARCH_MIPS diff --git a/runtime/vm/code_patcher_mips_test.cc b/runtime/vm/code_patcher_mips_test.cc index 833cfbf1409..c5b51b56a42 100644 --- a/runtime/vm/code_patcher_mips_test.cc +++ b/runtime/vm/code_patcher_mips_test.cc @@ -18,30 +18,6 @@ namespace dart { -CODEGEN_TEST_GENERATE(NativePatchStaticCall, test) { - SequenceNode* node_seq = test->node_sequence(); - const String& native_name = - String::ZoneHandle(Symbols::New("TestStaticCallPatching")); - NativeFunction native_function = - reinterpret_cast(TestStaticCallPatching); - test->function().set_is_native(true); - node_seq->Add(new ReturnNode(Scanner::kDummyTokenIndex, - new NativeBodyNode(Scanner::kDummyTokenIndex, - test->function(), - native_name, - native_function))); -} - -CODEGEN_TEST2_GENERATE(PatchStaticCall, function, test) { - SequenceNode* node_seq = test->node_sequence(); - ArgumentListNode* arguments = new ArgumentListNode(Scanner::kDummyTokenIndex); - node_seq->Add(new ReturnNode(Scanner::kDummyTokenIndex, - new StaticCallNode(Scanner::kDummyTokenIndex, - function, arguments))); -} - -CODEGEN_TEST2_RUN(PatchStaticCall, NativePatchStaticCall, Instance::null()); - #define __ assembler-> ASSEMBLER_TEST_GENERATE(IcDataAccess, assembler) { diff --git a/runtime/vm/code_patcher_x64.cc b/runtime/vm/code_patcher_x64.cc index 07ebb0ba89a..a5d0e0aec01 100644 --- a/runtime/vm/code_patcher_x64.cc +++ b/runtime/vm/code_patcher_x64.cc @@ -15,15 +15,14 @@ namespace dart { -// A Dart instance call passes the ic-data in RBX. -// The expected pattern of a dart instance call: +// The expected pattern of a Dart unoptimized call (static and instance): // 00: 48 bb imm64 mov RBX, ic-data // 10: 49 bb imm64 mov R11, target_address // 20: 41 ff d3 call R11 // 23 <- return address -class InstanceCall : public ValueObject { +class UnoptimizedCall : public ValueObject { public: - explicit InstanceCall(uword return_address) + explicit UnoptimizedCall(uword return_address) : start_(return_address - kCallPatternSize) { ASSERT(IsValid(return_address)); ASSERT((kCallPatternSize - 10) == Assembler::kCallExternalLabelSize); @@ -56,10 +55,42 @@ class InstanceCall : public ValueObject { private: uword start_; + DISALLOW_IMPLICIT_CONSTRUCTORS(UnoptimizedCall); +}; + + +class InstanceCall : public UnoptimizedCall { + public: + explicit InstanceCall(uword return_address) + : UnoptimizedCall(return_address) { +#if defined(DEBUG) + ICData& test_ic_data = ICData::Handle(); + test_ic_data ^= ic_data(); + ASSERT(test_ic_data.num_args_tested() > 0); +#endif // DEBUG + } + + private: DISALLOW_IMPLICIT_CONSTRUCTORS(InstanceCall); }; +class UnoptimizedStaticCall : public UnoptimizedCall { + public: + explicit UnoptimizedStaticCall(uword return_address) + : UnoptimizedCall(return_address) { +#if defined(DEBUG) + ICData& test_ic_data = ICData::Handle(); + test_ic_data ^= ic_data(); + ASSERT(test_ic_data.num_args_tested() == 0); +#endif // DEBUG + } + + private: + DISALLOW_IMPLICIT_CONSTRUCTORS(UnoptimizedStaticCall); +}; + + // The expected pattern of a dart static call: // mov R10, arguments_descriptor_array (10 bytes) (optional in polym. calls) // mov R11, target_address (10 bytes) @@ -183,6 +214,16 @@ intptr_t CodePatcher::InstanceCallSizeInBytes() { } +RawFunction* CodePatcher::GetUnoptimizedStaticCallTargetAt( + uword return_address, const Code& code) { + ASSERT(code.ContainsInstructionAt(return_address)); + UnoptimizedStaticCall static_call(return_address); + ICData& ic_data = ICData::Handle(); + ic_data ^= static_call.ic_data(); + return ic_data.GetTargetAt(0); +} + + void CodePatcher::InsertCallAt(uword start, uword target) { // The inserted call should not overlap the lazy deopt jump code. ASSERT(start + ShortCallPattern::InstructionLength() <= target); diff --git a/runtime/vm/code_patcher_x64_test.cc b/runtime/vm/code_patcher_x64_test.cc index 41eceb0e424..202c27a3b0b 100644 --- a/runtime/vm/code_patcher_x64_test.cc +++ b/runtime/vm/code_patcher_x64_test.cc @@ -18,30 +18,6 @@ namespace dart { -CODEGEN_TEST_GENERATE(NativePatchStaticCall, test) { - SequenceNode* node_seq = test->node_sequence(); - const String& native_name = - String::ZoneHandle(Symbols::New("TestStaticCallPatching")); - NativeFunction native_function = - reinterpret_cast(TestStaticCallPatching); - test->function().set_is_native(true); - node_seq->Add(new ReturnNode(Scanner::kDummyTokenIndex, - new NativeBodyNode(Scanner::kDummyTokenIndex, - test->function(), - native_name, - native_function))); -} - -CODEGEN_TEST2_GENERATE(PatchStaticCall, function, test) { - SequenceNode* node_seq = test->node_sequence(); - ArgumentListNode* arguments = new ArgumentListNode(Scanner::kDummyTokenIndex); - node_seq->Add(new ReturnNode(Scanner::kDummyTokenIndex, - new StaticCallNode(Scanner::kDummyTokenIndex, - function, arguments))); -} - -CODEGEN_TEST2_RUN(PatchStaticCall, NativePatchStaticCall, Instance::null()); - #define __ assembler-> ASSEMBLER_TEST_GENERATE(IcDataAccess, assembler) { diff --git a/runtime/vm/debugger.cc b/runtime/vm/debugger.cc index 7b3d9898642..bbdb5a09824 100644 --- a/runtime/vm/debugger.cc +++ b/runtime/vm/debugger.cc @@ -1585,8 +1585,9 @@ void Debugger::SignalBpReached() { } else if (bpt->breakpoint_kind_ == PcDescriptors::kFuncCall) { func_to_instrument = bpt->function(); const Code& code = Code::Handle(func_to_instrument.CurrentCode()); - const Function& callee = - Function::Handle(code.GetStaticCallTargetFunctionAt(bpt->pc_)); + ASSERT(!code.is_optimized()); + const Function& callee = Function::Handle( + CodePatcher::GetUnoptimizedStaticCallTargetAt(bpt->pc_, code)); ASSERT(!callee.IsNull()); if (IsDebuggable(callee)) { func_to_instrument = callee.raw(); diff --git a/runtime/vm/flow_graph_compiler.cc b/runtime/vm/flow_graph_compiler.cc index a9d19442d7c..d9b0faf0ead 100644 --- a/runtime/vm/flow_graph_compiler.cc +++ b/runtime/vm/flow_graph_compiler.cc @@ -616,8 +616,13 @@ void FlowGraphCompiler::GenerateStaticCall(intptr_t deopt_id, const Array& arguments_descriptor = Array::ZoneHandle(ArgumentsDescriptor::New(argument_count, argument_names)); - EmitStaticCall(function, arguments_descriptor, argument_count, - deopt_id, token_pos, locs); + if (is_optimizing()) { + EmitStaticCall(function, arguments_descriptor, argument_count, + deopt_id, token_pos, locs); + } else { + EmitUnoptimizedStaticCall(function, arguments_descriptor, argument_count, + deopt_id, token_pos, locs); + } } diff --git a/runtime/vm/flow_graph_compiler.h b/runtime/vm/flow_graph_compiler.h index a1c79fe07b2..c9e1e28d9d0 100644 --- a/runtime/vm/flow_graph_compiler.h +++ b/runtime/vm/flow_graph_compiler.h @@ -486,6 +486,13 @@ class FlowGraphCompiler : public ValueObject { intptr_t token_pos, LocationSummary* locs); + void EmitUnoptimizedStaticCall(const Function& function, + const Array& arguments_descriptor, + intptr_t argument_count, + intptr_t deopt_id, + intptr_t token_pos, + LocationSummary* locs); + // Type checking helper methods. void CheckClassIds(Register class_id_reg, const GrowableArray& class_ids, diff --git a/runtime/vm/flow_graph_compiler_arm.cc b/runtime/vm/flow_graph_compiler_arm.cc index 4df8476dfe3..b470d032eb2 100644 --- a/runtime/vm/flow_graph_compiler_arm.cc +++ b/runtime/vm/flow_graph_compiler_arm.cc @@ -1367,6 +1367,30 @@ void FlowGraphCompiler::EmitStaticCall(const Function& function, } +void FlowGraphCompiler::EmitUnoptimizedStaticCall( + const Function& target_function, + const Array& arguments_descriptor, + intptr_t argument_count, + intptr_t deopt_id, + intptr_t token_pos, + LocationSummary* locs) { + const ICData& ic_data = ICData::ZoneHandle( + ICData::New(parsed_function().function(), // Caller function. + String::Handle(target_function.name()), + arguments_descriptor, + deopt_id, + 0)); // No arguments checked. + ic_data.AddTarget(target_function); + __ LoadObject(R5, ic_data); + GenerateDartCall(deopt_id, + token_pos, + &StubCode::UnoptimizedStaticCallLabel(), + PcDescriptors::kFuncCall, + locs); + __ Drop(argument_count); +} + + void FlowGraphCompiler::EmitEqualityRegConstCompare(Register reg, const Object& obj, bool needs_number_check, diff --git a/runtime/vm/flow_graph_compiler_ia32.cc b/runtime/vm/flow_graph_compiler_ia32.cc index 8908d9b73c4..cc549ec7393 100644 --- a/runtime/vm/flow_graph_compiler_ia32.cc +++ b/runtime/vm/flow_graph_compiler_ia32.cc @@ -1446,6 +1446,30 @@ void FlowGraphCompiler::EmitStaticCall(const Function& function, } +void FlowGraphCompiler::EmitUnoptimizedStaticCall( + const Function& target_function, + const Array& arguments_descriptor, + intptr_t argument_count, + intptr_t deopt_id, + intptr_t token_pos, + LocationSummary* locs) { + const ICData& ic_data = ICData::ZoneHandle( + ICData::New(parsed_function().function(), // Caller function. + String::Handle(target_function.name()), + arguments_descriptor, + deopt_id, + 0)); // No arguments checked. + ic_data.AddTarget(target_function); + __ LoadObject(ECX, ic_data); + GenerateDartCall(deopt_id, + token_pos, + &StubCode::UnoptimizedStaticCallLabel(), + PcDescriptors::kFuncCall, + locs); + __ Drop(argument_count); +} + + void FlowGraphCompiler::EmitEqualityRegConstCompare(Register reg, const Object& obj, bool needs_number_check, diff --git a/runtime/vm/flow_graph_compiler_mips.cc b/runtime/vm/flow_graph_compiler_mips.cc index fae164bc67c..3cdecb2bcc6 100644 --- a/runtime/vm/flow_graph_compiler_mips.cc +++ b/runtime/vm/flow_graph_compiler_mips.cc @@ -1423,6 +1423,30 @@ void FlowGraphCompiler::EmitStaticCall(const Function& function, } +void FlowGraphCompiler::EmitUnoptimizedStaticCall( + const Function& target_function, + const Array& arguments_descriptor, + intptr_t argument_count, + intptr_t deopt_id, + intptr_t token_pos, + LocationSummary* locs) { + const ICData& ic_data = ICData::ZoneHandle( + ICData::New(parsed_function().function(), // Caller function. + String::Handle(target_function.name()), + arguments_descriptor, + deopt_id, + 0)); // No arguments checked. + ic_data.AddTarget(target_function); + __ LoadObject(S5, ic_data); + GenerateDartCall(deopt_id, + token_pos, + &StubCode::UnoptimizedStaticCallLabel(), + PcDescriptors::kFuncCall, + locs); + __ Drop(argument_count); +} + + void FlowGraphCompiler::EmitEqualityRegConstCompare(Register reg, const Object& obj, bool needs_number_check, diff --git a/runtime/vm/flow_graph_compiler_x64.cc b/runtime/vm/flow_graph_compiler_x64.cc index 75d5386ab73..7ea319f2ddb 100644 --- a/runtime/vm/flow_graph_compiler_x64.cc +++ b/runtime/vm/flow_graph_compiler_x64.cc @@ -1441,6 +1441,30 @@ void FlowGraphCompiler::EmitStaticCall(const Function& function, } +void FlowGraphCompiler::EmitUnoptimizedStaticCall( + const Function& target_function, + const Array& arguments_descriptor, + intptr_t argument_count, + intptr_t deopt_id, + intptr_t token_pos, + LocationSummary* locs) { + const ICData& ic_data = ICData::ZoneHandle( + ICData::New(parsed_function().function(), // Caller function. + String::Handle(target_function.name()), + arguments_descriptor, + deopt_id, + 0)); // No arguments checked. + ic_data.AddTarget(target_function); + __ LoadObject(RBX, ic_data); + GenerateDartCall(deopt_id, + token_pos, + &StubCode::UnoptimizedStaticCallLabel(), + PcDescriptors::kFuncCall, + locs); + __ Drop(argument_count); +} + + void FlowGraphCompiler::EmitEqualityRegConstCompare(Register reg, const Object& obj, bool needs_number_check, diff --git a/runtime/vm/native_entry_test.cc b/runtime/vm/native_entry_test.cc index 217c080d725..c8276f162fa 100644 --- a/runtime/vm/native_entry_test.cc +++ b/runtime/vm/native_entry_test.cc @@ -87,25 +87,4 @@ void TestNonNullSmiSum(Dart_NativeArguments args) { Dart_ExitScope(); } - -// Test code patching. -void TestStaticCallPatching(Dart_NativeArguments args) { - Dart_EnterScope(); - DartFrameIterator iterator; - iterator.NextFrame(); // Skip native call. - StackFrame* static_caller_frame = iterator.NextFrame(); - const Code& code = Code::Handle(static_caller_frame->LookupDartCode()); - uword target_address = - CodePatcher::GetStaticCallTargetAt(static_caller_frame->pc(), code); - const Function& target_function = - Function::Handle(code.GetStaticCallTargetFunctionAt( - static_caller_frame->pc())); - EXPECT(String::Handle(target_function.name()). - Equals(String::Handle(String::New("NativePatchStaticCall")))); - const uword function_entry_address = - Code::Handle(target_function.CurrentCode()).EntryPoint(); - EXPECT_EQ(function_entry_address, target_address); - Dart_ExitScope(); -} - } // namespace dart diff --git a/runtime/vm/native_entry_test.h b/runtime/vm/native_entry_test.h index 0d4923aab53..2c345487f70 100644 --- a/runtime/vm/native_entry_test.h +++ b/runtime/vm/native_entry_test.h @@ -12,7 +12,6 @@ namespace dart { void TestSmiSub(Dart_NativeArguments args); void TestSmiSum(Dart_NativeArguments args); void TestNonNullSmiSum(Dart_NativeArguments args); -void TestStaticCallPatching(Dart_NativeArguments args); } // namespace dart diff --git a/runtime/vm/object.cc b/runtime/vm/object.cc index 7373d7f1780..25f0cdc0329 100644 --- a/runtime/vm/object.cc +++ b/runtime/vm/object.cc @@ -8529,6 +8529,25 @@ bool ICData::HasCheck(const GrowableArray& cids) const { #endif // DEBUG +// Used for unoptimized static calls when no class-ids are checked. +void ICData::AddTarget(const Function& target) const { + ASSERT(num_args_tested() == 0); + // Can add only once. + const intptr_t old_num = NumberOfChecks(); + ASSERT(old_num == 0); + Array& data = Array::Handle(ic_data()); + const intptr_t new_len = data.Length() + TestEntryLength(); + data = Array::Grow(data, new_len, Heap::kOld); + set_ic_data(data); + WriteSentinel(data); + intptr_t data_pos = old_num * TestEntryLength(); + ASSERT(!target.IsNull()); + data.SetAt(data_pos++, target); + const Smi& value = Smi::Handle(Smi::New(0)); + data.SetAt(data_pos, value); +} + + void ICData::AddCheck(const GrowableArray& class_ids, const Function& target) const { DEBUG_ASSERT(!HasCheck(class_ids)); @@ -8799,7 +8818,7 @@ RawICData* ICData::New(const Function& function, intptr_t deopt_id, intptr_t num_args_tested) { ASSERT(Object::icdata_class() != Class::null()); - ASSERT(num_args_tested > 0); + ASSERT(num_args_tested >= 0); ICData& result = ICData::Handle(); { // IC data objects are long living objects, allocate them in old generation. diff --git a/runtime/vm/object.h b/runtime/vm/object.h index 92546eaf446..167f5a312b3 100644 --- a/runtime/vm/object.h +++ b/runtime/vm/object.h @@ -3237,6 +3237,9 @@ class ICData : public Object { return OFFSET_OF(RawICData, is_closure_call_); } + // Used for unoptimized static calls when no class-ids are checked. + void AddTarget(const Function& target) const; + // Adding checks. // Adds one more class test to ICData. Length of 'classes' must be equal to diff --git a/runtime/vm/object_test.cc b/runtime/vm/object_test.cc index 087cc70a77d..d545860cbe1 100644 --- a/runtime/vm/object_test.cc +++ b/runtime/vm/object_test.cc @@ -2544,6 +2544,13 @@ TEST_CASE(ICData) { EXPECT_EQ(kSmiCid, test_class_ids[0]); EXPECT_EQ(kSmiCid, test_class_ids[1]); EXPECT_EQ(target1.raw(), test_target.raw()); + + // Check ICData for unoptimized static calls. + const intptr_t kNumArgsChecked = 0; + const ICData& scall_icdata = ICData::Handle( + ICData::New(function, target_name, args_descriptor, 57, kNumArgsChecked)); + scall_icdata.AddTarget(target1); + EXPECT_EQ(target1.raw(), scall_icdata.GetTargetAt(0)); } diff --git a/runtime/vm/raw_object.h b/runtime/vm/raw_object.h index 9b235f28cc3..b29a35b7279 100644 --- a/runtime/vm/raw_object.h +++ b/runtime/vm/raw_object.h @@ -1025,7 +1025,7 @@ class RawICData : public RawObject { RawFunction* function_; // Parent/calling function of this IC. RawString* target_name_; // Name of target function. RawArray* args_descriptor_; // Arguments descriptor. - RawArray* ic_data_; // Contains test class-ids and target functions. + RawArray* ic_data_; // Contains class-ids, target and count. RawObject** to() { return reinterpret_cast(&ptr()->ic_data_); } diff --git a/runtime/vm/stub_code.h b/runtime/vm/stub_code.h index 1a3b791c0d4..d634fe128eb 100644 --- a/runtime/vm/stub_code.h +++ b/runtime/vm/stub_code.h @@ -61,6 +61,7 @@ class RawCode; V(ThreeArgsOptimizedCheckInlineCache) \ V(ClosureCallInlineCache) \ V(MegamorphicCall) \ + V(UnoptimizedStaticCall) \ V(OptimizeFunction) \ V(BreakpointDynamic) \ V(EqualityWithNullArg) \ diff --git a/runtime/vm/stub_code_arm.cc b/runtime/vm/stub_code_arm.cc index 300122de848..6e26bd9bd38 100644 --- a/runtime/vm/stub_code_arm.cc +++ b/runtime/vm/stub_code_arm.cc @@ -1593,29 +1593,95 @@ void StubCode::GenerateMegamorphicCallStub(Assembler* assembler) { } +// Intermediary stub between a static call and its target. ICData contains +// the target function and the call count. +// R5: ICData +void StubCode::GenerateUnoptimizedStaticCallStub(Assembler* assembler) { + GenerateUsageCounterIncrement(assembler, R6); +#if defined(DEBUG) + { Label ok; + // Check that the IC data array has NumberOfArgumentsChecked() == 0. + // 'num_args_tested' is stored as an untagged int. + __ ldr(R6, FieldAddress(R5, ICData::num_args_tested_offset())); + __ CompareImmediate(R6, 0); + __ b(&ok, EQ); + __ Stop("Incorrect IC data for unoptimized static call"); + __ Bind(&ok); + } +#endif // DEBUG + + // R5: IC data object (preserved). + __ ldr(R6, FieldAddress(R5, ICData::ic_data_offset())); + // R6: ic_data_array with entries: target functions and count. + __ AddImmediate(R6, R6, Array::data_offset() - kHeapObjectTag); + // R6: points directly to the first ic data array element. + const intptr_t target_offset = ICData::TargetIndexFor(0) * kWordSize; + const intptr_t count_offset = ICData::CountIndexFor(0) * kWordSize; + + // Increment count for this call. + Label increment_done; + __ LoadFromOffset(kLoadWord, R1, R6, count_offset); + __ adds(R1, R1, ShifterOperand(Smi::RawValue(1))); + __ StoreToOffset(kStoreWord, R1, R6, count_offset); + __ b(&increment_done, VC); // No overflow. + __ LoadImmediate(R1, Smi::RawValue(Smi::kMaxValue)); + __ StoreToOffset(kStoreWord, R1, R6, count_offset); + __ Bind(&increment_done); + + Label target_is_compiled; + // Get function and call it, if possible. + __ LoadFromOffset(kLoadWord, R1, R6, target_offset); + __ ldr(R0, FieldAddress(R1, Function::code_offset())); + __ CompareImmediate(R0, reinterpret_cast(Object::null())); + __ b(&target_is_compiled, NE); + // R1: function. + + __ EnterStubFrame(); + // Preserve target function and IC data object. + __ PushList((1 << R1) | (1 << R5)); + __ Push(R1); // Pass function. + __ CallRuntime(kCompileFunctionRuntimeEntry); + __ Drop(1); // Discard argument. + __ PopList((1 << R1) | (1 << R5)); // Restore function and IC data. + __ LeaveStubFrame(); + // R0: target function. + __ ldr(R0, FieldAddress(R1, Function::code_offset())); + + __ Bind(&target_is_compiled); + // R0: target code. + __ ldr(R0, FieldAddress(R0, Code::instructions_offset())); + __ AddImmediate(R0, Instructions::HeaderSize() - kHeapObjectTag); + // Load arguments descriptor into R4. + __ ldr(R4, FieldAddress(R5, ICData::arguments_descriptor_offset())); + __ bx(R0); +} + + void StubCode::GenerateBreakpointRuntimeStub(Assembler* assembler) { __ Unimplemented("BreakpointRuntime stub"); } // LR: return address (Dart code). -// R4: arguments descriptor array. +// R5: IC data (unoptimized static call). void StubCode::GenerateBreakpointStaticStub(Assembler* assembler) { // Create a stub frame as we are pushing some objects on the stack before // calling into the runtime. __ EnterStubFrame(); __ LoadImmediate(R0, reinterpret_cast(Object::null())); // Preserve arguments descriptor and make room for result. - __ PushList((1 << R0) | (1 << R4)); + __ PushList((1 << R0) | (1 << R5)); __ CallRuntime(kBreakpointStaticHandlerRuntimeEntry); // Pop code object result and restore arguments descriptor. - __ PopList((1 << R0) | (1 << R4)); + __ PopList((1 << R0) | (1 << R5)); __ LeaveStubFrame(); // Now call the static function. The breakpoint handler function // ensures that the call target is compiled. __ ldr(R0, FieldAddress(R0, Code::instructions_offset())); __ AddImmediate(R0, Instructions::HeaderSize() - kHeapObjectTag); + // Load arguments descriptor into R4. + __ ldr(R4, FieldAddress(R5, ICData::arguments_descriptor_offset())); __ bx(R0); } diff --git a/runtime/vm/stub_code_ia32.cc b/runtime/vm/stub_code_ia32.cc index 82659626862..3797ffd1fb4 100644 --- a/runtime/vm/stub_code_ia32.cc +++ b/runtime/vm/stub_code_ia32.cc @@ -1581,7 +1581,7 @@ void StubCode::GenerateNArgsCheckInlineCacheStub(Assembler* assembler, const intptr_t count_offset = ICData::CountIndexFor(num_args) * kWordSize; __ movl(EAX, Address(EBX, target_offset)); __ addl(Address(EBX, count_offset), Immediate(Smi::RawValue(1))); - __ j(NO_OVERFLOW, &call_target_function); + __ j(NO_OVERFLOW, &call_target_function, Assembler::kNearJump); __ movl(Address(EBX, count_offset), Immediate(Smi::RawValue(Smi::kMaxValue))); @@ -1680,6 +1680,66 @@ void StubCode::GenerateMegamorphicCallStub(Assembler* assembler) { GenerateNArgsCheckInlineCacheStub(assembler, 1); } +// Intermediary stub between a static call and its target. ICData contains +// the target function and the call count. +// ECX: ICData +void StubCode::GenerateUnoptimizedStaticCallStub(Assembler* assembler) { + GenerateUsageCounterIncrement(assembler, EBX); +#if defined(DEBUG) + { Label ok; + // Check that the IC data array has NumberOfArgumentsChecked() == 0. + // 'num_args_tested' is stored as an untagged int. + __ movl(EBX, FieldAddress(ECX, ICData::num_args_tested_offset())); + __ cmpl(EBX, Immediate(0)); + __ j(EQUAL, &ok, Assembler::kNearJump); + __ Stop("Incorrect IC data for unoptimized static call"); + __ Bind(&ok); + } +#endif // DEBUG + + // ECX: IC data object (preserved). + __ movl(EBX, FieldAddress(ECX, ICData::ic_data_offset())); + // EBX: ic_data_array with entries: target functions and count. + __ leal(EBX, FieldAddress(EBX, Array::data_offset())); + // EBX: points directly to the first ic data array element. + const intptr_t target_offset = ICData::TargetIndexFor(0) * kWordSize; + const intptr_t count_offset = ICData::CountIndexFor(0) * kWordSize; + + // Increment count for this call. + Label increment_done; + __ addl(Address(EBX, count_offset), Immediate(Smi::RawValue(1))); + __ j(NO_OVERFLOW, &increment_done, Assembler::kNearJump); + __ movl(Address(EBX, count_offset), Immediate(Smi::RawValue(Smi::kMaxValue))); + __ Bind(&increment_done); + + const Immediate& raw_null = + Immediate(reinterpret_cast(Object::null())); + Label target_is_compiled; + // Get function and call it, if possible. + __ movl(EDI, Address(EBX, target_offset)); + __ movl(EAX, FieldAddress(EDI, Function::code_offset())); + __ cmpl(EAX, raw_null); + __ j(NOT_EQUAL, &target_is_compiled, Assembler::kNearJump); + __ EnterStubFrame(); + __ pushl(EDI); // Preserve target function. + __ pushl(ECX); // Preserve IC data object. + __ pushl(EDI); // Pass function. + __ CallRuntime(kCompileFunctionRuntimeEntry); + __ popl(EAX); // Discard argument. + __ popl(ECX); // Restore IC data object. + __ popl(EDI); // Restore target function. + __ LeaveFrame(); + __ movl(EAX, FieldAddress(EDI, Function::code_offset())); + + __ Bind(&target_is_compiled); + // EAX: Target code. + __ movl(EAX, FieldAddress(EAX, Code::instructions_offset())); + __ addl(EAX, Immediate(Instructions::HeaderSize() - kHeapObjectTag)); + // Load arguments descriptor into EDX. + __ movl(EDX, FieldAddress(ECX, ICData::arguments_descriptor_offset())); + __ jmp(EAX); +} + // EDX, EXC: May contain arguments to runtime stub. void StubCode::GenerateBreakpointRuntimeStub(Assembler* assembler) { @@ -1701,21 +1761,23 @@ void StubCode::GenerateBreakpointRuntimeStub(Assembler* assembler) { } -// EDX: Arguments descriptor array. +// ECX: ICData (unoptimized static call). // TOS(0): return address (Dart code). void StubCode::GenerateBreakpointStaticStub(Assembler* assembler) { // Create a stub frame as we are pushing some objects on the stack before // calling into the runtime. __ EnterStubFrame(); - __ pushl(EDX); // Preserve arguments descriptor. + __ pushl(ECX); // Preserve ICData for unoptimized call. const Immediate& raw_null = Immediate(reinterpret_cast(Object::null())); __ pushl(raw_null); // Room for result. __ CallRuntime(kBreakpointStaticHandlerRuntimeEntry); __ popl(EAX); // Code object. - __ popl(EDX); // Restore arguments descriptor. + __ popl(ECX); // Restore ICData. __ LeaveFrame(); + // Load arguments descriptor into EDX. + __ movl(EDX, FieldAddress(ECX, ICData::arguments_descriptor_offset())); // Now call the static function. The breakpoint handler function // ensures that the call target is compiled. // Note that we can't just jump to the CallStatic function stub diff --git a/runtime/vm/stub_code_mips.cc b/runtime/vm/stub_code_mips.cc index f1aef49c385..c9240ad0c7b 100644 --- a/runtime/vm/stub_code_mips.cc +++ b/runtime/vm/stub_code_mips.cc @@ -1798,13 +1798,83 @@ void StubCode::GenerateMegamorphicCallStub(Assembler* assembler) { } +// Intermediary stub between a static call and its target. ICData contains +// the target function and the call count. +// S5: ICData +void StubCode::GenerateUnoptimizedStaticCallStub(Assembler* assembler) { + GenerateUsageCounterIncrement(assembler, T0); + __ TraceSimMsg("UnoptimizedStaticCallStub"); +#if defined(DEBUG) + { Label ok; + // Check that the IC data array has NumberOfArgumentsChecked() == 0. + // 'num_args_tested' is stored as an untagged int. + __ lw(T0, FieldAddress(S5, ICData::num_args_tested_offset())); + __ beq(T0, ZR, &ok); + __ Stop("Incorrect IC data for unoptimized static call"); + __ Bind(&ok); + } +#endif // DEBUG + + // S5: IC data object (preserved). + __ lw(T0, FieldAddress(S5, ICData::ic_data_offset())); + // T0: ic_data_array with entries: target functions and count. + __ AddImmediate(T0, Array::data_offset() - kHeapObjectTag); + // T0: points directly to the first ic data array element. + const intptr_t target_offset = ICData::TargetIndexFor(0) * kWordSize; + const intptr_t count_offset = ICData::CountIndexFor(0) * kWordSize; + + // Increment count for this call. + Label increment_done; + __ lw(T4, Address(T0, count_offset)); + __ AddImmediateDetectOverflow(T4, T4, Smi::RawValue(1), T5, T6); + __ bgez(T5, &increment_done); // No overflow. + __ delay_slot()->sw(T4, Address(T0, count_offset)); + + __ LoadImmediate(T1, Smi::RawValue(Smi::kMaxValue)); + __ sw(T1, Address(T0, count_offset)); + + __ Bind(&increment_done); + + Label target_is_compiled; + // Get function and call it, if possible. + __ lw(T3, Address(T0, target_offset)); + __ lw(T4, FieldAddress(T3, Function::code_offset())); + __ LoadImmediate(TMP, reinterpret_cast(Object::null())); + __ bne(T4, TMP, &target_is_compiled); + + __ EnterStubFrame(); + // Preserve target function and IC data object. + // Two preserved registers, one argument (function) => 3 slots. + __ addiu(SP, SP, Immediate(-3 * kWordSize)); + __ sw(S5, Address(SP, 2 * kWordSize)); // Preserve IC data. + __ sw(T3, Address(SP, 1 * kWordSize)); // Preserve function. + __ sw(T3, Address(SP, 0 * kWordSize)); // Function argument. + __ CallRuntime(kCompileFunctionRuntimeEntry); + __ lw(T3, Address(SP, 1 * kWordSize)); // Restore function. + __ lw(S5, Address(SP, 2 * kWordSize)); // Restore IC data. + __ addiu(SP, SP, Immediate(3 * kWordSize)); + // T3: target function. + __ lw(T4, FieldAddress(T3, Function::code_offset())); + __ LeaveStubFrame(); + + __ Bind(&target_is_compiled); + // T4: target code. + __ lw(T3, FieldAddress(T4, Code::instructions_offset())); + __ AddImmediate(T3, Instructions::HeaderSize() - kHeapObjectTag); + __ jr(T3); + // Load arguments descriptor into S4. + __ delay_slot()-> + lw(S4, FieldAddress(S5, ICData::arguments_descriptor_offset())); +} + + void StubCode::GenerateBreakpointRuntimeStub(Assembler* assembler) { __ Unimplemented("BreakpointRuntime stub"); } // RA: return address (Dart code). -// S4: Arguments descriptor array. +// S5: IC data (unoptimized static call). void StubCode::GenerateBreakpointStaticStub(Assembler* assembler) { __ TraceSimMsg("BreakpointStaticStub"); // Create a stub frame as we are pushing some objects on the stack before @@ -1812,13 +1882,13 @@ void StubCode::GenerateBreakpointStaticStub(Assembler* assembler) { __ EnterStubFrame(); // Preserve arguments descriptor and make room for result. __ addiu(SP, SP, Immediate(-2 * kWordSize)); - __ sw(S4, Address(SP, 1 * kWordSize)); + __ sw(S5, Address(SP, 1 * kWordSize)); __ LoadImmediate(TMP, reinterpret_cast(Object::null())); __ sw(TMP, Address(SP, 0 * kWordSize)); __ CallRuntime(kBreakpointStaticHandlerRuntimeEntry); // Pop code object result and restore arguments descriptor. __ lw(T0, Address(SP, 0 * kWordSize)); - __ lw(S4, Address(SP, 1 * kWordSize)); + __ lw(S5, Address(SP, 1 * kWordSize)); __ addiu(SP, SP, Immediate(2 * kWordSize)); __ LeaveStubFrame(); @@ -1826,6 +1896,8 @@ void StubCode::GenerateBreakpointStaticStub(Assembler* assembler) { // ensures that the call target is compiled. __ lw(T0, FieldAddress(T0, Code::instructions_offset())); __ AddImmediate(T0, Instructions::HeaderSize() - kHeapObjectTag); + // Load arguments descriptor into S4. + __ lw(S4, FieldAddress(S5, ICData::arguments_descriptor_offset())); __ jr(T0); } diff --git a/runtime/vm/stub_code_x64.cc b/runtime/vm/stub_code_x64.cc index 74905ddda3b..97e2bbce326 100644 --- a/runtime/vm/stub_code_x64.cc +++ b/runtime/vm/stub_code_x64.cc @@ -1562,7 +1562,7 @@ void StubCode::GenerateNArgsCheckInlineCacheStub(Assembler* assembler, const intptr_t count_offset = ICData::CountIndexFor(num_args) * kWordSize; __ movq(RAX, Address(R12, target_offset)); __ addq(Address(R12, count_offset), Immediate(Smi::RawValue(1))); - __ j(NO_OVERFLOW, &call_target_function); + __ j(NO_OVERFLOW, &call_target_function, Assembler::kNearJump); __ movq(Address(R12, count_offset), Immediate(Smi::RawValue(Smi::kMaxValue))); @@ -1660,6 +1660,69 @@ void StubCode::GenerateMegamorphicCallStub(Assembler* assembler) { } +// Intermediary stub between a static call and its target. ICData contains +// the target function and the call count. +// RBX: ICData +void StubCode::GenerateUnoptimizedStaticCallStub(Assembler* assembler) { + GenerateUsageCounterIncrement(assembler, RCX); +#if defined(DEBUG) + { Label ok; + // Check that the IC data array has NumberOfArgumentsChecked() == 0. + // 'num_args_tested' is stored as an untagged int. + __ movq(RCX, FieldAddress(RBX, ICData::num_args_tested_offset())); + __ cmpq(RCX, Immediate(0)); + __ j(EQUAL, &ok, Assembler::kNearJump); + __ Stop("Incorrect IC data for unoptimized static call"); + __ Bind(&ok); + } +#endif // DEBUG + + // RBX: IC data object (preserved). + __ movq(R12, FieldAddress(RBX, ICData::ic_data_offset())); + // R12: ic_data_array with entries: target functions and count. + __ leaq(R12, FieldAddress(R12, Array::data_offset())); + // R12: points directly to the first ic data array element. + const intptr_t target_offset = ICData::TargetIndexFor(0) * kWordSize; + const intptr_t count_offset = ICData::CountIndexFor(0) * kWordSize; + + // Increment count for this call. + Label increment_done; + __ addq(Address(R12, count_offset), Immediate(Smi::RawValue(1))); + __ j(NO_OVERFLOW, &increment_done, Assembler::kNearJump); + __ movq(Address(R12, count_offset), + Immediate(Smi::RawValue(Smi::kMaxValue))); + __ Bind(&increment_done); + + const Immediate& raw_null = + Immediate(reinterpret_cast(Object::null())); + Label target_is_compiled; + // Get function and call it, if possible. + __ movq(R13, Address(R12, target_offset)); + __ movq(RAX, FieldAddress(R13, Function::code_offset())); + __ cmpq(RAX, raw_null); + __ j(NOT_EQUAL, &target_is_compiled, Assembler::kNearJump); + + __ EnterStubFrame(); + __ pushq(R13); // Preserve target function. + __ pushq(RBX); // Preserve IC data object. + __ pushq(R13); // Pass function. + __ CallRuntime(kCompileFunctionRuntimeEntry); + __ popq(RAX); // Discard argument. + __ popq(RBX); // Restore IC data object. + __ popq(R13); // Restore target function. + __ LeaveFrame(); + __ movq(RAX, FieldAddress(R13, Function::code_offset())); + + __ Bind(&target_is_compiled); + // RAX: Target code. + __ movq(RAX, FieldAddress(RAX, Code::instructions_offset())); + __ addq(RAX, Immediate(Instructions::HeaderSize() - kHeapObjectTag)); + // Load arguments descriptor into R10. + __ movq(R10, FieldAddress(RBX, ICData::arguments_descriptor_offset())); + __ jmp(RAX); +} + + // RBX, R10: May contain arguments to runtime stub. // TOS(0): return address (Dart code). void StubCode::GenerateBreakpointRuntimeStub(Assembler* assembler) { @@ -1681,18 +1744,21 @@ void StubCode::GenerateBreakpointRuntimeStub(Assembler* assembler) { } +// RBX: ICData (unoptimized static call) // TOS(0): return address (Dart code). void StubCode::GenerateBreakpointStaticStub(Assembler* assembler) { const Immediate& raw_null = Immediate(reinterpret_cast(Object::null())); __ EnterStubFrame(); - __ pushq(R10); // Preserve arguments descriptor. + __ pushq(RBX); // Preserve IC data for unoptimized call. __ pushq(raw_null); // Room for result. __ CallRuntime(kBreakpointStaticHandlerRuntimeEntry); __ popq(RAX); // Code object. - __ popq(R10); // Restore arguments descriptor. + __ popq(RBX); // Restore IC data. __ LeaveFrame(); + // Load arguments descriptor into R10. + __ movq(R10, FieldAddress(RBX, ICData::arguments_descriptor_offset())); // Now call the static function. The breakpoint handler function // ensures that the call target is compiled. __ movq(RBX, FieldAddress(RAX, Code::instructions_offset()));