diff --git a/runtime/vm/compiler/relocation.cc b/runtime/vm/compiler/relocation.cc index e2f44aa1398..428a84f15a4 100644 --- a/runtime/vm/compiler/relocation.cc +++ b/runtime/vm/compiler/relocation.cc @@ -81,13 +81,8 @@ void CodeRelocator::Relocate(bool is_vm_isolate) { auto& current_caller = Code::Handle(zone); auto& call_targets = Array::Handle(zone); - // Do one linear pass over all code objects and determine: - // - // * the maximum instruction size - // * the maximum number of calls - // * the maximum offset into a target instruction - // - FindLargestInstruction(); + auto& next_caller = Code::Handle(zone); + auto& next_caller_targets = Array::Handle(zone); // Emit all instructions and do relocations on the way. for (intptr_t i = 0; i < code_objects_->length(); ++i) { @@ -106,8 +101,14 @@ void CodeRelocator::Relocate(bool is_vm_isolate) { // If we have forward/backwards calls which are almost out-of-range, we'll // create trampolines now. - BuildTrampolinesForAlmostOutOfRangeCalls( - /*force=*/(i == (code_objects_->length() - 1))); + if (i < (code_objects_->length() - 1)) { + next_caller = (*code_objects_)[i + 1]; + next_caller_targets = next_caller.static_calls_target_table(); + } else { + next_caller = Code::null(); + next_caller_targets = Array::null(); + } + BuildTrampolinesForAlmostOutOfRangeCalls(next_caller, next_caller_targets); } // We're guaranteed to have all calls resolved, since @@ -143,40 +144,6 @@ void CodeRelocator::Relocate(bool is_vm_isolate) { // however we might need it to write information into V8 snapshot profile. } -void CodeRelocator::FindLargestInstruction() { - auto zone = thread_->zone(); - auto& current_caller = Code::Handle(zone); - auto& call_targets = Array::Handle(zone); - - for (intptr_t i = 0; i < code_objects_->length(); ++i) { - current_caller = (*code_objects_)[i]; - const intptr_t size = - ImageWriter::SizeInSnapshot(current_caller.instructions()); - if (size > max_instructions_size_) { - max_instructions_size_ = size; - } - - call_targets = current_caller.static_calls_target_table(); - if (!call_targets.IsNull()) { - intptr_t num_calls = 0; - StaticCallsTable calls(call_targets); - for (auto call : calls) { - kind_type_and_offset_ = call.Get(); - const auto kind = - Code::KindField::decode(kind_type_and_offset_.Value()); - if (kind == Code::kCallViaCode) { - continue; - } - num_calls++; - } - - if (num_calls > max_calls_) { - max_calls_ = num_calls; - } - } - } -} - bool CodeRelocator::AddInstructionsToText(CodePtr code) { InstructionsPtr instructions = Code::InstructionsOf(code); @@ -418,11 +385,11 @@ bool CodeRelocator::IsTargetInRangeFor(UnresolvedCall* unresolved_call, const auto forward_distance = target_text_offset - unresolved_call->text_offset; if (unresolved_call->is_tail_call) { - return TailCallDistanceLimits::Lower() < forward_distance && - forward_distance < TailCallDistanceLimits::Upper(); + return TailCallDistanceLimits::Lower() <= forward_distance && + forward_distance <= TailCallDistanceLimits::Upper(); } else { - return CallDistanceLimits::Lower() < forward_distance && - forward_distance < CallDistanceLimits::Upper(); + return CallDistanceLimits::Lower() <= forward_distance && + forward_distance <= CallDistanceLimits::Upper(); } } @@ -478,21 +445,37 @@ CodePtr CodeRelocator::GetTarget(const StaticCallsTableEntry& call) { return destination_.ptr(); } -void CodeRelocator::BuildTrampolinesForAlmostOutOfRangeCalls(bool force) { +void CodeRelocator::BuildTrampolinesForAlmostOutOfRangeCalls( + const Code& next_caller, + const Array& next_caller_targets) { + const bool all_functions_emitted = next_caller.IsNull(); + + uword next_size = 0; + uword next_call_count = 0; + if (!all_functions_emitted) { + next_size = ImageWriter::SizeInSnapshot(next_caller.instructions()); + if (!next_caller_targets.IsNull()) { + StaticCallsTable calls(next_caller_targets); + next_call_count = calls.Length(); + } + } + while (!all_unresolved_calls_.IsEmpty()) { UnresolvedCall* unresolved_call = all_unresolved_calls_.First(); - // If we can emit another instructions object without causing the unresolved - // forward calls to become out-of-range, we'll not resolve it yet (maybe the - // target function will come very soon and we don't need a trampoline at - // all). - const intptr_t future_boundary = - next_text_offset_ + max_instructions_size_ + - kTrampolineSize * - (unresolved_calls_by_destination_.Length() + max_calls_); - if (IsTargetInRangeFor(unresolved_call, future_boundary) && - !FLAG_always_generate_trampolines_for_testing && !force) { - break; + if (!all_functions_emitted) { + // If we can emit another instructions object without causing the + // unresolved forward calls to become out-of-range, we'll not resolve it + // yet (maybe the target function will come very soon and we don't need + // a trampoline at all). + const intptr_t future_boundary = + next_text_offset_ + next_size + + kTrampolineSize * + (unresolved_calls_by_destination_.Length() + next_call_count - 1); + if (IsTargetInRangeFor(unresolved_call, future_boundary) && + !FLAG_always_generate_trampolines_for_testing) { + break; + } } // We have a "critical" [unresolved_call] we have to resolve. If an diff --git a/runtime/vm/compiler/relocation.h b/runtime/vm/compiler/relocation.h index bd9cc15958a..d3bf2947b10 100644 --- a/runtime/vm/compiler/relocation.h +++ b/runtime/vm/compiler/relocation.h @@ -183,7 +183,9 @@ class CodeRelocator : public StackResource { intptr_t destination_text); void ResolveTrampoline(UnresolvedTrampoline* unresolved_trampoline); - void BuildTrampolinesForAlmostOutOfRangeCalls(bool force); + void BuildTrampolinesForAlmostOutOfRangeCalls( + const Code& next_caller, + const Array& next_caller_targets); intptr_t FindDestinationInText(const InstructionsPtr destination, intptr_t offset_into_target); diff --git a/runtime/vm/compiler/relocation_test.cc b/runtime/vm/compiler/relocation_test.cc index 3da5aa7ba54..fa73d3a902b 100644 --- a/runtime/vm/compiler/relocation_test.cc +++ b/runtime/vm/compiler/relocation_test.cc @@ -23,6 +23,20 @@ DECLARE_FLAG(int, lower_pc_relative_call_distance); DECLARE_FLAG(int, upper_pc_relative_call_distance); struct RelocatorTestHelper { + const intptr_t kTrampolineSize = + Utils::RoundUp(PcRelativeTrampolineJumpPattern::kLengthInBytes, + compiler::target::Instructions::kBarePayloadAlignment); + + // The callers on arm/arm64 have to save LR before calling, so the call + // instruction will be 4 byte sinto the instruction stream. +#if defined(TARGET_ARCH_ARM64) + static const intptr_t kOffsetOfCall = 4; +#elif defined(TARGET_ARCH_ARM) + static const intptr_t kOffsetOfCall = 4; +#else + static const intptr_t kOffsetOfCall = 0; +#endif + explicit RelocatorTestHelper(Thread* thread) : thread(thread), locker(thread, thread->isolate_group()->program_lock()), @@ -67,11 +81,6 @@ struct RelocatorTestHelper { EmitCodeFor(code, [&](compiler::Assembler* assembler) { #if defined(TARGET_ARCH_ARM64) - // TODO(kustermann): Remove conservative approximation in relocator and - // make tests precise. - __ mov(R0, R0); - __ mov(R0, R0); - __ mov(R0, R0); SPILLS_RETURN_ADDRESS_FROM_LR_TO_REGISTER( __ stp(LR, R1, compiler::Address(CSP, -2 * kWordSize, @@ -260,7 +269,18 @@ struct RelocatorTestHelper { ISOLATE_UNIT_TEST_CASE(CodeRelocator_DirectForwardCall) { RelocatorTestHelper helper(thread); - helper.CreateInstructions({32, 36, 32}); + const intptr_t fmax = FLAG_upper_pc_relative_call_distance; + + // The gap is 8 bytes smaller than what could be directly forward-called, + // because the relocator's decision when to insert a trampoline is purely + // based on whether unresolved calls can reach such a trampoline if the next + // instruction is emitted (not taking into account that the next instruction + // might actually make some of those unresolved calls resolved). + helper.CreateInstructions({ + 16, // caller (call instruction @helper.kOffsetOfCall) + fmax - (16 - helper.kOffsetOfCall) - 8, // 8 bytes less than maximum gap + 8 // forward call target + }); helper.EmitPcRelativeCallFunction(0, 2); helper.EmitReturn42Function(2); helper.BuildImageAndRunTest( @@ -280,8 +300,13 @@ ISOLATE_UNIT_TEST_CASE(CodeRelocator_DirectForwardCall) { ISOLATE_UNIT_TEST_CASE(CodeRelocator_OutOfRangeForwardCall) { RelocatorTestHelper helper(thread); - helper.CreateInstructions( - {32, FLAG_upper_pc_relative_call_distance - 32 + 4, 32}); + const intptr_t fmax = FLAG_upper_pc_relative_call_distance; + + helper.CreateInstructions({ + 16, // caller (call instruction @helper.kOffsetOfCall) + fmax - (16 - helper.kOffsetOfCall) + 4, // 4 bytes above maximum gap + 8 // forwards call target + }); helper.EmitPcRelativeCallFunction(0, 2); helper.EmitReturn42Function(2); helper.BuildImageAndRunTest([&](const GrowableArray& @@ -305,7 +330,13 @@ ISOLATE_UNIT_TEST_CASE(CodeRelocator_OutOfRangeForwardCall) { ISOLATE_UNIT_TEST_CASE(CodeRelocator_DirectBackwardCall) { RelocatorTestHelper helper(thread); - helper.CreateInstructions({32, 32, 32}); + const intptr_t bmax = -FLAG_lower_pc_relative_call_distance; + + helper.CreateInstructions({ + 8, // backwards call target + bmax - 8 - helper.kOffsetOfCall, // maximize out backwards call range + 16 // caller (call instruction @helper.kOffsetOfCall) + }); helper.EmitReturn42Function(0); helper.EmitPcRelativeCallFunction(2, 0); helper.BuildImageAndRunTest( @@ -325,58 +356,73 @@ ISOLATE_UNIT_TEST_CASE(CodeRelocator_DirectBackwardCall) { ISOLATE_UNIT_TEST_CASE(CodeRelocator_OutOfRangeBackwardCall) { RelocatorTestHelper helper(thread); - helper.CreateInstructions({32, 32, 32, 32 + 4, 32, 32, 32, 32, 32}); + const intptr_t bmax = -FLAG_lower_pc_relative_call_distance; + const intptr_t fmax = FLAG_upper_pc_relative_call_distance; + + helper.CreateInstructions({ + 8, // backward call target + bmax - 8 - helper.kOffsetOfCall + 4, // 4 bytes exceeding backwards range + 16, // caller (call instruction @helper.kOffsetOfCall) + fmax - (16 - helper.kOffsetOfCall) - + 4, // 4 bytes less than forward range + 4, + 4, // out-of-range, so trampoline has to be inserted before this + }); helper.EmitReturn42Function(0); - helper.EmitPcRelativeCallFunction(4, 0); + helper.EmitPcRelativeCallFunction(2, 0); helper.BuildImageAndRunTest([&](const GrowableArray& commands, uword* entry_point) { - EXPECT_EQ(10, commands.length()); + EXPECT_EQ(7, commands.length()); // This is the backwards call target. EXPECT_EQ(ImageWriterCommand::InsertInstructionOfCode, commands[0].op); EXPECT_EQ(ImageWriterCommand::InsertInstructionOfCode, commands[1].op); - EXPECT_EQ(ImageWriterCommand::InsertInstructionOfCode, commands[2].op); - EXPECT_EQ(ImageWriterCommand::InsertInstructionOfCode, commands[3].op); // This makes an out-of-range backwards call. The relocator will make the // call go to a trampoline instead. It will delay insertion of the // trampoline until it almost becomes out-of-range. + EXPECT_EQ(ImageWriterCommand::InsertInstructionOfCode, commands[2].op); + EXPECT_EQ(ImageWriterCommand::InsertInstructionOfCode, commands[3].op); EXPECT_EQ(ImageWriterCommand::InsertInstructionOfCode, commands[4].op); - EXPECT_EQ(ImageWriterCommand::InsertInstructionOfCode, commands[5].op); - EXPECT_EQ(ImageWriterCommand::InsertInstructionOfCode, commands[6].op); // This is the last change the relocator thinks it can ensure the // out-of-range call above can call a trampoline - so it injets it here and // no later. - EXPECT_EQ(ImageWriterCommand::InsertBytesOfTrampoline, commands[7].op); - EXPECT_EQ(ImageWriterCommand::InsertInstructionOfCode, commands[8].op); - EXPECT_EQ(ImageWriterCommand::InsertInstructionOfCode, commands[9].op); + EXPECT_EQ(ImageWriterCommand::InsertBytesOfTrampoline, commands[5].op); + EXPECT_EQ(ImageWriterCommand::InsertInstructionOfCode, commands[6].op); - *entry_point = commands[4].expected_offset; + *entry_point = commands[2].expected_offset; }); } ISOLATE_UNIT_TEST_CASE(CodeRelocator_OutOfRangeBackwardCall2) { RelocatorTestHelper helper(thread); - helper.CreateInstructions({32, 32, 32, 32 + 4, 32}); + const intptr_t bmax = -FLAG_lower_pc_relative_call_distance; + + helper.CreateInstructions({ + 8, // backwards call target + bmax - 8 - helper.kOffsetOfCall + 4, // 4 bytes exceeding backwards range + 16, // caller (call instruction @helper.kOffsetOfCall) + 4, + }); helper.EmitReturn42Function(0); - helper.EmitPcRelativeCallFunction(4, 0); + helper.EmitPcRelativeCallFunction(2, 0); helper.BuildImageAndRunTest( [&](const GrowableArray& commands, uword* entry_point) { - EXPECT_EQ(6, commands.length()); + EXPECT_EQ(5, commands.length()); // This is the backwards call target. EXPECT_EQ(ImageWriterCommand::InsertInstructionOfCode, commands[0].op); EXPECT_EQ(ImageWriterCommand::InsertInstructionOfCode, commands[1].op); - EXPECT_EQ(ImageWriterCommand::InsertInstructionOfCode, commands[2].op); - EXPECT_EQ(ImageWriterCommand::InsertInstructionOfCode, commands[3].op); // This makes an out-of-range backwards call. The relocator will make // the call go to a trampoline instead. It will delay insertion of the - // trampoline until it almost becomes out-of-range. - EXPECT_EQ(ImageWriterCommand::InsertInstructionOfCode, commands[4].op); + // trampoline until it almost becomes out-of-range (or in this case no + // more instructions follow). + EXPECT_EQ(ImageWriterCommand::InsertInstructionOfCode, commands[2].op); + EXPECT_EQ(ImageWriterCommand::InsertInstructionOfCode, commands[3].op); // There's no other instructions coming, so the relocator will resolve // any pending out-of-range calls by inserting trampolines at the end. - EXPECT_EQ(ImageWriterCommand::InsertBytesOfTrampoline, commands[5].op); + EXPECT_EQ(ImageWriterCommand::InsertBytesOfTrampoline, commands[4].op); *entry_point = commands[4].expected_offset; });