From bc16959fc7b27f151bbfcd4944076f9aecbc9089 Mon Sep 17 00:00:00 2001 From: Samir Jindel Date: Wed, 21 Aug 2019 13:33:37 +0000 Subject: [PATCH] Reland "[vm/ffi] Dispatch native callbacks through trampolines if we can't ensure callbacks will always be executable." The original revision is in patchset 1. Three bugs are fixed: 1. Fix SIMARM_X64 build: no need to generate trampolines for AOT or simulated JIT. 2. Hot-reload: Fix hot-reload: don't invalidate Code for force-optimized functions. 3. Windows: Provide shadow space to runtime routines. Cq-Include-Trybots: luci.dart.try:vm-kernel-precomp-mac-debug-simarm_x64-try,vm-kernel-precomp-mac-release-simarm_x64-try,vm-kernel-reload-linux-debug-x64-try,vm-kernel-reload-rollback-linux-debug-x64-try,vm-kernel-win-debug-ia32-try,vm-kernel-win-debug-x64-try,vm-kernel-win-release-ia32-try,vm-kernel-win-release-x64-try Change-Id: I326009cfacb51a84e9de4ddf9ff2d6d415460f91 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/113829 Commit-Queue: Samir Jindel Reviewed-by: Martin Kustermann --- runtime/bin/ffi_test/ffi_test_functions.cc | 37 ++- runtime/lib/ffi.cc | 11 +- runtime/vm/bitfield.h | 10 +- runtime/vm/code_comments.cc | 23 ++ runtime/vm/code_comments.h | 43 ++++ .../vm/compiler/assembler/assembler_arm.cc | 72 ++++-- runtime/vm/compiler/assembler/assembler_arm.h | 10 +- .../vm/compiler/assembler/assembler_arm64.cc | 54 +++-- .../vm/compiler/assembler/assembler_arm64.h | 8 +- .../vm/compiler/assembler/assembler_ia32.cc | 50 ++-- .../vm/compiler/assembler/assembler_ia32.h | 8 +- .../vm/compiler/assembler/assembler_x64.cc | 57 +++-- runtime/vm/compiler/assembler/assembler_x64.h | 7 +- runtime/vm/compiler/assembler/disassembler.cc | 16 +- runtime/vm/compiler/assembler/disassembler.h | 10 +- runtime/vm/compiler/backend/il.cc | 8 +- runtime/vm/compiler/backend/il_arm.cc | 47 ++-- runtime/vm/compiler/backend/il_arm64.cc | 51 ++--- runtime/vm/compiler/backend/il_ia32.cc | 32 +-- runtime/vm/compiler/backend/il_x64.cc | 37 ++- runtime/vm/compiler/backend/locations.h | 22 +- runtime/vm/compiler/ffi.cc | 14 +- runtime/vm/compiler/ffi.h | 4 +- runtime/vm/compiler/runtime_api.cc | 6 +- runtime/vm/compiler/runtime_api.h | 2 +- .../vm/compiler/runtime_offsets_extracted.h | 214 +++++++++--------- runtime/vm/compiler/runtime_offsets_list.h | 1 - runtime/vm/compiler/stub_code_compiler.h | 24 ++ runtime/vm/compiler/stub_code_compiler_arm.cc | 129 +++++++++-- .../vm/compiler/stub_code_compiler_arm64.cc | 140 ++++++++++-- .../vm/compiler/stub_code_compiler_ia32.cc | 101 +++++++-- runtime/vm/compiler/stub_code_compiler_x64.cc | 130 +++++++++-- runtime/vm/constants.h | 16 ++ runtime/vm/constants_arm.h | 2 + runtime/vm/constants_arm64.h | 2 + runtime/vm/constants_ia32.h | 4 + runtime/vm/ffi_callback_trampolines.cc | 93 ++++++++ runtime/vm/ffi_callback_trampolines.h | 74 ++++++ runtime/vm/isolate.cc | 10 +- runtime/vm/isolate.h | 41 ++-- runtime/vm/isolate_reload.cc | 6 +- runtime/vm/object.cc | 38 +--- runtime/vm/runtime_entry.cc | 56 +++-- runtime/vm/runtime_entry.h | 3 +- runtime/vm/runtime_entry_list.h | 1 - runtime/vm/stack_frame_arm.h | 3 + runtime/vm/stack_frame_arm64.h | 3 + runtime/vm/stack_frame_ia32.h | 3 + runtime/vm/stack_frame_x64.h | 3 + runtime/vm/stub_code_list.h | 1 - runtime/vm/thread.cc | 30 ++- runtime/vm/thread.h | 19 +- runtime/vm/vm_sources.gni | 4 + tests/ffi/function_callbacks_test.dart | 24 ++ tests/ffi/function_gc_test.dart | 6 +- 55 files changed, 1362 insertions(+), 458 deletions(-) create mode 100644 runtime/vm/code_comments.cc create mode 100644 runtime/vm/code_comments.h create mode 100644 runtime/vm/ffi_callback_trampolines.cc create mode 100644 runtime/vm/ffi_callback_trampolines.h diff --git a/runtime/bin/ffi_test/ffi_test_functions.cc b/runtime/bin/ffi_test/ffi_test_functions.cc index 4c203a381fd..6e6223ef9ad 100644 --- a/runtime/bin/ffi_test/ffi_test_functions.cc +++ b/runtime/bin/ffi_test/ffi_test_functions.cc @@ -576,29 +576,42 @@ DART_EXPORT void* UnprotectCodeOtherThread(void* isolate, return nullptr; } -DART_EXPORT void* UnprotectCode() { +struct HelperThreadState { std::mutex mutex; std::condition_variable cvar; - std::unique_lock lock(mutex); // locks the mutex - std::thread* helper = new std::thread(UnprotectCodeOtherThread, - Dart_CurrentIsolate(), &cvar, &mutex); + std::unique_ptr helper; +}; - cvar.wait(lock); +DART_EXPORT void* TestUnprotectCode(void (*fn)(void*)) { + HelperThreadState* state = new HelperThreadState; - return helper; + { + std::unique_lock lock(state->mutex); // locks the mutex + state->helper.reset(new std::thread(UnprotectCodeOtherThread, + Dart_CurrentIsolate(), &state->cvar, + &state->mutex)); + + state->cvar.wait(lock); + } + + if (fn != nullptr) { + fn(state); + return nullptr; + } else { + return state; + } } -DART_EXPORT void WaitForHelper(void* helper) { - std::thread* thread = reinterpret_cast(helper); - thread->join(); - delete thread; +DART_EXPORT void WaitForHelper(HelperThreadState* helper) { + helper->helper->join(); + delete helper; } #else // Our version of VSC++ doesn't support std::thread yet. -DART_EXPORT void* UnprotectCode() { +DART_EXPORT void WaitForHelper(void* helper) {} +DART_EXPORT void* TestUnprotectCode(void (*fn)(void)) { return nullptr; } -DART_EXPORT void WaitForHelper(void* helper) {} #endif //////////////////////////////////////////////////////////////////////////////// diff --git a/runtime/lib/ffi.cc b/runtime/lib/ffi.cc index 72d6c09991f..347cd0557e0 100644 --- a/runtime/lib/ffi.cc +++ b/runtime/lib/ffi.cc @@ -436,7 +436,10 @@ static uword CompileNativeCallback(const Function& c_signature, const Function& dart_target, const Instance& exceptional_return) { Thread* const thread = Thread::Current(); - const int32_t callback_id = thread->AllocateFfiCallbackId(); + + uword entry_point = 0; + const int32_t callback_id = thread->AllocateFfiCallbackId(&entry_point); + ASSERT(NativeCallbackTrampolines::Enabled() == (entry_point != 0)); // Create a new Function named 'FfiCallback' and stick it in the 'dart:ffi' // library. Note that these functions will never be invoked by Dart, so it @@ -501,7 +504,11 @@ static uword CompileNativeCallback(const Function& c_signature, thread->SetFfiCallbackCode(callback_id, code); - return code.EntryPoint(); + if (entry_point != 0) { + return entry_point; + } else { + return code.EntryPoint(); + } } #endif diff --git a/runtime/vm/bitfield.h b/runtime/vm/bitfield.h index 7d15b0a0d69..8b417ddfc85 100644 --- a/runtime/vm/bitfield.h +++ b/runtime/vm/bitfield.h @@ -28,18 +28,20 @@ class BitField { } // Returns a S mask of the bit field. - static S mask() { return (kUwordOne << size) - 1; } + static constexpr S mask() { return (kUwordOne << size) - 1; } // Returns a S mask of the bit field which can be applied directly to // to the raw unshifted bits. - static S mask_in_place() { return ((kUwordOne << size) - 1) << position; } + static constexpr S mask_in_place() { + return ((kUwordOne << size) - 1) << position; + } // Returns the shift count needed to right-shift the bit field to // the least-significant bits. - static int shift() { return position; } + static constexpr int shift() { return position; } // Returns the size of the bit field. - static int bitsize() { return size; } + static constexpr int bitsize() { return size; } // Returns an S with the bit field value encoded. static S encode(T value) { diff --git a/runtime/vm/code_comments.cc b/runtime/vm/code_comments.cc new file mode 100644 index 00000000000..b4a8e033855 --- /dev/null +++ b/runtime/vm/code_comments.cc @@ -0,0 +1,23 @@ +// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#include "vm/code_comments.h" + +namespace dart { + +#if !defined(DART_PRECOMPILED_RUNTIME) && !defined(PRODUCT) +const Code::Comments& CreateCommentsFrom(compiler::Assembler* assembler) { + const auto& comments = assembler->comments(); + Code::Comments& result = Code::Comments::New(comments.length()); + + for (intptr_t i = 0; i < comments.length(); i++) { + result.SetPCOffsetAt(i, comments[i]->pc_offset()); + result.SetCommentAt(i, comments[i]->comment()); + } + + return result; +} +#endif // !defined(DART_PRECOMPILED_RUNTIME) && !defined(PRODUCT) + +} // namespace dart diff --git a/runtime/vm/code_comments.h b/runtime/vm/code_comments.h new file mode 100644 index 00000000000..d7d14805f39 --- /dev/null +++ b/runtime/vm/code_comments.h @@ -0,0 +1,43 @@ +// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#ifndef RUNTIME_VM_CODE_COMMENTS_H_ +#define RUNTIME_VM_CODE_COMMENTS_H_ + +#include "vm/code_observers.h" +#include "vm/compiler/assembler/assembler.h" +#include "vm/object.h" + +namespace dart { + +#if !defined(DART_PRECOMPILED_RUNTIME) && !defined(PRODUCT) + +class CodeCommentsWrapper final : public CodeComments { + public: + explicit CodeCommentsWrapper(const Code::Comments& comments) + : comments_(comments), string_(String::Handle()) {} + + intptr_t Length() const override { return comments_.Length(); } + + intptr_t PCOffsetAt(intptr_t i) const override { + return comments_.PCOffsetAt(i); + } + + const char* CommentAt(intptr_t i) const override { + string_ = comments_.CommentAt(i); + return string_.ToCString(); + } + + private: + const Code::Comments& comments_; + String& string_; +}; + +const Code::Comments& CreateCommentsFrom(compiler::Assembler* assembler); + +#endif // !defined(DART_PRECOMPILED_RUNTIME) && !defined(PRODUCT) + +} // namespace dart + +#endif // RUNTIME_VM_CODE_COMMENTS_H_ diff --git a/runtime/vm/compiler/assembler/assembler_arm.cc b/runtime/vm/compiler/assembler/assembler_arm.cc index c6594268a7b..0e6e40c2c2d 100644 --- a/runtime/vm/compiler/assembler/assembler_arm.cc +++ b/runtime/vm/compiler/assembler/assembler_arm.cc @@ -190,6 +190,10 @@ void Assembler::and_(Register rd, Register rn, Operand o, Condition cond) { EmitType01(cond, o.type(), AND, 0, rn, rd, o); } +void Assembler::ands(Register rd, Register rn, Operand o, Condition cond) { + EmitType01(cond, o.type(), AND, 1, rn, rd, o); +} + void Assembler::eor(Register rd, Register rn, Operand o, Condition cond) { EmitType01(cond, o.type(), EOR, 0, rn, rd, o); } @@ -541,20 +545,7 @@ void Assembler::strex(Register rd, Register rt, Register rn, Condition cond) { Emit(encoding); } -void Assembler::TransitionGeneratedToNative(Register destination_address, - Register exit_frame_fp, - Register addr, - Register state) { - // Save exit frame information to enable stack walking. - StoreToOffset(kWord, exit_frame_fp, THR, - target::Thread::top_exit_frame_info_offset()); - - // Mark that the thread is executing native code. - StoreToOffset(kWord, destination_address, THR, - target::Thread::vm_tag_offset()); - LoadImmediate(state, target::Thread::native_execution_state()); - StoreToOffset(kWord, state, THR, target::Thread::execution_state_offset()); - +void Assembler::EnterSafepoint(Register addr, Register state) { if (FLAG_use_slow_path || TargetCPUFeatures::arm_version() == ARMv5TE) { EnterSafepointSlowly(); } else { @@ -585,7 +576,27 @@ void Assembler::EnterSafepointSlowly() { blx(TMP); } -void Assembler::TransitionNativeToGenerated(Register addr, Register state) { +void Assembler::TransitionGeneratedToNative(Register destination_address, + Register exit_frame_fp, + Register addr, + Register state, + bool enter_safepoint) { + // Save exit frame information to enable stack walking. + StoreToOffset(kWord, exit_frame_fp, THR, + target::Thread::top_exit_frame_info_offset()); + + // Mark that the thread is executing native code. + StoreToOffset(kWord, destination_address, THR, + target::Thread::vm_tag_offset()); + LoadImmediate(state, target::Thread::native_execution_state()); + StoreToOffset(kWord, state, THR, target::Thread::execution_state_offset()); + + if (enter_safepoint) { + EnterSafepoint(addr, state); + } +} + +void Assembler::ExitSafepoint(Register addr, Register state) { if (FLAG_use_slow_path || TargetCPUFeatures::arm_version() == ARMv5TE) { ExitSafepointSlowly(); } else { @@ -608,6 +619,31 @@ void Assembler::TransitionNativeToGenerated(Register addr, Register state) { Bind(&done); } +} + +void Assembler::ExitSafepointSlowly() { + ldr(TMP, Address(THR, target::Thread::exit_safepoint_stub_offset())); + ldr(TMP, FieldAddress(TMP, target::Code::entry_point_offset())); + blx(TMP); +} + +void Assembler::TransitionNativeToGenerated(Register addr, + Register state, + bool exit_safepoint) { + if (exit_safepoint) { + ExitSafepoint(addr, state); + } else { +#if defined(DEBUG) + // Ensure we've already left the safepoint. + LoadImmediate(state, 1 << target::Thread::safepoint_state_inside_bit()); + ldr(TMP, Address(THR, target::Thread::safepoint_state_offset())); + ands(TMP, TMP, Operand(state)); // Is-at-safepoint is the LSB. + Label ok; + b(&ok, ZERO); + Breakpoint(); + Bind(&ok); +#endif + } // Mark that the thread is executing Dart code. LoadImmediate(state, target::Thread::vm_tag_compiled_id()); @@ -621,12 +657,6 @@ void Assembler::TransitionNativeToGenerated(Register addr, Register state) { target::Thread::top_exit_frame_info_offset()); } -void Assembler::ExitSafepointSlowly() { - ldr(TMP, Address(THR, target::Thread::exit_safepoint_stub_offset())); - ldr(TMP, FieldAddress(TMP, target::Code::entry_point_offset())); - blx(TMP); -} - void Assembler::clrex() { ASSERT(TargetCPUFeatures::arm_version() != ARMv5TE); int32_t encoding = (kSpecialCondition << kConditionShift) | B26 | B24 | B22 | diff --git a/runtime/vm/compiler/assembler/assembler_arm.h b/runtime/vm/compiler/assembler/assembler_arm.h index e72b34a1581..7a552326983 100644 --- a/runtime/vm/compiler/assembler/assembler_arm.h +++ b/runtime/vm/compiler/assembler/assembler_arm.h @@ -399,6 +399,7 @@ class Assembler : public AssemblerBase { // Data-processing instructions. void and_(Register rd, Register rn, Operand o, Condition cond = AL); + void ands(Register rd, Register rn, Operand o, Condition cond = AL); void eor(Register rd, Register rn, Operand o, Condition cond = AL); @@ -535,8 +536,13 @@ class Assembler : public AssemblerBase { void TransitionGeneratedToNative(Register destination_address, Register exit_frame_fp, Register scratch0, - Register scratch1); - void TransitionNativeToGenerated(Register scratch0, Register scratch1); + Register scratch1, + bool enter_safepoint); + void TransitionNativeToGenerated(Register scratch0, + Register scratch1, + bool exit_safepoint); + void EnterSafepoint(Register scratch0, Register scratch1); + void ExitSafepoint(Register scratch0, Register scratch1); // Miscellaneous instructions. void clrex(); diff --git a/runtime/vm/compiler/assembler/assembler_arm64.cc b/runtime/vm/compiler/assembler/assembler_arm64.cc index 42429b40ddf..63211333f9c 100644 --- a/runtime/vm/compiler/assembler/assembler_arm64.cc +++ b/runtime/vm/compiler/assembler/assembler_arm64.cc @@ -56,6 +56,11 @@ void Assembler::Emit(int32_t value) { buffer_.Emit(value); } +void Assembler::Emit64(int64_t value) { + AssemblerBuffer::EnsureCapacity ensured(&buffer_); + buffer_.Emit(value); +} + int32_t Assembler::BindImm19Branch(int64_t position, int64_t dest) { if (use_far_branches() && !CanEncodeImm19BranchOffset(dest)) { // Far branches are enabled, and we can't encode the branch offset in @@ -1301,21 +1306,10 @@ void Assembler::LeaveDartFrame(RestorePP restore_pp) { LeaveFrame(); } -void Assembler::TransitionGeneratedToNative(Register destination, - Register new_exit_frame, - Register state) { +void Assembler::EnterSafepoint(Register state) { Register addr = TMP2; ASSERT(addr != state); - // Save exit frame information to enable stack walking. - StoreToOffset(new_exit_frame, THR, - target::Thread::top_exit_frame_info_offset()); - - // Mark that the thread is executing native code. - StoreToOffset(destination, THR, target::Thread::vm_tag_offset()); - LoadImmediate(state, target::Thread::native_execution_state()); - StoreToOffset(state, THR, target::Thread::execution_state_offset()); - Label slow_path, done, retry; if (!FLAG_use_slow_path) { movz(addr, Immediate(target::Thread::safepoint_state_offset()), 0); @@ -1339,7 +1333,25 @@ void Assembler::TransitionGeneratedToNative(Register destination, Bind(&done); } -void Assembler::TransitionNativeToGenerated(Register state) { +void Assembler::TransitionGeneratedToNative(Register destination, + Register new_exit_frame, + Register state, + bool enter_safepoint) { + // Save exit frame information to enable stack walking. + StoreToOffset(new_exit_frame, THR, + target::Thread::top_exit_frame_info_offset()); + + // Mark that the thread is executing native code. + StoreToOffset(destination, THR, target::Thread::vm_tag_offset()); + LoadImmediate(state, target::Thread::native_execution_state()); + StoreToOffset(state, THR, target::Thread::execution_state_offset()); + + if (enter_safepoint) { + EnterSafepoint(state); + } +} + +void Assembler::ExitSafepoint(Register state) { Register addr = TMP2; ASSERT(addr != state); @@ -1364,6 +1376,22 @@ void Assembler::TransitionNativeToGenerated(Register state) { blr(addr); Bind(&done); +} + +void Assembler::TransitionNativeToGenerated(Register state, + bool exit_safepoint) { + if (exit_safepoint) { + ExitSafepoint(state); + } else { +#if defined(DEBUG) + // Ensure we've already left the safepoint. + ldr(TMP, Address(THR, target::Thread::safepoint_state_offset())); + Label ok; + tbz(&ok, TMP, target::Thread::safepoint_state_inside_bit()); + Breakpoint(); + Bind(&ok); +#endif + } // Mark that the thread is executing Dart code. LoadImmediate(state, target::Thread::vm_tag_compiled_id()); diff --git a/runtime/vm/compiler/assembler/assembler_arm64.h b/runtime/vm/compiler/assembler/assembler_arm64.h index 749b7966973..e8c565e830f 100644 --- a/runtime/vm/compiler/assembler/assembler_arm64.h +++ b/runtime/vm/compiler/assembler/assembler_arm64.h @@ -513,6 +513,7 @@ class Assembler : public AssemblerBase { // Emit data (e.g encoded instruction or immediate) in instruction stream. void Emit(int32_t value); + void Emit64(int64_t value); // On some other platforms, we draw a distinction between safe and unsafe // smis. @@ -1547,8 +1548,11 @@ class Assembler : public AssemblerBase { void TransitionGeneratedToNative(Register destination_address, Register new_exit_frame, - Register scratch); - void TransitionNativeToGenerated(Register scratch); + Register scratch, + bool enter_safepoint); + void TransitionNativeToGenerated(Register scratch, bool exit_safepoint); + void EnterSafepoint(Register scratch); + void ExitSafepoint(Register scratch); void CheckCodePointer(); void RestoreCodePointer(); diff --git a/runtime/vm/compiler/assembler/assembler_ia32.cc b/runtime/vm/compiler/assembler/assembler_ia32.cc index 84c87b10d08..5834616fb87 100644 --- a/runtime/vm/compiler/assembler/assembler_ia32.cc +++ b/runtime/vm/compiler/assembler/assembler_ia32.cc @@ -2141,18 +2141,7 @@ void Assembler::BranchOnMonomorphicCheckedEntryJIT(Label* label) { } } -void Assembler::TransitionGeneratedToNative(Register destination_address, - Register new_exit_frame, - Register scratch) { - // Save exit frame information to enable stack walking. - movl(Address(THR, target::Thread::top_exit_frame_info_offset()), - new_exit_frame); - - // Mark that the thread is executing native code. - movl(VMTagAddress(), destination_address); - movl(Address(THR, target::Thread::execution_state_offset()), - Immediate(target::Thread::native_execution_state())); - +void Assembler::EnterSafepoint(Register scratch) { // Compare and swap the value at Thread::safepoint_state from unacquired to // acquired. On success, jump to 'success'; otherwise, fallthrough. Label done; @@ -2175,7 +2164,25 @@ void Assembler::TransitionGeneratedToNative(Register destination_address, Bind(&done); } -void Assembler::TransitionNativeToGenerated(Register scratch) { +void Assembler::TransitionGeneratedToNative(Register destination_address, + Register new_exit_frame, + Register scratch, + bool enter_safepoint) { + // Save exit frame information to enable stack walking. + movl(Address(THR, target::Thread::top_exit_frame_info_offset()), + new_exit_frame); + + // Mark that the thread is executing native code. + movl(VMTagAddress(), destination_address); + movl(Address(THR, target::Thread::execution_state_offset()), + Immediate(target::Thread::native_execution_state())); + + if (enter_safepoint) { + EnterSafepoint(scratch); + } +} + +void Assembler::ExitSafepoint(Register scratch) { // Compare and swap the value at Thread::safepoint_state from acquired to // unacquired. On success, jump to 'success'; otherwise, fallthrough. Label done; @@ -2196,6 +2203,23 @@ void Assembler::TransitionNativeToGenerated(Register scratch) { call(scratch); Bind(&done); +} + +void Assembler::TransitionNativeToGenerated(Register scratch, + bool exit_safepoint) { + if (exit_safepoint) { + ExitSafepoint(scratch); + } else { +#if defined(DEBUG) + // Ensure we've already left the safepoint. + movl(scratch, Address(THR, target::Thread::safepoint_state_offset())); + andl(scratch, Immediate(1 << target::Thread::safepoint_state_inside_bit())); + Label ok; + j(ZERO, &ok); + Breakpoint(); + Bind(&ok); +#endif + } // Mark that the thread is executing Dart code. movl(Assembler::VMTagAddress(), diff --git a/runtime/vm/compiler/assembler/assembler_ia32.h b/runtime/vm/compiler/assembler/assembler_ia32.h index d5193ef373a..586b7dd5546 100644 --- a/runtime/vm/compiler/assembler/assembler_ia32.h +++ b/runtime/vm/compiler/assembler/assembler_ia32.h @@ -664,11 +664,13 @@ class Assembler : public AssemblerBase { // Require a temporary register 'tmp'. // Clobber all non-CPU registers (e.g. XMM registers and the "FPU stack"). // However XMM0 is saved for convenience. - void TransitionGeneratedToNative(Register destination_address, Register new_exit_frame, - Register scratch); - void TransitionNativeToGenerated(Register scratch); + Register scratch, + bool enter_safepoint); + void TransitionNativeToGenerated(Register scratch, bool exit_safepoint); + void EnterSafepoint(Register scratch); + void ExitSafepoint(Register scratch); // Create a frame for calling into runtime that preserves all volatile // registers. Frame's RSP is guaranteed to be correctly aligned and diff --git a/runtime/vm/compiler/assembler/assembler_x64.cc b/runtime/vm/compiler/assembler/assembler_x64.cc index 053bff1005e..e5f9e7dc303 100644 --- a/runtime/vm/compiler/assembler/assembler_x64.cc +++ b/runtime/vm/compiler/assembler/assembler_x64.cc @@ -160,16 +160,7 @@ void Assembler::setcc(Condition condition, ByteRegister dst) { EmitUint8(0xC0 + (dst & 0x07)); } -void Assembler::TransitionGeneratedToNative(Register destination_address, - Register new_exit_frame) { - // Save exit frame information to enable stack walking. - movq(Address(THR, target::Thread::top_exit_frame_info_offset()), - new_exit_frame); - - movq(Assembler::VMTagAddress(), destination_address); - movq(Address(THR, target::Thread::execution_state_offset()), - Immediate(target::Thread::native_execution_state())); - +void Assembler::EnterSafepoint() { // Compare and swap the value at Thread::safepoint_state from unacquired to // acquired. If the CAS fails, go to a slow-path stub. Label done; @@ -186,14 +177,32 @@ void Assembler::TransitionGeneratedToNative(Register destination_address, movq(TMP, Address(THR, target::Thread::enter_safepoint_stub_offset())); movq(TMP, FieldAddress(TMP, target::Code::entry_point_offset())); - // Use call instead of CFunctionCall to prevent having to clean up shadow - // space afterwards. This is possible because safepoint stub has no arguments. + + // Use call instead of CallCFunction to avoid having to clean up shadow space + // afterwards. This is possible because the safepoint stub does not use the + // shadow space as scratch and has no arguments. call(TMP); Bind(&done); } -void Assembler::TransitionNativeToGenerated() { +void Assembler::TransitionGeneratedToNative(Register destination_address, + Register new_exit_frame, + bool enter_safepoint) { + // Save exit frame information to enable stack walking. + movq(Address(THR, target::Thread::top_exit_frame_info_offset()), + new_exit_frame); + + movq(Assembler::VMTagAddress(), destination_address); + movq(Address(THR, target::Thread::execution_state_offset()), + Immediate(target::Thread::native_execution_state())); + + if (enter_safepoint) { + EnterSafepoint(); + } +} + +void Assembler::LeaveSafepoint() { // Compare and swap the value at Thread::safepoint_state from acquired to // unacquired. On success, jump to 'success'; otherwise, fallthrough. Label done; @@ -210,11 +219,29 @@ void Assembler::TransitionNativeToGenerated() { movq(TMP, Address(THR, target::Thread::exit_safepoint_stub_offset())); movq(TMP, FieldAddress(TMP, target::Code::entry_point_offset())); - // Use call instead of CFunctionCall to prevent having to clean up shadow - // space afterwards. This is possible because safepoint stub has no arguments. + + // Use call instead of CallCFunction to avoid having to clean up shadow space + // afterwards. This is possible because the safepoint stub does not use the + // shadow space as scratch and has no arguments. call(TMP); Bind(&done); +} + +void Assembler::TransitionNativeToGenerated(bool leave_safepoint) { + if (leave_safepoint) { + LeaveSafepoint(); + } else { +#if defined(DEBUG) + // Ensure we've already left the safepoint. + movq(TMP, Address(THR, target::Thread::safepoint_state_offset())); + andq(TMP, Immediate((1 << target::Thread::safepoint_state_inside_bit()))); + Label ok; + j(ZERO, &ok); + Breakpoint(); + Bind(&ok); +#endif + } movq(Assembler::VMTagAddress(), Immediate(target::Thread::vm_tag_compiled_id())); diff --git a/runtime/vm/compiler/assembler/assembler_x64.h b/runtime/vm/compiler/assembler/assembler_x64.h index 363654007fd..b519aae9d1e 100644 --- a/runtime/vm/compiler/assembler/assembler_x64.h +++ b/runtime/vm/compiler/assembler/assembler_x64.h @@ -306,9 +306,12 @@ class Assembler : public AssemblerBase { void setcc(Condition condition, ByteRegister dst); + void EnterSafepoint(); + void LeaveSafepoint(); void TransitionGeneratedToNative(Register destination_address, - Register new_exit_frame); - void TransitionNativeToGenerated(); + Register new_exit_frame, + bool enter_safepoint); + void TransitionNativeToGenerated(bool leave_safepoint); // Register-register, register-address and address-register instructions. #define RR(width, name, ...) \ diff --git a/runtime/vm/compiler/assembler/disassembler.cc b/runtime/vm/compiler/assembler/disassembler.cc index e39e721c5d2..b7aa53dc2a2 100644 --- a/runtime/vm/compiler/assembler/disassembler.cc +++ b/runtime/vm/compiler/assembler/disassembler.cc @@ -156,9 +156,11 @@ void DisassembleToMemory::Print(const char* format, ...) { void Disassembler::Disassemble(uword start, uword end, DisassemblyFormatter* formatter, - const Code& code) { - const Code::Comments& comments = - code.IsNull() ? Code::Comments::New(0) : code.comments(); + const Code& code, + const Code::Comments* comments) { + if (comments == nullptr) { + comments = code.IsNull() ? &Code::Comments::New(0) : &code.comments(); + } ASSERT(formatter != NULL); char hex_buffer[kHexadecimalBufferSize]; // Instruction in hexadecimal form. char human_buffer[kUserReadableBufferSize]; // Human-readable instruction. @@ -169,14 +171,14 @@ void Disassembler::Disassemble(uword start, while (pc < end) { const intptr_t offset = pc - start; const intptr_t old_comment_finger = comment_finger; - while (comment_finger < comments.Length() && - comments.PCOffsetAt(comment_finger) <= offset) { + while (comment_finger < comments->Length() && + comments->PCOffsetAt(comment_finger) <= offset) { formatter->Print( " ;; %s\n", - String::Handle(comments.CommentAt(comment_finger)).ToCString()); + String::Handle(comments->CommentAt(comment_finger)).ToCString()); comment_finger++; } - if (old_comment_finger != comment_finger) { + if (old_comment_finger != comment_finger && !code.IsNull()) { char str[4000]; BufferFormatter f(str, sizeof(str)); // Comment emitted, emit inlining information. diff --git a/runtime/vm/compiler/assembler/disassembler.h b/runtime/vm/compiler/assembler/disassembler.h index a58339a9f00..e21068667fc 100644 --- a/runtime/vm/compiler/assembler/disassembler.h +++ b/runtime/vm/compiler/assembler/disassembler.h @@ -118,7 +118,8 @@ class Disassembler : public AllStatic { static void Disassemble(uword start, uword end, DisassemblyFormatter* formatter, - const Code& code); + const Code& code, + const Code::Comments* comments = nullptr); static void Disassemble(uword start, uword end, @@ -126,6 +127,13 @@ class Disassembler : public AllStatic { Disassemble(start, end, formatter, Code::Handle()); } + static void Disassemble(uword start, + uword end, + DisassemblyFormatter* formatter, + const Code::Comments* comments) { + Disassemble(start, end, formatter, Code::Handle(), comments); + } + static void Disassemble(uword start, uword end, const Code& code) { #if !defined(PRODUCT) || defined(FORCE_INCLUDE_DISASSEMBLER) DisassembleToStdout stdout_formatter; diff --git a/runtime/vm/compiler/backend/il.cc b/runtime/vm/compiler/backend/il.cc index 8cbf68ac966..31a008f3055 100644 --- a/runtime/vm/compiler/backend/il.cc +++ b/runtime/vm/compiler/backend/il.cc @@ -3981,10 +3981,10 @@ void FunctionEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) { __ Bind(compiler->GetJumpLabel(this)); } -// In the AOT compiler we want to reduce code size, so generate no -// fall-through code in [FlowGraphCompiler::CompileGraph()]. -// (As opposed to here where we don't check for the return value of -// [Intrinsify]). + // In the AOT compiler we want to reduce code size, so generate no + // fall-through code in [FlowGraphCompiler::CompileGraph()]. + // (As opposed to here where we don't check for the return value of + // [Intrinsify]). const Function& function = compiler->parsed_function().function(); if (function.IsDynamicFunction()) { compiler->SpecialStatsBegin(CombinedCodeStatistics::kTagCheckedEntry); diff --git a/runtime/vm/compiler/backend/il_arm.cc b/runtime/vm/compiler/backend/il_arm.cc index ace771a7796..e747e3e2c89 100644 --- a/runtime/vm/compiler/backend/il_arm.cc +++ b/runtime/vm/compiler/backend/il_arm.cc @@ -1038,12 +1038,13 @@ void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { // Update information in the thread object and enter a safepoint. const Register tmp = locs()->temp(1).reg(); if (CanExecuteGeneratedCodeInSafepoint()) { - __ TransitionGeneratedToNative(branch, FPREG, saved_fp, tmp); + __ TransitionGeneratedToNative(branch, FPREG, saved_fp, tmp, + /*enter_safepoint=*/true); __ blx(branch); // Update information in the thread object and leave the safepoint. - __ TransitionNativeToGenerated(saved_fp, tmp); + __ TransitionNativeToGenerated(saved_fp, tmp, /*leave_safepoint=*/true); } else { // We cannot trust that this code will be executable within a safepoint. // Therefore we delegate the responsibility of entering/exiting the @@ -1091,9 +1092,11 @@ void NativeReturnInstr::EmitNativeCode(FlowGraphCompiler* compiler) { __ Pop(vm_tag_reg); - // Reset the exit frame info to - // old_exit_frame_reg *before* entering the safepoint. - __ TransitionGeneratedToNative(vm_tag_reg, old_exit_frame_reg, tmp, tmp1); + // If we were called by a trampoline, it will enter the safepoint on our + // behalf. + __ TransitionGeneratedToNative( + vm_tag_reg, old_exit_frame_reg, tmp, tmp1, + /*enter_safepoint=*/!NativeCallbackTrampolines::Enabled()); __ PopNativeCalleeSavedRegisters(); @@ -1160,18 +1163,20 @@ void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) { __ PushNativeCalleeSavedRegisters(); - // Load the thread object. - // TODO(35765): Fix linking issue on AOT. - // TOOD(35934): Exclude native callbacks from snapshots. + // Load the thread object. If we were called by a trampoline, the thread is + // already loaded. // - // Create another frame to align the frame before continuing in "native" code. - { + // TODO(35765): Fix linking issue on AOT. + if (!NativeCallbackTrampolines::Enabled()) { + // Create another frame to align the frame before continuing in "native" + // code. __ EnterFrame(1 << FP, 0); __ ReserveAlignedFrameSpace(0); + __ LoadImmediate(R0, callback_id_); __ LoadImmediate( - R0, reinterpret_cast(DLRT_GetThreadForNativeCallback)); - __ blx(R0); + R1, reinterpret_cast(DLRT_GetThreadForNativeCallback)); + __ blx(R1); __ mov(THR, compiler::Operand(R0)); __ LeaveFrame(1 << FP); @@ -1189,28 +1194,22 @@ void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) { __ LoadImmediate(R0, 0); __ StoreToOffset(kWord, R0, THR, top_resource_offset); - // Save top exit frame info. Don't set it to 0 yet -- - // TransitionNativeToGenerated will handle that *after* leaving the safepoint. + // Save top exit frame info. Don't set it to 0 yet, + // TransitionNativeToGenerated will handle that. __ LoadFromOffset(kWord, R0, THR, compiler::target::Thread::top_exit_frame_info_offset()); __ Push(R0); __ EmitEntryFrameVerification(R0); - __ TransitionNativeToGenerated(/*scratch0=*/R0, /*scratch1=*/R1); + // Either DLRT_GetThreadForNativeCallback or the callback trampoline (caller) + // will leave the safepoint for us. + __ TransitionNativeToGenerated(/*scratch0=*/R0, /*scratch1=*/R1, + /*exit_safepoint=*/false); // Now that the safepoint has ended, we can touch Dart objects without // handles. - // Otherwise we'll clobber the argument sent from the caller. - ASSERT(CallingConventions::ArgumentRegisters[0] != TMP && - CallingConventions::ArgumentRegisters[0] != TMP2 && - CallingConventions::ArgumentRegisters[0] != R1); - __ LoadImmediate(CallingConventions::ArgumentRegisters[0], callback_id_); - __ LoadFromOffset(kWord, R1, THR, - compiler::target::Thread::verify_callback_entry_offset()); - __ blx(R1); - // Load the code object. __ LoadFromOffset(kWord, R0, THR, compiler::target::Thread::callback_code_offset()); diff --git a/runtime/vm/compiler/backend/il_arm64.cc b/runtime/vm/compiler/backend/il_arm64.cc index 59f3811e429..ced71ddb8e4 100644 --- a/runtime/vm/compiler/backend/il_arm64.cc +++ b/runtime/vm/compiler/backend/il_arm64.cc @@ -914,7 +914,8 @@ void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { if (CanExecuteGeneratedCodeInSafepoint()) { // Update information in the thread object and enter a safepoint. - __ TransitionGeneratedToNative(branch, FPREG, temp); + __ TransitionGeneratedToNative(branch, FPREG, temp, + /*enter_safepoint=*/true); // We are entering runtime code, so the C stack pointer must be restored // from the stack limit to the top of the stack. @@ -926,7 +927,7 @@ void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { __ mov(SP, CSP); // Update information in the thread object and leave the safepoint. - __ TransitionNativeToGenerated(temp); + __ TransitionNativeToGenerated(temp, /*leave_safepoint=*/true); } else { // We cannot trust that this code will be executable within a safepoint. // Therefore we delegate the responsibility of entering/exiting the @@ -977,9 +978,14 @@ void NativeReturnInstr::EmitNativeCode(FlowGraphCompiler* compiler) { __ Pop(vm_tag_reg); - // Reset the exit frame info to - // old_exit_frame_reg *before* entering the safepoint. - __ TransitionGeneratedToNative(vm_tag_reg, old_exit_frame_reg, tmp); + // Reset the exit frame info to old_exit_frame_reg *before* entering the + // safepoint. + // + // If we were called by a trampoline, it will enter the safepoint on our + // behalf. + __ TransitionGeneratedToNative( + vm_tag_reg, old_exit_frame_reg, tmp, + /*enter_safepoint=*/!NativeCallbackTrampolines::Enabled()); __ PopNativeCalleeSavedRegisters(); @@ -1044,18 +1050,20 @@ void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) { __ PushNativeCalleeSavedRegisters(); - // Load the thread object. - // TODO(35765): Fix linking issue on AOT. - // TOOD(35934): Exclude native callbacks from snapshots. + // Load the thread object. If we were called by a trampoline, the thread is + // already loaded. // - // Create another frame to align the frame before continuing in "native" code. - { + // TODO(35765): Fix linking issue on AOT. + if (!NativeCallbackTrampolines::Enabled()) { + // Create another frame to align the frame before continuing in "native" + // code. __ EnterFrame(0); __ ReserveAlignedFrameSpace(0); + __ LoadImmediate(R0, callback_id_); __ LoadImmediate( - R0, reinterpret_cast(DLRT_GetThreadForNativeCallback)); - __ blr(R0); + R1, reinterpret_cast(DLRT_GetThreadForNativeCallback)); + __ blr(R1); __ mov(THR, R0); __ LeaveFrame(); @@ -1075,8 +1083,8 @@ void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) { __ Push(R0); __ StoreToOffset(ZR, THR, compiler::target::Thread::top_resource_offset()); - // Save the top exit frame info. We don't set it to 0 yet in Thread because we - // need to leave the safepoint first. + // Save the top exit frame info. We don't set it to 0 yet: + // TransitionNativeToGenerated will handle that. __ LoadFromOffset(R0, THR, compiler::target::Thread::top_exit_frame_info_offset()); __ Push(R0); @@ -1085,22 +1093,13 @@ void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) { // correct offset from FP. __ EmitEntryFrameVerification(); - // TransitionNativeToGenerated will reset top exit frame info to 0 *after* - // leaving the safepoint. - __ TransitionNativeToGenerated(R0); + // Either DLRT_GetThreadForNativeCallback or the callback trampoline (caller) + // will leave the safepoint for us. + __ TransitionNativeToGenerated(R0, /*exit_safepoint=*/false); // Now that the safepoint has ended, we can touch Dart objects without // handles. - // Otherwise we'll clobber the argument sent from the caller. - ASSERT(CallingConventions::ArgumentRegisters[0] != TMP && - CallingConventions::ArgumentRegisters[0] != TMP2 && - CallingConventions::ArgumentRegisters[0] != R1); - __ LoadImmediate(CallingConventions::ArgumentRegisters[0], callback_id_); - __ LoadFromOffset(R1, THR, - compiler::target::Thread::verify_callback_entry_offset()); - __ blr(R1); - // Load the code object. __ LoadFromOffset(R0, THR, compiler::target::Thread::callback_code_offset()); __ LoadFieldFromOffset(R0, R0, diff --git a/runtime/vm/compiler/backend/il_ia32.cc b/runtime/vm/compiler/backend/il_ia32.cc index 487ec60495f..1729f425d4d 100644 --- a/runtime/vm/compiler/backend/il_ia32.cc +++ b/runtime/vm/compiler/backend/il_ia32.cc @@ -174,7 +174,12 @@ void NativeReturnInstr::EmitNativeCode(FlowGraphCompiler* compiler) { // This will reset the exit frame info to old_exit_frame_reg *before* entering // the safepoint. - __ TransitionGeneratedToNative(vm_tag_reg, old_exit_frame_reg, tmp); + // + // If we were called by a trampoline, it will enter the safepoint on our + // behalf. + __ TransitionGeneratedToNative( + vm_tag_reg, old_exit_frame_reg, tmp, + /*enter_safepoint=*/!NativeCallbackTrampolines::Enabled()); // Move XMM0 into ST0 if needed. if (return_in_st0) { @@ -954,9 +959,10 @@ void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { __ movl(compiler::Address(FPREG, kSavedCallerPcSlotFromFp * kWordSize), tmp); if (CanExecuteGeneratedCodeInSafepoint()) { - __ TransitionGeneratedToNative(branch, FPREG, tmp); + __ TransitionGeneratedToNative(branch, FPREG, tmp, + /*enter_safepoint=*/true); __ call(branch); - __ TransitionNativeToGenerated(tmp); + __ TransitionNativeToGenerated(tmp, /*leave_safepoint=*/true); } else { // We cannot trust that this code will be executable within a safepoint. // Therefore we delegate the responsibility of entering/exiting the @@ -1030,13 +1036,14 @@ void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) { __ pushl(EDI); // Load the thread object. - // TOOD(35934): Exclude native callbacks from snapshots. // Linking in AOT is not relevant here since we don't support AOT for IA32. // Create another frame to align the frame before continuing in "native" code. - { + // If we were called by a trampoline, it has already loaded the thread. + if (!NativeCallbackTrampolines::Enabled()) { __ EnterFrame(0); - __ ReserveAlignedFrameSpace(0); + __ ReserveAlignedFrameSpace(compiler::target::kWordSize); + __ movl(compiler::Address(SPREG, 0), compiler::Immediate(callback_id_)); __ movl(EAX, compiler::Immediate(reinterpret_cast( DLRT_GetThreadForNativeCallback))); __ call(EAX); @@ -1064,18 +1071,11 @@ void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) { // correct offset from FP. __ EmitEntryFrameVerification(); - // TransitionNativeToGenerated will reset top exit frame info to 0 *after* - // leaving the safepoint. - __ TransitionNativeToGenerated(EAX); + // Either DLRT_GetThreadForNativeCallback or the callback trampoline (caller) + // will leave the safepoint for us. + __ TransitionNativeToGenerated(EAX, /*exit_safepoint=*/false); // Now that the safepoint has ended, we can hold Dart objects with bare hands. - // TODO(35934): fix linking issue - __ pushl(compiler::Immediate(callback_id_)); - __ movl(EAX, - compiler::Address( - THR, compiler::target::Thread::verify_callback_entry_offset())); - __ call(EAX); - __ popl(EAX); // Load the code object. __ movl(EAX, compiler::Address( diff --git a/runtime/vm/compiler/backend/il_x64.cc b/runtime/vm/compiler/backend/il_x64.cc index 598ff98461f..d65b9844dc2 100644 --- a/runtime/vm/compiler/backend/il_x64.cc +++ b/runtime/vm/compiler/backend/il_x64.cc @@ -160,9 +160,11 @@ void NativeReturnInstr::EmitNativeCode(FlowGraphCompiler* compiler) { __ popq(vm_tag_reg); - // TransitionGeneratedToNative will reset the exit frame info to - // old_exit_frame_reg *before* entering the safepoint. - __ TransitionGeneratedToNative(vm_tag_reg, old_exit_frame_reg); + // If we were called by a trampoline, it will enter the safepoint on our + // behalf. + __ TransitionGeneratedToNative( + vm_tag_reg, old_exit_frame_reg, + /*enter_safepoint=*/!NativeCallbackTrampolines::Enabled()); // Restore C++ ABI callee-saved registers. __ PopRegisters(CallingConventions::kCalleeSaveCpuRegisters, @@ -967,12 +969,13 @@ void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) { if (CanExecuteGeneratedCodeInSafepoint()) { // Update information in the thread object and enter a safepoint. - __ TransitionGeneratedToNative(target_address, FPREG); + __ TransitionGeneratedToNative(target_address, FPREG, + /*enter_safepoint=*/true); __ CallCFunction(target_address); // Update information in the thread object and leave the safepoint. - __ TransitionNativeToGenerated(); + __ TransitionNativeToGenerated(/*leave_safepoint=*/true); } else { // We cannot trust that this code will be executable within a safepoint. // Therefore we delegate the responsibility of entering/exiting the @@ -1054,16 +1057,18 @@ void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) { // Load the thread object. // TODO(35765): Fix linking issue on AOT. - // TOOD(35934): Exclude native callbacks from snapshots. // // Create another frame to align the frame before continuing in "native" code. - { + // If we were called by a trampoline, it has already loaded the thread. + if (!NativeCallbackTrampolines::Enabled()) { __ EnterFrame(0); __ ReserveAlignedFrameSpace(0); + COMPILE_ASSERT(RAX != CallingConventions::kArg1Reg); + __ movq(CallingConventions::kArg1Reg, compiler::Immediate(callback_id_)); __ movq(RAX, compiler::Immediate(reinterpret_cast( DLRT_GetThreadForNativeCallback))); - __ call(RAX); + __ CallCFunction(RAX); __ movq(THR, RAX); __ LeaveFrame(); @@ -1088,19 +1093,9 @@ void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) { // correct offset from FP. __ EmitEntryFrameVerification(); - // TransitionNativeToGenerated will reset top exit frame info to 0 *after* - // leaving the safepoint. - __ TransitionNativeToGenerated(); - - // Now that the safepoint has ended, we can touch Dart objects without - // handles. - // Otherwise we'll clobber the argument sent from the caller. - COMPILE_ASSERT(RAX != CallingConventions::kArg1Reg); - __ movq(CallingConventions::kArg1Reg, compiler::Immediate(callback_id_)); - __ movq(RAX, - compiler::Address( - THR, compiler::target::Thread::verify_callback_entry_offset())); - __ call(RAX); + // Either DLRT_GetThreadForNativeCallback or the callback trampoline (caller) + // will leave the safepoint for us. + __ TransitionNativeToGenerated(/*exit_safepoint=*/false); // Load the code object. __ movq(RAX, compiler::Address( diff --git a/runtime/vm/compiler/backend/locations.h b/runtime/vm/compiler/backend/locations.h index 198e9367395..638fc842a3b 100644 --- a/runtime/vm/compiler/backend/locations.h +++ b/runtime/vm/compiler/backend/locations.h @@ -608,6 +608,26 @@ class RegisterSet : public ValueObject { #endif } +#if !defined(TARGET_ARCH_DBC) + void AddAllArgumentRegisters() { + // All (native) arguments are passed on the stack in IA32. +#if !defined(TARGET_ARCH_IA32) + for (intptr_t i = 0; i < kNumberOfCpuRegisters; ++i) { + const Register reg = static_cast(i); + if (IsArgumentRegister(reg)) { + Add(Location::RegisterLocation(reg)); + } + } + for (intptr_t i = 0; i < kNumberOfFpuRegisters; ++i) { + const FpuRegister reg = static_cast(i); + if (IsFpuArgumentRegister(reg)) { + Add(Location::FpuRegisterLocation(reg)); + } + } +#endif + } +#endif + void Add(Location loc, Representation rep = kTagged) { if (loc.IsRegister()) { cpu_registers_.Add(loc.reg()); @@ -840,7 +860,7 @@ class FrameRebase : public ValueObject { FrameRebase(Register old_base, Register new_base, intptr_t stack_delta) : old_base_(old_base), new_base_(new_base), stack_delta_(stack_delta) {} - Location Rebase(Location loc) { + Location Rebase(Location loc) const { if (loc.IsPairLocation()) { return Location::Pair(Rebase(loc.Component(0)), Rebase(loc.Component(1))); } diff --git a/runtime/vm/compiler/ffi.cc b/runtime/vm/compiler/ffi.cc index 030aeea7cd3..5313922c2e7 100644 --- a/runtime/vm/compiler/ffi.cc +++ b/runtime/vm/compiler/ffi.cc @@ -9,6 +9,7 @@ #include "platform/globals.h" #include "vm/compiler/backend/locations.h" #include "vm/compiler/runtime_api.h" +#include "vm/compiler/stub_code_compiler.h" #include "vm/growable_array.h" #include "vm/object_store.h" #include "vm/stack_frame.h" @@ -354,6 +355,7 @@ class ArgumentAllocator : public ValueObject { intptr_t stack_height_in_slots = 0; }; +#if !defined(TARGET_ARCH_DBC) ZoneGrowableArray* CallbackArgumentTranslator::TranslateArgumentLocations( const ZoneGrowableArray& arg_locs) { @@ -398,10 +400,17 @@ Location CallbackArgumentTranslator::TranslateArgument(Location arg) { // saved argument registers and stack arguments. Also add slots for the // shadow space if present (factored into // kCallbackSlotsBeforeSavedArguments). + // + // Finally, if we are using NativeCallbackTrampolines, factor in the extra + // stack space corresponding to those trampolines' frames (above the entry + // frame). + intptr_t stack_delta = kCallbackSlotsBeforeSavedArguments; + if (NativeCallbackTrampolines::Enabled()) { + stack_delta += StubCodeCompiler::kNativeCallbackTrampolineStackDelta; + } FrameRebase rebase( /*old_base=*/SPREG, /*new_base=*/SPREG, - /*stack_delta=*/argument_slots_required_ + - kCallbackSlotsBeforeSavedArguments); + /*stack_delta=*/argument_slots_required_ + stack_delta); return rebase.Rebase(arg); } @@ -415,6 +424,7 @@ Location CallbackArgumentTranslator::TranslateArgument(Location arg) { argument_slots_used_ += 8 / target::kWordSize; return result; } +#endif // !defined(TARGET_ARCH_DBC) // Takes a list of argument representations, and converts it to a list of // argument locations based on calling convention. diff --git a/runtime/vm/compiler/ffi.h b/runtime/vm/compiler/ffi.h index 0e5ff05e3e8..5c743d4dff6 100644 --- a/runtime/vm/compiler/ffi.h +++ b/runtime/vm/compiler/ffi.h @@ -119,7 +119,7 @@ class FfiSignatureDescriptor : public ValueObject { static const intptr_t kOffsetArgumentLocations = 3; }; -#endif // defined(TARGET_ARCH_DBC) +#else // defined(TARGET_ARCH_DBC) // This classes translates the ABI location of arguments into the locations they // will inhabit after entry-frame setup in the invocation of a native callback. @@ -146,6 +146,8 @@ class CallbackArgumentTranslator : public ValueObject { intptr_t argument_slots_required_ = 0; }; +#endif // defined(TARGET_ARCH_DBC) + bool IsAsFunctionInternal(Zone* zone, Isolate* isolate, const Function& func); } // namespace ffi diff --git a/runtime/vm/compiler/runtime_api.cc b/runtime/vm/compiler/runtime_api.cc index 22e9f07d3e2..1d651dcece6 100644 --- a/runtime/vm/compiler/runtime_api.cc +++ b/runtime/vm/compiler/runtime_api.cc @@ -291,7 +291,6 @@ bool RawObject::IsTypedDataClassId(intptr_t cid) { return dart::RawObject::IsTypedDataClassId(cid); } - const word Class::kNoTypeArguments = dart::Class::kNoTypeArguments; classid_t Class::GetId(const dart::Class& handle) { @@ -528,6 +527,11 @@ uword Thread::safepoint_state_acquired() { return dart::Thread::safepoint_state_acquired(); } +intptr_t Thread::safepoint_state_inside_bit() { + COMPILE_ASSERT(dart::Thread::AtSafepointField::bitsize() == 1); + return dart::Thread::AtSafepointField::shift(); +} + uword Thread::generated_execution_state() { return dart::Thread::ExecutionState::kThreadInGenerated; } diff --git a/runtime/vm/compiler/runtime_api.h b/runtime/vm/compiler/runtime_api.h index 8e85af1fd4b..cb38030c69c 100644 --- a/runtime/vm/compiler/runtime_api.h +++ b/runtime/vm/compiler/runtime_api.h @@ -651,13 +651,13 @@ class Thread : public AllStatic { static word write_barrier_wrappers_thread_offset(Register regno); static word array_write_barrier_entry_point_offset(); static word write_barrier_entry_point_offset(); - static word verify_callback_entry_offset(); static word vm_tag_offset(); static uword vm_tag_compiled_id(); static word safepoint_state_offset(); static uword safepoint_state_unacquired(); static uword safepoint_state_acquired(); + static intptr_t safepoint_state_inside_bit(); static word execution_state_offset(); static uword vm_execution_state(); diff --git a/runtime/vm/compiler/runtime_offsets_extracted.h b/runtime/vm/compiler/runtime_offsets_extracted.h index 507f2d96cf0..021c6ddd51b 100644 --- a/runtime/vm/compiler/runtime_offsets_extracted.h +++ b/runtime/vm/compiler/runtime_offsets_extracted.h @@ -191,11 +191,11 @@ static constexpr dart::compiler::target::word String_hash_offset = 8; static constexpr dart::compiler::target::word String_length_offset = 4; static constexpr dart::compiler::target::word SubtypeTestCache_cache_offset = 4; static constexpr dart::compiler::target::word - Thread_AllocateArray_entry_point_offset = 288; + Thread_AllocateArray_entry_point_offset = 284; static constexpr dart::compiler::target::word Thread_active_exception_offset = - 624; + 616; static constexpr dart::compiler::target::word Thread_active_stacktrace_offset = - 628; + 620; static constexpr dart::compiler::target::word Thread_array_write_barrier_code_offset = 112; static constexpr dart::compiler::target::word @@ -203,14 +203,14 @@ static constexpr dart::compiler::target::word static constexpr dart::compiler::target::word Thread_async_stack_trace_offset = 84; static constexpr dart::compiler::target::word - Thread_auto_scope_native_wrapper_entry_point_offset = 248; + Thread_auto_scope_native_wrapper_entry_point_offset = 244; static constexpr dart::compiler::target::word Thread_bool_false_offset = 104; static constexpr dart::compiler::target::word Thread_bool_true_offset = 100; static constexpr dart::compiler::target::word Thread_call_to_runtime_entry_point_offset = 200; static constexpr dart::compiler::target::word Thread_call_to_runtime_stub_offset = 132; -static constexpr dart::compiler::target::word Thread_dart_stream_offset = 656; +static constexpr dart::compiler::target::word Thread_dart_stream_offset = 648; static constexpr dart::compiler::target::word Thread_optimize_entry_offset = 228; static constexpr dart::compiler::target::word Thread_optimize_stub_offset = 156; @@ -219,36 +219,36 @@ static constexpr dart::compiler::target::word Thread_deoptimize_entry_offset = static constexpr dart::compiler::target::word Thread_deoptimize_stub_offset = 160; static constexpr dart::compiler::target::word Thread_double_abs_address_offset = - 268; + 264; static constexpr dart::compiler::target::word - Thread_double_negate_address_offset = 264; + Thread_double_negate_address_offset = 260; static constexpr dart::compiler::target::word Thread_end_offset = 60; static constexpr dart::compiler::target::word Thread_enter_safepoint_stub_offset = 180; static constexpr dart::compiler::target::word Thread_execution_state_offset = - 640; + 632; static constexpr dart::compiler::target::word Thread_exit_safepoint_stub_offset = 184; static constexpr dart::compiler::target::word Thread_call_native_through_safepoint_stub_offset = 188; static constexpr dart::compiler::target::word - Thread_call_native_through_safepoint_entry_point_offset = 240; + Thread_call_native_through_safepoint_entry_point_offset = 236; static constexpr dart::compiler::target::word Thread_fix_allocation_stub_code_offset = 120; static constexpr dart::compiler::target::word Thread_fix_callers_target_code_offset = 116; static constexpr dart::compiler::target::word - Thread_float_absolute_address_offset = 280; + Thread_float_absolute_address_offset = 276; static constexpr dart::compiler::target::word - Thread_float_negate_address_offset = 276; + Thread_float_negate_address_offset = 272; static constexpr dart::compiler::target::word Thread_float_not_address_offset = - 272; + 268; static constexpr dart::compiler::target::word - Thread_float_zerow_address_offset = 284; + Thread_float_zerow_address_offset = 280; static constexpr dart::compiler::target::word Thread_global_object_pool_offset = - 632; + 624; static constexpr dart::compiler::target::word - Thread_interpret_call_entry_point_offset = 252; + Thread_interpret_call_entry_point_offset = 248; static constexpr dart::compiler::target::word Thread_invoke_dart_code_from_bytecode_stub_offset = 128; static constexpr dart::compiler::target::word @@ -269,7 +269,7 @@ static constexpr dart::compiler::target::word static constexpr dart::compiler::target::word Thread_monomorphic_miss_stub_offset = 152; static constexpr dart::compiler::target::word - Thread_no_scope_native_wrapper_entry_point_offset = 244; + Thread_no_scope_native_wrapper_entry_point_offset = 240; static constexpr dart::compiler::target::word Thread_null_error_shared_with_fpu_regs_entry_point_offset = 208; static constexpr dart::compiler::target::word @@ -280,10 +280,10 @@ static constexpr dart::compiler::target::word Thread_null_error_shared_without_fpu_regs_stub_offset = 136; static constexpr dart::compiler::target::word Thread_object_null_offset = 96; static constexpr dart::compiler::target::word - Thread_predefined_symbols_address_offset = 256; -static constexpr dart::compiler::target::word Thread_resume_pc_offset = 636; + Thread_predefined_symbols_address_offset = 252; +static constexpr dart::compiler::target::word Thread_resume_pc_offset = 628; static constexpr dart::compiler::target::word Thread_safepoint_state_offset = - 644; + 636; static constexpr dart::compiler::target::word Thread_slow_type_test_stub_offset = 172; static constexpr dart::compiler::target::word Thread_stack_limit_offset = 36; @@ -312,9 +312,7 @@ static constexpr dart::compiler::target::word Thread_write_barrier_entry_point_offset = 192; static constexpr dart::compiler::target::word Thread_write_barrier_mask_offset = 44; -static constexpr dart::compiler::target::word - Thread_verify_callback_entry_offset = 236; -static constexpr dart::compiler::target::word Thread_callback_code_offset = 648; +static constexpr dart::compiler::target::word Thread_callback_code_offset = 640; static constexpr dart::compiler::target::word TimelineStream_enabled_offset = 8; static constexpr dart::compiler::target::word TwoByteString_data_offset = 12; static constexpr dart::compiler::target::word Type_arguments_offset = 16; @@ -349,8 +347,8 @@ static constexpr dart::compiler::target::word Code_function_entry_point_offset[] = {4, 8}; static constexpr dart::compiler::target::word Thread_write_barrier_wrappers_thread_offset[] = { - 588, 592, 596, 600, 604, -1, 608, 612, - 616, 620, -1, -1, -1, -1, -1, -1}; + 580, 584, 588, 592, 596, -1, 600, 604, + 608, 612, -1, -1, -1, -1, -1, -1}; static constexpr dart::compiler::target::word Array_header_size = 12; static constexpr dart::compiler::target::word Context_header_size = 12; static constexpr dart::compiler::target::word Double_InstanceSize = 16; @@ -550,11 +548,11 @@ static constexpr dart::compiler::target::word String_hash_offset = 4; static constexpr dart::compiler::target::word String_length_offset = 8; static constexpr dart::compiler::target::word SubtypeTestCache_cache_offset = 8; static constexpr dart::compiler::target::word - Thread_AllocateArray_entry_point_offset = 568; + Thread_AllocateArray_entry_point_offset = 560; static constexpr dart::compiler::target::word Thread_active_exception_offset = - 1256; + 1240; static constexpr dart::compiler::target::word Thread_active_stacktrace_offset = - 1264; + 1248; static constexpr dart::compiler::target::word Thread_array_write_barrier_code_offset = 216; static constexpr dart::compiler::target::word @@ -562,14 +560,14 @@ static constexpr dart::compiler::target::word static constexpr dart::compiler::target::word Thread_async_stack_trace_offset = 168; static constexpr dart::compiler::target::word - Thread_auto_scope_native_wrapper_entry_point_offset = 488; + Thread_auto_scope_native_wrapper_entry_point_offset = 480; static constexpr dart::compiler::target::word Thread_bool_false_offset = 200; static constexpr dart::compiler::target::word Thread_bool_true_offset = 192; static constexpr dart::compiler::target::word Thread_call_to_runtime_entry_point_offset = 392; static constexpr dart::compiler::target::word Thread_call_to_runtime_stub_offset = 256; -static constexpr dart::compiler::target::word Thread_dart_stream_offset = 1320; +static constexpr dart::compiler::target::word Thread_dart_stream_offset = 1304; static constexpr dart::compiler::target::word Thread_optimize_entry_offset = 448; static constexpr dart::compiler::target::word Thread_optimize_stub_offset = 304; @@ -578,36 +576,36 @@ static constexpr dart::compiler::target::word Thread_deoptimize_entry_offset = static constexpr dart::compiler::target::word Thread_deoptimize_stub_offset = 312; static constexpr dart::compiler::target::word Thread_double_abs_address_offset = - 528; + 520; static constexpr dart::compiler::target::word - Thread_double_negate_address_offset = 520; + Thread_double_negate_address_offset = 512; static constexpr dart::compiler::target::word Thread_end_offset = 120; static constexpr dart::compiler::target::word Thread_enter_safepoint_stub_offset = 352; static constexpr dart::compiler::target::word Thread_execution_state_offset = - 1288; + 1272; static constexpr dart::compiler::target::word Thread_exit_safepoint_stub_offset = 360; static constexpr dart::compiler::target::word Thread_call_native_through_safepoint_stub_offset = 368; static constexpr dart::compiler::target::word - Thread_call_native_through_safepoint_entry_point_offset = 472; + Thread_call_native_through_safepoint_entry_point_offset = 464; static constexpr dart::compiler::target::word Thread_fix_allocation_stub_code_offset = 232; static constexpr dart::compiler::target::word Thread_fix_callers_target_code_offset = 224; static constexpr dart::compiler::target::word - Thread_float_absolute_address_offset = 552; + Thread_float_absolute_address_offset = 544; static constexpr dart::compiler::target::word - Thread_float_negate_address_offset = 544; + Thread_float_negate_address_offset = 536; static constexpr dart::compiler::target::word Thread_float_not_address_offset = - 536; + 528; static constexpr dart::compiler::target::word - Thread_float_zerow_address_offset = 560; + Thread_float_zerow_address_offset = 552; static constexpr dart::compiler::target::word Thread_global_object_pool_offset = - 1272; + 1256; static constexpr dart::compiler::target::word - Thread_interpret_call_entry_point_offset = 496; + Thread_interpret_call_entry_point_offset = 488; static constexpr dart::compiler::target::word Thread_invoke_dart_code_from_bytecode_stub_offset = 248; static constexpr dart::compiler::target::word @@ -628,7 +626,7 @@ static constexpr dart::compiler::target::word static constexpr dart::compiler::target::word Thread_monomorphic_miss_stub_offset = 296; static constexpr dart::compiler::target::word - Thread_no_scope_native_wrapper_entry_point_offset = 480; + Thread_no_scope_native_wrapper_entry_point_offset = 472; static constexpr dart::compiler::target::word Thread_null_error_shared_with_fpu_regs_entry_point_offset = 408; static constexpr dart::compiler::target::word @@ -639,10 +637,10 @@ static constexpr dart::compiler::target::word Thread_null_error_shared_without_fpu_regs_stub_offset = 264; static constexpr dart::compiler::target::word Thread_object_null_offset = 184; static constexpr dart::compiler::target::word - Thread_predefined_symbols_address_offset = 504; -static constexpr dart::compiler::target::word Thread_resume_pc_offset = 1280; + Thread_predefined_symbols_address_offset = 496; +static constexpr dart::compiler::target::word Thread_resume_pc_offset = 1264; static constexpr dart::compiler::target::word Thread_safepoint_state_offset = - 1296; + 1280; static constexpr dart::compiler::target::word Thread_slow_type_test_stub_offset = 336; static constexpr dart::compiler::target::word Thread_stack_limit_offset = 72; @@ -671,10 +669,8 @@ static constexpr dart::compiler::target::word Thread_write_barrier_entry_point_offset = 376; static constexpr dart::compiler::target::word Thread_write_barrier_mask_offset = 88; -static constexpr dart::compiler::target::word - Thread_verify_callback_entry_offset = 464; static constexpr dart::compiler::target::word Thread_callback_code_offset = - 1304; + 1288; static constexpr dart::compiler::target::word TimelineStream_enabled_offset = 16; static constexpr dart::compiler::target::word TwoByteString_data_offset = 16; @@ -710,8 +706,8 @@ static constexpr dart::compiler::target::word Code_function_entry_point_offset[] = {8, 16}; static constexpr dart::compiler::target::word Thread_write_barrier_wrappers_thread_offset[] = { - 1168, 1176, 1184, 1192, -1, -1, 1200, 1208, - 1216, 1224, 1232, -1, 1240, 1248, -1, -1}; + 1152, 1160, 1168, 1176, -1, -1, 1184, 1192, + 1200, 1208, 1216, -1, 1224, 1232, -1, -1}; static constexpr dart::compiler::target::word Array_header_size = 24; static constexpr dart::compiler::target::word Context_header_size = 24; static constexpr dart::compiler::target::word Double_InstanceSize = 16; @@ -909,11 +905,11 @@ static constexpr dart::compiler::target::word String_hash_offset = 8; static constexpr dart::compiler::target::word String_length_offset = 4; static constexpr dart::compiler::target::word SubtypeTestCache_cache_offset = 4; static constexpr dart::compiler::target::word - Thread_AllocateArray_entry_point_offset = 288; + Thread_AllocateArray_entry_point_offset = 284; static constexpr dart::compiler::target::word Thread_active_exception_offset = - 588; + 580; static constexpr dart::compiler::target::word Thread_active_stacktrace_offset = - 592; + 584; static constexpr dart::compiler::target::word Thread_array_write_barrier_code_offset = 112; static constexpr dart::compiler::target::word @@ -921,14 +917,14 @@ static constexpr dart::compiler::target::word static constexpr dart::compiler::target::word Thread_async_stack_trace_offset = 84; static constexpr dart::compiler::target::word - Thread_auto_scope_native_wrapper_entry_point_offset = 248; + Thread_auto_scope_native_wrapper_entry_point_offset = 244; static constexpr dart::compiler::target::word Thread_bool_false_offset = 104; static constexpr dart::compiler::target::word Thread_bool_true_offset = 100; static constexpr dart::compiler::target::word Thread_call_to_runtime_entry_point_offset = 200; static constexpr dart::compiler::target::word Thread_call_to_runtime_stub_offset = 132; -static constexpr dart::compiler::target::word Thread_dart_stream_offset = 620; +static constexpr dart::compiler::target::word Thread_dart_stream_offset = 612; static constexpr dart::compiler::target::word Thread_optimize_entry_offset = 228; static constexpr dart::compiler::target::word Thread_optimize_stub_offset = 156; @@ -937,36 +933,36 @@ static constexpr dart::compiler::target::word Thread_deoptimize_entry_offset = static constexpr dart::compiler::target::word Thread_deoptimize_stub_offset = 160; static constexpr dart::compiler::target::word Thread_double_abs_address_offset = - 268; + 264; static constexpr dart::compiler::target::word - Thread_double_negate_address_offset = 264; + Thread_double_negate_address_offset = 260; static constexpr dart::compiler::target::word Thread_end_offset = 60; static constexpr dart::compiler::target::word Thread_enter_safepoint_stub_offset = 180; static constexpr dart::compiler::target::word Thread_execution_state_offset = - 604; + 596; static constexpr dart::compiler::target::word Thread_exit_safepoint_stub_offset = 184; static constexpr dart::compiler::target::word Thread_call_native_through_safepoint_stub_offset = 188; static constexpr dart::compiler::target::word - Thread_call_native_through_safepoint_entry_point_offset = 240; + Thread_call_native_through_safepoint_entry_point_offset = 236; static constexpr dart::compiler::target::word Thread_fix_allocation_stub_code_offset = 120; static constexpr dart::compiler::target::word Thread_fix_callers_target_code_offset = 116; static constexpr dart::compiler::target::word - Thread_float_absolute_address_offset = 280; + Thread_float_absolute_address_offset = 276; static constexpr dart::compiler::target::word - Thread_float_negate_address_offset = 276; + Thread_float_negate_address_offset = 272; static constexpr dart::compiler::target::word Thread_float_not_address_offset = - 272; + 268; static constexpr dart::compiler::target::word - Thread_float_zerow_address_offset = 284; + Thread_float_zerow_address_offset = 280; static constexpr dart::compiler::target::word Thread_global_object_pool_offset = - 596; + 588; static constexpr dart::compiler::target::word - Thread_interpret_call_entry_point_offset = 252; + Thread_interpret_call_entry_point_offset = 248; static constexpr dart::compiler::target::word Thread_invoke_dart_code_from_bytecode_stub_offset = 128; static constexpr dart::compiler::target::word @@ -987,7 +983,7 @@ static constexpr dart::compiler::target::word static constexpr dart::compiler::target::word Thread_monomorphic_miss_stub_offset = 152; static constexpr dart::compiler::target::word - Thread_no_scope_native_wrapper_entry_point_offset = 244; + Thread_no_scope_native_wrapper_entry_point_offset = 240; static constexpr dart::compiler::target::word Thread_null_error_shared_with_fpu_regs_entry_point_offset = 208; static constexpr dart::compiler::target::word @@ -998,10 +994,10 @@ static constexpr dart::compiler::target::word Thread_null_error_shared_without_fpu_regs_stub_offset = 136; static constexpr dart::compiler::target::word Thread_object_null_offset = 96; static constexpr dart::compiler::target::word - Thread_predefined_symbols_address_offset = 256; -static constexpr dart::compiler::target::word Thread_resume_pc_offset = 600; + Thread_predefined_symbols_address_offset = 252; +static constexpr dart::compiler::target::word Thread_resume_pc_offset = 592; static constexpr dart::compiler::target::word Thread_safepoint_state_offset = - 608; + 600; static constexpr dart::compiler::target::word Thread_slow_type_test_stub_offset = 172; static constexpr dart::compiler::target::word Thread_stack_limit_offset = 36; @@ -1030,9 +1026,7 @@ static constexpr dart::compiler::target::word Thread_write_barrier_entry_point_offset = 192; static constexpr dart::compiler::target::word Thread_write_barrier_mask_offset = 44; -static constexpr dart::compiler::target::word - Thread_verify_callback_entry_offset = 236; -static constexpr dart::compiler::target::word Thread_callback_code_offset = 612; +static constexpr dart::compiler::target::word Thread_callback_code_offset = 604; static constexpr dart::compiler::target::word TimelineStream_enabled_offset = 8; static constexpr dart::compiler::target::word TwoByteString_data_offset = 12; static constexpr dart::compiler::target::word Type_arguments_offset = 16; @@ -1264,11 +1258,11 @@ static constexpr dart::compiler::target::word String_hash_offset = 4; static constexpr dart::compiler::target::word String_length_offset = 8; static constexpr dart::compiler::target::word SubtypeTestCache_cache_offset = 8; static constexpr dart::compiler::target::word - Thread_AllocateArray_entry_point_offset = 568; + Thread_AllocateArray_entry_point_offset = 560; static constexpr dart::compiler::target::word Thread_active_exception_offset = - 1344; + 1328; static constexpr dart::compiler::target::word Thread_active_stacktrace_offset = - 1352; + 1336; static constexpr dart::compiler::target::word Thread_array_write_barrier_code_offset = 216; static constexpr dart::compiler::target::word @@ -1276,14 +1270,14 @@ static constexpr dart::compiler::target::word static constexpr dart::compiler::target::word Thread_async_stack_trace_offset = 168; static constexpr dart::compiler::target::word - Thread_auto_scope_native_wrapper_entry_point_offset = 488; + Thread_auto_scope_native_wrapper_entry_point_offset = 480; static constexpr dart::compiler::target::word Thread_bool_false_offset = 200; static constexpr dart::compiler::target::word Thread_bool_true_offset = 192; static constexpr dart::compiler::target::word Thread_call_to_runtime_entry_point_offset = 392; static constexpr dart::compiler::target::word Thread_call_to_runtime_stub_offset = 256; -static constexpr dart::compiler::target::word Thread_dart_stream_offset = 1408; +static constexpr dart::compiler::target::word Thread_dart_stream_offset = 1392; static constexpr dart::compiler::target::word Thread_optimize_entry_offset = 448; static constexpr dart::compiler::target::word Thread_optimize_stub_offset = 304; @@ -1292,36 +1286,36 @@ static constexpr dart::compiler::target::word Thread_deoptimize_entry_offset = static constexpr dart::compiler::target::word Thread_deoptimize_stub_offset = 312; static constexpr dart::compiler::target::word Thread_double_abs_address_offset = - 528; + 520; static constexpr dart::compiler::target::word - Thread_double_negate_address_offset = 520; + Thread_double_negate_address_offset = 512; static constexpr dart::compiler::target::word Thread_end_offset = 120; static constexpr dart::compiler::target::word Thread_enter_safepoint_stub_offset = 352; static constexpr dart::compiler::target::word Thread_execution_state_offset = - 1376; + 1360; static constexpr dart::compiler::target::word Thread_exit_safepoint_stub_offset = 360; static constexpr dart::compiler::target::word Thread_call_native_through_safepoint_stub_offset = 368; static constexpr dart::compiler::target::word - Thread_call_native_through_safepoint_entry_point_offset = 472; + Thread_call_native_through_safepoint_entry_point_offset = 464; static constexpr dart::compiler::target::word Thread_fix_allocation_stub_code_offset = 232; static constexpr dart::compiler::target::word Thread_fix_callers_target_code_offset = 224; static constexpr dart::compiler::target::word - Thread_float_absolute_address_offset = 552; + Thread_float_absolute_address_offset = 544; static constexpr dart::compiler::target::word - Thread_float_negate_address_offset = 544; + Thread_float_negate_address_offset = 536; static constexpr dart::compiler::target::word Thread_float_not_address_offset = - 536; + 528; static constexpr dart::compiler::target::word - Thread_float_zerow_address_offset = 560; + Thread_float_zerow_address_offset = 552; static constexpr dart::compiler::target::word Thread_global_object_pool_offset = - 1360; + 1344; static constexpr dart::compiler::target::word - Thread_interpret_call_entry_point_offset = 496; + Thread_interpret_call_entry_point_offset = 488; static constexpr dart::compiler::target::word Thread_invoke_dart_code_from_bytecode_stub_offset = 248; static constexpr dart::compiler::target::word @@ -1342,7 +1336,7 @@ static constexpr dart::compiler::target::word static constexpr dart::compiler::target::word Thread_monomorphic_miss_stub_offset = 296; static constexpr dart::compiler::target::word - Thread_no_scope_native_wrapper_entry_point_offset = 480; + Thread_no_scope_native_wrapper_entry_point_offset = 472; static constexpr dart::compiler::target::word Thread_null_error_shared_with_fpu_regs_entry_point_offset = 408; static constexpr dart::compiler::target::word @@ -1353,10 +1347,10 @@ static constexpr dart::compiler::target::word Thread_null_error_shared_without_fpu_regs_stub_offset = 264; static constexpr dart::compiler::target::word Thread_object_null_offset = 184; static constexpr dart::compiler::target::word - Thread_predefined_symbols_address_offset = 504; -static constexpr dart::compiler::target::word Thread_resume_pc_offset = 1368; + Thread_predefined_symbols_address_offset = 496; +static constexpr dart::compiler::target::word Thread_resume_pc_offset = 1352; static constexpr dart::compiler::target::word Thread_safepoint_state_offset = - 1384; + 1368; static constexpr dart::compiler::target::word Thread_slow_type_test_stub_offset = 336; static constexpr dart::compiler::target::word Thread_stack_limit_offset = 72; @@ -1385,10 +1379,8 @@ static constexpr dart::compiler::target::word Thread_write_barrier_entry_point_offset = 376; static constexpr dart::compiler::target::word Thread_write_barrier_mask_offset = 88; -static constexpr dart::compiler::target::word - Thread_verify_callback_entry_offset = 464; static constexpr dart::compiler::target::word Thread_callback_code_offset = - 1392; + 1376; static constexpr dart::compiler::target::word TimelineStream_enabled_offset = 16; static constexpr dart::compiler::target::word TwoByteString_data_offset = 16; @@ -1424,9 +1416,9 @@ static constexpr dart::compiler::target::word Code_function_entry_point_offset[] = {8, 16}; static constexpr dart::compiler::target::word Thread_write_barrier_wrappers_thread_offset[] = { - 1168, 1176, 1184, 1192, 1200, 1208, 1216, 1224, 1232, 1240, 1248, - 1256, 1264, 1272, 1280, -1, -1, -1, -1, 1288, 1296, 1304, - 1312, 1320, 1328, 1336, -1, -1, -1, -1, -1, -1}; + 1152, 1160, 1168, 1176, 1184, 1192, 1200, 1208, 1216, 1224, 1232, + 1240, 1248, 1256, 1264, -1, -1, -1, -1, 1272, 1280, 1288, + 1296, 1304, 1312, 1320, -1, -1, -1, -1, -1, -1}; static constexpr dart::compiler::target::word Array_header_size = 24; static constexpr dart::compiler::target::word Context_header_size = 24; static constexpr dart::compiler::target::word Double_InstanceSize = 16; @@ -1628,23 +1620,23 @@ static constexpr dart::compiler::target::word SubtypeTestCache_cache_offset = 8; static constexpr dart::compiler::target::word Thread_AllocateArray_entry_point_offset = 296; static constexpr dart::compiler::target::word Thread_active_exception_offset = - 896; + 888; static constexpr dart::compiler::target::word Thread_active_stacktrace_offset = - 904; + 896; static constexpr dart::compiler::target::word Thread_async_stack_trace_offset = 168; static constexpr dart::compiler::target::word Thread_auto_scope_native_wrapper_entry_point_offset = 216; static constexpr dart::compiler::target::word Thread_bool_false_offset = 200; static constexpr dart::compiler::target::word Thread_bool_true_offset = 192; -static constexpr dart::compiler::target::word Thread_dart_stream_offset = 960; +static constexpr dart::compiler::target::word Thread_dart_stream_offset = 952; static constexpr dart::compiler::target::word Thread_double_abs_address_offset = 256; static constexpr dart::compiler::target::word Thread_double_negate_address_offset = 248; static constexpr dart::compiler::target::word Thread_end_offset = 120; static constexpr dart::compiler::target::word Thread_execution_state_offset = - 928; + 920; static constexpr dart::compiler::target::word Thread_float_absolute_address_offset = 280; static constexpr dart::compiler::target::word @@ -1654,7 +1646,7 @@ static constexpr dart::compiler::target::word Thread_float_not_address_offset = static constexpr dart::compiler::target::word Thread_float_zerow_address_offset = 288; static constexpr dart::compiler::target::word Thread_global_object_pool_offset = - 912; + 904; static constexpr dart::compiler::target::word Thread_isolate_offset = 96; static constexpr dart::compiler::target::word Thread_marking_stack_block_offset = 144; @@ -1663,9 +1655,9 @@ static constexpr dart::compiler::target::word static constexpr dart::compiler::target::word Thread_object_null_offset = 184; static constexpr dart::compiler::target::word Thread_predefined_symbols_address_offset = 232; -static constexpr dart::compiler::target::word Thread_resume_pc_offset = 920; +static constexpr dart::compiler::target::word Thread_resume_pc_offset = 912; static constexpr dart::compiler::target::word Thread_safepoint_state_offset = - 936; + 928; static constexpr dart::compiler::target::word Thread_stack_limit_offset = 72; static constexpr dart::compiler::target::word Thread_stack_overflow_flags_offset = 80; @@ -1680,7 +1672,7 @@ static constexpr dart::compiler::target::word static constexpr dart::compiler::target::word Thread_vm_tag_offset = 160; static constexpr dart::compiler::target::word Thread_write_barrier_mask_offset = 88; -static constexpr dart::compiler::target::word Thread_callback_code_offset = 944; +static constexpr dart::compiler::target::word Thread_callback_code_offset = 936; static constexpr dart::compiler::target::word TimelineStream_enabled_offset = 16; static constexpr dart::compiler::target::word TwoByteString_data_offset = 16; @@ -1913,23 +1905,23 @@ static constexpr dart::compiler::target::word SubtypeTestCache_cache_offset = 4; static constexpr dart::compiler::target::word Thread_AllocateArray_entry_point_offset = 152; static constexpr dart::compiler::target::word Thread_active_exception_offset = - 452; + 448; static constexpr dart::compiler::target::word Thread_active_stacktrace_offset = - 456; + 452; static constexpr dart::compiler::target::word Thread_async_stack_trace_offset = 84; static constexpr dart::compiler::target::word Thread_auto_scope_native_wrapper_entry_point_offset = 112; static constexpr dart::compiler::target::word Thread_bool_false_offset = 104; static constexpr dart::compiler::target::word Thread_bool_true_offset = 100; -static constexpr dart::compiler::target::word Thread_dart_stream_offset = 484; +static constexpr dart::compiler::target::word Thread_dart_stream_offset = 480; static constexpr dart::compiler::target::word Thread_double_abs_address_offset = 132; static constexpr dart::compiler::target::word Thread_double_negate_address_offset = 128; static constexpr dart::compiler::target::word Thread_end_offset = 60; static constexpr dart::compiler::target::word Thread_execution_state_offset = - 468; + 464; static constexpr dart::compiler::target::word Thread_float_absolute_address_offset = 144; static constexpr dart::compiler::target::word @@ -1939,7 +1931,7 @@ static constexpr dart::compiler::target::word Thread_float_not_address_offset = static constexpr dart::compiler::target::word Thread_float_zerow_address_offset = 148; static constexpr dart::compiler::target::word Thread_global_object_pool_offset = - 460; + 456; static constexpr dart::compiler::target::word Thread_isolate_offset = 48; static constexpr dart::compiler::target::word Thread_marking_stack_block_offset = 72; @@ -1948,9 +1940,9 @@ static constexpr dart::compiler::target::word static constexpr dart::compiler::target::word Thread_object_null_offset = 96; static constexpr dart::compiler::target::word Thread_predefined_symbols_address_offset = 120; -static constexpr dart::compiler::target::word Thread_resume_pc_offset = 464; +static constexpr dart::compiler::target::word Thread_resume_pc_offset = 460; static constexpr dart::compiler::target::word Thread_safepoint_state_offset = - 472; + 468; static constexpr dart::compiler::target::word Thread_stack_limit_offset = 36; static constexpr dart::compiler::target::word Thread_stack_overflow_flags_offset = 40; @@ -1965,7 +1957,7 @@ static constexpr dart::compiler::target::word static constexpr dart::compiler::target::word Thread_vm_tag_offset = 80; static constexpr dart::compiler::target::word Thread_write_barrier_mask_offset = 44; -static constexpr dart::compiler::target::word Thread_callback_code_offset = 476; +static constexpr dart::compiler::target::word Thread_callback_code_offset = 472; static constexpr dart::compiler::target::word TimelineStream_enabled_offset = 8; static constexpr dart::compiler::target::word TwoByteString_data_offset = 12; static constexpr dart::compiler::target::word Type_arguments_offset = 16; diff --git a/runtime/vm/compiler/runtime_offsets_list.h b/runtime/vm/compiler/runtime_offsets_list.h index 29588ec2d45..f67e7291ba2 100644 --- a/runtime/vm/compiler/runtime_offsets_list.h +++ b/runtime/vm/compiler/runtime_offsets_list.h @@ -217,7 +217,6 @@ NOT_IN_DBC(FIELD(Thread, write_barrier_code_offset)) \ NOT_IN_DBC(FIELD(Thread, write_barrier_entry_point_offset)) \ FIELD(Thread, write_barrier_mask_offset) \ - NOT_IN_DBC(FIELD(Thread, verify_callback_entry_offset)) \ FIELD(Thread, callback_code_offset) \ FIELD(TimelineStream, enabled_offset) \ FIELD(TwoByteString, data_offset) \ diff --git a/runtime/vm/compiler/stub_code_compiler.h b/runtime/vm/compiler/stub_code_compiler.h index b7d033218a7..c7ce24af9c4 100644 --- a/runtime/vm/compiler/stub_code_compiler.h +++ b/runtime/vm/compiler/stub_code_compiler.h @@ -65,6 +65,30 @@ class StubCodeCompiler : public AllStatic { static void GenerateUsageCounterIncrement(Assembler* assembler, Register temp_reg); static void GenerateOptimizedUsageCounterIncrement(Assembler* assembler); + +#if defined(TARGET_ARCH_X64) + static constexpr intptr_t kNativeCallbackTrampolineSize = 10; + static constexpr intptr_t kNativeCallbackSharedStubSize = 217; + static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 2; +#elif defined(TARGET_ARCH_IA32) + static constexpr intptr_t kNativeCallbackTrampolineSize = 10; + static constexpr intptr_t kNativeCallbackSharedStubSize = 90; + static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 2; +#elif defined(TARGET_ARCH_ARM) + static constexpr intptr_t kNativeCallbackTrampolineSize = 12; + static constexpr intptr_t kNativeCallbackSharedStubSize = 140; + static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 4; +#elif defined(TARGET_ARCH_ARM64) + static constexpr intptr_t kNativeCallbackTrampolineSize = 12; + static constexpr intptr_t kNativeCallbackSharedStubSize = 284; + static constexpr intptr_t kNativeCallbackTrampolineStackDelta = 2; +#endif + +#if !defined(TARGET_ARCH_DBC) + static void GenerateJITCallbackTrampolines(Assembler* assembler, + intptr_t next_callback_id); +#endif + #endif // !defined(DART_PRECOMPILED_RUNTIME) }; diff --git a/runtime/vm/compiler/stub_code_compiler_arm.cc b/runtime/vm/compiler/stub_code_compiler_arm.cc index 39974a27260..2ab5d441a24 100644 --- a/runtime/vm/compiler/stub_code_compiler_arm.cc +++ b/runtime/vm/compiler/stub_code_compiler_arm.cc @@ -322,32 +322,131 @@ void StubCodeCompiler::GenerateCallNativeThroughSafepointStub( // TransitionGeneratedToNative might clobber LR if it takes the slow path. __ mov(R4, Operand(LR)); - __ TransitionGeneratedToNative(R8, FPREG, R9 /*volatile*/, NOTFP); + __ TransitionGeneratedToNative(R8, FPREG, R9 /*volatile*/, NOTFP, + /*enter_safepoint=*/true); __ blx(R8); - __ TransitionNativeToGenerated(R9 /*volatile*/, NOTFP); + __ TransitionNativeToGenerated(R9 /*volatile*/, NOTFP, + /*exit_safepoint=*/true); __ bx(R4); } -void StubCodeCompiler::GenerateVerifyCallbackStub(Assembler* assembler) { - __ EnterFrame(1 << FP | 1 << LR, 0); - __ ReserveAlignedFrameSpace(0); +#if !defined(DART_PRECOMPILER) +void StubCodeCompiler::GenerateJITCallbackTrampolines( + Assembler* assembler, + intptr_t next_callback_id) { +#if defined(USING_SIMULATOR) + // TODO(37299): FFI is not support in SIMARM. + __ Breakpoint(); +#else + Label done; - // First argument is already set up by the caller. + // TMP is volatile and not used for passing any arguments. + COMPILE_ASSERT(!IsCalleeSavedRegister(TMP) && !IsArgumentRegister(TMP)); + + for (intptr_t i = 0; + i < NativeCallbackTrampolines::NumCallbackTrampolinesPerPage(); ++i) { + // We don't use LoadImmediate because we need the trampoline size to be + // fixed independently of the callback ID. + // + // PC points two instructions ahead of the current one -- directly where we + // store the callback ID. + __ ldr(TMP, Address(PC, 0)); + __ b(&done); + __ Emit(next_callback_id + i); + } + + ASSERT(__ CodeSize() == + kNativeCallbackTrampolineSize * + NativeCallbackTrampolines::NumCallbackTrampolinesPerPage()); + + __ Bind(&done); + + const intptr_t shared_stub_start = __ CodeSize(); + + // Save THR (callee-saved), R4 & R5 (temporaries, callee-saved), and LR. + COMPILE_ASSERT(StubCodeCompiler::kNativeCallbackTrampolineStackDelta == 4); + __ PushList((1 << LR) | (1 << THR) | (1 << R4) | (1 << R5)); + + // Don't rely on TMP being preserved by assembler macros anymore. + __ mov(R4, Operand(TMP)); + + COMPILE_ASSERT(IsCalleeSavedRegister(R4)); + COMPILE_ASSERT(!IsArgumentRegister(THR)); + + RegisterSet argument_registers; + argument_registers.AddAllArgumentRegisters(); + __ PushRegisters(argument_registers); + + // Load the thread, verify the callback ID and exit the safepoint. // - // Second argument is the return address of the caller. - __ mov(CallingConventions::ArgumentRegisters[1], Operand(LR)); - ASSERT(R2 != CallingConventions::ArgumentRegisters[0] && - R2 != CallingConventions::ArgumentRegisters[1]); - __ LoadFromOffset(kWord, R2, THR, - kVerifyCallbackIsolateRuntimeEntry.OffsetFromThread()); - __ blx(R2); + // We exit the safepoint inside DLRT_GetThreadForNativeCallbackTrampoline + // in order to safe code size on this shared stub. + { + __ EnterFrame(1 << FP, 0); + __ ReserveAlignedFrameSpace(0); - __ LeaveFrame(1 << FP | 1 << LR); - __ Ret(); + __ mov(R0, Operand(R4)); + + // Since DLRT_GetThreadForNativeCallbackTrampoline can theoretically be + // loaded anywhere, we use the same trick as before to ensure a predictable + // instruction sequence. + Label call; + __ ldr(R1, Address(PC, 0)); + __ b(&call); + __ Emit( + reinterpret_cast(&DLRT_GetThreadForNativeCallbackTrampoline)); + + __ Bind(&call); + __ blx(R1); + __ mov(THR, Operand(R0)); + + __ LeaveFrame(1 << FP); + } + + __ PopRegisters(argument_registers); + + COMPILE_ASSERT(!IsArgumentRegister(R8)); + + // Load the code object. + __ LoadFromOffset(kWord, R5, THR, + compiler::target::Thread::callback_code_offset()); + __ LoadFieldFromOffset(kWord, R5, R5, + compiler::target::GrowableObjectArray::data_offset()); + __ ldr(R5, __ ElementAddressForRegIndex( + /*is_load=*/true, + /*external=*/false, + /*array_cid=*/kArrayCid, + /*index, smi-tagged=*/compiler::target::kWordSize * 2, + /*array=*/R5, + /*index=*/R4)); + __ LoadFieldFromOffset(kWord, R5, R5, + compiler::target::Code::entry_point_offset()); + + // On entry to the function, there will be four extra slots on the stack: + // saved THR, R4, R5 and the return address. The target will know to skip + // them. + __ blx(R5); + + // EnterSafepoint clobbers R4, R5 and TMP, all saved or volatile. + __ EnterSafepoint(R4, R5); + + // Returns. + __ PopList((1 << PC) | (1 << THR) | (1 << R4) | (1 << R5)); + + ASSERT((__ CodeSize() - shared_stub_start) == kNativeCallbackSharedStubSize); + ASSERT(__ CodeSize() <= VirtualMemory::PageSize()); + +#if defined(DEBUG) + while (__ CodeSize() < VirtualMemory::PageSize()) { + __ Breakpoint(); + } +#endif +#endif } +#endif // !defined(DART_PRECOMPILER) void StubCodeCompiler::GenerateNullErrorSharedWithoutFPURegsStub( Assembler* assembler) { diff --git a/runtime/vm/compiler/stub_code_compiler_arm64.cc b/runtime/vm/compiler/stub_code_compiler_arm64.cc index 9961c181770..c565ba58ccc 100644 --- a/runtime/vm/compiler/stub_code_compiler_arm64.cc +++ b/runtime/vm/compiler/stub_code_compiler_arm64.cc @@ -266,7 +266,8 @@ void StubCodeCompiler::GenerateCallNativeThroughSafepointStub( COMPILE_ASSERT((1 << R19) & kAbiPreservedCpuRegs); __ mov(R19, LR); - __ TransitionGeneratedToNative(R8, FPREG, R9 /*volatile*/); + __ TransitionGeneratedToNative(R8, FPREG, R9 /*volatile*/, + /*enter_safepoint=*/true); __ mov(CSP, SP); #if defined(DEBUG) @@ -282,27 +283,136 @@ void StubCodeCompiler::GenerateCallNativeThroughSafepointStub( __ blr(R8); __ mov(SP, CSP); - __ TransitionNativeToGenerated(R9); + __ TransitionNativeToGenerated(R9, /*leave_safepoint=*/true); __ ret(R19); } -void StubCodeCompiler::GenerateVerifyCallbackStub(Assembler* assembler) { - __ EnterFrame(0); - __ ReserveAlignedFrameSpace(0); +#if !defined(DART_PRECOMPILER) +void StubCodeCompiler::GenerateJITCallbackTrampolines( + Assembler* assembler, + intptr_t next_callback_id) { +#if !defined(HOST_ARCH_ARM64) + // TODO(37299): FFI is not support in SIMARM64. + __ Breakpoint(); +#else + Label done; - // First argument is already set up by the caller. + // R8 is volatile and not used for passing any arguments. + COMPILE_ASSERT(!IsCalleeSavedRegister(R8) && !IsArgumentRegister(R8)); + for (intptr_t i = 0; + i < NativeCallbackTrampolines::NumCallbackTrampolinesPerPage(); ++i) { + // We don't use LoadImmediate because we need the trampoline size to be + // fixed independently of the callback ID. + // + // Instead we paste the callback ID directly in the code load it + // PC-relative. + __ ldr(R8, compiler::Address::PC(2 * Instr::kInstrSize)); + __ b(&done); + __ Emit(next_callback_id + i); + } + + ASSERT(__ CodeSize() == + kNativeCallbackTrampolineSize * + NativeCallbackTrampolines::NumCallbackTrampolinesPerPage()); + + __ Bind(&done); + + const intptr_t shared_stub_start = __ CodeSize(); + + // The load of the callback ID might have incorrect higher-order bits, since + // we only emit a 32-bit callback ID. + __ uxtw(R8, R8); + + // Save THR (callee-saved) and LR on real real C stack (CSP). Keeps it + // aligned. + COMPILE_ASSERT(StubCodeCompiler::kNativeCallbackTrampolineStackDelta == 2); + __ stp(THR, LR, Address(CSP, -2 * target::kWordSize, Address::PairPreIndex)); + + COMPILE_ASSERT(!IsArgumentRegister(THR)); + + RegisterSet all_registers; + all_registers.AddAllArgumentRegisters(); + + // The call below might clobber R8 (volatile, holding callback_id). + all_registers.Add(Location::RegisterLocation(R8)); + + // Load the thread, verify the callback ID and exit the safepoint. // - // Second argument is the return address of the caller. - __ mov(CallingConventions::ArgumentRegisters[1], LR); - __ LoadFromOffset(R2, THR, - kVerifyCallbackIsolateRuntimeEntry.OffsetFromThread()); - __ mov(CSP, SP); - __ blr(R2); - __ mov(SP, CSP); + // We exit the safepoint inside DLRT_GetThreadForNativeCallbackTrampoline + // in order to safe code size on this shared stub. + { + __ mov(SP, CSP); - __ LeaveFrame(); - __ Ret(); + __ EnterFrame(0); + __ PushRegisters(all_registers); + + __ EnterFrame(0); + __ ReserveAlignedFrameSpace(0); + + __ mov(CSP, SP); + + // Since DLRT_GetThreadForNativeCallbackTrampoline can theoretically be + // loaded anywhere, we use the same trick as before to ensure a predictable + // instruction sequence. + Label call; + __ mov(R0, R8); + __ ldr(R1, compiler::Address::PC(2 * Instr::kInstrSize)); + __ b(&call); + + __ Emit64( + reinterpret_cast(&DLRT_GetThreadForNativeCallbackTrampoline)); + + __ Bind(&call); + __ blr(R1); + __ mov(THR, R0); + + __ LeaveFrame(); + + __ PopRegisters(all_registers); + __ LeaveFrame(); + + __ mov(CSP, SP); + } + + COMPILE_ASSERT(!IsCalleeSavedRegister(R9) && !IsArgumentRegister(R9)); + + // Load the code object. + __ LoadFromOffset(R9, THR, compiler::target::Thread::callback_code_offset()); + __ LoadFieldFromOffset(R9, R9, + compiler::target::GrowableObjectArray::data_offset()); + __ ldr(R9, __ ElementAddressForRegIndex( + /*is_load=*/true, + /*external=*/false, + /*array_cid=*/kArrayCid, + /*index, smi-tagged=*/compiler::target::kWordSize * 2, + /*array=*/R9, + /*index=*/R8)); + __ LoadFieldFromOffset(R9, R9, compiler::target::Code::entry_point_offset()); + + // Clobbers all volatile registers, including the callback ID in R8. + // Resets CSP and SP, important for EnterSafepoint below. + __ blr(R9); + + // EnterSafepoint clobbers TMP, TMP2 and R8 -- all volatile and not holding + // return values. + __ EnterSafepoint(R8); + + // Pop LR and THR from the real stack (CSP). + __ ldp(THR, LR, Address(CSP, 2 * target::kWordSize, Address::PairPostIndex)); + + __ ret(); + + ASSERT((__ CodeSize() - shared_stub_start) == kNativeCallbackSharedStubSize); + ASSERT(__ CodeSize() <= VirtualMemory::PageSize()); + +#if defined(DEBUG) + while (__ CodeSize() < VirtualMemory::PageSize()) { + __ Breakpoint(); + } +#endif +#endif // !defined(HOST_ARCH_ARM64) } +#endif // !defined(DART_PRECOMPILER) // R1: The extracted method. // R4: The type_arguments_field_offset (or 0) diff --git a/runtime/vm/compiler/stub_code_compiler_ia32.cc b/runtime/vm/compiler/stub_code_compiler_ia32.cc index b770097b29f..202ee75bb5e 100644 --- a/runtime/vm/compiler/stub_code_compiler_ia32.cc +++ b/runtime/vm/compiler/stub_code_compiler_ia32.cc @@ -192,34 +192,99 @@ void StubCodeCompiler::GenerateCallNativeThroughSafepointStub( Assembler* assembler) { __ popl(EBX); - __ TransitionGeneratedToNative(EAX, FPREG, ECX /*volatile*/); + __ TransitionGeneratedToNative(EAX, FPREG, ECX /*volatile*/, + /*enter_safepoint=*/true); __ call(EAX); - __ TransitionNativeToGenerated(ECX /*volatile*/); + __ TransitionNativeToGenerated(ECX /*volatile*/, /*leave_safepoint=*/true); __ jmp(EBX); } -void StubCodeCompiler::GenerateVerifyCallbackStub(Assembler* assembler) { - __ EnterFrame(0); - __ ReserveAlignedFrameSpace(0); +void StubCodeCompiler::GenerateJITCallbackTrampolines( + Assembler* assembler, + intptr_t next_callback_id) { + Label done; - // The return address needs to be the second argument to - // VerifyCallbackIsolate. - __ movl(EAX, Address(FPREG, 4)); - __ pushl(EAX); + // EAX is volatile and doesn't hold any arguments. + COMPILE_ASSERT(!IsArgumentRegister(EAX) && !IsCalleeSavedRegister(EAX)); - // Argument to the stub is callback ID, which is also the first argument to - // VerifyCallbackIsolate. - __ movl(EAX, Address(FPREG, 8)); - __ pushl(EAX); + for (intptr_t i = 0; + i < NativeCallbackTrampolines::NumCallbackTrampolinesPerPage(); ++i) { + __ movl(EAX, compiler::Immediate(next_callback_id + i)); + __ jmp(&done); + } - // Call the VerifyCallbackIsolate runtime entry. - __ movl(EAX, - Address(THR, kVerifyCallbackIsolateRuntimeEntry.OffsetFromThread())); - __ call(EAX); + ASSERT(__ CodeSize() == + kNativeCallbackTrampolineSize * + NativeCallbackTrampolines::NumCallbackTrampolinesPerPage()); + + __ Bind(&done); + + const intptr_t shared_stub_start = __ CodeSize(); + + // Save THR which is callee-saved. + __ pushl(THR); + + // THR & return address + COMPILE_ASSERT(StubCodeCompiler::kNativeCallbackTrampolineStackDelta == 2); + + // Load the thread, verify the callback ID and exit the safepoint. + // + // We exit the safepoint inside DLRT_GetThreadForNativeCallbackTrampoline + // in order to safe code size on this shared stub. + { + __ EnterFrame(0); + __ ReserveAlignedFrameSpace(compiler::target::kWordSize); + + __ movl(compiler::Address(SPREG, 0), EAX); + __ movl(EAX, compiler::Immediate(reinterpret_cast( + DLRT_GetThreadForNativeCallbackTrampoline))); + __ call(EAX); + __ movl(THR, EAX); + __ movl(EAX, compiler::Address(SPREG, 0)); + + __ LeaveFrame(); + } + + COMPILE_ASSERT(!IsCalleeSavedRegister(ECX) && !IsArgumentRegister(ECX)); + COMPILE_ASSERT(ECX != THR); + + // Load the target from the thread. + __ movl(ECX, compiler::Address( + THR, compiler::target::Thread::callback_code_offset())); + __ movl(ECX, compiler::FieldAddress( + ECX, compiler::target::GrowableObjectArray::data_offset())); + __ movl(ECX, __ ElementAddressForRegIndex( + /*external=*/false, + /*array_cid=*/kArrayCid, + /*index, smi-tagged=*/compiler::target::kWordSize * 2, + /*array=*/ECX, + /*index=*/EAX)); + __ movl(ECX, compiler::FieldAddress( + ECX, compiler::target::Code::entry_point_offset())); + + // On entry to the function, there will be two extra slots on the stack: + // the saved THR and the return address. The target will know to skip them. + __ call(ECX); + + // EnterSafepoint takes care to not clobber *any* registers (besides scratch). + __ EnterSafepoint(/*scratch=*/ECX); + + // Restore THR (callee-saved). + __ popl(THR); - __ LeaveFrame(); __ ret(); + + // 'kNativeCallbackSharedStubSize' is an upper bound because the exact + // instruction size can vary slightly based on OS calling conventions. + ASSERT((__ CodeSize() - shared_stub_start) <= kNativeCallbackSharedStubSize); + ASSERT(__ CodeSize() <= VirtualMemory::PageSize()); + +#if defined(DEBUG) + while (__ CodeSize() < VirtualMemory::PageSize()) { + __ Breakpoint(); + } +#endif } void StubCodeCompiler::GenerateNullErrorSharedWithoutFPURegsStub( diff --git a/runtime/vm/compiler/stub_code_compiler_x64.cc b/runtime/vm/compiler/stub_code_compiler_x64.cc index ae8191775b4..7ac6ced12e1 100644 --- a/runtime/vm/compiler/stub_code_compiler_x64.cc +++ b/runtime/vm/compiler/stub_code_compiler_x64.cc @@ -238,20 +238,6 @@ void StubCodeCompiler::GenerateExitSafepointStub(Assembler* assembler) { __ ret(); } -void StubCodeCompiler::GenerateVerifyCallbackStub(Assembler* assembler) { - // SP points to return address, which needs to be the second argument to - // VerifyCallbackIsolate. - __ movq(CallingConventions::kArg2Reg, Address(SPREG, 0)); - - __ EnterFrame(0); - __ ReserveAlignedFrameSpace(0); - __ movq(RAX, - Address(THR, kVerifyCallbackIsolateRuntimeEntry.OffsetFromThread())); - __ CallCFunction(RAX); - __ LeaveFrame(); - __ ret(); -} - // Calls native code within a safepoint. // // On entry: @@ -263,18 +249,130 @@ void StubCodeCompiler::GenerateVerifyCallbackStub(Assembler* assembler) { // RBX, R12 clobbered void StubCodeCompiler::GenerateCallNativeThroughSafepointStub( Assembler* assembler) { - __ TransitionGeneratedToNative(RBX, FPREG); + __ TransitionGeneratedToNative(RBX, FPREG, /*enter_safepoint=*/true); __ popq(R12); __ CallCFunction(RBX); - __ TransitionNativeToGenerated(); + __ TransitionNativeToGenerated(/*leave_safepoint=*/true); // Faster than jmp because it doesn't confuse the branch predictor. __ pushq(R12); __ ret(); } +#if !defined(DART_PRECOMPILER) +void StubCodeCompiler::GenerateJITCallbackTrampolines( + Assembler* assembler, + intptr_t next_callback_id) { + Label done; + + // RAX is volatile and not used for passing any arguments. + COMPILE_ASSERT(!IsCalleeSavedRegister(RAX) && !IsArgumentRegister(RAX)); + + for (intptr_t i = 0; + i < NativeCallbackTrampolines::NumCallbackTrampolinesPerPage(); ++i) { + __ movq(RAX, compiler::Immediate(next_callback_id + i)); + __ jmp(&done); + } + + ASSERT(__ CodeSize() == + kNativeCallbackTrampolineSize * + NativeCallbackTrampolines::NumCallbackTrampolinesPerPage()); + + __ Bind(&done); + + const intptr_t shared_stub_start = __ CodeSize(); + + // Save THR which is callee-saved. + __ pushq(THR); + + // 2 = THR & return address + COMPILE_ASSERT(2 == StubCodeCompiler::kNativeCallbackTrampolineStackDelta); + + // Save the callback ID. + __ pushq(RAX); + + // Save all registers which might hold arguments. + __ PushRegisters(CallingConventions::kArgumentRegisters, + CallingConventions::kFpuArgumentRegisters); + + // Load the thread, verify the callback ID and exit the safepoint. + // + // We exit the safepoint inside DLRT_GetThreadForNativeCallbackTrampoline + // in order to save code size on this shared stub. + { + __ EnterFrame(0); + __ ReserveAlignedFrameSpace(0); + + COMPILE_ASSERT(RAX != CallingConventions::kArg1Reg); + __ movq(CallingConventions::kArg1Reg, RAX); + __ movq(RAX, compiler::Immediate(reinterpret_cast( + DLRT_GetThreadForNativeCallbackTrampoline))); + __ CallCFunction(RAX); + __ movq(THR, RAX); + + __ LeaveFrame(); + } + + // Restore the arguments. + __ PopRegisters(CallingConventions::kArgumentRegisters, + CallingConventions::kFpuArgumentRegisters); + + // Restore the callback ID. + __ popq(RAX); + + // Current state: + // + // Stack: + // + // + // + // + // Registers: Like entry, except RAX == callback_id and THR == thread + // All argument registers are untouched. + + COMPILE_ASSERT(!IsCalleeSavedRegister(TMP) && !IsArgumentRegister(TMP)); + + // Load the target from the thread. + __ movq(TMP, compiler::Address( + THR, compiler::target::Thread::callback_code_offset())); + __ movq(TMP, compiler::FieldAddress( + TMP, compiler::target::GrowableObjectArray::data_offset())); + __ movq(TMP, __ ElementAddressForRegIndex( + /*external=*/false, + /*array_cid=*/kArrayCid, + /*index, smi-tagged=*/compiler::target::kWordSize * 2, + /*array=*/TMP, + /*index=*/RAX)); + __ movq(TMP, compiler::FieldAddress( + TMP, compiler::target::Code::entry_point_offset())); + + // On entry to the function, there will be two extra slots on the stack: + // the saved THR and the return address. The target will know to skip them. + __ call(TMP); + + // EnterSafepoint takes care to not clobber *any* registers (besides TMP). + __ EnterSafepoint(); + + // Restore THR (callee-saved). + __ popq(THR); + + __ ret(); + + // 'kNativeCallbackSharedStubSize' is an upper bound because the exact + // instruction size can vary slightly based on OS calling conventions. + ASSERT((__ CodeSize() - shared_stub_start) <= kNativeCallbackSharedStubSize); + ASSERT(__ CodeSize() <= VirtualMemory::PageSize()); + +#if defined(DEBUG) + while (__ CodeSize() < VirtualMemory::PageSize()) { + __ Breakpoint(); + } +#endif +} +#endif // !defined(DART_PRECOMPILER) + // RBX: The extracted method. // RDX: The type_arguments_field_offset (or 0) void StubCodeCompiler::GenerateBuildMethodExtractorStub( diff --git a/runtime/vm/constants.h b/runtime/vm/constants.h index 7fe6c15bc2f..5e58117c9dd 100644 --- a/runtime/vm/constants.h +++ b/runtime/vm/constants.h @@ -85,6 +85,22 @@ class RegisterNames { #endif // !defined(HOST_ARCH_EQUALS_TARGET_ARCH) }; +#if !defined(TARGET_ARCH_DBC) + +static constexpr bool IsArgumentRegister(Register reg) { + return ((1 << reg) & CallingConventions::kArgumentRegisters) != 0; +} + +static constexpr bool IsFpuArgumentRegister(FpuRegister reg) { + return ((1 << reg) & CallingConventions::kFpuArgumentRegisters) != 0; +} + +static constexpr bool IsCalleeSavedRegister(Register reg) { + return ((1 << reg) & CallingConventions::kCalleeSaveCpuRegisters) != 0; +} + +#endif // !defined(TARGET_ARCH_DBC) + } // namespace dart #endif // RUNTIME_VM_CONSTANTS_H_ diff --git a/runtime/vm/constants_arm.h b/runtime/vm/constants_arm.h index 15254c5991e..a1f578ccb8f 100644 --- a/runtime/vm/constants_arm.h +++ b/runtime/vm/constants_arm.h @@ -349,6 +349,8 @@ class CallingConventions { static constexpr bool kArgumentIntRegXorFpuReg = false; + static constexpr intptr_t kCalleeSaveCpuRegisters = kAbiPreservedCpuRegs; + // Whether floating-point values should be passed as integers ("softfp" vs // "hardfp"). Android and iOS always use the "softfp" calling convention, even // when hardfp support is present. diff --git a/runtime/vm/constants_arm64.h b/runtime/vm/constants_arm64.h index 75ecfea9381..dca40c3ade5 100644 --- a/runtime/vm/constants_arm64.h +++ b/runtime/vm/constants_arm64.h @@ -198,6 +198,8 @@ class CallingConventions { static const bool kArgumentIntRegXorFpuReg = false; + static constexpr intptr_t kCalleeSaveCpuRegisters = kAbiPreservedCpuRegs; + // Whether floating-point values should be passed as integers ("softfp" vs // "hardfp"). static constexpr bool kAbiSoftFP = false; diff --git a/runtime/vm/constants_ia32.h b/runtime/vm/constants_ia32.h index d2cbb9351bb..561b7f5e246 100644 --- a/runtime/vm/constants_ia32.h +++ b/runtime/vm/constants_ia32.h @@ -130,12 +130,16 @@ class CallingConventions { public: static const Register ArgumentRegisters[]; static const intptr_t kArgumentRegisters = 0; + static const intptr_t kFpuArgumentRegisters = 0; static const intptr_t kNumArgRegs = 0; static const XmmRegister FpuArgumentRegisters[]; static const intptr_t kXmmArgumentRegisters = 0; static const intptr_t kNumFpuArgRegs = 0; + static constexpr intptr_t kCalleeSaveCpuRegisters = + (1 << EDI) | (1 << ESI) | (1 << EBX); + static const bool kArgumentIntRegXorFpuReg = false; // Whether floating-point values should be passed as integers ("softfp" vs diff --git a/runtime/vm/ffi_callback_trampolines.cc b/runtime/vm/ffi_callback_trampolines.cc new file mode 100644 index 00000000000..5516ddbf157 --- /dev/null +++ b/runtime/vm/ffi_callback_trampolines.cc @@ -0,0 +1,93 @@ +// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#include "vm/ffi_callback_trampolines.h" +#include "vm/code_comments.h" +#include "vm/code_observers.h" +#include "vm/compiler/assembler/assembler.h" +#include "vm/compiler/assembler/disassembler.h" +#include "vm/exceptions.h" + +namespace dart { + +DECLARE_FLAG(bool, disassemble_stubs); + +#if !defined(DART_PRECOMPILED_RUNTIME) && !defined(TARGET_ARCH_DBC) +uword NativeCallbackTrampolines::AllocateTrampoline() { +#if defined(DART_PRECOMPILER) + ASSERT(!Enabled()); + UNREACHABLE(); +#else + + // Callback IDs are limited to 32-bits for trampoline compactness. + if (kWordSize == 8 && + !Utils::IsInt(32, next_callback_id_ + NumCallbackTrampolinesPerPage())) { + Exceptions::ThrowOOM(); + } + + if (trampolines_left_on_page_ == 0) { + VirtualMemory* const memory = VirtualMemory::AllocateAligned( + /*size=*/VirtualMemory::PageSize(), + /*alignment=*/VirtualMemory::PageSize(), + /*is_executable=*/false, /*name=*/"Dart VM FFI callback trampolines"); + + if (memory == nullptr) { + Exceptions::ThrowOOM(); + } + + trampoline_pages_.Add(memory); + + compiler::Assembler assembler(/*object_pool_builder=*/nullptr); + compiler::StubCodeCompiler::GenerateJITCallbackTrampolines( + &assembler, next_callback_id_); + + MemoryRegion region(memory->address(), memory->size()); + assembler.FinalizeInstructions(region); + + memory->Protect(VirtualMemory::kReadExecute); + +#if !defined(PRODUCT) + const char* name = "FfiJitCallbackTrampolines"; + ASSERT(!Thread::Current()->IsAtSafepoint()); + if (CodeObservers::AreActive()) { + const auto& comments = CreateCommentsFrom(&assembler); + CodeCommentsWrapper wrapper(comments); + CodeObservers::NotifyAll(name, + /*base=*/memory->start(), + /*prologue_offset=*/0, + /*size=*/assembler.CodeSize(), + /*optimized=*/false, // not really relevant + &wrapper); + } +#endif +#if !defined(PRODUCT) || defined(FORCE_INCLUDE_DISASSEMBLER) + if (FLAG_disassemble_stubs && FLAG_support_disassembler) { + DisassembleToStdout formatter; + THR_Print( + "Code for native callback trampolines " + "[%" Pd " -> %" Pd "]: {\n", + next_callback_id_, + next_callback_id_ + NumCallbackTrampolinesPerPage() - 1); + const auto& comments = CreateCommentsFrom(&assembler); + Disassembler::Disassemble(memory->start(), + memory->start() + assembler.CodeSize(), + &formatter, &comments); + } +#endif + + next_callback_trampoline_ = memory->start(); + trampolines_left_on_page_ = NumCallbackTrampolinesPerPage(); + } + + trampolines_left_on_page_--; + next_callback_id_++; + const uword entrypoint = next_callback_trampoline_; + next_callback_trampoline_ += + compiler::StubCodeCompiler::kNativeCallbackTrampolineSize; + return entrypoint; +#endif // defined(DART_PRECOMPILER) +} +#endif // !defined(DART_PRECOMPILED_RUNTIME) && !defined(TARGET_ARCH_DBC) + +} // namespace dart diff --git a/runtime/vm/ffi_callback_trampolines.h b/runtime/vm/ffi_callback_trampolines.h new file mode 100644 index 00000000000..65c2a37ece6 --- /dev/null +++ b/runtime/vm/ffi_callback_trampolines.h @@ -0,0 +1,74 @@ +// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#include "platform/allocation.h" +#include "platform/growable_array.h" +#include "vm/compiler/stub_code_compiler.h" +#include "vm/flag_list.h" +#include "vm/virtual_memory.h" + +#ifndef RUNTIME_VM_FFI_CALLBACK_TRAMPOLINES_H_ +#define RUNTIME_VM_FFI_CALLBACK_TRAMPOLINES_H_ + +namespace dart { + +#if !defined(DART_PRECOMPILED_RUNTIME) && !defined(TARGET_ARCH_DBC) +// In JIT mode, when write-protection is enabled without dual-mapping, we cannot +// rely on Instructions generated in the Isolate's heap to be executable while +// native code is running in a safepoint. This means that native code cannot +// directly invoke FFI callback trampolines. +// +// To solve this, we create trampolines tied to consecutive sequences of +// callback IDs which leave the safepoint before invoking the FFI callback, +// and re-enter the safepoint on return from the callback. +// +// Since we can never map these trampolines RX -> RW, we eagerly generate as +// many as will fit on a single page, since pages are the smallest granularity +// of memory protection. +// +// See also: +// - StubCodeCompiler::GenerateJITCallbackTrampolines +// - {NativeEntryInstr, NativeReturnInstr}::EmitNativeCode +DECLARE_FLAG(bool, write_protect_code); + +class NativeCallbackTrampolines : public ValueObject { + public: + static bool Enabled() { return !FLAG_precompiled_mode; } + + static intptr_t NumCallbackTrampolinesPerPage() { + return (VirtualMemory::PageSize() - + compiler::StubCodeCompiler::kNativeCallbackSharedStubSize) / + compiler::StubCodeCompiler::kNativeCallbackTrampolineSize; + } + + NativeCallbackTrampolines() {} + ~NativeCallbackTrampolines() { + // Unmap all the trampoline pages. 'VirtualMemory's are new-allocated. + for (intptr_t i = 0; i < trampoline_pages_.length(); ++i) { + delete trampoline_pages_[i]; + } + } + + // For each callback ID, we have an entry in Thread::ffi_callback_code_ and + // a trampoline here. These arrays must be kept in sync and this method is + // exposed to assert that. + intptr_t next_callback_id() const { return next_callback_id_; } + + // Allocates a callback trampoline corresponding to the callback id + // 'next_callback_id()'. Returns an entrypoint to the trampoline. + uword AllocateTrampoline(); + + private: + MallocGrowableArray trampoline_pages_; + uword next_callback_trampoline_ = 0; + intptr_t trampolines_left_on_page_ = 0; + intptr_t next_callback_id_ = 0; + + DISALLOW_COPY_AND_ASSIGN(NativeCallbackTrampolines); +}; +#endif // !defined(DART_PRECOMPILED_RUNTIME) && !defined(TARGET_ARCH_DBC) + +} // namespace dart + +#endif // RUNTIME_VM_FFI_CALLBACK_TRAMPOLINES_H_ diff --git a/runtime/vm/isolate.cc b/runtime/vm/isolate.cc index a240829b85f..a31c59bfe80 100644 --- a/runtime/vm/isolate.cc +++ b/runtime/vm/isolate.cc @@ -53,6 +53,11 @@ #include "vm/timeline_analysis.h" #include "vm/visitor.h" +#if !defined(DART_PRECOMPILED_RUNTIME) +#include "vm/compiler/assembler/assembler.h" +#include "vm/compiler/stub_code_compiler.h" +#endif + namespace dart { DECLARE_FLAG(bool, print_metrics); @@ -73,7 +78,7 @@ static void DeterministicModeHandler(bool value) { FLAG_background_compilation = false; // Timing dependent. FLAG_concurrent_mark = false; // Timing dependent. FLAG_concurrent_sweep = false; // Timing dependent. - FLAG_random_seed = 0x44617274; // "Dart" + FLAG_random_seed = 0x44617274; // "Dart" #if !defined(PRODUCT) && !defined(DART_PRECOMPILED_RUNTIME) FLAG_load_deferred_eagerly = true; #else @@ -1052,6 +1057,9 @@ Isolate::Isolate(IsolateGroup* isolate_group, ic_miss_code_(Code::null()), class_table_(), store_buffer_(new StoreBuffer()), +#if !defined(TARGET_ARCH_DBC) && !defined(DART_PRECOMPILED_RUNTIME) + native_callback_trampolines_(), +#endif #if !defined(PRODUCT) last_resume_timestamp_(OS::GetCurrentTimeMillis()), vm_tag_counters_(), diff --git a/runtime/vm/isolate.h b/runtime/vm/isolate.h index 12d341ceec4..e8df2bb27cd 100644 --- a/runtime/vm/isolate.h +++ b/runtime/vm/isolate.h @@ -19,6 +19,7 @@ #include "vm/class_table.h" #include "vm/constants_kbc.h" #include "vm/exceptions.h" +#include "vm/ffi_callback_trampolines.h" #include "vm/fixed_cache.h" #include "vm/growable_array.h" #include "vm/handles.h" @@ -32,6 +33,7 @@ #include "vm/thread.h" #include "vm/thread_stack_resource.h" #include "vm/token_position.h" +#include "vm/virtual_memory.h" namespace dart { @@ -404,6 +406,12 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry { void set_init_callback_data(void* value) { init_callback_data_ = value; } void* init_callback_data() const { return init_callback_data_; } +#if !defined(TARGET_ARCH_DBC) && !defined(DART_PRECOMPILED_RUNTIME) + NativeCallbackTrampolines* native_callback_trampolines() { + return &native_callback_trampolines_; + } +#endif + Dart_EnvironmentCallback environment_callback() const { return environment_callback_; } @@ -898,7 +906,7 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry { #if defined(PRODUCT) void set_use_osr(bool use_osr) { ASSERT(!use_osr); } -#else // defined(PRODUCT) +#else // defined(PRODUCT) void set_use_osr(bool use_osr) { isolate_flags_ = UseOsrBit::update(use_osr, isolate_flags_); } @@ -1029,6 +1037,10 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry { Heap* heap_ = nullptr; IsolateGroup* isolate_group_ = nullptr; +#if !defined(DART_PRECOMPILED_RUNTIME) && !defined(TARGET_ARCH_DBC) + NativeCallbackTrampolines native_callback_trampolines_; +#endif + #define ISOLATE_FLAG_BITS(V) \ V(ErrorsFatal) \ V(IsRunnable) \ @@ -1085,23 +1097,14 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry { VMTagCounters vm_tag_counters_; // We use 6 list entries for each pending service extension calls. - enum { - kPendingHandlerIndex = 0, - kPendingMethodNameIndex, - kPendingKeysIndex, - kPendingValuesIndex, - kPendingReplyPortIndex, - kPendingIdIndex, - kPendingEntrySize - }; + enum {kPendingHandlerIndex = 0, kPendingMethodNameIndex, kPendingKeysIndex, + kPendingValuesIndex, kPendingReplyPortIndex, kPendingIdIndex, + kPendingEntrySize}; RawGrowableObjectArray* pending_service_extension_calls_; // We use 2 list entries for each registered extension handler. - enum { - kRegisteredNameIndex = 0, - kRegisteredHandlerIndex, - kRegisteredEntrySize - }; + enum {kRegisteredNameIndex = 0, kRegisteredHandlerIndex, + kRegisteredEntrySize}; RawGrowableObjectArray* registered_service_extension_handlers_; Metric* metrics_list_head_ = nullptr; @@ -1214,13 +1217,13 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry { REUSABLE_HANDLE_LIST(REUSABLE_FRIEND_DECLARATION) #undef REUSABLE_FRIEND_DECLARATION - friend class Become; // VisitObjectPointers + friend class Become; // VisitObjectPointers friend class GCCompactor; // VisitObjectPointers - friend class GCMarker; // VisitObjectPointers + friend class GCMarker; // VisitObjectPointers friend class SafepointHandler; - friend class ObjectGraph; // VisitObjectPointers + friend class ObjectGraph; // VisitObjectPointers friend class HeapSnapshotWriter; // VisitObjectPointers - friend class Scavenger; // VisitObjectPointers + friend class Scavenger; // VisitObjectPointers friend class HeapIterationScope; // VisitObjectPointers friend class ServiceIsolate; friend class Thread; diff --git a/runtime/vm/isolate_reload.cc b/runtime/vm/isolate_reload.cc index 473f50af777..3d55a15ee32 100644 --- a/runtime/vm/isolate_reload.cc +++ b/runtime/vm/isolate_reload.cc @@ -1933,7 +1933,11 @@ class InvalidationCollector : public ObjectVisitor { } const Object& handle = Object::Handle(zone_, obj); if (handle.IsFunction()) { - functions_->Add(&Function::Cast(handle)); + const auto& func = Function::Cast(handle); + if (!func.ForceOptimize()) { + // Force-optimized functions cannot deoptimize. + functions_->Add(&func); + } } else if (handle.IsKernelProgramInfo()) { kernel_infos_->Add(&KernelProgramInfo::Cast(handle)); } diff --git a/runtime/vm/object.cc b/runtime/vm/object.cc index b9322cf0e46..80cd90750ed 100644 --- a/runtime/vm/object.cc +++ b/runtime/vm/object.cc @@ -12,6 +12,7 @@ #include "vm/bit_vector.h" #include "vm/bootstrap.h" #include "vm/class_finalizer.h" +#include "vm/code_comments.h" #include "vm/code_observers.h" #include "vm/compiler/aot/precompiler.h" #include "vm/compiler/assembler/assembler.h" @@ -14896,42 +14897,6 @@ RawCode* Code::New(intptr_t pointer_offsets_length) { } #if !defined(DART_PRECOMPILED_RUNTIME) -#if !defined(PRODUCT) -class CodeCommentsWrapper final : public CodeComments { - public: - explicit CodeCommentsWrapper(const Code::Comments& comments) - : comments_(comments), string_(String::Handle()) {} - - intptr_t Length() const override { return comments_.Length(); } - - intptr_t PCOffsetAt(intptr_t i) const override { - return comments_.PCOffsetAt(i); - } - - const char* CommentAt(intptr_t i) const override { - string_ = comments_.CommentAt(i); - return string_.ToCString(); - } - - private: - const Code::Comments& comments_; - String& string_; -}; - -static const Code::Comments& CreateCommentsFrom( - compiler::Assembler* assembler) { - const auto& comments = assembler->comments(); - Code::Comments& result = Code::Comments::New(comments.length()); - - for (intptr_t i = 0; i < comments.length(); i++) { - result.SetPCOffsetAt(i, comments[i]->pc_offset()); - result.SetCommentAt(i, comments[i]->comment()); - } - - return result; -} -#endif - RawCode* Code::FinalizeCodeAndNotify(const Function& function, FlowGraphCompiler* compiler, compiler::Assembler* assembler, @@ -15130,7 +15095,6 @@ void Code::NotifyCodeObservers(const char* name, } #endif } - #endif // !defined(DART_PRECOMPILED_RUNTIME) bool Code::SlowFindRawCodeVisitor::FindObject(RawObject* raw_obj) const { diff --git a/runtime/vm/runtime_entry.cc b/runtime/vm/runtime_entry.cc index 4a01769112b..8fd02900302 100644 --- a/runtime/vm/runtime_entry.cc +++ b/runtime/vm/runtime_entry.cc @@ -1875,10 +1875,10 @@ DEFINE_RUNTIME_ENTRY(NoSuchMethodFromCallStub, 4) { Class& cls = Class::Handle(zone, receiver.clazz()); Function& function = Function::Handle(zone); -// Dart distinguishes getters and regular methods and allows their calls -// to mix with conversions, and its selectors are independent of arity. So do -// a zigzagged lookup to see if this call failed because of an arity mismatch, -// need for conversion, or there really is no such method. + // Dart distinguishes getters and regular methods and allows their calls + // to mix with conversions, and its selectors are independent of arity. So do + // a zigzagged lookup to see if this call failed because of an arity mismatch, + // need for conversion, or there really is no such method. #define NO_SUCH_METHOD() \ const Object& result = Object::Handle( \ @@ -3066,7 +3066,8 @@ extern "C" void DFLRT_ExitSafepoint(NativeArguments __unusable_) { DEFINE_RAW_LEAF_RUNTIME_ENTRY(ExitSafepoint, 0, false, &DFLRT_ExitSafepoint); // Not registered as a runtime entry because we can't use Thread to look it up. -extern "C" Thread* DLRT_GetThreadForNativeCallback() { +static Thread* GetThreadForNativeCallback(uword callback_id, + uword return_address) { Thread* const thread = Thread::Current(); if (thread == nullptr) { FATAL("Cannot invoke native callback outside an isolate."); @@ -3077,17 +3078,46 @@ extern "C" Thread* DLRT_GetThreadForNativeCallback() { if (!thread->IsMutatorThread()) { FATAL("Native callbacks must be invoked on the mutator thread."); } + + // Set the execution state to VM while waiting for the safepoint to end. + // This isn't strictly necessary but enables tests to check that we're not + // in native code anymore. See tests/ffi/function_gc_test.dart for example. + thread->set_execution_state(Thread::kThreadInVM); + + thread->ExitSafepoint(); + thread->VerifyCallbackIsolate(callback_id, return_address); + return thread; } -extern "C" void DLRT_VerifyCallbackIsolate(int32_t callback_id, - uword return_address) { - Thread::Current()->VerifyCallbackIsolate(callback_id, return_address); +#if defined(HOST_OS_WINDOWS) +#pragma intrinsic(_ReturnAddress) +#endif + +// This is called directly by NativeEntryInstr. At the moment we enter this +// routine, the caller is generated code in the Isolate heap. Therefore we check +// that the return address (caller) corresponds to the declared callback ID's +// code within this Isolate. +extern "C" Thread* DLRT_GetThreadForNativeCallback(uword callback_id) { + CHECK_STACK_ALIGNMENT; +#if defined(HOST_OS_WINDOWS) + void* return_address = _ReturnAddress(); +#else + void* return_address = __builtin_return_address(0); +#endif + return GetThreadForNativeCallback(callback_id, + reinterpret_cast(return_address)); +} + +// This is called by a native callback trampoline +// (see StubCodeCompiler::GenerateJITCallbackTrampolines). There is no need to +// check the return address because the trampoline will use the callback ID to +// look up the generated code. We still check that the callback ID is valid for +// this isolate. +extern "C" Thread* DLRT_GetThreadForNativeCallbackTrampoline( + uword callback_id) { + CHECK_STACK_ALIGNMENT; + return GetThreadForNativeCallback(callback_id, 0); } -DEFINE_RAW_LEAF_RUNTIME_ENTRY( - VerifyCallbackIsolate, - 1, - false /* is_float */, - reinterpret_cast(&DLRT_VerifyCallbackIsolate)); } // namespace dart diff --git a/runtime/vm/runtime_entry.h b/runtime/vm/runtime_entry.h index f072c149975..4206b1e974c 100644 --- a/runtime/vm/runtime_entry.h +++ b/runtime/vm/runtime_entry.h @@ -146,7 +146,8 @@ RUNTIME_ENTRY_LIST(DECLARE_RUNTIME_ENTRY) LEAF_RUNTIME_ENTRY_LIST(DECLARE_LEAF_RUNTIME_ENTRY) // Expected to be called inside a safepoint. -extern "C" Thread* DLRT_GetThreadForNativeCallback(); +extern "C" Thread* DLRT_GetThreadForNativeCallback(uword callback_id); +extern "C" Thread* DLRT_GetThreadForNativeCallbackTrampoline(uword callback_id); const char* DeoptReasonToCString(ICData::DeoptReasonId deopt_reason); diff --git a/runtime/vm/runtime_entry_list.h b/runtime/vm/runtime_entry_list.h index 31bbcb6aea2..fc61f648dad 100644 --- a/runtime/vm/runtime_entry_list.h +++ b/runtime/vm/runtime_entry_list.h @@ -87,7 +87,6 @@ namespace dart { RawSmi*) \ V(void, EnterSafepoint) \ V(void, ExitSafepoint) \ - V(void, VerifyCallbackIsolate, int32_t, uword) } // namespace dart diff --git a/runtime/vm/stack_frame_arm.h b/runtime/vm/stack_frame_arm.h index cf95a6703b2..64139897627 100644 --- a/runtime/vm/stack_frame_arm.h +++ b/runtime/vm/stack_frame_arm.h @@ -59,6 +59,9 @@ COMPILE_ASSERT(kAbiPreservedFpuRegCount == 4); // passed on stack and arguments saved in callback prologue. // // 2 = return adddress (1) + saved frame pointer (1). +// +// If NativeCallbackTrampolines::Enabled(), then +// kNativeCallbackTrampolineStackDelta must be added as well. constexpr intptr_t kCallbackSlotsBeforeSavedArguments = 2; } // namespace dart diff --git a/runtime/vm/stack_frame_arm64.h b/runtime/vm/stack_frame_arm64.h index b1b2288beda..abaa49a7144 100644 --- a/runtime/vm/stack_frame_arm64.h +++ b/runtime/vm/stack_frame_arm64.h @@ -53,6 +53,9 @@ COMPILE_ASSERT(kAbiPreservedFpuRegCount == 8); // arguments passed on stack and arguments saved in callback prologue. // // 2 = return adddress (1) + saved frame pointer (1). +// +// If NativeCallbackTrampolines::Enabled(), then +// kNativeCallbackTrampolineStackDelta must be added as well. constexpr intptr_t kCallbackSlotsBeforeSavedArguments = 2; } // namespace dart diff --git a/runtime/vm/stack_frame_ia32.h b/runtime/vm/stack_frame_ia32.h index 70717c28adc..896fb050058 100644 --- a/runtime/vm/stack_frame_ia32.h +++ b/runtime/vm/stack_frame_ia32.h @@ -48,6 +48,9 @@ static const int kExitLinkSlotFromEntryFp = -7; // All arguments are passed on the stack, so none need to be saved. Therefore // there is no frame for holding the saved arguments. +// +// If NativeCallbackTrampolines::Enabled(), then +// kNativeCallbackTrampolineStackDelta must be added as well. constexpr intptr_t kCallbackSlotsBeforeSavedArguments = 0; } // namespace dart diff --git a/runtime/vm/stack_frame_x64.h b/runtime/vm/stack_frame_x64.h index 193db62e3f1..1507960f794 100644 --- a/runtime/vm/stack_frame_x64.h +++ b/runtime/vm/stack_frame_x64.h @@ -58,6 +58,9 @@ static const int kExitLinkSlotFromEntryFp = -10; // passed on stack and arguments saved in callback prologue. 2 = return adddress // (1) + saved frame pointer (1). Also add slots for the shadow space, if // present. +// +// If NativeCallbackTrampolines::Enabled(), then +// kNativeCallbackTrampolineStackDelta must be added as well. constexpr intptr_t kCallbackSlotsBeforeSavedArguments = 2 + CallingConventions::kShadowSpaceBytes / kWordSize; diff --git a/runtime/vm/stub_code_list.h b/runtime/vm/stub_code_list.h index bb35068c7eb..0ca76d29a63 100644 --- a/runtime/vm/stub_code_list.h +++ b/runtime/vm/stub_code_list.h @@ -78,7 +78,6 @@ namespace dart { V(OneArgOptimizedCheckInlineCacheWithExactnessCheck) \ V(EnterSafepoint) \ V(ExitSafepoint) \ - V(VerifyCallback) \ V(CallNativeThroughSafepoint) #else diff --git a/runtime/vm/thread.cc b/runtime/vm/thread.cc index f9bcb736c39..41a2afa31b6 100644 --- a/runtime/vm/thread.cc +++ b/runtime/vm/thread.cc @@ -5,6 +5,7 @@ #include "vm/thread.h" #include "vm/dart_api_state.h" +#include "vm/ffi_callback_trampolines.h" #include "vm/growable_array.h" #include "vm/heap/safepoint.h" #include "vm/isolate.h" @@ -940,14 +941,25 @@ DisableThreadInterruptsScope::~DisableThreadInterruptsScope() { } const intptr_t kInitialCallbackIdsReserved = 1024; -int32_t Thread::AllocateFfiCallbackId() { +int32_t Thread::AllocateFfiCallbackId(uword* trampoline) { Zone* Z = isolate()->current_zone(); if (ffi_callback_code_ == GrowableObjectArray::null()) { ffi_callback_code_ = GrowableObjectArray::New(kInitialCallbackIdsReserved); } const auto& array = GrowableObjectArray::Handle(Z, ffi_callback_code_); array.Add(Code::Handle(Z, Code::null())); - return array.Length() - 1; + const int32_t id = array.Length() - 1; + + // Allocate a native callback trampoline if necessary. +#if !defined(DART_PRECOMPILED_RUNTIME) && !defined(TARGET_ARCH_DBC) + if (NativeCallbackTrampolines::Enabled()) { + auto* const tramps = isolate()->native_callback_trampolines(); + ASSERT(tramps->next_callback_id() == id); + *trampoline = tramps->AllocateTrampoline(); + } +#endif + + return id; } void Thread::SetFfiCallbackCode(int32_t callback_id, const Code& code) { @@ -972,11 +984,15 @@ void Thread::VerifyCallbackIsolate(int32_t callback_id, uword entry) { FATAL("Cannot invoke callback on incorrect isolate."); } - RawObject** const code_array = - Array::DataOf(GrowableObjectArray::NoSafepointData(array)); - const RawCode* const code = Code::RawCast(code_array[callback_id]); - if (!Code::ContainsInstructionAt(code, entry)) { - FATAL("Cannot invoke callback on incorrect isolate."); + if (entry != 0) { + RawObject** const code_array = + Array::DataOf(GrowableObjectArray::NoSafepointData(array)); + // RawCast allocates handles in ASSERTs. + const RawCode* const code = + reinterpret_cast(code_array[callback_id]); + if (!Code::ContainsInstructionAt(code, entry)) { + FATAL("Cannot invoke callback on incorrect isolate."); + } } } diff --git a/runtime/vm/thread.h b/runtime/vm/thread.h index 1413ee11aa9..894197a1bf1 100644 --- a/runtime/vm/thread.h +++ b/runtime/vm/thread.h @@ -69,6 +69,12 @@ class TypeParameter; class TypeUsageInfo; class Zone; +namespace compiler { +namespace target { +class Thread; +} // namespace target +} // namespace compiler + #define REUSABLE_HANDLE_LIST(V) \ V(AbstractType) \ V(Array) \ @@ -170,7 +176,6 @@ class Zone; 0) \ V(uword, optimize_entry_, StubCode::OptimizeFunction().EntryPoint(), 0) \ V(uword, deoptimize_entry_, StubCode::Deoptimize().EntryPoint(), 0) \ - V(uword, verify_callback_entry_, StubCode::VerifyCallback().EntryPoint(), 0) \ V(uword, call_native_through_safepoint_entry_point_, \ StubCode::CallNativeThroughSafepoint().EntryPoint(), 0) #endif @@ -781,11 +786,15 @@ class Thread : public ThreadState { } } - int32_t AllocateFfiCallbackId(); + int32_t AllocateFfiCallbackId(uword* trampoline); void SetFfiCallbackCode(int32_t callback_id, const Code& code); - // Ensure that 'entry' points within the code of the callback identified by - // 'callback_id'. Aborts otherwise. + // Ensure that 'callback_id' refers to a valid callback in this isolate. + // + // If "entry != 0", additionally checks that entry is inside the instructions + // of this callback. + // + // Aborts if any of these conditions fails. void VerifyCallbackIsolate(int32_t callback_id, uword entry); Thread* next() const { return next_; } @@ -933,6 +942,7 @@ class Thread : public ThreadState { #undef REUSABLE_HANDLE_SCOPE_VARIABLE #endif // defined(DEBUG) + // Generated code assumes that AtSafepointField is the LSB. class AtSafepointField : public BitField {}; class SafepointRequestedField : public BitField {}; class BlockedForSafepointField : public BitField {}; @@ -985,6 +995,7 @@ class Thread : public ThreadState { friend class StackZone; friend class ThreadRegistry; friend class CompilerState; + friend class compiler::target::Thread; DISALLOW_COPY_AND_ASSIGN(Thread); }; diff --git a/runtime/vm/vm_sources.gni b/runtime/vm/vm_sources.gni index 2a07f21fd1f..fcc057f39ac 100644 --- a/runtime/vm/vm_sources.gni +++ b/runtime/vm/vm_sources.gni @@ -26,6 +26,8 @@ vm_sources = [ "class_table.h", "clustered_snapshot.cc", "clustered_snapshot.h", + "code_comments.h", + "code_comments.cc", "code_descriptors.cc", "code_descriptors.h", "code_entry_kind.h", @@ -99,6 +101,8 @@ vm_sources = [ "elf.h", "exceptions.cc", "exceptions.h", + "ffi_callback_trampolines.cc", + "ffi_callback_trampolines.h", "finalizable_data.h", "fixed_cache.h", "flag_list.h", diff --git a/tests/ffi/function_callbacks_test.dart b/tests/ffi/function_callbacks_test.dart index 1b25dc12b67..43779382f08 100644 --- a/tests/ffi/function_callbacks_test.dart +++ b/tests/ffi/function_callbacks_test.dart @@ -5,6 +5,7 @@ // Dart test program for testing dart:ffi function pointers with callbacks. // // VMOptions=--enable-testing-pragmas +// VMOptions=--enable-testing-pragmas --write-protect-code --no-dual-map-code // SharedObjects=ffi_test_functions library FfiTest; @@ -172,6 +173,12 @@ void testGC() { triggerGc(); } +typedef WaitForHelper = Void Function(Pointer); +void waitForHelper(Pointer helper) { + print("helper: $helper"); + testLibrary.lookupFunction("WaitForHelper")(helper); +} + final List testcases = [ Test("SimpleAddition", Pointer.fromFunction(simpleAddition, 0)), Test("IntComputation", Pointer.fromFunction(intComputation, 0)), @@ -195,6 +202,7 @@ final List testcases = [ throwExceptionPointer, Pointer.fromAddress(42))), Test("ThrowException", Pointer.fromFunction(throwExceptionInt, 42)), Test("GC", Pointer.fromFunction(testGC, null)), + Test("UnprotectCode", Pointer.fromFunction(waitForHelper, null)), ]; testCallbackWrongThread() => @@ -244,4 +252,20 @@ void main() async { testCallbackOutsideIsolate(); //# 02: ok await testCallbackWrongIsolate(); //# 03: ok } + + testManyCallbacks(); //# 04: ok } + +void testManyCallbacks() { + // Create enough callbacks (1000) to overflow one page of the JIT callback + // trampolines. The use of distinct exceptional return values forces separate + // trampolines. + final List pointers = []; + for (int i = 0; i < 1000; ++i) { + pointers.add(Pointer.fromFunction(simpleAddition, i)); + } + + for (final pointer in pointers) { + Test("SimpleAddition", pointer).run(); + } +} \ No newline at end of file diff --git a/tests/ffi/function_gc_test.dart b/tests/ffi/function_gc_test.dart index 00724a345e3..b212b32ab12 100644 --- a/tests/ffi/function_gc_test.dart +++ b/tests/ffi/function_gc_test.dart @@ -111,12 +111,12 @@ void testRegress37069() { } final unprotectCode = ffiTestFunctions.lookupFunction< - ffi.Pointer Function(), - ffi.Pointer Function()>("UnprotectCode"); + ffi.Pointer Function(ffi.Pointer), + ffi.Pointer Function(ffi.Pointer)>("TestUnprotectCode"); final waitForHelper = ffiTestFunctions.lookupFunction< ffi.Void Function(ffi.Pointer), void Function(ffi.Pointer)>("WaitForHelper"); void testWriteProtection() { - waitForHelper(unprotectCode()); + waitForHelper(unprotectCode(ffi.nullptr)); }