[vm/ffi] FFI callbacks on ARM32.

Please see go/dart-ffi-callbacks for design context and motivation.

Change-Id: Ie5edcb8837157c679954a670fb19d545e22fec69
Cq-Include-Trybots: luci.dart.try:vm-kernel-linux-debug-simdbc64-try, vm-kernel-linux-release-simdbc64-try, vm-kernel-mac-debug-simdbc64-try, vm-kernel-mac-release-simdbc64-try, vm-kernel-reload-mac-debug-simdbc64-try, vm-kernel-reload-mac-release-simdbc64-try, vm-kernel-linux-debug-ia32-try, vm-dartkb-linux-debug-simarm64-try, vm-kernel-win-debug-ia32-try, vm-ffi-android-debug-arm-try
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/101828
Commit-Queue: Samir Jindel <sjindel@google.com>
Reviewed-by: Aart Bik <ajcbik@google.com>
Reviewed-by: Daco Harkes <dacoharkes@google.com>

runtime/lib/ffi.cc

@@ -552,12 +552,11 @@ static uword CompileNativeCallback(const Function& c_signature,
                                    const Function& dart_target) {
 #if defined(DART_PRECOMPILED_RUNTIME) || defined(DART_PRECOMPILER)
   UNREACHABLE();
-#elif !defined(TARGET_ARCH_X64) && !defined(TARGET_ARCH_IA32) &&               \
-    !defined(TARGET_ARCH_ARM64)
+#elif defined(TARGET_ARCH_DBC)
   // https://github.com/dart-lang/sdk/issues/35774
   // FFI is supported, but callbacks are not.
   Exceptions::ThrowUnsupportedError(
-      "FFI callbacks are currently supported on Intel and 64-bit ARM only.");
+      "FFI callbacks are not yet supported on DBC.");
 #else
   Thread* const thread = Thread::Current();
   const int32_t callback_id = thread->AllocateFfiCallbackId();

runtime/vm/compiler/assembler/assembler_arm.cc

@@ -548,10 +548,12 @@ void Assembler::strex(Register rd, Register rt, Register rn, Condition cond) {
 }
 
 void Assembler::TransitionGeneratedToNative(Register destination_address,
+                                            Register exit_frame_fp,
                                             Register addr,
                                             Register state) {
   // Save exit frame information to enable stack walking.
-  StoreToOffset(kWord, FP, THR, Thread::top_exit_frame_info_offset());
+  StoreToOffset(kWord, exit_frame_fp, THR,
+                Thread::top_exit_frame_info_offset());
 
   // Mark that the thread is executing native code.
   StoreToOffset(kWord, destination_address, THR, Thread::vm_tag_offset());
@@ -2496,6 +2498,33 @@ void Assembler::PopRegisters(const RegisterSet& regs) {
   }
 }
 
+void Assembler::PushNativeCalleeSavedRegisters() {
+  // Save new context and C++ ABI callee-saved registers.
+  PushList(kAbiPreservedCpuRegs);
+
+  const DRegister firstd = EvenDRegisterOf(kAbiFirstPreservedFpuReg);
+  if (TargetCPUFeatures::vfp_supported()) {
+    ASSERT(2 * kAbiPreservedFpuRegCount < 16);
+    // Save FPU registers. 2 D registers per Q register.
+    vstmd(DB_W, SP, firstd, 2 * kAbiPreservedFpuRegCount);
+  } else {
+    sub(SP, SP, Operand(kAbiPreservedFpuRegCount * kFpuRegisterSize));
+  }
+}
+
+void Assembler::PopNativeCalleeSavedRegisters() {
+  const DRegister firstd = EvenDRegisterOf(kAbiFirstPreservedFpuReg);
+  // Restore C++ ABI callee-saved registers.
+  if (TargetCPUFeatures::vfp_supported()) {
+    // Restore FPU registers. 2 D registers per Q register.
+    vldmd(IA_W, SP, firstd, 2 * kAbiPreservedFpuRegCount);
+  } else {
+    AddImmediate(SP, kAbiPreservedFpuRegCount * kFpuRegisterSize);
+  }
+  // Restore CPU registers.
+  PopList(kAbiPreservedCpuRegs);
+}
+
 void Assembler::MoveRegister(Register rd, Register rm, Condition cond) {
   if (rd != rm) {
     mov(rd, Operand(rm), cond);
@@ -3215,6 +3244,23 @@ void Assembler::ReserveAlignedFrameSpace(intptr_t frame_space) {
   }
 }
 
+void Assembler::EmitEntryFrameVerification(Register scratch) {
+#if defined(DEBUG)
+  Label done;
+  ASSERT(!constant_pool_allowed());
+  LoadImmediate(scratch,
+                compiler::target::frame_layout.exit_link_slot_from_entry_fp *
+                    compiler::target::kWordSize);
+  add(scratch, scratch, Operand(FPREG));
+  cmp(scratch, Operand(SPREG));
+  b(&done, EQ);
+
+  Breakpoint();
+
+  Bind(&done);
+#endif
+}
+
 void Assembler::EnterCallRuntimeFrame(intptr_t frame_space) {
   Comment("EnterCallRuntimeFrame");
   // Preserve volatile CPU registers and PP.

runtime/vm/compiler/assembler/assembler_arm.h

@@ -510,9 +510,12 @@ class Assembler : public AssemblerBase {
   void ldrex(Register rd, Register rn, Condition cond = AL);
   void strex(Register rd, Register rt, Register rn, Condition cond = AL);
 
-  // Requires two temporary registers 'scratch0' and 'scratch1' (in addition to
-  // TMP).
+  // Emit code to transition between generated and native modes.
+  //
+  // These require that CSP and SP are equal and aligned and require two scratch
+  // registers (in addition to TMP).
   void TransitionGeneratedToNative(Register destination_address,
+                                   Register exit_frame_fp,
                                    Register scratch0,
                                    Register scratch1);
   void TransitionNativeToGenerated(Register scratch0, Register scratch1);
@@ -917,6 +920,12 @@ class Assembler : public AssemblerBase {
   void PushRegisters(const RegisterSet& regs);
   void PopRegisters(const RegisterSet& regs);
 
+  // Push all registers which are callee-saved according to the ARM ABI.
+  void PushNativeCalleeSavedRegisters();
+
+  // Pop all registers which are callee-saved according to the ARM ABI.
+  void PopNativeCalleeSavedRegisters();
+
   void CompareRegisters(Register rn, Register rm) { cmp(rn, Operand(rm)); }
   void BranchIf(Condition condition, Label* label) { b(label, condition); }
 
@@ -1003,6 +1012,13 @@ class Assembler : public AssemblerBase {
   void Ret();
   void ReserveAlignedFrameSpace(intptr_t frame_space);
 
+  // In debug mode, this generates code to check that:
+  //   FP + kExitLinkSlotFromEntryFp == SP
+  // or triggers breakpoint otherwise.
+  //
+  // Requires a scratch register in addition to the assembler temporary.
+  void EmitEntryFrameVerification(Register scratch);
+
   // Create a frame for calling into runtime that preserves all volatile
   // registers.  Frame's SP is guaranteed to be correctly aligned and
   // frame_space bytes are reserved under it.

runtime/vm/compiler/backend/il.cc

@@ -3960,17 +3960,6 @@ LocationSummary* NativeEntryInstr::MakeLocationSummary(Zone* zone,
   UNREACHABLE();
 }
 
-#if !defined(TARGET_ARCH_X64) && !defined(TARGET_ARCH_IA32) &&                 \
-    !defined(TARGET_ARCH_ARM64)
-void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
-  UNREACHABLE();
-}
-
-void NativeReturnInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
-  UNREACHABLE();
-}
-#endif
-
 LocationSummary* OsrEntryInstr::MakeLocationSummary(Zone* zone,
                                                     bool optimizing) const {
   UNREACHABLE();
@@ -5533,11 +5522,6 @@ LocationSummary* FfiCallInstr::MakeLocationSummary(Zone* zone,
   return summary;
 }
 
-LocationSummary* NativeReturnInstr::MakeLocationSummary(Zone* zone,
-                                                        bool opt) const {
-  UNREACHABLE();
-}
-
 #endif  // !defined(TARGET_ARCH_DBC)
 
 Representation FfiCallInstr::representation() const {

runtime/vm/compiler/backend/il_arm.cc

@@ -1023,7 +1023,8 @@ void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
                                  RawPcDescriptors::Kind::kOther, locs());
 
   // Update information in the thread object and enter a safepoint.
-  __ TransitionGeneratedToNative(branch, saved_fp, locs()->temp(1).reg());
+  __ TransitionGeneratedToNative(branch, FPREG, saved_fp,
+                                 locs()->temp(1).reg());
 
   __ blx(branch);
 
@@ -1044,6 +1045,174 @@ void FfiCallInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
   __ PopRegister(TMP);
 }
 
+void NativeReturnInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
+  __ LeaveDartFrame();
+
+  // The dummy return address is in LR, no need to pop it as on Intel.
+
+  // These can be anything besides the return registers (R0 and R1) and THR
+  // (R10).
+  const Register vm_tag_reg = R2, old_exit_frame_reg = R3, tmp = R4, tmp1 = R5;
+
+  __ Pop(old_exit_frame_reg);
+
+  // Restore top_resource.
+  __ Pop(tmp);
+  __ StoreToOffset(kWord, tmp, THR,
+                   compiler::target::Thread::top_resource_offset());
+
+  __ Pop(vm_tag_reg);
+
+  // Reset the exit frame info to
+  // old_exit_frame_reg *before* entering the safepoint.
+  __ TransitionGeneratedToNative(vm_tag_reg, old_exit_frame_reg, tmp, tmp1);
+
+  __ PopNativeCalleeSavedRegisters();
+
+  // Leave the entry frame.
+  __ LeaveFrame(1 << LR | 1 << FP);
+
+  // Leave the dummy frame holding the pushed arguments.
+  __ LeaveFrame(1 << LR | 1 << FP);
+
+  __ Ret();
+
+  // For following blocks.
+  __ set_constant_pool_allowed(true);
+}
+
+void NativeEntryInstr::SaveArgument(FlowGraphCompiler* compiler,
+                                    Location loc) const {
+  if (loc.IsPairLocation()) {
+    // Save higher-order component first, so bytes are in little-endian layout
+    // overall.
+    for (intptr_t i : {1, 0}) {
+      SaveArgument(compiler, loc.Component(i));
+    }
+    return;
+  }
+
+  if (loc.HasStackIndex()) return;
+
+  if (loc.IsRegister()) {
+    __ Push(loc.reg());
+  } else if (loc.IsFpuRegister()) {
+    const DRegister src = EvenDRegisterOf(loc.fpu_reg());
+    __ SubImmediateSetFlags(SPREG, SPREG, 8, AL);
+    __ StoreDToOffset(src, SPREG, 0);
+  } else {
+    UNREACHABLE();
+  }
+}
+
+void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
+  if (FLAG_precompiled_mode) {
+    UNREACHABLE();
+  }
+
+  // Constant pool cannot be used until we enter the actual Dart frame.
+  __ set_constant_pool_allowed(false);
+
+  __ Bind(compiler->GetJumpLabel(this));
+
+  // Create a dummy frame holding the pushed arguments. This simplifies
+  // NativeReturnInstr::EmitNativeCode.
+  __ EnterFrame((1 << FP) | (1 << LR), 0);
+
+  // Save the argument registers, in reverse order.
+  for (intptr_t i = argument_locations_->length(); i-- > 0;) {
+    SaveArgument(compiler, argument_locations_->At(i));
+  }
+
+  // Enter the entry frame.
+  __ EnterFrame((1 << FP) | (1 << LR), 0);
+
+  // Save a space for the code object.
+  __ PushImmediate(0);
+
+  __ PushNativeCalleeSavedRegisters();
+
+  // Load the thread object.
+  // TODO(35765): Fix linking issue on AOT.
+  // TOOD(35934): Exclude native callbacks from snapshots.
+  //
+  // Create another frame to align the frame before continuing in "native" code.
+  {
+    __ EnterFrame(1 << FP, 0);
+    __ ReserveAlignedFrameSpace(0);
+
+    __ LoadImmediate(
+        R0, reinterpret_cast<int64_t>(DLRT_GetThreadForNativeCallback));
+    __ blx(R0);
+    __ mov(THR, Operand(R0));
+
+    __ LeaveFrame(1 << FP);
+  }
+
+  // Save the current VMTag on the stack.
+  __ LoadFromOffset(kWord, R0, THR, compiler::target::Thread::vm_tag_offset());
+  __ Push(R0);
+
+  // Save top resource.
+  const intptr_t top_resource_offset =
+      compiler::target::Thread::top_resource_offset();
+  __ LoadFromOffset(kWord, R0, THR, top_resource_offset);
+  __ Push(R0);
+  __ LoadImmediate(R0, 0);
+  __ StoreToOffset(kWord, R0, THR, top_resource_offset);
+
+  // Save top exit frame info. Don't set it to 0 yet --
+  // TransitionNativeToGenerated will handle that *after* leaving the safepoint.
+  __ LoadFromOffset(kWord, R0, THR,
+                    compiler::target::Thread::top_exit_frame_info_offset());
+  __ Push(R0);
+
+  __ EmitEntryFrameVerification(R0);
+
+  __ TransitionNativeToGenerated(/*scratch0=*/R0, /*scratch1=*/R1);
+
+  // Now that the safepoint has ended, we can touch Dart objects without
+  // handles.
+
+  // Otherwise we'll clobber the argument sent from the caller.
+  ASSERT(CallingConventions::ArgumentRegisters[0] != TMP &&
+         CallingConventions::ArgumentRegisters[0] != TMP2 &&
+         CallingConventions::ArgumentRegisters[0] != R1);
+  __ LoadImmediate(CallingConventions::ArgumentRegisters[0], callback_id_);
+  __ LoadFromOffset(
+      kWord, R1, THR,
+      compiler::target::Thread::verify_callback_isolate_entry_point_offset());
+  __ blx(R1);
+
+  // Load the code object.
+  __ LoadFromOffset(kWord, R0, THR,
+                    compiler::target::Thread::callback_code_offset());
+  __ LoadFieldFromOffset(kWord, R0, R0,
+                         compiler::target::GrowableObjectArray::data_offset());
+  __ LoadFieldFromOffset(kWord, CODE_REG, R0,
+                         compiler::target::Array::data_offset() +
+                             callback_id_ * compiler::target::kWordSize);
+
+  // Put the code object in the reserved slot.
+  __ StoreToOffset(kWord, CODE_REG, FPREG,
+                   kPcMarkerSlotFromFp * compiler::target::kWordSize);
+  if (FLAG_precompiled_mode && FLAG_use_bare_instructions) {
+    __ ldr(PP,
+           Address(THR, compiler::target::Thread::global_object_pool_offset()));
+  } else {
+    __ LoadImmediate(PP, 0);  // GC safe value into PP.
+  }
+
+  // Load a dummy return address which suggests that we are inside of
+  // InvokeDartCodeStub. This is how the stack walker detects an entry frame.
+  __ LoadFromOffset(kWord, LR, THR,
+                    compiler::target::Thread::invoke_dart_code_stub_offset());
+  __ LoadFieldFromOffset(kWord, LR, LR,
+                         compiler::target::Code::entry_point_offset());
+
+  FunctionEntryInstr::EmitNativeCode(compiler);
+}
+
 LocationSummary* OneByteStringFromCharCodeInstr::MakeLocationSummary(
     Zone* zone,
     bool opt) const {

runtime/vm/compiler/backend/il_arm64.cc

@@ -949,7 +949,7 @@ void NativeReturnInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
 
   // The dummy return address is in LR, no need to pop it as on Intel.
 
-  // These can be anything besides the return register (R0).
+  // These can be anything besides the return register (R0) and THR (R26).
   const Register vm_tag_reg = R1, old_exit_frame_reg = R2, tmp = R3;
 
   __ Pop(old_exit_frame_reg);

runtime/vm/compiler/backend/il_dbc.cc

@@ -64,7 +64,8 @@ DECLARE_FLAG(int, optimization_counter_threshold);
   M(UnaryInt64Op)                                                              \
   M(CheckedSmiOp)                                                              \
   M(CheckedSmiComparison)                                                      \
-  M(SimdOp)
+  M(SimdOp)                                                                    \
+  M(NativeReturn)
 
 // Location summaries actually are not used by the unoptimizing DBC compiler
 // because we don't allocate any registers.
@@ -2168,6 +2169,10 @@ EMIT_NATIVE_CODE(CheckArrayBound, 2) {
                           (licm_hoisted_ ? ICData::kHoisted : 0));
 }
 
+void NativeEntryInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
+  UNREACHABLE();
+}
+
 }  // namespace dart
 
 #endif  // defined TARGET_ARCH_DBC

runtime/vm/compiler/stub_code_compiler_arm.cc

@@ -301,7 +301,21 @@ void StubCodeCompiler::GenerateExitSafepointStub(Assembler* assembler) {
 }
 
 void StubCodeCompiler::GenerateVerifyCallbackStub(Assembler* assembler) {
-  __ Breakpoint();
+  __ EnterFrame(1 << FP | 1 << LR, 0);
+  __ ReserveAlignedFrameSpace(0);
+
+  // First argument is already set up by the caller.
+  //
+  // Second argument is the return address of the caller.
+  __ mov(CallingConventions::ArgumentRegisters[1], Operand(LR));
+  ASSERT(R2 != CallingConventions::ArgumentRegisters[0] &&
+         R2 != CallingConventions::ArgumentRegisters[1]);
+  __ LoadFromOffset(kWord, R2, THR,
+                    kVerifyCallbackIsolateRuntimeEntry.OffsetFromThread());
+  __ blx(R2);
+
+  __ LeaveFrame(1 << FP | 1 << LR);
+  __ Ret();
 }
 
 void StubCodeCompiler::GenerateNullErrorSharedWithoutFPURegsStub(
@@ -1045,17 +1059,7 @@ void StubCodeCompiler::GenerateInvokeDartCodeStub(Assembler* assembler) {
   __ ldr(IP, Address(R3, target::Thread::invoke_dart_code_stub_offset()));
   __ Push(IP);
 
-  // Save new context and C++ ABI callee-saved registers.
-  __ PushList(kAbiPreservedCpuRegs);
-
-  const DRegister firstd = EvenDRegisterOf(kAbiFirstPreservedFpuReg);
-  if (TargetCPUFeatures::vfp_supported()) {
-    ASSERT(2 * kAbiPreservedFpuRegCount < 16);
-    // Save FPU registers. 2 D registers per Q register.
-    __ vstmd(DB_W, SP, firstd, 2 * kAbiPreservedFpuRegCount);
-  } else {
-    __ sub(SP, SP, Operand(kAbiPreservedFpuRegCount * kFpuRegisterSize));
-  }
+  __ PushNativeCalleeSavedRegisters();
 
   // Set up THR, which caches the current thread in Dart code.
   if (THR != R3) {
@@ -1086,6 +1090,8 @@ void StubCodeCompiler::GenerateInvokeDartCodeStub(Assembler* assembler) {
 #endif
   __ Push(R9);
 
+  __ EmitEntryFrameVerification(R9);
+
   // Mark that the thread is executing Dart code. Do this after initializing the
   // exit link for the profiler.
   __ LoadImmediate(R9, VMTag::kDartCompiledTagId);
@@ -1149,15 +1155,8 @@ void StubCodeCompiler::GenerateInvokeDartCodeStub(Assembler* assembler) {
   __ Pop(R4);
   __ StoreToOffset(kWord, R4, THR, target::Thread::vm_tag_offset());
 
-  // Restore C++ ABI callee-saved registers.
-  if (TargetCPUFeatures::vfp_supported()) {
-    // Restore FPU registers. 2 D registers per Q register.
-    __ vldmd(IA_W, SP, firstd, 2 * kAbiPreservedFpuRegCount);
-  } else {
-    __ AddImmediate(SP, kAbiPreservedFpuRegCount * kFpuRegisterSize);
-  }
-  // Restore CPU registers.
-  __ PopList(kAbiPreservedCpuRegs);
+  __ PopNativeCalleeSavedRegisters();
+
   __ set_constant_pool_allowed(false);
 
   // Restore the frame pointer and return.

runtime/vm/message.cc

@@ -4,6 +4,8 @@
 
 #include "vm/message.h"
 
+#include <utility>
+
 #include "vm/dart_entry.h"
 #include "vm/json_stream.h"
 #include "vm/object.h"
@@ -11,7 +13,7 @@
 
 namespace dart {
 
-const Dart_Port Message::kIllegalPort;
+const Dart_Port Message::kIllegalPort = 0;
 
 Message::Message(Dart_Port dest_port,
                  uint8_t* snapshot,

runtime/vm/message.h

@@ -6,6 +6,7 @@
 #define RUNTIME_VM_MESSAGE_H_
 
 #include <memory>
+#include <utility>
 
 #include "platform/assert.h"
 #include "vm/allocation.h"
@@ -42,7 +43,7 @@ class Message {
   } OOBMsgTag;
 
   // A port number which is never used.
-  static const Dart_Port kIllegalPort = 0;
+  static const Dart_Port kIllegalPort;
 
   // A new message to be sent between two isolates. The data handed to this
   // message will be disposed by calling free() once the message object is

runtime/vm/stack_frame.h

@@ -470,14 +470,6 @@ DART_FORCE_INLINE static uword LocalVarAddress(uword fp, intptr_t index) {
   return fp + LocalVarIndex(0, index) * kWordSize;
 }
 
-#if !defined(TARGET_ARCH_X64) && !defined(TARGET_ARCH_IA32) &&                 \
-    !defined(TARGET_ARCH_ARM64)
-// For FFI native -> Dart callbacks, the number of stack slots between arguments
-// passed on stack and arguments saved in callback prologue. This placeholder
-// here is for unsupported architectures.
-constexpr intptr_t kCallbackSlotsBeforeSavedArguments = -1;
-#endif
-
 }  // namespace dart
 
 #endif  // RUNTIME_VM_STACK_FRAME_H_

runtime/vm/stack_frame_arm.h

@@ -55,6 +55,12 @@ COMPILE_ASSERT(kAbiPreservedCpuRegCount == 7);
 COMPILE_ASSERT(kAbiPreservedFpuRegCount == 4);
 #endif
 
+// For FFI native -> Dart callbacks, the number of stack slots between arguments
+// passed on stack and arguments saved in callback prologue.
+//
+// 2 = return adddress (1) + saved frame pointer (1).
+constexpr intptr_t kCallbackSlotsBeforeSavedArguments = 2;
+
 }  // namespace dart
 
 #endif  // RUNTIME_VM_STACK_FRAME_ARM_H_

runtime/vm/stack_frame_dbc.h

@@ -81,6 +81,10 @@ static const int kExitLinkSlotFromEntryFp = 0;
 // on all other architectures.
 static const uword kInterruptStackLimit = 0;
 
+// TODO(37140): For FFI native -> Dart callbacks, the number of stack slots
+// between arguments passed on stack and arguments saved in callback prologue.
+constexpr intptr_t kCallbackSlotsBeforeSavedArguments = -1;
+
 }  // namespace dart
 
 #endif  // RUNTIME_VM_STACK_FRAME_DBC_H_

tests/ffi/ffi.status

@@ -24,9 +24,6 @@ negative_function_test: Skip
 [ $arch == x64 || $arch == arm64 || $arch == simdbc64 ]
 enable_structs_test: SkipByDesign  # Tests that structs don't work on 32-bit systems.
 
-[ $arch != x64 && $arch != ia32 && $arch != arm64 ]
-function_callbacks_test: Skip # Issue 35761
-
 [ $runtime == dart_precompiled ]
 *: Skip # AOT is not yet supported: dartbug.com/35765
 
@@ -44,3 +41,6 @@ function_callbacks_test: Skip # Issue 35761
 
 [ $system != android && $arch == arm ]
 *: Skip # "hardfp" calling convention is not yet supported (iOS is also supported but not tested): dartbug.com/36309
+
+[ $arch == simdbc64 ]
+function_callbacks_test: Skip  # Issue 37140