[vm, ffi] Better handle errors that are not unhandled exceptions during FFI callbacks.

Before this change, an error reaching an FFI callback would attempt to execute the normal invocation stub from the beginning in the FFI callback's frame, which quickly crashes. After this change, the runtime recognizes this marker use of the invocation stub and returns to the FFI callback function instead.

TEST=ffi/unwind
Bug: https://github.com/dart-lang/sdk/issues/39487
Change-Id: I477cfcfc236e6cf518ebfe52860ba49e466ebf8b
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/409562
Reviewed-by: Daco Harkes <dacoharkes@google.com>
Commit-Queue: Ryan Macnak <rmacnak@google.com>

runtime/bin/ffi_test/ffi_test_functions.cc

@@ -928,6 +928,13 @@ DART_EXPORT intptr_t TestSimpleAddition(intptr_t (*add)(int, int)) {
   return 0;
 }
 
+DART_EXPORT intptr_t TestUnwindError(intptr_t (*add)(int, int)) {
+  const intptr_t result = add(10, 20);
+  printf("result %" PRIdPTR "\n", result);
+  CHECK_EQ(result, 42);
+  return 0;
+}
+
 //// Following tests are copied from above, with the role of Dart and C++ code
 //// reversed.
 

runtime/bin/ffi_test/ffi_test_functions_vmspecific.cc

@@ -1469,4 +1469,9 @@ DART_EXPORT void ManyHandles(Dart_Handle o0,
 }
 #undef CHECK_STRING
 
+DART_EXPORT Dart_Handle TestUnwindErrorThroughHandle(Dart_Handle (*add)(int,
+                                                                        int)) {
+  return add(10, 20);
+}
+
 }  // namespace dart

runtime/vm/compiler/stub_code_compiler_arm.cc

@@ -3108,7 +3108,8 @@ void StubCodeCompiler::GenerateJumpToFrameStub() {
 //
 // The arguments are stored in the Thread object.
 // Does not return.
-void StubCodeCompiler::GenerateRunExceptionHandlerStub() {
+static void GenerateRunExceptionHandler(Assembler* assembler,
+                                        bool unbox_exception) {
   WRITES_RETURN_ADDRESS_TO_LR(
       __ LoadFromOffset(LR, THR, target::Thread::resume_pc_offset()));
 
@@ -3120,6 +3121,15 @@ void StubCodeCompiler::GenerateRunExceptionHandlerStub() {
   // Exception object.
   __ LoadFromOffset(R0, THR, target::Thread::active_exception_offset());
   __ StoreToOffset(R2, THR, target::Thread::active_exception_offset());
+  if (unbox_exception) {
+    compiler::Label not_smi, done;
+    __ BranchIfNotSmi(R0, &not_smi);
+    __ SmiUntag(R0);
+    __ Jump(&done);
+    __ Bind(&not_smi);
+    __ ldr(R0, FieldAddress(R0, Mint::value_offset()));
+    __ Bind(&done);
+  }
 
   // StackTrace object.
   __ LoadFromOffset(R1, THR, target::Thread::active_stacktrace_offset());
@@ -3129,6 +3139,14 @@ void StubCodeCompiler::GenerateRunExceptionHandlerStub() {
       __ bx(LR));  // Jump to the exception handler code.
 }
 
+void StubCodeCompiler::GenerateRunExceptionHandlerStub() {
+  GenerateRunExceptionHandler(assembler, false);
+}
+
+void StubCodeCompiler::GenerateRunExceptionHandlerUnboxStub() {
+  GenerateRunExceptionHandler(assembler, true);
+}
+
 // Deoptimize a frame on the call stack before rewinding.
 // The arguments are stored in the Thread object.
 // No result.

runtime/vm/compiler/stub_code_compiler_arm64.cc

@@ -3494,7 +3494,8 @@ void StubCodeCompiler::GenerateJumpToFrameStub() {
 //
 // The arguments are stored in the Thread object.
 // Does not return.
-void StubCodeCompiler::GenerateRunExceptionHandlerStub() {
+static void GenerateRunExceptionHandler(Assembler* assembler,
+                                        bool unbox_exception) {
   WRITES_RETURN_ADDRESS_TO_LR(
       __ LoadFromOffset(LR, THR, target::Thread::resume_pc_offset()));
 
@@ -3506,6 +3507,15 @@ void StubCodeCompiler::GenerateRunExceptionHandlerStub() {
   // Exception object.
   __ LoadFromOffset(R0, THR, target::Thread::active_exception_offset());
   __ StoreToOffset(R2, THR, target::Thread::active_exception_offset());
+  if (unbox_exception) {
+    compiler::Label not_smi, done;
+    __ BranchIfNotSmi(R0, &not_smi);
+    __ SmiUntag(R0);
+    __ Jump(&done);
+    __ Bind(&not_smi);
+    __ ldr(R0, FieldAddress(R0, Mint::value_offset()));
+    __ Bind(&done);
+  }
 
   // StackTrace object.
   __ LoadFromOffset(R1, THR, target::Thread::active_stacktrace_offset());
@@ -3514,6 +3524,14 @@ void StubCodeCompiler::GenerateRunExceptionHandlerStub() {
   __ ret();  // Jump to the exception handler code.
 }
 
+void StubCodeCompiler::GenerateRunExceptionHandlerStub() {
+  GenerateRunExceptionHandler(assembler, false);
+}
+
+void StubCodeCompiler::GenerateRunExceptionHandlerUnboxStub() {
+  GenerateRunExceptionHandler(assembler, true);
+}
+
 // Deoptimize a frame on the call stack before rewinding.
 // The arguments are stored in the Thread object.
 // No result.

runtime/vm/compiler/stub_code_compiler_ia32.cc

@@ -2987,7 +2987,8 @@ void StubCodeCompiler::GenerateJumpToFrameStub() {
 //
 // The arguments are stored in the Thread object.
 // No result.
-void StubCodeCompiler::GenerateRunExceptionHandlerStub() {
+static void GenerateRunExceptionHandler(Assembler* assembler,
+                                        bool unbox_exception) {
   ASSERT(kExceptionObjectReg == EAX);
   ASSERT(kStackTraceObjectReg == EDX);
   __ movl(EBX, Address(THR, target::Thread::resume_pc_offset()));
@@ -2999,6 +3000,17 @@ void StubCodeCompiler::GenerateRunExceptionHandlerStub() {
   Address exception_addr(THR, target::Thread::active_exception_offset());
   __ movl(kExceptionObjectReg, exception_addr);
   __ movl(exception_addr, ECX);
+  if (unbox_exception) {
+    compiler::Label not_smi, done;
+    __ BranchIfNotSmi(kExceptionObjectReg, &not_smi,
+                      compiler::Assembler::kNearJump);
+    __ SmiUntag(kExceptionObjectReg);
+    __ jmp(&done, compiler::Assembler::kNearJump);
+    __ Bind(&not_smi);
+    __ movl(kExceptionObjectReg,
+            compiler::FieldAddress(kExceptionObjectReg, Mint::value_offset()));
+    __ Bind(&done);
+  }
 
   // Load the stacktrace from the current thread.
   Address stacktrace_addr(THR, target::Thread::active_stacktrace_offset());
@@ -3008,6 +3020,14 @@ void StubCodeCompiler::GenerateRunExceptionHandlerStub() {
   __ jmp(EBX);  // Jump to continuation point.
 }
 
+void StubCodeCompiler::GenerateRunExceptionHandlerStub() {
+  GenerateRunExceptionHandler(assembler, false);
+}
+
+void StubCodeCompiler::GenerateRunExceptionHandlerUnboxStub() {
+  GenerateRunExceptionHandler(assembler, true);
+}
+
 // Deoptimize a frame on the call stack before rewinding.
 // The arguments are stored in the Thread object.
 // No result.

runtime/vm/compiler/stub_code_compiler_riscv.cc

@@ -2967,11 +2967,21 @@ void StubCodeCompiler::GenerateJumpToFrameStub() {
 //
 // The arguments are stored in the Thread object.
 // Does not return.
-void StubCodeCompiler::GenerateRunExceptionHandlerStub() {
+static void GenerateRunExceptionHandler(Assembler* assembler,
+                                        bool unbox_exception) {
   // Exception object.
   ASSERT(kExceptionObjectReg == A0);
   __ LoadFromOffset(A0, THR, target::Thread::active_exception_offset());
   __ StoreToOffset(NULL_REG, THR, target::Thread::active_exception_offset());
+  if (unbox_exception) {
+    compiler::Label not_smi, done;
+    __ BranchIfNotSmi(A0, &not_smi);
+    __ SmiUntag(A0);
+    __ Jump(&done);
+    __ Bind(&not_smi);
+    __ lx(A0, FieldAddress(A0, Mint::value_offset()));
+    __ Bind(&done);
+  }
 
   // StackTrace object.
   ASSERT(kStackTraceObjectReg == A1);
@@ -2982,6 +2992,14 @@ void StubCodeCompiler::GenerateRunExceptionHandlerStub() {
   __ ret();  // Jump to the exception handler code.
 }
 
+void StubCodeCompiler::GenerateRunExceptionHandlerStub() {
+  GenerateRunExceptionHandler(assembler, false);
+}
+
+void StubCodeCompiler::GenerateRunExceptionHandlerUnboxStub() {
+  GenerateRunExceptionHandler(assembler, true);
+}
+
 // Deoptimize a frame on the call stack before rewinding.
 // The arguments are stored in the Thread object.
 // No result.

runtime/vm/compiler/stub_code_compiler_x64.cc

@@ -3392,7 +3392,8 @@ void StubCodeCompiler::GenerateJumpToFrameStub() {
 //
 // The arguments are stored in the Thread object.
 // No result.
-void StubCodeCompiler::GenerateRunExceptionHandlerStub() {
+static void GenerateRunExceptionHandler(Assembler* assembler,
+                                        bool unbox_exception) {
   ASSERT(kExceptionObjectReg == RAX);
   ASSERT(kStackTraceObjectReg == RDX);
   __ movq(CallingConventions::kArg1Reg,
@@ -3407,6 +3408,17 @@ void StubCodeCompiler::GenerateRunExceptionHandlerStub() {
   Address exception_addr(THR, target::Thread::active_exception_offset());
   __ movq(kExceptionObjectReg, exception_addr);
   __ movq(exception_addr, TMP);
+  if (unbox_exception) {
+    compiler::Label not_smi, done;
+    __ BranchIfNotSmi(kExceptionObjectReg, &not_smi,
+                      compiler::Assembler::kNearJump);
+    __ SmiUntagAndSignExtend(kExceptionObjectReg);
+    __ jmp(&done, compiler::Assembler::kNearJump);
+    __ Bind(&not_smi);
+    __ movq(kExceptionObjectReg,
+            compiler::FieldAddress(kExceptionObjectReg, Mint::value_offset()));
+    __ Bind(&done);
+  }
 
   // Load the stacktrace from the current thread.
   Address stacktrace_addr(THR, target::Thread::active_stacktrace_offset());
@@ -3416,6 +3428,14 @@ void StubCodeCompiler::GenerateRunExceptionHandlerStub() {
   __ jmp(CallingConventions::kArg1Reg);  // Jump to continuation point.
 }
 
+void StubCodeCompiler::GenerateRunExceptionHandlerStub() {
+  GenerateRunExceptionHandler(assembler, false);
+}
+
+void StubCodeCompiler::GenerateRunExceptionHandlerUnboxStub() {
+  GenerateRunExceptionHandler(assembler, true);
+}
+
 // Deoptimize a frame on the call stack before rewinding.
 // The arguments are stored in the Thread object.
 // No result.

runtime/vm/exceptions.cc

@@ -12,6 +12,7 @@
 #include "lib/stacktrace.h"
 
 #include "vm/dart_api_impl.h"
+#include "vm/dart_api_state.h"
 #include "vm/dart_entry.h"
 #include "vm/datastream.h"
 #include "vm/debugger.h"
@@ -538,24 +539,6 @@ CatchEntryMoves* CatchEntryMovesMapReader::ReadCompressedCatchEntryMovesSuffix(
   return moves;
 }
 
-static void FindErrorHandler(uword* handler_pc,
-                             uword* handler_sp,
-                             uword* handler_fp) {
-  StackFrameIterator frames(ValidationPolicy::kDontValidateFrames,
-                            Thread::Current(),
-                            StackFrameIterator::kNoCrossThreadIteration);
-  StackFrame* frame = frames.NextFrame();
-  ASSERT(frame != nullptr);
-  while (!frame->IsEntryFrame()) {
-    frame = frames.NextFrame();
-    ASSERT(frame != nullptr);
-  }
-  ASSERT(frame->IsEntryFrame());
-  *handler_pc = frame->pc();
-  *handler_sp = frame->sp();
-  *handler_fp = frame->fp();
-}
-
 static void ClearLazyDeopts(Thread* thread, uword frame_pointer) {
   if (thread->pending_deopts().HasPendingDeopts()) {
     // We may be jumping over frames scheduled for lazy deopt. Remove these
@@ -590,19 +573,40 @@ static void ClearLazyDeopts(Thread* thread, uword frame_pointer) {
   }
 }
 
+enum ExceptionType { kPassObject, kPassHandle, kPassUnboxed };
+
 static void JumpToExceptionHandler(Thread* thread,
                                    uword program_counter,
                                    uword stack_pointer,
                                    uword frame_pointer,
                                    const Object& exception_object,
-                                   const Object& stacktrace_object) {
+                                   const Object& stacktrace_object,
+                                   ExceptionType type = kPassObject) {
   bool clear_deopt = false;
   uword remapped_pc = thread->pending_deopts().RemapExceptionPCForDeopt(
       program_counter, frame_pointer, &clear_deopt);
-  thread->set_active_exception(exception_object);
+  uword run_exception_pc = StubCode::RunExceptionHandler().EntryPoint();
+  switch (type) {
+    case kPassObject:
+      thread->set_active_exception(exception_object);
+      break;
+    case kPassHandle: {
+      LocalHandle* handle =
+          thread->api_top_scope()->local_handles()->AllocateHandle();
+      handle->set_ptr(exception_object.ptr());
+      thread->set_active_exception(handle);
+      break;
+    }
+    case kPassUnboxed: {
+      thread->set_active_exception(exception_object);
+      run_exception_pc = StubCode::RunExceptionHandlerUnbox().EntryPoint();
+      break;
+    }
+    default:
+      UNREACHABLE();
+  }
   thread->set_active_stacktrace(stacktrace_object);
   thread->set_resume_pc(remapped_pc);
-  uword run_exception_pc = StubCode::RunExceptionHandler().EntryPoint();
   Exceptions::JumpToFrame(thread, run_exception_pc, stack_pointer,
                           frame_pointer, clear_deopt);
 }
@@ -1012,39 +1016,46 @@ void Exceptions::PropagateError(const Error& error) {
     const Instance& stk = Instance::Handle(zone, uhe.stacktrace());
     Exceptions::ReThrow(thread, exc, stk);
   } else {
+    const Instance& stk = StackTrace::Handle(zone);  // Null stacktrace.
     // Return to the invocation stub and return this error object.  The
     // C++ code which invoked this dart sequence can check and do the
     // appropriate thing.
-    uword handler_pc = 0;
-    uword handler_sp = 0;
-    uword handler_fp = 0;
-    FindErrorHandler(&handler_pc, &handler_sp, &handler_fp);
-    JumpToExceptionHandler(thread, handler_pc, handler_sp, handler_fp, error,
-                           StackTrace::Handle(zone));  // Null stacktrace.
+    StackFrameIterator frames(ValidationPolicy::kDontValidateFrames, thread,
+                              StackFrameIterator::kNoCrossThreadIteration);
+    StackFrame* frame = frames.NextFrame();
+    StackFrame* prev = frame;
+    ASSERT(frame != nullptr);
+    while (!frame->IsEntryFrame()) {
+      prev = frame;
+      frame = frames.NextFrame();
+      ASSERT(frame != nullptr);
+    }
+    if (frame->pc() == StubCode::InvokeDartCode().EntryPoint()) {
+      // This is an FFI callback using the invocation stub as a marker. Real use
+      // of invocation stub would be in the middle, not the entry point. Use the
+      // callback's exceptional return value instead of the error unless the
+      // return type is Dart_Handle.
+      ASSERT(prev->IsDartFrame());
+      frame = prev;
+      const Function& func =
+          Function::Handle(zone, frame->LookupDartFunction());
+      ASSERT(func.IsFfiCallbackTrampoline());
+      if (func.FfiCSignatureReturnsHandle()) {
+        JumpToExceptionHandler(thread, frame->pc(), frame->sp(), frame->fp(),
+                               error, stk, kPassHandle);
+      } else {
+        const Instance& val =
+            Instance::Handle(zone, func.FfiCallbackExceptionalReturn());
+        JumpToExceptionHandler(thread, frame->pc(), frame->sp(), frame->fp(),
+                               val, stk, kPassUnboxed);
+      }
+    }
+    JumpToExceptionHandler(thread, frame->pc(), frame->sp(), frame->fp(), error,
+                           stk);
   }
   UNREACHABLE();
 }
 
-void Exceptions::PropagateToEntry(const Error& error) {
-  Thread* thread = Thread::Current();
-  Zone* zone = thread->zone();
-  ASSERT(thread->top_exit_frame_info() != 0);
-  Instance& stacktrace = Instance::Handle(zone);
-  if (error.IsUnhandledException()) {
-    const UnhandledException& uhe = UnhandledException::Cast(error);
-    stacktrace = uhe.stacktrace();
-  } else {
-    stacktrace = Exceptions::CurrentStackTrace();
-  }
-  uword handler_pc = 0;
-  uword handler_sp = 0;
-  uword handler_fp = 0;
-  FindErrorHandler(&handler_pc, &handler_sp, &handler_fp);
-  JumpToExceptionHandler(thread, handler_pc, handler_sp, handler_fp, error,
-                         stacktrace);
-  UNREACHABLE();
-}
-
 void Exceptions::ThrowByType(ExceptionType type, const Array& arguments) {
   Thread* thread = Thread::Current();
   const Object& result =

runtime/vm/object.cc

@@ -8786,6 +8786,13 @@ bool Function::FfiCSignatureReturnsStruct() const {
   return true;
 }
 
+bool Function::FfiCSignatureReturnsHandle() const {
+  ASSERT(IsFfiCallbackTrampoline());
+  const auto& c_signature = FunctionType::Handle(FfiCSignature());
+  const auto& type = AbstractType::Handle(c_signature.result_type());
+  return type.type_class_id() == kFfiHandleCid;
+}
+
 int32_t Function::FfiCallbackId() const {
   ASSERT(IsFfiCallbackTrampoline());
 

runtime/vm/object.h

@@ -3024,6 +3024,7 @@ class Function : public Object {
 
   bool FfiCSignatureContainsHandles() const;
   bool FfiCSignatureReturnsStruct() const;
+  bool FfiCSignatureReturnsHandle() const;
 
   // Can only be called on FFI trampolines.
   int32_t FfiCallbackId() const;

runtime/vm/stub_code_list.h

@@ -32,6 +32,7 @@ namespace dart {
   V(GetCStackPointer)                                                          \
   V(JumpToFrame)                                                               \
   V(RunExceptionHandler)                                                       \
+  V(RunExceptionHandlerUnbox)                                                  \
   V(DeoptForRewind)                                                            \
   V(WriteBarrier)                                                              \
   V(WriteBarrierWrappers)                                                      \

runtime/vm/thread.cc

@@ -229,6 +229,11 @@ void Thread::set_active_exception(const Object& value) {
   active_exception_ = value.ptr();
 }
 
+void Thread::set_active_exception(LocalHandle* value) {
+  active_exception_ = ObjectPtr(reinterpret_cast<uword>(value));
+  ASSERT(active_exception_.IsImmediateObject());  // GC won't try to visit this.
+}
+
 void Thread::set_active_stacktrace(const Object& value) {
   active_stacktrace_ = value.ptr();
 }

runtime/vm/thread.h

@@ -54,6 +54,7 @@ class Interpreter;
 class Isolate;
 class IsolateGroup;
 class Library;
+class LocalHandle;
 class Object;
 class OSThread;
 class JSONObject;
@@ -874,6 +875,7 @@ class Thread : public ThreadState {
 
   ObjectPtr active_exception() const { return active_exception_; }
   void set_active_exception(const Object& value);
+  void set_active_exception(LocalHandle* value);
   static intptr_t active_exception_offset() {
     return OFFSET_OF(Thread, active_exception_);
   }

tests/ffi/callback_unwind_error_test.dart

@@ -9,21 +9,37 @@ import "dart:isolate";
 
 import "callback_tests_utils.dart";
 
-typedef SimpleAdditionType = Int32 Function(Int32, Int32);
-int simpleAddition(int x, int y) {
-  print("simpleAddition($x, $y)");
+typedef Type = Int32 Function(Int32, Int32);
+int unwindError(int x, int y) {
+  print("unwindError($x, $y)");
   Isolate.current.kill(priority: Isolate.immediate);
   return x + y;
 }
 
 final testcases = [
-  CallbackTest(
-    "SimpleAddition",
-    Pointer.fromFunction<SimpleAdditionType>(simpleAddition, 0),
-  ),
+  CallbackTest("UnwindError", Pointer.fromFunction<Type>(unwindError, 42)),
 ];
 
-void main() {
+void child(_) {
   testcases.forEach((t) => t.run());
   throw "Should not be reached";
 }
+
+void main() {
+  var onExit = new RawReceivePort();
+  var onError = new RawReceivePort();
+  onExit.handler = ((msg) {
+    print("Child exited");
+    onExit.close();
+    onError.close();
+  });
+  onError.handler = ((msg) {
+    throw "Child error: $msg";
+  });
+  Isolate.spawn(
+    child,
+    null,
+    onError: onError.sendPort,
+    onExit: onExit.sendPort,
+  );
+}

tests/ffi/vmspecific_callback_unwind_error_through_handle_test.dart

@@ -0,0 +1,51 @@
+// Copyright (c) 2025, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+// SharedObjects=ffi_test_functions
+
+import "dart:ffi";
+import "dart:isolate";
+
+import "dylib_utils.dart";
+
+final ffiTestFunctions = dlopenPlatformSpecific("ffi_test_functions");
+
+Object unwindErrorThroughHandle(int x, int y) {
+  print("unwindErrorThroughHandle($x, $y)");
+  Isolate.current.kill(priority: Isolate.immediate);
+  return x + y;
+}
+
+typedef CallbackType = Handle Function(Int32, Int32);
+typedef CalloutCType = Handle Function(Pointer);
+typedef CalloutDartType = Object Function(Pointer);
+
+void child(_) {
+  var callout = ffiTestFunctions.lookupFunction<CalloutCType, CalloutDartType>(
+    "TestUnwindErrorThroughHandle",
+    isLeaf: false,
+  );
+  var callback = Pointer.fromFunction<CallbackType>(unwindErrorThroughHandle);
+  var result = callout(callback);
+  throw "Should not be reached";
+}
+
+void main() {
+  var onExit = new RawReceivePort();
+  var onError = new RawReceivePort();
+  onExit.handler = ((msg) {
+    print("Child exited");
+    onExit.close();
+    onError.close();
+  });
+  onError.handler = ((msg) {
+    throw "Child error: $msg";
+  });
+  Isolate.spawn(
+    child,
+    null,
+    onError: onError.sendPort,
+    onExit: onExit.sendPort,
+  );
+}