[vm, ffi] Better handle errors that are not unhandled exceptions during FFI callbacks.

Before this change, an error reaching an FFI callback would attempt to execute the normal invocation stub from the beginning in the FFI callback's frame, which quickly crashes. After this change, the runtime recognizes this marker use of the invocation stub and returns to the FFI callback function instead.

TEST=ffi/unwind
Bug: https://github.com/dart-lang/sdk/issues/39487
Change-Id: I477cfcfc236e6cf518ebfe52860ba49e466ebf8b
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/409562
Reviewed-by: Daco Harkes <dacoharkes@google.com>
Commit-Queue: Ryan Macnak <rmacnak@google.com>

runtime/vm/compiler/stub_code_compiler_ia32.cc

@@ -2987,7 +2987,8 @@ void StubCodeCompiler::GenerateJumpToFrameStub() {
 //
 // The arguments are stored in the Thread object.
 // No result.
-void StubCodeCompiler::GenerateRunExceptionHandlerStub() {
+static void GenerateRunExceptionHandler(Assembler* assembler,
+                                        bool unbox_exception) {
   ASSERT(kExceptionObjectReg == EAX);
   ASSERT(kStackTraceObjectReg == EDX);
   __ movl(EBX, Address(THR, target::Thread::resume_pc_offset()));
@@ -2999,6 +3000,17 @@ void StubCodeCompiler::GenerateRunExceptionHandlerStub() {
   Address exception_addr(THR, target::Thread::active_exception_offset());
   __ movl(kExceptionObjectReg, exception_addr);
   __ movl(exception_addr, ECX);
+  if (unbox_exception) {
+    compiler::Label not_smi, done;
+    __ BranchIfNotSmi(kExceptionObjectReg, &not_smi,
+                      compiler::Assembler::kNearJump);
+    __ SmiUntag(kExceptionObjectReg);
+    __ jmp(&done, compiler::Assembler::kNearJump);
+    __ Bind(&not_smi);
+    __ movl(kExceptionObjectReg,
+            compiler::FieldAddress(kExceptionObjectReg, Mint::value_offset()));
+    __ Bind(&done);
+  }
 
   // Load the stacktrace from the current thread.
   Address stacktrace_addr(THR, target::Thread::active_stacktrace_offset());
@@ -3008,6 +3020,14 @@ void StubCodeCompiler::GenerateRunExceptionHandlerStub() {
   __ jmp(EBX);  // Jump to continuation point.
 }
 
+void StubCodeCompiler::GenerateRunExceptionHandlerStub() {
+  GenerateRunExceptionHandler(assembler, false);
+}
+
+void StubCodeCompiler::GenerateRunExceptionHandlerUnboxStub() {
+  GenerateRunExceptionHandler(assembler, true);
+}
+
 // Deoptimize a frame on the call stack before rewinding.
 // The arguments are stored in the Thread object.
 // No result.