[vm/concurrency] Add Dart_CreateLightweightIsolate/Dart_RunLoopAsync APIs

Dart_CreateLightweightIsolate API:

  This API can be used by embedders to create a lightweight isolate
  (inside an existing isolate group).

  It is the analogous to Dart_CreateIsolateGroup - though taking a
  parent isolate as parameter (inside whose IG we create a new isolate)
  instead of taking kernel/snapshot data.

  Right now this API works on AOT and returns an error in JIT, both cases
  are covered by the test.

Dart_RunLoopAsync:

  The API can be used by embedders to transfer ownership of an isolate to
  the VM, which will take care of running the message handling loop and
  shuts the isolate down once the last receive port has been closed.

  It does allow listening to error/exit events generated by the message
  loop implementation as well as allows setting errors-are-fatal,
  effectively the same event-loop related parameters from the
  `Isolate.spawn()` API, just in Dart.

  It's the embedders responsibility to first launch initial dart code
  which will take care of responding to events (i.e. directly/indirectly
  open a receive port) - otherwise the isolate cannot be talked to and
  would immediately shut down.

Since our vm/cc tests do support running in AOT, we use a hybrid
approach to test the functionality: We let a normal Dart test call a
small C wrapper using FFI, to create a lightweight isolate, run it on a
new thread and join that thread.

TEST=vm/dart{,_2}/isolates/dart_api_create_lightweight_isolate_test

Issue https://github.com/dart-lang/sdk/issues/36097
Closes https://github.com/dart-lang/sdk/issues/44088

Change-Id: Id77ba928793fdb517f6cb7e8130df98a0366ddd6
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/170983
Commit-Queue: Martin Kustermann <kustermann@google.com>
Reviewed-by: Ryan Macnak <rmacnak@google.com>
Reviewed-by: Alexander Aprelev <aam@google.com>

runtime/bin/ffi_test/ffi_test_functions_vmspecific.cc

@@ -10,6 +10,7 @@
 #include <csignal>
 
 #include "platform/globals.h"
+#include "platform/memory_sanitizer.h"
 #if defined(HOST_OS_WINDOWS)
 #include <psapi.h>
 #include <windows.h>
@@ -47,6 +48,12 @@ namespace dart {
 
 #define CHECK_EQ(X, Y) CHECK((X) == (Y))
 
+#define ENSURE(X)                                                              \
+  if (!(X)) {                                                                  \
+    fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, "Check failed: " #X);   \
+    exit(1);                                                                   \
+  }
+
 ////////////////////////////////////////////////////////////////////////////////
 // Functions for stress-testing.
 
@@ -271,6 +278,89 @@ DART_EXPORT intptr_t TestCallbackWrongIsolate(void (*fn)()) {
 
 #endif  // defined(TARGET_OS_LINUX)
 
+DART_EXPORT void IGH_MsanUnpoison(void* start, intptr_t length) {
+  MSAN_UNPOISON(start, length);
+}
+
+DART_EXPORT Dart_Isolate IGH_CreateIsolate(const char* name, void* peer) {
+  struct Helper {
+    static void ShutdownCallback(void* ig_data, void* isolate_data) {
+      char* string = reinterpret_cast<char*>(isolate_data);
+      ENSURE(string[0] == 'a');
+      string[0] = 'x';
+    }
+    static void CleanupCallback(void* ig_data, void* isolate_data) {
+      char* string = reinterpret_cast<char*>(isolate_data);
+      ENSURE(string[2] == 'c');
+      string[2] = 'z';
+    }
+  };
+
+  Dart_Isolate parent = Dart_CurrentIsolate();
+  Dart_ExitIsolate();
+
+  char* error = nullptr;
+  Dart_Isolate child =
+      Dart_CreateIsolateInGroup(parent, name, &Helper::ShutdownCallback,
+                                &Helper::CleanupCallback, peer, &error);
+  if (child == nullptr) {
+    Dart_EnterIsolate(parent);
+    Dart_Handle error_obj = Dart_NewStringFromCString(error);
+    free(error);
+    Dart_ThrowException(error_obj);
+    return nullptr;
+  }
+  Dart_ExitIsolate();
+  Dart_EnterIsolate(parent);
+  return child;
+}
+
+DART_EXPORT void IGH_StartIsolate(Dart_Isolate child_isolate,
+                                  int64_t main_isolate_port,
+                                  const char* library_uri,
+                                  const char* function_name,
+                                  bool errors_are_fatal,
+                                  Dart_Port on_error_port,
+                                  Dart_Port on_exit_port) {
+  Dart_Isolate parent = Dart_CurrentIsolate();
+  Dart_ExitIsolate();
+  Dart_EnterIsolate(child_isolate);
+  {
+    Dart_EnterScope();
+
+    Dart_Handle library_name = Dart_NewStringFromCString(library_uri);
+    ENSURE(!Dart_IsError(library_name));
+
+    Dart_Handle library = Dart_LookupLibrary(library_name);
+    ENSURE(!Dart_IsError(library));
+
+    Dart_Handle fun = Dart_NewStringFromCString(function_name);
+    ENSURE(!Dart_IsError(fun));
+
+    Dart_Handle port = Dart_NewInteger(main_isolate_port);
+    ENSURE(!Dart_IsError(port));
+
+    Dart_Handle args[] = {
+        port,
+    };
+
+    Dart_Handle result = Dart_Invoke(library, fun, 1, args);
+    if (Dart_IsError(result)) {
+      fprintf(stderr, "Failed to invoke %s/%s in child isolate: %s\n",
+              library_uri, function_name, Dart_GetError(result));
+    }
+    ENSURE(!Dart_IsError(result));
+
+    Dart_ExitScope();
+  }
+
+  char* error = nullptr;
+  ENSURE(
+      Dart_RunLoopAsync(errors_are_fatal, on_error_port, on_exit_port, &error));
+
+  Dart_EnterIsolate(parent);
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 // Initialize `dart_api_dl.h`
 DART_EXPORT intptr_t InitDartApiDL(void* data) {

runtime/include/dart_api.h

@@ -1007,6 +1007,37 @@ Dart_CreateIsolateGroup(const char* script_uri,
                         void* isolate_group_data,
                         void* isolate_data,
                         char** error);
+/**
+ * Creates a new isolate inside the isolate group of [group_member].
+ *
+ * Requires there to be no current isolate.
+ *
+ * \param group_member An isolate from the same group into which the newly created
+ *   isolate should be born into. Other threads may not have entered / enter this
+ *   member isolate.
+ * \param name A short name for the isolate for debugging purposes.
+ * \param shutdown_callback A callback to be called when the isolate is being
+ *   shutdown (may be NULL).
+ * \param cleanup_callback A callback to be called when the isolate is being
+ *   cleaned up (may be NULL).
+ * \param isolate_data The embedder-specific data associated with this isolate.
+ * \param error Set to NULL if creation is successful, set to an error
+ *   message otherwise. The caller is responsible for calling free() on the
+ *   error message.
+ *
+ * \return The newly created isolate on success, or NULL if isolate creation
+ *   failed.
+ *
+ * If successful, the newly created isolate will become the current isolate.
+ */
+DART_EXPORT Dart_Isolate
+Dart_CreateIsolateInGroup(Dart_Isolate group_member,
+                          const char* name,
+                          Dart_IsolateShutdownCallback shutdown_callback,
+                          Dart_IsolateCleanupCallback cleanup_callback,
+                          void* child_isolate_data,
+                          char** error);
+
 /* TODO(turnidge): Document behavior when there is already a current
  * isolate. */
 
@@ -1483,6 +1514,31 @@ DART_EXPORT bool Dart_HasServiceMessages();
  *   error handle is returned.
  */
 DART_EXPORT DART_WARN_UNUSED_RESULT Dart_Handle Dart_RunLoop();
+
+/**
+ * Lets the VM run message processing for the isolate.
+ *
+ * This function expects there to a current isolate and the current isolate
+ * must not have an active api scope. The VM will take care of making the
+ * isolate runnable (if not already), handles its message loop and will take
+ * care of shutting the isolate down once it's done.
+ *
+ * \param errors_are_fatal Whether uncaught errors should be fatal.
+ * \param on_error_port A port to notify on uncaught errors (or ILLEGAL_PORT).
+ * \param on_exit_port A port to notify on exit (or ILLEGAL_PORT).
+ * \param error A non-NULL pointer which will hold an error message if the call
+ *   fails. The error has to be free()ed by the caller.
+ *
+ * \return If successfull the VM takes owernship of the isolate and takes care
+ *   of its message loop. If not successful the caller retains owernship of the
+ *   isolate.
+ */
+DART_EXPORT DART_WARN_UNUSED_RESULT bool Dart_RunLoopAsync(
+    bool errors_are_fatal,
+    Dart_Port on_error_port,
+    Dart_Port on_exit_port,
+    char** error);
+
 /* TODO(turnidge): Should this be removed from the public api? */
 
 /**

runtime/lib/isolate.cc

@@ -420,7 +420,7 @@ class SpawnIsolateTask : public ThreadPool::Task {
 
     child->set_origin_id(state_->origin_id());
     child->set_spawn_state(std::move(state_));
-    child->Run();
+    child->RunViaSpawnApi();
   }
 
   void FailedSpawn(const char* error) {

runtime/tests/vm/dart/isolates/dart_api_create_lightweight_isolate_test.dart

@@ -0,0 +1,225 @@
+// Copyright (c) 2020, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+// SharedObjects=ffi_test_functions
+// VMOptions=--enable-isolate-groups --disable-heap-verification
+
+import 'dart:async';
+import 'dart:ffi';
+import 'dart:io';
+import 'dart:isolate';
+
+import 'package:expect/expect.dart';
+import 'package:ffi/ffi.dart';
+
+import '../../../../../tests/ffi/dylib_utils.dart';
+
+final bool isAOT = Platform.executable.contains('dart_precompiled_runtime');
+final sdkRoot = Platform.script.resolve('../../../../../');
+
+class Isolate extends Struct {}
+
+abstract class FfiBindings {
+  static final ffiTestFunctions = dlopenPlatformSpecific("ffi_test_functions");
+
+  static final IGH_CreateIsolate = ffiTestFunctions.lookupFunction<
+      Pointer<Isolate> Function(Pointer<Utf8>, Pointer<Void>),
+      Pointer<Isolate> Function(
+          Pointer<Utf8>, Pointer<Void>)>('IGH_CreateIsolate');
+
+  static final IGH_StartIsolate = ffiTestFunctions.lookupFunction<
+      Pointer<Void> Function(Pointer<Isolate>, Int64, Pointer<Utf8>,
+          Pointer<Utf8>, IntPtr, Int64, Int64),
+      Pointer<Void> Function(Pointer<Isolate>, int, Pointer<Utf8>,
+          Pointer<Utf8>, int, int, int)>('IGH_StartIsolate');
+
+  static final Dart_CurrentIsolate = DynamicLibrary.executable()
+      .lookupFunction<Pointer<Isolate> Function(), Pointer<Isolate> Function()>(
+          "Dart_CurrentIsolate");
+
+  static final Dart_IsolateData = DynamicLibrary.executable().lookupFunction<
+      Pointer<Isolate> Function(Pointer<Isolate>),
+      Pointer<Isolate> Function(Pointer<Isolate>)>("Dart_IsolateData");
+
+  static final Dart_PostInteger = DynamicLibrary.executable()
+      .lookupFunction<IntPtr Function(Int64, Int64), int Function(int, int)>(
+          "Dart_PostInteger");
+
+  static Pointer<Isolate> createLightweightIsolate(
+      String name, Pointer<Void> peer) {
+    final cname = Utf8.toUtf8(name);
+    try {
+      final isolate = IGH_CreateIsolate(cname, peer);
+      Expect.isTrue(isolate.address != 0);
+      return isolate;
+    } finally {
+      free(cname);
+    }
+  }
+
+  static void invokeTopLevelAndRunLoopAsync(
+      Pointer<Isolate> isolate, SendPort sendPort, String name,
+      {bool? errorsAreFatal, SendPort? onError, SendPort? onExit}) {
+    final dartScript = sdkRoot.resolve(
+        'runtime/tests/vm/dart/isolates/dart_api_create_lightweight_isolate_test.dart');
+    final libraryUri = Utf8.toUtf8(dartScript.toString());
+    final functionName = Utf8.toUtf8(name);
+
+    IGH_StartIsolate(
+        isolate,
+        sendPort.nativePort,
+        libraryUri,
+        functionName,
+        errorsAreFatal == false ? 0 : 1,
+        onError != null ? onError.nativePort : 0,
+        onExit != null ? onExit.nativePort : 0);
+
+    free(libraryUri);
+    free(functionName);
+  }
+}
+
+void scheduleAsyncInvocation(void fun()) {
+  final rp = RawReceivePort();
+  rp.handler = (_) {
+    try {
+      fun();
+    } finally {
+      rp.close();
+    }
+  };
+  rp.sendPort.send(null);
+}
+
+Future withPeerPointer(fun(Pointer<Void> peer)) async {
+  final Pointer<Void> peer = Utf8.toUtf8('abc').cast();
+  try {
+    await fun(peer);
+  } catch (e, s) {
+    print('Exception: $e\nStack:$s');
+    rethrow;
+  } finally {
+    // The shutdown callback is called before the exit listeners are notified, so
+    // we can validate that a->x has been changed.
+    Expect.isTrue(Utf8.fromUtf8(peer.cast()).startsWith('xb'));
+
+    // The cleanup callback is called after after notifying exit listeners. So we
+    // wait a little here to ensure the write of the callback has arrived.
+    await Future.delayed(const Duration(milliseconds: 100));
+    Expect.equals('xbz', Utf8.fromUtf8(peer.cast()));
+    free(peer);
+  }
+}
+
+@pragma('vm:entry-point')
+void childTestIsolateData(int mainPort) {
+  final peerIsolateData =
+      FfiBindings.Dart_IsolateData(FfiBindings.Dart_CurrentIsolate());
+  FfiBindings.Dart_PostInteger(mainPort, peerIsolateData.address);
+}
+
+Future testIsolateData() async {
+  await withPeerPointer((Pointer<Void> peer) async {
+    final rp = ReceivePort();
+    final exit = ReceivePort();
+    final isolate = FfiBindings.createLightweightIsolate('debug-name', peer);
+    FfiBindings.invokeTopLevelAndRunLoopAsync(
+        isolate, rp.sendPort, 'childTestIsolateData',
+        onExit: exit.sendPort);
+
+    Expect.equals(peer.address, await rp.first);
+    await exit.first;
+
+    exit.close();
+    rp.close();
+  });
+}
+
+@pragma('vm:entry-point')
+void childTestMultipleErrors(int mainPort) {
+  scheduleAsyncInvocation(() {
+    for (int i = 0; i < 10; ++i) {
+      scheduleAsyncInvocation(() => throw 'error-$i');
+    }
+  });
+}
+
+Future testMultipleErrors() async {
+  await withPeerPointer((Pointer<Void> peer) async {
+    final rp = ReceivePort();
+    final accumulatedErrors = <dynamic>[];
+    final errors = ReceivePort()..listen(accumulatedErrors.add);
+    final exit = ReceivePort();
+    final isolate = FfiBindings.createLightweightIsolate('debug-name', peer);
+    FfiBindings.invokeTopLevelAndRunLoopAsync(
+        isolate, rp.sendPort, 'childTestMultipleErrors',
+        errorsAreFatal: false, onError: errors.sendPort, onExit: exit.sendPort);
+    await exit.first;
+    Expect.equals(10, accumulatedErrors.length);
+    for (int i = 0; i < 10; ++i) {
+      Expect.equals('error-$i', accumulatedErrors[i][0]);
+      Expect.isTrue(
+          accumulatedErrors[i][1].contains('childTestMultipleErrors'));
+    }
+
+    exit.close();
+    errors.close();
+    rp.close();
+  });
+}
+
+@pragma('vm:entry-point')
+void childTestFatalError(int mainPort) {
+  scheduleAsyncInvocation(() {
+    scheduleAsyncInvocation(() => throw 'error-0');
+    scheduleAsyncInvocation(() => throw 'error-1');
+  });
+}
+
+Future testFatalError() async {
+  await withPeerPointer((Pointer<Void> peer) async {
+    final rp = ReceivePort();
+    final accumulatedErrors = <dynamic>[];
+    final errors = ReceivePort()..listen(accumulatedErrors.add);
+    final exit = ReceivePort();
+    final isolate = FfiBindings.createLightweightIsolate('debug-name', peer);
+    FfiBindings.invokeTopLevelAndRunLoopAsync(
+        isolate, rp.sendPort, 'childTestFatalError',
+        errorsAreFatal: true, onError: errors.sendPort, onExit: exit.sendPort);
+    await exit.first;
+    Expect.equals(1, accumulatedErrors.length);
+    Expect.equals('error-0', accumulatedErrors[0][0]);
+    Expect.isTrue(accumulatedErrors[0][1].contains('childTestFatalError'));
+
+    exit.close();
+    errors.close();
+    rp.close();
+  });
+}
+
+Future testAot() async {
+  await testIsolateData();
+  await testMultipleErrors();
+  await testFatalError();
+}
+
+Future testJit() async {
+  dynamic exception;
+  try {
+    FfiBindings.createLightweightIsolate('debug-name', Pointer.fromAddress(0));
+  } catch (e) {
+    exception = e;
+  }
+  Expect.isTrue(exception
+      .toString()
+      .contains('Lightweight isolates are not yet ready in JIT mode'));
+}
+
+Future main(args) async {
+  if (isAOT) {
+    await testAot();
+  } else {
+    await testJit();
+  }
+}

runtime/tests/vm/dart_2/isolates/dart_api_create_lightweight_isolate_test.dart

@@ -0,0 +1,234 @@
+// Copyright (c) 2020, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+// SharedObjects=ffi_test_functions
+// VMOptions=--enable-isolate-groups --disable-heap-verification
+
+import 'dart:async';
+import 'dart:ffi';
+import 'dart:io';
+import 'dart:isolate';
+
+import 'package:expect/expect.dart';
+import 'package:ffi/ffi.dart';
+
+import '../../../../../tests/ffi/dylib_utils.dart';
+
+final bool isAOT = Platform.executable.contains('dart_precompiled_runtime');
+final sdkRoot = Platform.script.resolve('../../../../../');
+
+class Isolate extends Struct {}
+
+abstract class FfiBindings {
+  static final ffiTestFunctions = dlopenPlatformSpecific("ffi_test_functions");
+
+  static final IGH_MsanUnpoison = ffiTestFunctions.lookupFunction<
+      Pointer<Isolate> Function(Pointer<Void>, IntPtr),
+      Pointer<Isolate> Function(Pointer<Void>, int)>('IGH_MsanUnpoison');
+
+  static final IGH_CreateIsolate = ffiTestFunctions.lookupFunction<
+      Pointer<Isolate> Function(Pointer<Utf8>, Pointer<Void>),
+      Pointer<Isolate> Function(
+          Pointer<Utf8>, Pointer<Void>)>('IGH_CreateIsolate');
+
+  static final IGH_StartIsolate = ffiTestFunctions.lookupFunction<
+      Pointer<Void> Function(Pointer<Isolate>, Int64, Pointer<Utf8>,
+          Pointer<Utf8>, IntPtr, Int64, Int64),
+      Pointer<Void> Function(Pointer<Isolate>, int, Pointer<Utf8>,
+          Pointer<Utf8>, int, int, int)>('IGH_StartIsolate');
+
+  static final Dart_CurrentIsolate = DynamicLibrary.executable()
+      .lookupFunction<Pointer<Isolate> Function(), Pointer<Isolate> Function()>(
+          "Dart_CurrentIsolate");
+
+  static final Dart_IsolateData = DynamicLibrary.executable().lookupFunction<
+      Pointer<Isolate> Function(Pointer<Isolate>),
+      Pointer<Isolate> Function(Pointer<Isolate>)>("Dart_IsolateData");
+
+  static final Dart_PostInteger = DynamicLibrary.executable()
+      .lookupFunction<IntPtr Function(Int64, Int64), int Function(int, int)>(
+          "Dart_PostInteger");
+
+  static Pointer<Isolate> createLightweightIsolate(
+      String name, Pointer<Void> peer) {
+    final cname = Utf8.toUtf8(name);
+    IGH_MsanUnpoison(cname.cast(), name.length + 10);
+    try {
+      final isolate = IGH_CreateIsolate(cname, peer);
+      Expect.isTrue(isolate.address != 0);
+      return isolate;
+    } finally {
+      free(cname);
+    }
+  }
+
+  static void invokeTopLevelAndRunLoopAsync(
+      Pointer<Isolate> isolate, SendPort sendPort, String name,
+      {bool errorsAreFatal, SendPort onError, SendPort onExit}) {
+    final dartScriptUri = sdkRoot.resolve(
+        'runtime/tests/vm/dart_2/isolates/dart_api_create_lightweight_isolate_test.dart');
+    final dartScript = dartScriptUri.toString();
+    final libraryUri = Utf8.toUtf8(dartScript);
+    IGH_MsanUnpoison(libraryUri.cast(), dartScript.length + 1);
+    final functionName = Utf8.toUtf8(name);
+    IGH_MsanUnpoison(functionName.cast(), name.length + 1);
+
+    IGH_StartIsolate(
+        isolate,
+        sendPort.nativePort,
+        libraryUri,
+        functionName,
+        errorsAreFatal == false ? 0 : 1,
+        onError != null ? onError.nativePort : 0,
+        onExit != null ? onExit.nativePort : 0);
+
+    free(libraryUri);
+    free(functionName);
+  }
+}
+
+void scheduleAsyncInvocation(void fun()) {
+  final rp = RawReceivePort();
+  rp.handler = (_) {
+    try {
+      fun();
+    } finally {
+      rp.close();
+    }
+  };
+  rp.sendPort.send(null);
+}
+
+Future withPeerPointer(fun(Pointer<Void> peer)) async {
+  final Pointer<Void> peer = Utf8.toUtf8('abc').cast();
+  FfiBindings.IGH_MsanUnpoison(peer.cast(), 'abc'.length + 1);
+  try {
+    await fun(peer);
+  } catch (e, s) {
+    print('Exception: $e\nStack:$s');
+    rethrow;
+  } finally {
+    // The shutdown callback is called before the exit listeners are notified, so
+    // we can validate that a->x has been changed.
+    Expect.isTrue(Utf8.fromUtf8(peer.cast()).startsWith('xb'));
+
+    // The cleanup callback is called after after notifying exit listeners. So we
+    // wait a little here to ensure the write of the callback has arrived.
+    await Future.delayed(const Duration(milliseconds: 100));
+    Expect.equals('xbz', Utf8.fromUtf8(peer.cast()));
+    free(peer);
+  }
+}
+
+@pragma('vm:entry-point')
+void childTestIsolateData(int mainPort) {
+  final peerIsolateData =
+      FfiBindings.Dart_IsolateData(FfiBindings.Dart_CurrentIsolate());
+  FfiBindings.Dart_PostInteger(mainPort, peerIsolateData.address);
+}
+
+Future testIsolateData() async {
+  await withPeerPointer((Pointer<Void> peer) async {
+    final rp = ReceivePort();
+    final exit = ReceivePort();
+    final isolate = FfiBindings.createLightweightIsolate('debug-name', peer);
+    FfiBindings.invokeTopLevelAndRunLoopAsync(
+        isolate, rp.sendPort, 'childTestIsolateData',
+        onExit: exit.sendPort);
+
+    Expect.equals(peer.address, await rp.first);
+    await exit.first;
+
+    exit.close();
+    rp.close();
+  });
+}
+
+@pragma('vm:entry-point')
+void childTestMultipleErrors(int mainPort) {
+  scheduleAsyncInvocation(() {
+    for (int i = 0; i < 10; ++i) {
+      scheduleAsyncInvocation(() => throw 'error-$i');
+    }
+  });
+}
+
+Future testMultipleErrors() async {
+  await withPeerPointer((Pointer<Void> peer) async {
+    final rp = ReceivePort();
+    final accumulatedErrors = <dynamic>[];
+    final errors = ReceivePort()..listen(accumulatedErrors.add);
+    final exit = ReceivePort();
+    final isolate = FfiBindings.createLightweightIsolate('debug-name', peer);
+    FfiBindings.invokeTopLevelAndRunLoopAsync(
+        isolate, rp.sendPort, 'childTestMultipleErrors',
+        errorsAreFatal: false, onError: errors.sendPort, onExit: exit.sendPort);
+    await exit.first;
+    Expect.equals(10, accumulatedErrors.length);
+    for (int i = 0; i < 10; ++i) {
+      Expect.equals('error-$i', accumulatedErrors[i][0]);
+      Expect.isTrue(
+          accumulatedErrors[i][1].contains('childTestMultipleErrors'));
+    }
+
+    exit.close();
+    errors.close();
+    rp.close();
+  });
+}
+
+@pragma('vm:entry-point')
+void childTestFatalError(int mainPort) {
+  scheduleAsyncInvocation(() {
+    scheduleAsyncInvocation(() => throw 'error-0');
+    scheduleAsyncInvocation(() => throw 'error-1');
+  });
+}
+
+Future testFatalError() async {
+  await withPeerPointer((Pointer<Void> peer) async {
+    final rp = ReceivePort();
+    final accumulatedErrors = <dynamic>[];
+    final errors = ReceivePort()..listen(accumulatedErrors.add);
+    final exit = ReceivePort();
+    final isolate = FfiBindings.createLightweightIsolate('debug-name', peer);
+    FfiBindings.invokeTopLevelAndRunLoopAsync(
+        isolate, rp.sendPort, 'childTestFatalError',
+        errorsAreFatal: true, onError: errors.sendPort, onExit: exit.sendPort);
+    await exit.first;
+    Expect.equals(1, accumulatedErrors.length);
+    Expect.equals('error-0', accumulatedErrors[0][0]);
+    Expect.isTrue(accumulatedErrors[0][1].contains('childTestFatalError'));
+
+    exit.close();
+    errors.close();
+    rp.close();
+  });
+}
+
+Future testAot() async {
+  await testIsolateData();
+  await testMultipleErrors();
+  await testFatalError();
+}
+
+Future testJit() async {
+  dynamic exception;
+  try {
+    FfiBindings.createLightweightIsolate('debug-name', Pointer.fromAddress(0));
+  } catch (e) {
+    exception = e;
+  }
+  Expect.isTrue(exception
+      .toString()
+      .contains('Lightweight isolates are not yet ready in JIT mode'));
+}
+
+Future main(args) async {
+  if (isAOT) {
+    await testAot();
+  } else {
+    await testJit();
+  }
+}

runtime/tests/vm/vm.status

@@ -327,12 +327,14 @@ cc/Profiler_ToggleRecordAllocation: SkipByDesign
 cc/Profiler_TrivialRecordAllocation: SkipByDesign
 cc/Profiler_TypedArrayAllocation: SkipByDesign
 cc/Service_Profile: SkipByDesign
+dart/isolates/dart_api_create_lightweight_isolate_test: SkipByDesign # Test uses dart:ffi which is not supported on simulators.
 dart/isolates/thread_pool_test: SkipByDesign # Test uses dart:ffi which is not supported on simulators.
 dart/regress_41971_test: SkipByDesign # dart:ffi is not supported on simulator
 dart/sdk_hash_test: SkipSlow # gen_kernel is slow to run on simarm
 dart/unboxed_param_args_descriptor_test: SkipByDesign # FFI helper not supported on simulator
 dart/unboxed_param_tear_off_test: SkipByDesign # FFI helper not supported on simulator
 dart/unboxed_param_test: SkipByDesign # FFI helper not supported on simulator
+dart_2/isolates/dart_api_create_lightweight_isolate_test: SkipByDesign # Test uses dart:ffi which is not supported on simulators.
 dart_2/isolates/thread_pool_test: SkipByDesign # Test uses dart:ffi which is not supported on simulators.
 dart_2/regress_41971_test: SkipByDesign # dart:ffi is not supported on simulator
 dart_2/sdk_hash_test: SkipSlow # gen_kernel is slow to run on simarm

runtime/vm/dart.cc

@@ -1068,7 +1068,7 @@ void Dart::RunShutdownCallback() {
   Isolate* isolate = thread->isolate();
   void* isolate_group_data = isolate->group()->embedder_data();
   void* isolate_data = isolate->init_callback_data();
-  Dart_IsolateShutdownCallback callback = Isolate::ShutdownCallback();
+  Dart_IsolateShutdownCallback callback = isolate->on_shutdown_callback();
   if (callback != NULL) {
     TransitionVMToNative transition(thread);
     (callback)(isolate_group_data, isolate_data);

runtime/vm/dart_api_impl.cc

@@ -1563,6 +1563,39 @@ Dart_CreateIsolateGroupFromKernel(const char* script_uri,
   return isolate;
 }
 
+DART_EXPORT Dart_Isolate
+Dart_CreateIsolateInGroup(Dart_Isolate group_member,
+                          const char* name,
+                          Dart_IsolateShutdownCallback shutdown_callback,
+                          Dart_IsolateCleanupCallback cleanup_callback,
+                          void* child_isolate_data,
+                          char** error) {
+  CHECK_NO_ISOLATE(Isolate::Current());
+  auto member = reinterpret_cast<Isolate*>(group_member);
+  if (member->IsScheduled()) {
+    FATAL("The given member isolate (%s) must not have been entered.",
+          member->name());
+  }
+
+  *error = nullptr;
+
+  Isolate* isolate;
+#if defined(DART_PRECOMPILED_RUNTIME)
+  isolate = CreateWithinExistingIsolateGroupAOT(member->group(), name, error);
+  if (isolate != nullptr) {
+    isolate->set_origin_id(member->origin_id());
+    isolate->set_init_callback_data(child_isolate_data);
+    isolate->set_on_shutdown_callback(shutdown_callback);
+    isolate->set_on_cleanup_callback(cleanup_callback);
+  }
+#else
+  *error = Utils::StrDup("Lightweight isolates are not yet ready in JIT mode.");
+  isolate = nullptr;
+#endif
+
+  return Api::CastIsolate(isolate);
+}
+
 DART_EXPORT void Dart_ShutdownIsolate() {
   Thread* T = Thread::Current();
   Isolate* I = T->isolate();
@@ -2090,6 +2123,53 @@ DART_EXPORT Dart_Handle Dart_RunLoop() {
   return Api::Success();
 }
 
+DART_EXPORT bool Dart_RunLoopAsync(bool errors_are_fatal,
+                                   Dart_Port on_error_port,
+                                   Dart_Port on_exit_port,
+                                   char** error) {
+  auto thread = Thread::Current();
+  auto isolate = thread->isolate();
+  CHECK_ISOLATE(isolate);
+  *error = nullptr;
+
+  if (thread->api_top_scope() != nullptr) {
+    *error = Utils::StrDup("There must not be an active api scope.");
+    return false;
+  }
+
+  if (!isolate->is_runnable()) {
+    const char* error_msg = isolate->MakeRunnable();
+    if (error_msg != nullptr) {
+      *error = Utils::StrDup(error_msg);
+      return false;
+    }
+  }
+
+  isolate->SetErrorsFatal(errors_are_fatal);
+
+  if (on_error_port != ILLEGAL_PORT || on_exit_port != ILLEGAL_PORT) {
+    auto thread = Thread::Current();
+    TransitionNativeToVM transition(thread);
+    StackZone zone(thread);
+    HANDLESCOPE(thread);
+
+    if (on_error_port != ILLEGAL_PORT) {
+      const auto& port =
+          SendPort::Handle(thread->zone(), SendPort::New(on_error_port));
+      isolate->AddErrorListener(port);
+    }
+    if (on_exit_port != ILLEGAL_PORT) {
+      const auto& port =
+          SendPort::Handle(thread->zone(), SendPort::New(on_exit_port));
+      isolate->AddExitListener(port, Instance::null_instance());
+    }
+  }
+
+  Dart_ExitIsolate();
+  isolate->RunViaEmbedder();
+  return true;
+}
+
 DART_EXPORT Dart_Handle Dart_HandleMessage() {
   Thread* T = Thread::Current();
   Isolate* I = T->isolate();

runtime/vm/dart_api_impl_test.cc

@@ -7866,10 +7866,10 @@ VM_UNIT_TEST_CASE(DartAPI_IsolateShutdownRunDartCode) {
       "}\n";
 
   // Create an isolate.
-  Dart_Isolate isolate = TestCase::CreateTestIsolate();
+  auto isolate = reinterpret_cast<Isolate*>(TestCase::CreateTestIsolate());
   EXPECT(isolate != NULL);
 
-  Isolate::SetShutdownCallback(IsolateShutdownRunDartCodeTestCallback);
+  isolate->set_on_shutdown_callback(IsolateShutdownRunDartCodeTestCallback);
 
   {
     Dart_EnterScope();
@@ -7887,8 +7887,6 @@ VM_UNIT_TEST_CASE(DartAPI_IsolateShutdownRunDartCode) {
   // The shutdown callback has not been called.
   EXPECT_EQ(0, add_result);
 
-  EXPECT(isolate != NULL);
-
   // Shutdown the isolate.
   Dart_ShutdownIsolate();
 

runtime/vm/isolate.cc

@@ -1644,6 +1644,8 @@ Isolate::Isolate(IsolateGroup* isolate_group,
           reload_every_n_stack_overflow_checks_(FLAG_reload_every),
 #endif  // !defined(PRODUCT)
       start_time_micros_(OS::GetCurrentMonotonicMicros()),
+      on_shutdown_callback_(Isolate::ShutdownCallback()),
+      on_cleanup_callback_(Isolate::CleanupCallback()),
       random_(),
       mutex_(NOT_IN_PRODUCT("Isolate::mutex_")),
       constant_canonicalization_mutex_(
@@ -2048,8 +2050,6 @@ void Isolate::DeleteReloadContext() {
 #endif  // !defined(PRODUCT) && !defined(DART_PRECOMPILED_RUNTIME)
 
 const char* Isolate::MakeRunnable() {
-  ASSERT(Isolate::Current() == nullptr);
-
   MutexLocker ml(&mutex_);
   // Check if we are in a valid state to make the isolate runnable.
   if (is_runnable() == true) {
@@ -2417,11 +2417,18 @@ void Isolate::SetStickyError(ErrorPtr sticky_error) {
   sticky_error_ = sticky_error;
 }
 
-void Isolate::Run() {
+void Isolate::RunViaSpawnApi() {
+  ASSERT(spawn_state() != nullptr);
   message_handler()->Run(group()->thread_pool(), RunIsolate, ShutdownIsolate,
                          reinterpret_cast<uword>(this));
 }
 
+void Isolate::RunViaEmbedder() {
+  ASSERT(spawn_state() == nullptr);
+  message_handler()->Run(group()->thread_pool(), nullptr, ShutdownIsolate,
+                         reinterpret_cast<uword>(this));
+}
+
 void Isolate::AddClosureFunction(const Function& function) const {
   ASSERT(!Compiler::IsBackgroundCompilation());
   GrowableObjectArray& closures =
@@ -2634,7 +2641,7 @@ void Isolate::LowLevelCleanup(Isolate* isolate) {
   // Cache these two fields, since they are no longer available after the
   // `delete this` further down.
   IsolateGroup* isolate_group = isolate->isolate_group_;
-  Dart_IsolateCleanupCallback cleanup = Isolate::CleanupCallback();
+  Dart_IsolateCleanupCallback cleanup = isolate->on_cleanup_callback();
   auto callback_data = isolate->init_callback_data_;
 
   // From this point on the isolate is no longer visited by GC (which is ok,

runtime/vm/isolate.h

@@ -786,6 +786,8 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry<Isolate> {
     return thread == nullptr ? nullptr : thread->isolate();
   }
 
+  bool IsScheduled() { return scheduled_mutator_thread_ != nullptr; }
+
   // Register a newly introduced class.
   void RegisterClass(const Class& cls);
 #if defined(DEBUG)
@@ -857,6 +859,19 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry<Isolate> {
     message_notify_callback_ = value;
   }
 
+  void set_on_shutdown_callback(Dart_IsolateShutdownCallback value) {
+    on_shutdown_callback_ = value;
+  }
+  Dart_IsolateShutdownCallback on_shutdown_callback() {
+    return on_shutdown_callback_;
+  }
+  void set_on_cleanup_callback(Dart_IsolateCleanupCallback value) {
+    on_cleanup_callback_ = value;
+  }
+  Dart_IsolateCleanupCallback on_cleanup_callback() {
+    return on_cleanup_callback_;
+  }
+
   void bequeath(std::unique_ptr<Bequest> bequest) {
     bequest_ = std::move(bequest);
   }
@@ -924,7 +939,15 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry<Isolate> {
 
   const char* MakeRunnable();
   void MakeRunnableLocked();
-  void Run();
+
+  // Runs the isolate if it was created inside the VM as a response to
+  // invocation of Dart's `Isolate.spawn` api.
+  //
+  // It will wake up a potential await'er (e.g. `await Isolate.spawn()`).
+  void RunViaSpawnApi();
+
+  // Runs the isolate if it was created by the embedder.
+  void RunViaEmbedder();
 
   MessageHandler* message_handler() const { return message_handler_; }
   void set_message_handler(MessageHandler* value) { message_handler_ = value; }
@@ -1565,6 +1588,8 @@ class Isolate : public BaseIsolate, public IntrusiveDListEntry<Isolate> {
   // All other fields go here.
   int64_t start_time_micros_;
   Dart_MessageNotifyCallback message_notify_callback_ = nullptr;
+  Dart_IsolateShutdownCallback on_shutdown_callback_ = nullptr;
+  Dart_IsolateCleanupCallback on_cleanup_callback_ = nullptr;
   char* name_ = nullptr;
   Dart_Port main_port_ = 0;
   // Isolates created by Isolate.spawn have the same origin id.