[ffi] Add DynamicLibrary.openFromAssetId

Adds a new API `DynamicLibrary.openFromAssetId` to `dart:ffi` that
allows opening dynamically loaded libraries using their native asset IDs
instead of physical file paths.

Closes https://github.com/dart-lang/sdk/issues/63295

TEST=./tools/test.py -n vm-aot-linux-debug-x64 'ffi/native_assets/asset_*'
Cq-Include-Trybots: dart/try:vm-aot-linux-debug-arm64-try,vm-aot-linux-debug-x64-try,vm-aot-linux-debug-x64c-try,vm-aot-mac-debug-arm64-try,vm-aot-mac-debug-x64-try,vm-aot-obfuscate-linux-release-x64-try,vm-aot-optimization-level-linux-release-x64-try,vm-aot-win-debug-arm64-try,vm-aot-win-debug-x64-try,vm-aot-win-debug-x64c-try,vm-asan-linux-release-arm64-try,vm-asan-linux-release-x64-try,vm-asan-mac-release-arm64-try,vm-asan-win-release-x64-try,vm-dyn-linux-debug-x64-try,vm-dyn-mac-debug-arm64-try,vm-ffi-dyn-mac-debug-simarm64_arm64-try,vm-msan-linux-release-arm64-try,vm-msan-linux-release-x64-try,vm-tsan-linux-release-arm64-try,vm-tsan-linux-release-x64-try,vm-tsan-mac-release-arm64-try,vm-ubsan-linux-release-arm64-try,vm-ubsan-linux-release-x64-try,vm-ubsan-mac-release-arm64-try,vm-ubsan-win-release-x64-try
R=vegorov@google.com
Change-Id: Ic617cb0906d1688d2d080dae7d1e08ee58b4c8d6

CoreLibraryReviewExempt: VM-only
Change-Id: Ic617cb0906d1688d2d080dae7d1e08ee58b4c8d6
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/507180
Auto-Submit: Shikhar Soni <shikharsoni@google.com>
Reviewed-by: Daco Harkes <dacoharkes@google.com>
Commit-Queue: Daco Harkes <dacoharkes@google.com>

runtime/bin/native_assets_api_impl.cc

@@ -147,7 +147,7 @@ void* NativeAssets::DlopenSystem(const char* path, char** error) {
   return handle;
 }
 
-void* NativeAssets::DlopenProcess(char** error) {
+void* NativeAssets::DlopenProcess(char** /*error*/) {
 #if defined(DART_HOST_OS_LINUX) || defined(DART_HOST_OS_MACOS) ||              \
     defined(DART_HOST_OS_ANDROID) || defined(DART_HOST_OS_FUCHSIA)
   return RTLD_DEFAULT;

runtime/lib/ffi_dynamic_library.cc

@@ -43,6 +43,9 @@ DART_NORETURN static void SimulatorUnsupported() {
 DEFINE_NATIVE_ENTRY(Ffi_dl_open, 0, 1) {
   SimulatorUnsupported();
 }
+DEFINE_NATIVE_ENTRY(Ffi_dl_openFromAssetId, 0, 1) {
+  SimulatorUnsupported();
+}
 DEFINE_NATIVE_ENTRY(Ffi_dl_processLibrary, 0, 0) {
   SimulatorUnsupported();
 }
@@ -332,14 +335,17 @@ static char* AvailableAssetsToCString(Thread* const thread) {
 
 // If an error occurs populates |error| with an error message
 // (caller must free this message when it is no longer needed).
+// A successful process or executable lookup can return a nullptr handle, so
+// callers must use |asset_found| to distinguish success from an unknown asset.
 //
 // The |asset_location| is formatted as follows:
 // ['<path_type>', '<path (optional)>']
 // The |asset_location| is conform to: pkg/vm/lib/native_assets/validator.dart
-static void* FfiResolveAsset(Thread* const thread,
-                             const String& asset,
-                             const String& symbol,
-                             char** error) {
+static void* LoadAssetLibrary(Thread* const thread,
+                              const String& asset,
+                              bool* asset_found,
+                              char** error) {
+  *asset_found = false;
   void* handle = nullptr;
   NativeAssetsApi* native_assets_api =
       thread->isolate_group()->native_assets_api();
@@ -347,6 +353,8 @@ static void* FfiResolveAsset(Thread* const thread,
     // Let embedder resolve the asset id to asset path.
     NoActiveIsolateScope no_active_isolate_scope;
     handle = native_assets_api->dlopen(asset.ToCString(), error);
+    *asset_found = *error == nullptr;
+    return handle;
   }
   if (*error == nullptr && handle == nullptr) {
     // Fall back on VM reading ffi:native-assets from special library in kernel.
@@ -358,6 +366,7 @@ static void* FfiResolveAsset(Thread* const thread,
     if (asset_location.IsNull()) {
       return nullptr;
     }
+    *asset_found = true;
 
     const auto& asset_type =
         String::Cast(Object::Handle(zone, asset_location.At(0)));
@@ -413,10 +422,20 @@ static void* FfiResolveAsset(Thread* const thread,
       handle = native_assets_api->dlopen_process(error);
     }
   }
+  return handle;
+}
 
-  if (*error != nullptr) {
+static void* FfiResolveAsset(Thread* const thread,
+                             const String& asset,
+                             const String& symbol,
+                             char** error) {
+  bool asset_found = false;
+  void* handle = LoadAssetLibrary(thread, asset, &asset_found, error);
+  if (*error != nullptr || !asset_found) {
     return nullptr;
   }
+  NativeAssetsApi* native_assets_api =
+      thread->isolate_group()->native_assets_api();
   if (native_assets_api->dlsym == nullptr) {
     *error =
         OS::SCreate(/*use malloc*/ nullptr, "NativeAssetsApi::dlsym not set.");
@@ -427,6 +446,24 @@ static void* FfiResolveAsset(Thread* const thread,
   return result;
 }
 
+DEFINE_NATIVE_ENTRY(Ffi_dl_openFromAssetId, 0, 1) {
+  GET_NON_NULL_NATIVE_ARGUMENT(String, asset_id, arguments->NativeArgAt(0));
+  char* error = nullptr;
+  bool asset_found = false;
+  void* handle = LoadAssetLibrary(thread, asset_id, &asset_found, &error);
+  if (error != nullptr) {
+    const String& msg = String::Handle(String::New(error));
+    free(error);
+    Exceptions::ThrowArgumentError(msg);
+  }
+  if (!asset_found) {
+    const String& msg = String::Handle(String::NewFormatted(
+        "No asset with id '%s' found.", asset_id.ToCString()));
+    Exceptions::ThrowArgumentError(msg);
+  }
+  return DynamicLibrary::New(handle, true);
+}
+
 // Frees |error|.
 static void ThrowFfiResolveError(const String& symbol,
                                  const String& asset,

runtime/vm/bootstrap_natives.h

@@ -312,6 +312,7 @@ namespace dart {
   V(Ffi_deleteIsolateGroupNativeCallable, 1)                                   \
   V(Ffi_updateNativeCallableKeepIsolateAliveCounter, 1)                        \
   V(Ffi_dl_open, 1)                                                            \
+  V(Ffi_dl_openFromAssetId, 1)                                                 \
   V(Ffi_dl_close, 1)                                                           \
   V(Ffi_dl_lookup, 2)                                                          \
   V(Ffi_dl_getHandle, 1)                                                       \

sdk/lib/_internal/js_dev_runtime/patch/ffi_patch.dart

@@ -790,6 +790,10 @@ class DynamicLibrary {
   factory DynamicLibrary.open(String path) =>
       throw UnsupportedError('DynamicLibrary.open');
   @patch
+  factory DynamicLibrary.openFromAssetId(String assetId) =>
+      throw UnsupportedError('DynamicLibrary.openFromAssetId');
+
+  @patch
   Pointer<T> lookup<T extends NativeType>(String symbolName) =>
       throw UnsupportedError('DynamicLibrary.lookup');
   @patch

sdk/lib/_internal/vm/lib/ffi_dynamic_library_patch.dart

@@ -9,6 +9,8 @@ import 'dart:typed_data';
 
 @pragma("vm:external-name", "Ffi_dl_open")
 external DynamicLibrary _open(String path);
+@pragma("vm:external-name", "Ffi_dl_openFromAssetId")
+external DynamicLibrary _openFromAssetId(String assetId);
 @pragma("vm:external-name", "Ffi_dl_processLibrary")
 external DynamicLibrary _processLibrary();
 @pragma("vm:external-name", "Ffi_dl_executableLibrary")
@@ -22,6 +24,11 @@ final class DynamicLibrary {
     return _open(path);
   }
 
+  @patch
+  factory DynamicLibrary.openFromAssetId(String assetId) {
+    return _openFromAssetId(assetId);
+  }
+
   @patch
   factory DynamicLibrary.process() => _processLibrary();
 

sdk/lib/ffi/dynamic_library.dart

@@ -32,6 +32,19 @@ final class DynamicLibrary {
   /// which are equal (`==`), but not [identical].
   external factory DynamicLibrary.open(String path);
 
+  /// Loads a library registered under the given [assetId] in the native assets
+  /// mapping.
+  ///
+  /// The [assetId] must be registered in the active native assets mapping.
+  /// If the asset is statically linked into the current process or executable,
+  /// this call can still succeed, but [lookup] only succeeds for symbols
+  /// available for dynamic lookup.
+  ///
+  /// Calling this function multiple times with the same [assetId] only loads
+  /// the library once.
+  @Since('3.13')
+  external factory DynamicLibrary.openFromAssetId(String assetId);
+
   /// Looks up a symbol in the [DynamicLibrary] and returns its address in
   /// memory.
   ///

tests/ffi/native_assets/asset_absolute_test.dart

@@ -30,13 +30,13 @@ void main(List<String> args, Object? message) async {
   return await selfInvokingTest(
     doOnOuterInvocation: selfInvokes,
     doOnProcessInvocation: () async {
-      await runTests();
-      await testIsolateSpawn(runTests);
+      await runTests(args[1]);
+      await testIsolateSpawn(() => runTests(args[1]));
       await testIsolateSpawnUri(spawnUri: Platform.script, arguments: args);
     },
     doOnSpawnUriInvocation: () async {
-      await runTests();
-      await testIsolateSpawn(runTests);
+      await runTests(args[1]);
+      await testIsolateSpawn(() => runTests(args[1]));
     },
   )(args, message);
 }
@@ -50,22 +50,24 @@ Future<void> selfInvokes() async {
   await invokeSelf(
     selfSourceUri: selfSourceUri,
     runtime: Runtime.jit,
-    arguments: [runTestsArg],
+    arguments: [runTestsArg, selfSourceUri.toString()],
     nativeAssetsYaml: nativeAssetsYaml,
   );
   await invokeSelf(
     selfSourceUri: selfSourceUri,
     runtime: Runtime.aot,
-    arguments: [runTestsArg],
+    arguments: [runTestsArg, selfSourceUri.toString()],
     nativeAssetsYaml: nativeAssetsYaml,
     protobufAwareTreeshaking: true,
   );
 }
 
-Future<void> runTests() async {
+Future<void> runTests(String assetId) async {
   testFfiTestfunctionsDll();
+  testOpenFfiTestFunctionsAsset(assetId);
   testFfiTestFieldsDll();
   testNonExistingFunction();
+  testOpenFromAssetIdNotFound();
 }
 
 @Native<Int32 Function(Int32, Int32)>()

tests/ffi/native_assets/asset_executable_test.dart

@@ -29,13 +29,13 @@ main(List<String> args, Object? message) async {
   return await selfInvokingTest(
     doOnOuterInvocation: selfInvokes,
     doOnProcessInvocation: () async {
-      await runTests();
-      await testIsolateSpawn(runTests);
+      await runTests(args[1]);
+      await testIsolateSpawn(() => runTests(args[1]));
       await testIsolateSpawnUri(spawnUri: Platform.script, arguments: args);
     },
     doOnSpawnUriInvocation: () async {
-      await runTests();
-      await testIsolateSpawn(runTests);
+      await runTests(args[1]);
+      await testIsolateSpawn(() => runTests(args[1]));
     },
   )(args, message);
 }
@@ -49,21 +49,23 @@ Future<void> selfInvokes() async {
   await invokeSelf(
     selfSourceUri: selfSourceUri,
     runtime: Runtime.jit,
-    arguments: [runTestsArg],
+    arguments: [runTestsArg, selfSourceUri.toString()],
     nativeAssetsYaml: nativeAssetsYaml,
     protobufAwareTreeshaking: true,
   );
   await invokeSelf(
     selfSourceUri: selfSourceUri,
     runtime: Runtime.aot,
-    arguments: [runTestsArg],
+    arguments: [runTestsArg, selfSourceUri.toString()],
     nativeAssetsYaml: nativeAssetsYaml,
   );
 }
 
-Future<void> runTests() async {
+Future<void> runTests(String assetId) async {
   await testExecutable();
+  await testOpenExecutableAsset(assetId);
   testNonExistingFunction();
+  testOpenFromAssetIdNotFound();
 }
 
 typedef _PostInteger = Bool Function(Int64 port, Int64 message);

tests/ffi/native_assets/asset_library_annotation_test.dart

@@ -63,7 +63,9 @@ Future<void> selfInvokes() async {
 
 Future<void> runTests() async {
   testFfiTestfunctionsDll();
+  testOpenFfiTestFunctionsAsset(assetName);
   testFfiTestFieldsDll();
+  testOpenFromAssetIdNotFound();
 }
 
 @Native<Int32 Function(Int32, Int32)>()

tests/ffi/native_assets/asset_process_test.dart

@@ -28,13 +28,13 @@ main(List<String> args, Object? message) async {
   return await selfInvokingTest(
     doOnOuterInvocation: selfInvokes,
     doOnProcessInvocation: () async {
-      await runTests();
-      await testIsolateSpawn(runTests);
+      await runTests(args[1]);
+      await testIsolateSpawn(() => runTests(args[1]));
       await testIsolateSpawnUri(spawnUri: Platform.script, arguments: args);
     },
     doOnSpawnUriInvocation: () async {
-      await runTests();
-      await testIsolateSpawn(runTests);
+      await runTests(args[1]);
+      await testIsolateSpawn(() => runTests(args[1]));
     },
   )(args, message);
 }
@@ -52,28 +52,30 @@ Future<void> selfInvokes() async {
   await invokeSelf(
     selfSourceUri: selfSourceUri,
     runtime: Runtime.jit,
-    arguments: [runTestsArg],
+    arguments: [runTestsArg, selfSourceUri.toString()],
     nativeAssetsYaml: nativeAssetsYaml,
   );
   await invokeSelf(
     selfSourceUri: selfSourceUri,
     runtime: Runtime.appjit,
-    arguments: [runTestsArg],
+    arguments: [runTestsArg, selfSourceUri.toString()],
     nativeAssetsYaml: nativeAssetsYaml,
     protobufAwareTreeshaking: true,
   );
   await invokeSelf(
     selfSourceUri: selfSourceUri,
     runtime: Runtime.aot,
-    arguments: [runTestsArg],
+    arguments: [runTestsArg, selfSourceUri.toString()],
     nativeAssetsYaml: nativeAssetsYaml,
   );
 }
 
-Future<void> runTests() async {
+Future<void> runTests(String assetId) async {
   testProcessOrSystem();
+  testOpenProcessOrSystemAsset(assetId);
   testProcessOrSystemViaAddressOf();
   testNonExistingFunction();
+  testOpenFromAssetIdNotFound();
 }
 
 @Native<Pointer Function(IntPtr)>(symbol: 'malloc')

tests/ffi/native_assets/asset_relative_test.dart

@@ -31,13 +31,13 @@ void main(List<String> args, Object? message) async {
   return await selfInvokingTest(
     doOnOuterInvocation: selfInvokes,
     doOnProcessInvocation: () async {
-      await runTests();
-      await testIsolateSpawn(runTests);
+      await runTests(args[1]);
+      await testIsolateSpawn(() => runTests(args[1]));
       await testIsolateSpawnUri(spawnUri: Platform.script, arguments: args);
     },
     doOnSpawnUriInvocation: () async {
-      await runTests();
-      await testIsolateSpawn(runTests);
+      await runTests(args[1]);
+      await testIsolateSpawn(() => runTests(args[1]));
     },
   )(args, message);
 }
@@ -49,7 +49,7 @@ Future<void> selfInvokes() async {
     runtime: Runtime.jit,
     kernelCombine: KernelCombine.concatenation,
     relativePath: RelativePath.same,
-    arguments: [runTestsArg],
+    arguments: [runTestsArg, selfSourceUri.toString()],
     useSymlink: true,
     protobufAwareTreeshaking: true,
   );
@@ -57,7 +57,7 @@ Future<void> selfInvokes() async {
     selfSourceUri: selfSourceUri,
     runtime: Runtime.jit,
     relativePath: RelativePath.down,
-    arguments: [runTestsArg],
+    arguments: [runTestsArg, selfSourceUri.toString()],
     useSymlink: true,
     protobufAwareTreeshaking: false,
   );
@@ -69,7 +69,7 @@ Future<void> selfInvokes() async {
         ? AotCompile.assembly
         : AotCompile.elf,
     relativePath: RelativePath.up,
-    arguments: [runTestsArg],
+    arguments: [runTestsArg, selfSourceUri.toString()],
     useSymlink: true,
     protobufAwareTreeshaking: true,
   );
@@ -140,9 +140,11 @@ Future<void> invokeSelf({
   });
 }
 
-Future<void> runTests() async {
+Future<void> runTests(String assetId) async {
   testFfiTestfunctionsDll();
+  testOpenFfiTestFunctionsAsset(assetId);
   testNonExistingFunction();
+  testOpenFromAssetIdNotFound();
   testFfiTestFieldsDll();
 }
 

tests/ffi/native_assets/asset_system_test.dart

@@ -28,13 +28,13 @@ main(List<String> args, Object? message) async {
   return await selfInvokingTest(
     doOnOuterInvocation: selfInvokes,
     doOnProcessInvocation: () async {
-      await runTests();
-      await testIsolateSpawn(runTests);
+      await runTests(args[1]);
+      await testIsolateSpawn(() => runTests(args[1]));
       await testIsolateSpawnUri(spawnUri: Platform.script, arguments: args);
     },
     doOnSpawnUriInvocation: () async {
-      await runTests();
-      await testIsolateSpawn(runTests);
+      await runTests(args[1]);
+      await testIsolateSpawn(() => runTests(args[1]));
     },
   )(args, message);
 }
@@ -60,22 +60,24 @@ Future<void> selfInvokes() async {
   await invokeSelf(
     selfSourceUri: selfSourceUri,
     runtime: Runtime.jit,
-    arguments: [runTestsArg],
+    arguments: [runTestsArg, selfSourceUri.toString()],
     nativeAssetsYaml: nativeAssetsYaml,
   );
   await invokeSelf(
     selfSourceUri: selfSourceUri,
     runtime: Runtime.aot,
-    arguments: [runTestsArg],
+    arguments: [runTestsArg, selfSourceUri.toString()],
     nativeAssetsYaml: nativeAssetsYaml,
     protobufAwareTreeshaking: true,
   );
 }
 
-Future<void> runTests() async {
+Future<void> runTests(String assetId) async {
   testProcessOrSystem();
+  testOpenProcessOrSystemAsset(assetId);
   testProcessOrSystemViaAddressOf();
   testNonExistingFunction();
+  testOpenFromAssetIdNotFound();
 }
 
 @Native<Pointer Function(IntPtr)>()

tests/ffi/native_assets/helpers.dart

@@ -5,6 +5,7 @@
 // This file should be standalone (ignoring packages) because it is copied
 // with // OtherResources and used for compiling snapshots.
 
+import 'dart:async';
 import 'dart:convert';
 import 'dart:ffi';
 import 'dart:io';
@@ -608,3 +609,68 @@ void testNonExistingFunction() {
     addressOfError.message,
   );
 }
+
+void testOpenFfiTestFunctionsAsset(String assetId) {
+  final sumPlus42 = DynamicLibrary.openFromAssetId(assetId)
+      .lookupFunction<Int32 Function(Int32, Int32), int Function(int, int)>(
+        'SumPlus42',
+      );
+  Expect.equals(2 + 3 + 42, sumPlus42(2, 3));
+}
+
+void testOpenProcessOrSystemAsset(String assetId) {
+  final library = DynamicLibrary.openFromAssetId(assetId);
+  if (Platform.isWindows) {
+    final memAlloc = library
+        .lookupFunction<Pointer Function(Size), Pointer Function(int)>(
+          'CoTaskMemAlloc',
+        );
+    final memFree = library
+        .lookupFunction<Void Function(Pointer), void Function(Pointer)>(
+          'CoTaskMemFree',
+        );
+
+    final pointer = memAlloc(8);
+    Expect.notEquals(nullptr, pointer);
+    memFree(pointer);
+  } else {
+    final malloc = library
+        .lookupFunction<Pointer Function(IntPtr), Pointer Function(int)>(
+          'malloc',
+        );
+    final free = library
+        .lookupFunction<Void Function(Pointer), void Function(Pointer)>('free');
+
+    final pointer = malloc(8);
+    Expect.notEquals(nullptr, pointer);
+    free(pointer);
+  }
+}
+
+Future<void> testOpenExecutableAsset(String assetId) async {
+  final postInteger = DynamicLibrary.openFromAssetId(assetId)
+      .lookupFunction<Bool Function(Int64, Int64), bool Function(int, int)>(
+        'Dart_PostInteger',
+      );
+
+  const int message = 1337 * 42;
+  final completer = Completer();
+  final receivePort = ReceivePort()
+    ..listen((receivedMessage) => completer.complete(receivedMessage));
+
+  final success = postInteger(receivePort.sendPort.nativePort, message);
+  Expect.isTrue(success);
+
+  final postedMessage = await completer.future;
+  Expect.equals(message, postedMessage);
+
+  receivePort.close();
+}
+
+void testOpenFromAssetIdNotFound() {
+  final argumentError = Expect.throws<ArgumentError>(() {
+    DynamicLibrary.openFromAssetId(doesNotExistName);
+  });
+  Expect.contains(doesNotExistName, argumentError.message);
+  Expect.contains('No asset with id', argumentError.message);
+}