Add dart2bytecode snapshot and VM/AOT dynamic modules test configurations

Change-Id: I84f8dbc174dbac5a11ca84e248c7aecb3759aaad
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/380283
Reviewed-by: Slava Egorov <vegorov@google.com>
Reviewed-by: Sigmund Cherem <sigmund@google.com>
Commit-Queue: Alexander Markov <alexmarkov@google.com>

BUILD.gn

@@ -84,6 +84,7 @@ group("run_ffi_unit_tests") {
 }
 
 group("runtime_precompiled") {
+  import("runtime/runtime_args.gni")
   deps = [
     "runtime/bin:dart_precompiled_runtime",
     "runtime/bin:gen_snapshot",
@@ -94,6 +95,10 @@ group("runtime_precompiled") {
   if (is_linux || is_android) {
     deps += [ "runtime/bin:abstract_socket_test" ]
   }
+  if (dart_dynamic_modules) {
+    deps += [ "utils/dart2bytecode:dart2bytecode_snapshot" ]
+    deps += [ "utils/dynamic_module_runner:dynamic_module_runner_snapshot" ]
+  }
 }
 
 group("create_sdk") {

build/rbe/rewrapper_dart.py

@@ -564,6 +564,8 @@ trace to find the place to insert the appropriate support.
                 self.outputs.append(self.rebase(self.optarg))
             elif self.get_option(['--platform']):
                 self.extra_paths.add(self.rebase(self.optarg))
+            elif self.get_option(['--dynamic-interface']):
+                self.extra_paths.add(self.rebase(self.optarg))
             elif self.get_option(['--packages', '-D']):
                 pass
             elif arg in [

pkg/dynamic_modules/bin/dynamic_module_runner.dart

@@ -0,0 +1,12 @@
+// Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:dynamic_modules/dynamic_modules.dart' show loadModuleFromBytes;
+
+main(List<String> args) {
+  final bytes = File(args[0]).readAsBytesSync();
+  return loadModuleFromBytes(bytes);
+}

pkg/smith/lib/configuration.dart

@@ -821,6 +821,7 @@ class Compiler extends NamedEnum {
   static const dartkp = Compiler._('dartkp');
   static const specParser = Compiler._('spec_parser');
   static const fasta = Compiler._('fasta');
+  static const dart2bytecode = Compiler._('dart2bytecode');
 
   static final List<String> names = _all.keys.toList();
 
@@ -834,6 +835,7 @@ class Compiler extends NamedEnum {
     dartkp,
     specParser,
     fasta,
+    dart2bytecode,
   ], key: (compiler) => (compiler as Compiler).name);
 
   static Compiler find(String name) {
@@ -897,6 +899,8 @@ class Compiler extends NamedEnum {
         return const [Runtime.none];
       case Compiler.fasta:
         return const [Runtime.none];
+      case Compiler.dart2bytecode:
+        return const [Runtime.vm, Runtime.dartPrecompiled];
     }
 
     throw "unreachable";
@@ -922,6 +926,8 @@ class Compiler extends NamedEnum {
       case Compiler.specParser:
       case Compiler.fasta:
         return Runtime.none;
+      case Compiler.dart2bytecode:
+        return Runtime.dartPrecompiled;
     }
 
     throw "unreachable";

pkg/test_runner/lib/src/command.dart

@@ -193,7 +193,9 @@ class CompilationCommand extends ProcessCommand {
   CommandOutput createOutput(int exitCode, bool timedOut, List<int> stdout,
       List<int> stderr, Duration time, bool compilationSkipped,
       [int pid = 0]) {
-    if (displayName == 'precompiler' || displayName == 'app_jit') {
+    if (displayName == 'precompiler' ||
+        displayName == 'app_jit' ||
+        displayName == 'dart2bytecode') {
       return VMCommandOutput(
           this, exitCode, timedOut, stdout, stderr, time, pid);
     } else if (displayName == 'dart2wasm') {

pkg/test_runner/lib/src/compiler_configuration.dart

@@ -112,6 +112,9 @@ abstract class CompilerConfiguration {
 
       case Compiler.fasta:
         return FastaCompilerConfiguration(configuration);
+
+      case Compiler.dart2bytecode:
+        return BytecodeCompilerConfiguration(configuration);
     }
 
     throw "unreachable";
@@ -1581,3 +1584,110 @@ class FastaCompilerConfiguration extends CompilerConfiguration {
     return [];
   }
 }
+
+class BytecodeCompilerConfiguration extends CompilerConfiguration {
+  BytecodeCompilerConfiguration(super.configuration) : super._subclass();
+
+  @override
+  String computeCompilerPath() => dartAotRuntime();
+
+  @override
+  bool get runRuntimeDespiteMissingCompileTimeError => true;
+
+  String dartAotRuntime() => _useSdk
+      ? '${_configuration.buildDirectory}/dart-sdk/bin/dartaotruntime'
+      : '${_configuration.buildDirectory}/dart_precompiled_runtime';
+
+  String dart2bytecodeSnapshot() => _useSdk
+      ? '${_configuration.buildDirectory}/dart-sdk/bin/snapshots/dart2bytecode.dart.snapshot'
+      : '${_configuration.buildDirectory}/gen/dart2bytecode.dart.snapshot';
+
+  String platformKernelFile() => _useSdk
+      ? '${_configuration.buildDirectory}/dart-sdk/lib/_internal/vm_platform_strong.dill'
+      : '${_configuration.buildDirectory}/vm_platform_strong.dill';
+
+  String tempBytecodeFile(String tempDir) =>
+      Path('$tempDir/out.bytecode').toNativePath();
+
+  Command computeCompilationCommand(String tempDir, List<String> arguments,
+      Map<String, String> environmentOverrides) {
+    final bytecodeFile = tempBytecodeFile(tempDir);
+    final isProductMode = _configuration.configuration.mode == Mode.product;
+
+    final args = [
+      dart2bytecodeSnapshot(),
+      '--platform=${platformKernelFile()}',
+      '-o',
+      bytecodeFile,
+      arguments.where((name) => name.endsWith('.dart')).single,
+      ...arguments.where((name) =>
+          name.startsWith('-D') ||
+          name.startsWith('--define') ||
+          name.startsWith('--packages=') ||
+          name.startsWith('--enable-experiment=')),
+      '-Ddart.vm.product=$isProductMode',
+      if (_enableAsserts ||
+          arguments.contains('--enable-asserts') ||
+          arguments.contains('--enable_asserts'))
+        '--enable-asserts',
+    ];
+
+    return CompilationCommand(
+        'dart2bytecode',
+        bytecodeFile,
+        bootstrapDependencies(),
+        computeCompilerPath(),
+        args,
+        environmentOverrides,
+        alwaysCompile: !_useSdk);
+  }
+
+  @override
+  CommandArtifact computeCompilationArtifact(String tempDir,
+      List<String> arguments, Map<String, String> environmentOverrides) {
+    final commands = <Command>[
+      computeCompilationCommand(tempDir, arguments, environmentOverrides),
+    ];
+    return CommandArtifact(
+        commands, tempBytecodeFile(tempDir), 'application/dart-bytecode');
+  }
+
+  @override
+  List<String> computeCompilerArguments(
+      TestFile testFile, List<String> vmOptions, List<String> args) {
+    return [
+      ...testFile.sharedOptions,
+      ..._configuration.sharedOptions,
+      ..._experimentsArgument(_configuration, testFile),
+      ...args
+    ];
+  }
+
+  @override
+  List<String> computeRuntimeArguments(
+      RuntimeConfiguration runtimeConfiguration,
+      TestFile testFile,
+      List<String> vmOptions,
+      List<String> originalArguments,
+      CommandArtifact? artifact) {
+    var filename = artifact!.filename;
+
+    return [
+      if (_enableAsserts) '--enable_asserts',
+      ...vmOptions,
+      ...testFile.sharedOptions,
+      ..._configuration.sharedOptions,
+      ..._experimentsArgument(_configuration, testFile),
+      ..._replaceDartFiles(
+          originalArguments,
+          (_configuration.runtime == Runtime.dartPrecompiled)
+              ? '${_configuration.buildDirectory}/dynamic_module_runner.snapshot'
+              : Platform.script
+                  .resolve(
+                      '../../../pkg/dynamic_modules/bin/dynamic_module_runner.dart')
+                  .toFilePath()),
+      filename,
+      ...testFile.dartOptions
+    ];
+  }
+}

pkg/test_runner/lib/src/runtime_configuration.dart

@@ -390,7 +390,8 @@ class StandaloneDartRuntimeConfiguration extends DartVmRuntimeConfiguration {
         type != 'application/dart' &&
         type != 'application/dart-snapshot' &&
         type != 'application/kernel-ir' &&
-        type != 'application/kernel-ir-fully-linked') {
+        type != 'application/kernel-ir-fully-linked' &&
+        type != 'application/dart-bytecode') {
       throw "Dart VM cannot run files of type '$type'.";
     }
     if (isCrashExpected) {
@@ -427,7 +428,9 @@ class DartPrecompiledRuntimeConfiguration extends DartVmRuntimeConfiguration {
       bool isCrashExpected) {
     var script = artifact?.filename;
     var type = artifact?.mimeType;
-    if (script != null && type != 'application/dart-precompiled') {
+    if (script != null &&
+        type != 'application/dart-precompiled' &&
+        type != 'application/dart-bytecode') {
       throw "dart_precompiled cannot run files of type '$type'.";
     }
 

sdk/BUILD.gn

@@ -42,6 +42,7 @@ declare_args() {
 # ......utils/gen_snapshot or utils/gen_snapshot.exe (if not on ia32)
 # ......snapshots/
 # ........analysis_server.dart.snapshot
+# ........dart2bytecode.snapshot (if dart_dynamic_modules)
 # ........dart2js.dart.snapshot
 # ........dart2wasm_product.snapshot (if not on ia32)
 # ........dartdev.dart.snapshot (app-jit snapshot or kernel dill file)
@@ -143,6 +144,12 @@ if (dart_snapshot_kind == "app-jit") {
         "../utils/kernel-service:kernel-service_snapshot",
       ] ]
 }
+if (dart_dynamic_modules) {
+  _platform_sdk_snapshots += [ [
+        "dart2bytecode",
+        "../utils/dart2bytecode:dart2bytecode",
+      ] ]
+}
 
 _full_sdk_snapshots = _platform_sdk_snapshots + [
                         [

tools/bots/test_matrix.json

@@ -566,6 +566,16 @@
         "compiler": "app_jitk"
       }
     },
+    "vm-dyn-(linux|mac|win)-(debug|product|release)-(x64|x64c|simarm|simarm64|simarm64c)": {
+      "options": {
+        "compiler": "dart2bytecode"
+      }
+    },
+    "vm-aot-dyn-(linux|mac|win)-(debug|release|product)-(x64|x64c|simarm|simarm64|simarm64c)": {
+      "options": {
+        "compiler": "dart2bytecode"
+      }
+    },
     "ddc-(linux|win)-chrome": {
       "options": {
         "checked": true,

utils/dart2bytecode/BUILD.gn

@@ -0,0 +1,24 @@
+# Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+import("../../runtime/runtime_args.gni")
+import("../aot_snapshot.gni")
+
+group("dart2bytecode") {
+  public_deps = [ ":dart2bytecode_snapshot" ]
+}
+
+aot_snapshot("dart2bytecode_snapshot") {
+  main_dart = "../../pkg/dart2bytecode/bin/dart2bytecode.dart"
+  name = "dart2bytecode"
+
+  output = "$root_gen_dir/dart2bytecode.dart.snapshot"
+
+  # dartaotruntime has dart_product_config applied to it, so it is built in
+  # product mode in both release and product builds, and is only built in debug
+  # mode in debug builds. The following line ensures that the dartaotruntime and
+  # dart2bytecode.dart.snapshot in an SDK build are always compatible with
+  # each other.
+  force_product_mode = !dart_debug
+}

utils/dynamic_module_runner/BUILD.gn

@@ -0,0 +1,19 @@
+# Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+import("../aot_snapshot.gni")
+
+_dart_root = get_path_info("../..", "abspath")
+
+group("dynamic_module_runner") {
+  public_deps = [ ":dynamic_module_runner_snapshot" ]
+}
+
+aot_snapshot("dynamic_module_runner_snapshot") {
+  main_dart = "../../pkg/dynamic_modules/bin/dynamic_module_runner.dart"
+  name = "dynamic_module_runner"
+  gen_kernel_args =
+      [ "--dynamic-interface=" + rebase_path(
+            "$_dart_root/utils/dynamic_module_runner/dynamic_interface.yaml") ]
+}

utils/dynamic_module_runner/dynamic_interface.yaml

@@ -0,0 +1,90 @@
+# Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+extendable:
+  - library: 'dart:*'
+  - library: 'dart:core'
+    class: '_Enum'
+
+can-be-overridden:
+  - library: 'dart:*'
+
+callable:
+  - library: 'dart:*'
+  # These private classes and members are used directly by the compiler.
+  - library: 'dart:core'
+    class: '_Enum'
+    member: ''
+  - library: 'dart:core'
+    class: '_GrowableList'
+  - library: 'dart:core'
+    class: '_GrowableList'
+    member: '_literal1'
+  - library: 'dart:core'
+    class: '_GrowableList'
+    member: '_literal2'
+  - library: 'dart:core'
+    class: '_GrowableList'
+    member: '_literal3'
+  - library: 'dart:core'
+    class: '_GrowableList'
+    member: '_literal4'
+  - library: 'dart:core'
+    class: '_GrowableList'
+    member: '_literal5'
+  - library: 'dart:core'
+    class: '_GrowableList'
+    member: '_literal6'
+  - library: 'dart:core'
+    class: '_GrowableList'
+    member: '_literal7'
+  - library: 'dart:core'
+    class: '_GrowableList'
+    member: '_literal8'
+  - library: 'dart:core'
+    class: '_GrowableList'
+    member: ''
+  - library: 'dart:core'
+    class: '_GrowableList'
+    member: 'empty'
+  - library: 'dart:core'
+    class: '_GrowableList'
+    member: 'filled'
+  - library: 'dart:core'
+    class: '_GrowableList'
+    member: 'generate'
+  - library: 'dart:core'
+    class: '_List'
+  - library: 'dart:core'
+    class: '_List'
+    member: ''
+  - library: 'dart:core'
+    class: '_List'
+    member: 'empty'
+  - library: 'dart:core'
+    class: '_List'
+    member: 'filled'
+  - library: 'dart:core'
+    class: '_List'
+    member: 'generate'
+  - library: 'dart:collection'
+    class: '_Map'
+  - library: 'dart:collection'
+    class: '_Map'
+    member: ''
+  - library: 'dart:collection'
+    class: '_Set'
+  - library: 'dart:collection'
+    class: '_Set'
+    member: ''
+  - library: 'dart:async'
+    class: '_StreamIterator'
+  - library: 'dart:async'
+    class: '_StreamIterator'
+    member: ''
+  - library: 'dart:async'
+    class: '_StreamIterator'
+    member: '_subscription'
+  - library: 'dart:async'
+    member: '_asyncStarMoveNextHelper'