From 3d8829fad2a2ebfe4bfa25990c9308dcaf9a5df0 Mon Sep 17 00:00:00 2001 From: Alexander Markov Date: Thu, 15 Aug 2024 15:28:25 +0000 Subject: [PATCH] Add dart2bytecode snapshot and VM/AOT dynamic modules test configurations Change-Id: I84f8dbc174dbac5a11ca84e248c7aecb3759aaad Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/380283 Reviewed-by: Slava Egorov Reviewed-by: Sigmund Cherem Commit-Queue: Alexander Markov --- BUILD.gn | 5 + build/rbe/rewrapper_dart.py | 2 + .../bin/dynamic_module_runner.dart | 12 ++ pkg/smith/lib/configuration.dart | 6 + pkg/test_runner/lib/src/command.dart | 4 +- .../lib/src/compiler_configuration.dart | 110 ++++++++++++++++++ .../lib/src/runtime_configuration.dart | 7 +- sdk/BUILD.gn | 7 ++ tools/bots/test_matrix.json | 10 ++ utils/dart2bytecode/BUILD.gn | 24 ++++ utils/dynamic_module_runner/BUILD.gn | 19 +++ .../dynamic_interface.yaml | 90 ++++++++++++++ 12 files changed, 293 insertions(+), 3 deletions(-) create mode 100644 pkg/dynamic_modules/bin/dynamic_module_runner.dart create mode 100644 utils/dart2bytecode/BUILD.gn create mode 100644 utils/dynamic_module_runner/BUILD.gn create mode 100644 utils/dynamic_module_runner/dynamic_interface.yaml diff --git a/BUILD.gn b/BUILD.gn index c2ddf40491e..1d94b1dcf1f 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -84,6 +84,7 @@ group("run_ffi_unit_tests") { } group("runtime_precompiled") { + import("runtime/runtime_args.gni") deps = [ "runtime/bin:dart_precompiled_runtime", "runtime/bin:gen_snapshot", @@ -94,6 +95,10 @@ group("runtime_precompiled") { if (is_linux || is_android) { deps += [ "runtime/bin:abstract_socket_test" ] } + if (dart_dynamic_modules) { + deps += [ "utils/dart2bytecode:dart2bytecode_snapshot" ] + deps += [ "utils/dynamic_module_runner:dynamic_module_runner_snapshot" ] + } } group("create_sdk") { diff --git a/build/rbe/rewrapper_dart.py b/build/rbe/rewrapper_dart.py index 62faf8a568f..7e3aec5718e 100644 --- a/build/rbe/rewrapper_dart.py +++ b/build/rbe/rewrapper_dart.py @@ -564,6 +564,8 @@ trace to find the place to insert the appropriate support. self.outputs.append(self.rebase(self.optarg)) elif self.get_option(['--platform']): self.extra_paths.add(self.rebase(self.optarg)) + elif self.get_option(['--dynamic-interface']): + self.extra_paths.add(self.rebase(self.optarg)) elif self.get_option(['--packages', '-D']): pass elif arg in [ diff --git a/pkg/dynamic_modules/bin/dynamic_module_runner.dart b/pkg/dynamic_modules/bin/dynamic_module_runner.dart new file mode 100644 index 00000000000..bde4f95862f --- /dev/null +++ b/pkg/dynamic_modules/bin/dynamic_module_runner.dart @@ -0,0 +1,12 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:io'; + +import 'package:dynamic_modules/dynamic_modules.dart' show loadModuleFromBytes; + +main(List args) { + final bytes = File(args[0]).readAsBytesSync(); + return loadModuleFromBytes(bytes); +} diff --git a/pkg/smith/lib/configuration.dart b/pkg/smith/lib/configuration.dart index 44e6eea6a0f..04bdeb308ce 100644 --- a/pkg/smith/lib/configuration.dart +++ b/pkg/smith/lib/configuration.dart @@ -821,6 +821,7 @@ class Compiler extends NamedEnum { static const dartkp = Compiler._('dartkp'); static const specParser = Compiler._('spec_parser'); static const fasta = Compiler._('fasta'); + static const dart2bytecode = Compiler._('dart2bytecode'); static final List names = _all.keys.toList(); @@ -834,6 +835,7 @@ class Compiler extends NamedEnum { dartkp, specParser, fasta, + dart2bytecode, ], key: (compiler) => (compiler as Compiler).name); static Compiler find(String name) { @@ -897,6 +899,8 @@ class Compiler extends NamedEnum { return const [Runtime.none]; case Compiler.fasta: return const [Runtime.none]; + case Compiler.dart2bytecode: + return const [Runtime.vm, Runtime.dartPrecompiled]; } throw "unreachable"; @@ -922,6 +926,8 @@ class Compiler extends NamedEnum { case Compiler.specParser: case Compiler.fasta: return Runtime.none; + case Compiler.dart2bytecode: + return Runtime.dartPrecompiled; } throw "unreachable"; diff --git a/pkg/test_runner/lib/src/command.dart b/pkg/test_runner/lib/src/command.dart index f3cb2b8bfb6..fc53abd3b73 100644 --- a/pkg/test_runner/lib/src/command.dart +++ b/pkg/test_runner/lib/src/command.dart @@ -193,7 +193,9 @@ class CompilationCommand extends ProcessCommand { CommandOutput createOutput(int exitCode, bool timedOut, List stdout, List stderr, Duration time, bool compilationSkipped, [int pid = 0]) { - if (displayName == 'precompiler' || displayName == 'app_jit') { + if (displayName == 'precompiler' || + displayName == 'app_jit' || + displayName == 'dart2bytecode') { return VMCommandOutput( this, exitCode, timedOut, stdout, stderr, time, pid); } else if (displayName == 'dart2wasm') { diff --git a/pkg/test_runner/lib/src/compiler_configuration.dart b/pkg/test_runner/lib/src/compiler_configuration.dart index 70e4cb900f9..0278dcdeaee 100644 --- a/pkg/test_runner/lib/src/compiler_configuration.dart +++ b/pkg/test_runner/lib/src/compiler_configuration.dart @@ -112,6 +112,9 @@ abstract class CompilerConfiguration { case Compiler.fasta: return FastaCompilerConfiguration(configuration); + + case Compiler.dart2bytecode: + return BytecodeCompilerConfiguration(configuration); } throw "unreachable"; @@ -1581,3 +1584,110 @@ class FastaCompilerConfiguration extends CompilerConfiguration { return []; } } + +class BytecodeCompilerConfiguration extends CompilerConfiguration { + BytecodeCompilerConfiguration(super.configuration) : super._subclass(); + + @override + String computeCompilerPath() => dartAotRuntime(); + + @override + bool get runRuntimeDespiteMissingCompileTimeError => true; + + String dartAotRuntime() => _useSdk + ? '${_configuration.buildDirectory}/dart-sdk/bin/dartaotruntime' + : '${_configuration.buildDirectory}/dart_precompiled_runtime'; + + String dart2bytecodeSnapshot() => _useSdk + ? '${_configuration.buildDirectory}/dart-sdk/bin/snapshots/dart2bytecode.dart.snapshot' + : '${_configuration.buildDirectory}/gen/dart2bytecode.dart.snapshot'; + + String platformKernelFile() => _useSdk + ? '${_configuration.buildDirectory}/dart-sdk/lib/_internal/vm_platform_strong.dill' + : '${_configuration.buildDirectory}/vm_platform_strong.dill'; + + String tempBytecodeFile(String tempDir) => + Path('$tempDir/out.bytecode').toNativePath(); + + Command computeCompilationCommand(String tempDir, List arguments, + Map environmentOverrides) { + final bytecodeFile = tempBytecodeFile(tempDir); + final isProductMode = _configuration.configuration.mode == Mode.product; + + final args = [ + dart2bytecodeSnapshot(), + '--platform=${platformKernelFile()}', + '-o', + bytecodeFile, + arguments.where((name) => name.endsWith('.dart')).single, + ...arguments.where((name) => + name.startsWith('-D') || + name.startsWith('--define') || + name.startsWith('--packages=') || + name.startsWith('--enable-experiment=')), + '-Ddart.vm.product=$isProductMode', + if (_enableAsserts || + arguments.contains('--enable-asserts') || + arguments.contains('--enable_asserts')) + '--enable-asserts', + ]; + + return CompilationCommand( + 'dart2bytecode', + bytecodeFile, + bootstrapDependencies(), + computeCompilerPath(), + args, + environmentOverrides, + alwaysCompile: !_useSdk); + } + + @override + CommandArtifact computeCompilationArtifact(String tempDir, + List arguments, Map environmentOverrides) { + final commands = [ + computeCompilationCommand(tempDir, arguments, environmentOverrides), + ]; + return CommandArtifact( + commands, tempBytecodeFile(tempDir), 'application/dart-bytecode'); + } + + @override + List computeCompilerArguments( + TestFile testFile, List vmOptions, List args) { + return [ + ...testFile.sharedOptions, + ..._configuration.sharedOptions, + ..._experimentsArgument(_configuration, testFile), + ...args + ]; + } + + @override + List computeRuntimeArguments( + RuntimeConfiguration runtimeConfiguration, + TestFile testFile, + List vmOptions, + List originalArguments, + CommandArtifact? artifact) { + var filename = artifact!.filename; + + return [ + if (_enableAsserts) '--enable_asserts', + ...vmOptions, + ...testFile.sharedOptions, + ..._configuration.sharedOptions, + ..._experimentsArgument(_configuration, testFile), + ..._replaceDartFiles( + originalArguments, + (_configuration.runtime == Runtime.dartPrecompiled) + ? '${_configuration.buildDirectory}/dynamic_module_runner.snapshot' + : Platform.script + .resolve( + '../../../pkg/dynamic_modules/bin/dynamic_module_runner.dart') + .toFilePath()), + filename, + ...testFile.dartOptions + ]; + } +} diff --git a/pkg/test_runner/lib/src/runtime_configuration.dart b/pkg/test_runner/lib/src/runtime_configuration.dart index 41d827ffa1f..a5b9a81a2ff 100644 --- a/pkg/test_runner/lib/src/runtime_configuration.dart +++ b/pkg/test_runner/lib/src/runtime_configuration.dart @@ -390,7 +390,8 @@ class StandaloneDartRuntimeConfiguration extends DartVmRuntimeConfiguration { type != 'application/dart' && type != 'application/dart-snapshot' && type != 'application/kernel-ir' && - type != 'application/kernel-ir-fully-linked') { + type != 'application/kernel-ir-fully-linked' && + type != 'application/dart-bytecode') { throw "Dart VM cannot run files of type '$type'."; } if (isCrashExpected) { @@ -427,7 +428,9 @@ class DartPrecompiledRuntimeConfiguration extends DartVmRuntimeConfiguration { bool isCrashExpected) { var script = artifact?.filename; var type = artifact?.mimeType; - if (script != null && type != 'application/dart-precompiled') { + if (script != null && + type != 'application/dart-precompiled' && + type != 'application/dart-bytecode') { throw "dart_precompiled cannot run files of type '$type'."; } diff --git a/sdk/BUILD.gn b/sdk/BUILD.gn index c664ad9e0e0..7ab08d8a946 100644 --- a/sdk/BUILD.gn +++ b/sdk/BUILD.gn @@ -42,6 +42,7 @@ declare_args() { # ......utils/gen_snapshot or utils/gen_snapshot.exe (if not on ia32) # ......snapshots/ # ........analysis_server.dart.snapshot +# ........dart2bytecode.snapshot (if dart_dynamic_modules) # ........dart2js.dart.snapshot # ........dart2wasm_product.snapshot (if not on ia32) # ........dartdev.dart.snapshot (app-jit snapshot or kernel dill file) @@ -143,6 +144,12 @@ if (dart_snapshot_kind == "app-jit") { "../utils/kernel-service:kernel-service_snapshot", ] ] } +if (dart_dynamic_modules) { + _platform_sdk_snapshots += [ [ + "dart2bytecode", + "../utils/dart2bytecode:dart2bytecode", + ] ] +} _full_sdk_snapshots = _platform_sdk_snapshots + [ [ diff --git a/tools/bots/test_matrix.json b/tools/bots/test_matrix.json index 0d81473ba54..980bf65b291 100644 --- a/tools/bots/test_matrix.json +++ b/tools/bots/test_matrix.json @@ -566,6 +566,16 @@ "compiler": "app_jitk" } }, + "vm-dyn-(linux|mac|win)-(debug|product|release)-(x64|x64c|simarm|simarm64|simarm64c)": { + "options": { + "compiler": "dart2bytecode" + } + }, + "vm-aot-dyn-(linux|mac|win)-(debug|release|product)-(x64|x64c|simarm|simarm64|simarm64c)": { + "options": { + "compiler": "dart2bytecode" + } + }, "ddc-(linux|win)-chrome": { "options": { "checked": true, diff --git a/utils/dart2bytecode/BUILD.gn b/utils/dart2bytecode/BUILD.gn new file mode 100644 index 00000000000..bc769e74ca4 --- /dev/null +++ b/utils/dart2bytecode/BUILD.gn @@ -0,0 +1,24 @@ +# Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. + +import("../../runtime/runtime_args.gni") +import("../aot_snapshot.gni") + +group("dart2bytecode") { + public_deps = [ ":dart2bytecode_snapshot" ] +} + +aot_snapshot("dart2bytecode_snapshot") { + main_dart = "../../pkg/dart2bytecode/bin/dart2bytecode.dart" + name = "dart2bytecode" + + output = "$root_gen_dir/dart2bytecode.dart.snapshot" + + # dartaotruntime has dart_product_config applied to it, so it is built in + # product mode in both release and product builds, and is only built in debug + # mode in debug builds. The following line ensures that the dartaotruntime and + # dart2bytecode.dart.snapshot in an SDK build are always compatible with + # each other. + force_product_mode = !dart_debug +} diff --git a/utils/dynamic_module_runner/BUILD.gn b/utils/dynamic_module_runner/BUILD.gn new file mode 100644 index 00000000000..42abc5ab7a8 --- /dev/null +++ b/utils/dynamic_module_runner/BUILD.gn @@ -0,0 +1,19 @@ +# Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. + +import("../aot_snapshot.gni") + +_dart_root = get_path_info("../..", "abspath") + +group("dynamic_module_runner") { + public_deps = [ ":dynamic_module_runner_snapshot" ] +} + +aot_snapshot("dynamic_module_runner_snapshot") { + main_dart = "../../pkg/dynamic_modules/bin/dynamic_module_runner.dart" + name = "dynamic_module_runner" + gen_kernel_args = + [ "--dynamic-interface=" + rebase_path( + "$_dart_root/utils/dynamic_module_runner/dynamic_interface.yaml") ] +} diff --git a/utils/dynamic_module_runner/dynamic_interface.yaml b/utils/dynamic_module_runner/dynamic_interface.yaml new file mode 100644 index 00000000000..aca7ac6ae21 --- /dev/null +++ b/utils/dynamic_module_runner/dynamic_interface.yaml @@ -0,0 +1,90 @@ +# Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. + +extendable: + - library: 'dart:*' + - library: 'dart:core' + class: '_Enum' + +can-be-overridden: + - library: 'dart:*' + +callable: + - library: 'dart:*' + # These private classes and members are used directly by the compiler. + - library: 'dart:core' + class: '_Enum' + member: '' + - library: 'dart:core' + class: '_GrowableList' + - library: 'dart:core' + class: '_GrowableList' + member: '_literal1' + - library: 'dart:core' + class: '_GrowableList' + member: '_literal2' + - library: 'dart:core' + class: '_GrowableList' + member: '_literal3' + - library: 'dart:core' + class: '_GrowableList' + member: '_literal4' + - library: 'dart:core' + class: '_GrowableList' + member: '_literal5' + - library: 'dart:core' + class: '_GrowableList' + member: '_literal6' + - library: 'dart:core' + class: '_GrowableList' + member: '_literal7' + - library: 'dart:core' + class: '_GrowableList' + member: '_literal8' + - library: 'dart:core' + class: '_GrowableList' + member: '' + - library: 'dart:core' + class: '_GrowableList' + member: 'empty' + - library: 'dart:core' + class: '_GrowableList' + member: 'filled' + - library: 'dart:core' + class: '_GrowableList' + member: 'generate' + - library: 'dart:core' + class: '_List' + - library: 'dart:core' + class: '_List' + member: '' + - library: 'dart:core' + class: '_List' + member: 'empty' + - library: 'dart:core' + class: '_List' + member: 'filled' + - library: 'dart:core' + class: '_List' + member: 'generate' + - library: 'dart:collection' + class: '_Map' + - library: 'dart:collection' + class: '_Map' + member: '' + - library: 'dart:collection' + class: '_Set' + - library: 'dart:collection' + class: '_Set' + member: '' + - library: 'dart:async' + class: '_StreamIterator' + - library: 'dart:async' + class: '_StreamIterator' + member: '' + - library: 'dart:async' + class: '_StreamIterator' + member: '_subscription' + - library: 'dart:async' + member: '_asyncStarMoveNextHelper'