From 6c7604291c8520ffacfafe75b3f39974d6d7efdb Mon Sep 17 00:00:00 2001 From: Aske Simon Christensen Date: Tue, 10 May 2022 10:06:06 +0000 Subject: [PATCH] [dart2wasm] Build platform dill and compile dart2wasm to AOT snapshots This adds a --platform= option to dart2wasm to read the SDK libraries from that dill file instead of compiling them from source every time. If the option is not given, the SDK libraries are compiled, like before. Also adds a "dart2wasm" build target, which will build the dart2wasm platform dill and compile dart2wasm to two AOT snapshots (with and without asserts). The dart2wasm scripts in sdk/bin are updated to run via these snapshots and use this platform dill. This speeds up test runs for the dart2wasm-hostasserts-linux-x64-d8 configuration by approximately 45x. Change-Id: If2c7750a6eb39725310745f887792784d0dfc583 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/243624 Commit-Queue: Aske Simon Christensen Reviewed-by: William Hesse Reviewed-by: Joshua Litt --- BUILD.gn | 9 ++ .../lib/src/messages/codes_generated.dart | 2 +- pkg/dart2wasm/bin/dart2wasm.dart | 9 +- pkg/dart2wasm/dart2wasm.md | 1 + pkg/dart2wasm/lib/compile.dart | 8 +- pkg/dart2wasm/lib/target.dart | 2 + pkg/front_end/messages.yaml | 2 +- pkg/front_end/pubspec.yaml | 2 + .../test/spell_checking_list_common.txt | 2 + .../tool/_fasta/additional_targets.dart | 3 + sdk/bin/dart2wasm | 28 ++++- tools/bots/test_matrix.json | 3 +- utils/aot_snapshot.gni | 118 ++++++++++++++++++ utils/dart2wasm/BUILD.gn | 38 ++++++ 14 files changed, 215 insertions(+), 12 deletions(-) create mode 100644 utils/aot_snapshot.gni create mode 100644 utils/dart2wasm/BUILD.gn diff --git a/BUILD.gn b/BUILD.gn index 4b587beb70a..aad296d41da 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -103,6 +103,15 @@ group("dart2js") { deps = [ "utils/compiler:dart2js" ] } +group("dart2wasm") { + deps = [ + ":runtime_precompiled", + "utils/dart2wasm:compile_dart2wasm_platform", + "utils/dart2wasm:dart2wasm_asserts_snapshot", + "utils/dart2wasm:dart2wasm_snapshot", + ] +} + group("dartanalyzer") { deps = [ "utils/dartanalyzer" ] } diff --git a/pkg/_fe_analyzer_shared/lib/src/messages/codes_generated.dart b/pkg/_fe_analyzer_shared/lib/src/messages/codes_generated.dart index 36c0cb7a3fe..fc99416b47f 100644 --- a/pkg/_fe_analyzer_shared/lib/src/messages/codes_generated.dart +++ b/pkg/_fe_analyzer_shared/lib/src/messages/codes_generated.dart @@ -4124,7 +4124,7 @@ const MessageCode messageFastaUsageLong = Read the SDK platform from , which should be in Dill/Kernel IR format and contain the Dart SDK. - --target=dart2js|dart2js_server|dart_runner|dartdevc|flutter|flutter_runner|none|vm + --target=dart2js|dart2js_server|dart2wasm|dart_runner|dartdevc|flutter|flutter_runner|none|vm Specify the target configuration. --enable-asserts diff --git a/pkg/dart2wasm/bin/dart2wasm.dart b/pkg/dart2wasm/bin/dart2wasm.dart index 6a77001cef6..4a382a93de4 100644 --- a/pkg/dart2wasm/bin/dart2wasm.dart +++ b/pkg/dart2wasm/bin/dart2wasm.dart @@ -40,6 +40,7 @@ Never usage(String message) { print(""); print("Options:"); print(" --dart-sdk="); + print(" --platform="); print(""); for (String option in boolOptionMap.keys) { print(" --[no-]$option"); @@ -55,6 +56,7 @@ Never usage(String message) { Future main(List args) async { Uri sdkPath = Platform.script.resolve("../../../sdk"); + Uri? platformPath = null; TranslatorOptions options = TranslatorOptions(); List nonOptions = []; void Function(TranslatorOptions, int)? intOptionFun = null; @@ -65,6 +67,9 @@ Future main(List args) async { } else if (arg.startsWith("--dart-sdk=")) { String path = arg.substring("--dart-sdk=".length); sdkPath = Uri.file(Directory(path).absolute.path); + } else if (arg.startsWith("--platform=")) { + String path = arg.substring("--platform=".length); + platformPath = Uri.file(Directory(path).absolute.path); } else if (arg.startsWith("--no-")) { var optionFun = boolOptionMap[arg.substring(5)]; if (optionFun == null) usage("Unknown option $arg"); @@ -95,8 +100,8 @@ Future main(List args) async { String output = nonOptions[1]; Uri mainUri = resolveInputUri(input); - Uint8List? module = await compileToModule(mainUri, sdkPath, options, - (message) => printDiagnosticMessage(message, print)); + Uint8List? module = await compileToModule(mainUri, sdkPath, platformPath, + options, (message) => printDiagnosticMessage(message, print)); if (module == null) { exitCode = 1; diff --git a/pkg/dart2wasm/dart2wasm.md b/pkg/dart2wasm/dart2wasm.md index 8cc6b91c785..6d3ab61ccd5 100644 --- a/pkg/dart2wasm/dart2wasm.md +++ b/pkg/dart2wasm/dart2wasm.md @@ -11,6 +11,7 @@ where *options* include: | Option | Default | Description | | --------------------------------------- | ------- | ----------- | | `--dart-sdk=`*path* | relative to script | The location of the `sdk` directory inside the Dart SDK, containing the core library sources. +| `--platform=`*path* | none | The location of the platform `dill` file containing the compiled core libraries. | `--`[`no-`]`export-all` | no | Export all functions; otherwise, just export `main`. | `--`[`no-`]`import-shared-memory` | no | Import a shared memory buffer. If this is on, `--shared-memory-max-pages` must also be specified. | `--`[`no-`]`inlining` | no | Inline small functions. diff --git a/pkg/dart2wasm/lib/compile.dart b/pkg/dart2wasm/lib/compile.dart index 6e2c88b7c37..f8c473916b2 100644 --- a/pkg/dart2wasm/lib/compile.dart +++ b/pkg/dart2wasm/lib/compile.dart @@ -33,6 +33,7 @@ import 'package:dart2wasm/translator.dart'; Future compileToModule( Uri mainUri, Uri sdkRoot, + Uri? platformDill, TranslatorOptions options, void Function(DiagnosticMessage) handleDiagnosticMessage) async { var succeeded = true; @@ -46,13 +47,18 @@ Future compileToModule( Target target = WasmTarget(); CompilerOptions compilerOptions = CompilerOptions() ..target = target - ..compileSdk = true ..sdkRoot = sdkRoot ..environmentDefines = {} ..verbose = false ..onDiagnostic = diagnosticMessageHandler ..nnbdMode = NnbdMode.Strong; + if (platformDill != null) { + compilerOptions.sdkSummary = platformDill; + } else { + compilerOptions.compileSdk = true; + } + CompilerResult? compilerResult = await kernelForProgram(mainUri, compilerOptions); if (compilerResult == null || !succeeded) { diff --git a/pkg/dart2wasm/lib/target.dart b/pkg/dart2wasm/lib/target.dart index 7d710972ff9..5c7c4eacc83 100644 --- a/pkg/dart2wasm/lib/target.dart +++ b/pkg/dart2wasm/lib/target.dart @@ -51,6 +51,8 @@ class WasmTarget extends Target { 'dart:nativewrappers', 'dart:js_util_wasm', 'dart:js_wasm', + 'dart:wasm', + 'dart:developer', ]; @override diff --git a/pkg/front_end/messages.yaml b/pkg/front_end/messages.yaml index 887e231cb84..abf6263e19c 100644 --- a/pkg/front_end/messages.yaml +++ b/pkg/front_end/messages.yaml @@ -1964,7 +1964,7 @@ FastaUsageLong: Read the SDK platform from , which should be in Dill/Kernel IR format and contain the Dart SDK. - --target=dart2js|dart2js_server|dart_runner|dartdevc|flutter|flutter_runner|none|vm + --target=dart2js|dart2js_server|dart2wasm|dart_runner|dartdevc|flutter|flutter_runner|none|vm Specify the target configuration. --enable-asserts diff --git a/pkg/front_end/pubspec.yaml b/pkg/front_end/pubspec.yaml index 4fff6ab1e24..47e743ce957 100644 --- a/pkg/front_end/pubspec.yaml +++ b/pkg/front_end/pubspec.yaml @@ -23,6 +23,8 @@ dev_dependencies: path: ../build_integration compiler: path: ../compiler + dart2wasm: + path: ../dart2wasm dart_style: ^2.0.0 dev_compiler: path: ../dev_compiler diff --git a/pkg/front_end/test/spell_checking_list_common.txt b/pkg/front_end/test/spell_checking_list_common.txt index c09bea15364..815a290c0bd 100644 --- a/pkg/front_end/test/spell_checking_list_common.txt +++ b/pkg/front_end/test/spell_checking_list_common.txt @@ -723,6 +723,7 @@ cycles cyclic dart dart2js +dart2wasm dartdevc data date @@ -3339,6 +3340,7 @@ warn warning warnings was +wasm wasn't way ways diff --git a/pkg/front_end/tool/_fasta/additional_targets.dart b/pkg/front_end/tool/_fasta/additional_targets.dart index 6e9399a2123..13c91c22f67 100644 --- a/pkg/front_end/tool/_fasta/additional_targets.dart +++ b/pkg/front_end/tool/_fasta/additional_targets.dart @@ -10,6 +10,8 @@ import 'package:compiler/src/kernel/dart2js_target.dart' show Dart2jsTarget; import 'package:dev_compiler/src/kernel/target.dart' show DevCompilerTarget; +import 'package:dart2wasm/target.dart' show WasmTarget; + import 'package:vm/target/install.dart' as vm_target_install show installAdditionalTargets; @@ -21,5 +23,6 @@ void installAdditionalTargets() { targets["dart2js_server"] = (TargetFlags flags) => new Dart2jsTarget("dart2js_server", flags); targets["dartdevc"] = (TargetFlags flags) => new DevCompilerTarget(flags); + targets["dart2wasm"] = (TargetFlags flags) => new WasmTarget(); vm_target_install.installAdditionalTargets(); } diff --git a/sdk/bin/dart2wasm b/sdk/bin/dart2wasm index 893fcda529e..1cebe5ab3a5 100755 --- a/sdk/bin/dart2wasm +++ b/sdk/bin/dart2wasm @@ -19,21 +19,39 @@ function follow_links() { PROG_NAME="$(follow_links "$BASH_SOURCE")" # Handle the case where dart-sdk/bin has been symlinked to. -BIN_DIR="$(cd "${PROG_NAME%/*}" ; pwd -P)" -SDK_DIR="$(cd "${BIN_DIR}/.." ; pwd -P)" +PROG_DIR="$(cd "${PROG_NAME%/*}" ; pwd -P)" +SDK_DIR="$(cd "${PROG_DIR}/../.." ; pwd -P)" +# Locate build directory, containing executables, snapshots and platform dill. +if [[ `uname` == 'Darwin' ]]; then + OUT_DIR="$SDK_DIR/xcodebuild" +else + OUT_DIR="$SDK_DIR/out" +fi +DART_CONFIGURATION=${DART_CONFIGURATION:-ReleaseX64} +BIN_DIR="$OUT_DIR/$DART_CONFIGURATION" + +DART_PRECOMPILED_RUNTIME="$BIN_DIR/dart_precompiled_runtime" + +# Point to SDK directory. SDK_ARG="--dart-sdk=$SDK_DIR" -DART="$BIN_DIR/dart" +# Point to built platform dill. +PLATFORM="$BIN_DIR/dart2wasm_platform.dill" +PLATFORM_ARG="--platform=$PLATFORM" unset EXTRA_VM_OPTIONS declare -a EXTRA_VM_OPTIONS +# Choose snapshot with or without asserts enabled. +SNAPSHOT_NAME="dart2wasm" case $0 in *_developer) EXTRA_VM_OPTIONS+=('--enable_asserts') + SNAPSHOT_NAME="dart2wasm_asserts" ;; esac +SNAPSHOT="$BIN_DIR/$SNAPSHOT_NAME.snapshot" # We allow extra vm options to be passed in through an environment variable. if [[ $DART_VM_OPTIONS ]]; then @@ -43,6 +61,4 @@ fi DART_ROOT="$(cd "${SDK_DIR}/.." ; pwd -P)" -DART2WASM_COMPILER="$DART_ROOT/pkg/dart2wasm/bin/dart2wasm.dart" - -exec "$DART" "--packages=$DART_ROOT/.packages" "${EXTRA_VM_OPTIONS[@]}" "$DART2WASM_COMPILER" "$SDK_ARG" "$@" +exec "$DART_PRECOMPILED_RUNTIME" "--packages=$DART_ROOT/.packages" "${EXTRA_VM_OPTIONS[@]}" "$SNAPSHOT" "$SDK_ARG" "$PLATFORM_ARG" "$@" diff --git a/tools/bots/test_matrix.json b/tools/bots/test_matrix.json index b24317379d0..0a34fecb4d7 100644 --- a/tools/bots/test_matrix.json +++ b/tools/bots/test_matrix.json @@ -3026,7 +3026,8 @@ "name": "build dart", "script": "tools/build.py", "arguments": [ - "runtime" + "runtime", + "dart2wasm" ] }, { diff --git a/utils/aot_snapshot.gni b/utils/aot_snapshot.gni new file mode 100644 index 00000000000..5527528363b --- /dev/null +++ b/utils/aot_snapshot.gni @@ -0,0 +1,118 @@ +# Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. + +import("../build/dart/dart_action.gni") +import("../sdk_args.gni") + +_dart_root = get_path_info("..", "abspath") + +template("aot_snapshot") { + assert(defined(invoker.main_dart), "Must specify 'main_dart'") + gen_kernel_args = [] + if (defined(invoker.gen_kernel_args)) { + gen_kernel_args = invoker.gen_kernel_args + } + gen_snapshot_args = [] + if (defined(invoker.gen_snapshot_args)) { + gen_snapshot_args = invoker.gen_snapshot_args + } + main_dart = invoker.main_dart + name = target_name + if (defined(invoker.name)) { + name = invoker.name + } + extra_deps = [] + if (defined(invoker.deps)) { + extra_deps += invoker.deps + } + extra_inputs = [ main_dart ] + if (defined(invoker.inputs)) { + extra_inputs += invoker.inputs + } + if (defined(invoker.dot_packages)) { + dot_packages = invoker.dot_packages + } else { + dot_packages = rebase_path("$_dart_root/.dart_tool/package_config.json") + } + output = "$root_out_dir/$name.snapshot" + if (defined(invoker.output)) { + output = invoker.output + } + + dill = "$target_gen_dir/$name.dart.dill" + + # Build the kernel file using the prebuilt VM to speed up the debug and + # simulator builds. + prebuilt_dart_action(target_name + "_dill") { + if (defined(invoker.pool)) { + pool = invoker.pool + } + deps = extra_deps + [ + "$_dart_root/runtime/vm:kernel_platform_files($host_toolchain)", + "$_dart_root/runtime/vm:vm_platform", + "$_dart_root/utils/gen_kernel:bootstrap_gen_kernel", + ] + gen_kernel_kernel = + get_label_info("$_dart_root/utils/gen_kernel:bootstrap_gen_kernel", + "target_gen_dir") + "/bootstrap_gen_kernel.dill" + platform_dill = "$root_out_dir/vm_platform_strong.dill" + + inputs = extra_inputs + [ + gen_kernel_kernel, + platform_dill, + main_dart, + dot_packages, + ] + output = dill + outputs = [ output ] + + depfile = "$output.d" + + vm_args = [ + # Ensure gen_kernel.dart will use this SDK hash when consuming/producing kernel. + "-Dsdk_hash=$sdk_hash", + ] + + script = gen_kernel_kernel + + args = [ + "--packages=" + rebase_path(dot_packages), + "--platform=" + rebase_path(platform_dill), + "--aot", + "--output=" + rebase_path(output, root_build_dir), + "--depfile=" + rebase_path(depfile), + + # Ensure the compiled appliation (e.g. kernel-service, frontend-server, + # ...) will use this SDK hash when consuming/producing kernel. + # + # (Instead of ensuring every user of the "application_snapshot" / + # "kernel_snapshot" passes this if needed, we always pass it) + "-Dsdk_hash=$sdk_hash", + ] + args += gen_kernel_args + args += [ rebase_path(main_dart) ] + } + + # Create a snapshot from kernel built above. + gen_snapshot_action(target_name) { + if (defined(invoker.pool)) { + pool = invoker.pool + } + deps = extra_deps + [ ":${target_name}_dill" ] + + inputs = extra_inputs + + outputs = [ output ] + + abs_output = rebase_path(output) + + vm_args = [ + "--deterministic", + "--snapshot-kind=app-aot-elf", + "--elf=$abs_output", + ] + gen_snapshot_args + + args = [ rebase_path(dill) ] + } +} diff --git a/utils/dart2wasm/BUILD.gn b/utils/dart2wasm/BUILD.gn new file mode 100644 index 00000000000..428922fc642 --- /dev/null +++ b/utils/dart2wasm/BUILD.gn @@ -0,0 +1,38 @@ +# Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. + +import("../aot_snapshot.gni") +import("../compile_platform.gni") + +sdk_root = "../../sdk" + +aot_snapshot("dart2wasm_snapshot") { + main_dart = "../../pkg/dart2wasm/bin/dart2wasm.dart" + name = "dart2wasm" +} + +aot_snapshot("dart2wasm_asserts_snapshot") { + main_dart = "../../pkg/dart2wasm/bin/dart2wasm.dart" + name = "dart2wasm_asserts" + gen_kernel_args = [ "--enable-asserts" ] + gen_snapshot_args = [ "--enable-asserts" ] +} + +compile_platform("compile_dart2wasm_platform") { + single_root_scheme = "org-dartlang-sdk" + single_root_base = rebase_path("$sdk_root/") + libraries_specification_uri = "org-dartlang-sdk:///lib/libraries.json" + + outputs = [ + "$root_out_dir/dart2wasm_platform.dill", + "$root_out_dir/dart2wasm_outline.dill", + ] + + args = [ + "--target=dart2wasm", + "--no-defines", + "dart:core", + "--nnbd-strong", + ] +}