diff --git a/pkg/dart2wasm/benchmark/self_compile_benchmark.dart b/pkg/dart2wasm/benchmark/self_compile_benchmark.dart index 6639785d56c..7b37b69fb8b 100644 --- a/pkg/dart2wasm/benchmark/self_compile_benchmark.dart +++ b/pkg/dart2wasm/benchmark/self_compile_benchmark.dart @@ -11,7 +11,7 @@ import 'filesystem_io.dart' if (dart.library.js_interop) 'filesystem_js.dart'; Future main(List args) async { final sw = Stopwatch()..start(); final fileSystem = WasmCompilerFileSystem(); - final result = await compile( + final result = await compileBenchmark( fileSystem, 'pkg/dart2wasm/benchmark/self_compile_benchmark.dart'); print('Dart2WasmSelfCompile(RunTimeRaw): ${sw.elapsed.inMilliseconds} ms.'); @@ -22,7 +22,7 @@ Future main(List args) async { } } -Future compile( +Future compileBenchmark( WasmCompilerFileSystem fileSystem, String mainFile) async { // Avoid CFE self-detecting whether `stdout`/`stderr` is terminal and supports // colors (as we don't have `dart:io` available when we run dart2wasm in a @@ -35,12 +35,12 @@ Future compile( options.librariesSpecPath = Uri.file('${fileSystem.sdkRoot}/sdk/lib/libraries.json'); - final result = await compileToModule( + final result = await compile( options, fileSystem, (mod) => Uri.parse('$mod.maps'), (diag) { print('Diagnostics: ${diag.severity} ${diag.plainTextFormatted}'); }); if (result is! CompilationSuccess) { throw 'Compilation Failed: $result'; } - return result; + return result as CodegenResult; } diff --git a/pkg/dart2wasm/lib/compile.dart b/pkg/dart2wasm/lib/compile.dart index f442f06cc96..08f0a249357 100644 --- a/pkg/dart2wasm/lib/compile.dart +++ b/pkg/dart2wasm/lib/compile.dart @@ -16,9 +16,12 @@ import 'package:front_end/src/api_unstable/vm.dart' kernelForProgram, CfeSeverity; import 'package:kernel/ast.dart'; +import 'package:kernel/binary/ast_from_binary.dart' + show BinaryBuilderWithMetadata; import 'package:kernel/class_hierarchy.dart'; import 'package:kernel/core_types.dart'; -import 'package:kernel/kernel.dart' show writeComponentToText; +import 'package:kernel/kernel.dart' + show writeComponentToText, loadComponentFromBytes; import 'package:kernel/library_index.dart'; import 'package:kernel/text/ast_to_text.dart'; import 'package:kernel/type_environment.dart'; @@ -41,6 +44,7 @@ import 'deferred_loading.dart'; import 'dry_run.dart'; import 'dynamic_module_kernel_metadata.dart'; import 'dynamic_modules.dart'; +import 'js/method_collector.dart' show JSMethods; import 'js/runtime_generator.dart' as js; import 'modules.dart'; import 'record_class_generator.dart'; @@ -48,6 +52,7 @@ import 'records.dart'; import 'target.dart' as wasm show Mode; import 'target.dart' hide Mode; import 'translator.dart'; +import 'util.dart'; sealed class CompilationResult {} @@ -57,12 +62,40 @@ class CompilationDryRunError extends CompilationDryRunResult {} class CompilationDryRunSuccess extends CompilationDryRunResult {} -class CompilationSuccess extends CompilationResult { +sealed class CompilationSuccess extends CompilationResult {} + +class CfeResult extends CompilationSuccess { + final Component component; + final CoreTypes coreTypes; + + CfeResult(this.component, this.coreTypes); +} + +class TfaResult extends CompilationSuccess { + final Component component; + final CoreTypes coreTypes; + final LibraryIndex libraryIndex; + final ModuleStrategy moduleStrategy; + final MainModuleMetadata mainModuleMetadata; + final JSMethods jsInteropMethods; + final Map recordClasses; + + TfaResult( + this.component, + this.coreTypes, + this.libraryIndex, + this.moduleStrategy, + this.mainModuleMetadata, + this.jsInteropMethods, + this.recordClasses); +} + +class CodegenResult extends CompilationSuccess { final Map wasmModules; final String jsRuntime; final String supportJs; - CompilationSuccess(this.wasmModules, this.jsRuntime, this.supportJs); + CodegenResult(this.wasmModules, this.jsRuntime, this.supportJs); } abstract class CompilationError extends CompilationResult {} @@ -124,20 +157,12 @@ const List _librariesToIndex = [ /// This value will be added to the Wasm module in `sourceMappingURL` section. /// When this argument is null the code generator does not generate source /// mappings. -Future compileToModule( +Future compile( compiler.WasmCompilerOptions options, FileSystem fileSystem, Uri Function(String moduleName)? sourceMapUrlGenerator, void Function(CfeDiagnosticMessage) handleDiagnosticMessage, {void Function(String, String)? writeFile}) async { - var hadCompileTimeError = false; - void diagnosticMessageHandler(CfeDiagnosticMessage message) { - if (message.severity == CfeSeverity.error) { - hadCompileTimeError = true; - } - handleDiagnosticMessage(message); - } - final wasm.Mode mode; if (options.translatorOptions.jsCompatibility) { mode = wasm.Mode.jsCompatibility; @@ -150,6 +175,68 @@ Future compileToModule( options.translatorOptions.enableExperimentalWasmInterop, removeAsserts: !options.translatorOptions.enableAsserts, mode: mode); + + if (options.multiRootScheme != null) { + fileSystem = MultiRootFileSystem( + options.multiRootScheme!, + options.multiRoots.isEmpty ? [Uri.base] : options.multiRoots, + fileSystem); + } + + CfeResult? cfeResult; + TfaResult? tfaResult; + CompilationResult? lastResult; + + for (final phase in options.phases) { + switch (phase) { + case compiler.CompilerPhase.cfe: + lastResult = await _runCfePhase( + options, target, fileSystem, handleDiagnosticMessage); + if (lastResult is! CfeResult) return lastResult; + cfeResult = lastResult; + + case compiler.CompilerPhase.tfa: + lastResult = await _runTfaPhase( + cfeResult ?? await _loadCfeResult(options), + options, + target, + fileSystem); + if (lastResult is! TfaResult) return lastResult; + tfaResult = lastResult; + + case compiler.CompilerPhase.codegen: + lastResult = await _runCodegenPhase( + tfaResult ?? await _loadTfaResult(options, target, fileSystem), + options, + fileSystem, + sourceMapUrlGenerator); + } + } + + return lastResult!; +} + +Future _loadCfeResult(compiler.WasmCompilerOptions options) async { + final component = + loadComponentFromBytes(await File.fromUri(options.mainUri).readAsBytes()); + final coreTypes = CoreTypes(component); + return CfeResult(component, coreTypes); +} + +Future _runCfePhase( + compiler.WasmCompilerOptions options, + WasmTarget target, + FileSystem fileSystem, + void Function(CfeDiagnosticMessage) handleDiagnosticMessage, + {void Function(String, String)? writeFile}) async { + var hadCompileTimeError = false; + void diagnosticMessageHandler(CfeDiagnosticMessage message) { + if (message.severity == CfeSeverity.error) { + hadCompileTimeError = true; + } + handleDiagnosticMessage(message); + } + CompilerOptions compilerOptions = CompilerOptions() ..target = target // This is a dummy directory that always exists. This option should be @@ -169,21 +256,6 @@ Future compileToModule( ..verbose = false ..onDiagnostic = diagnosticMessageHandler ..fileSystem = fileSystem; - if (options.multiRootScheme != null) { - compilerOptions.fileSystem = MultiRootFileSystem( - options.multiRootScheme!, - options.multiRoots.isEmpty ? [Uri.base] : options.multiRoots, - compilerOptions.fileSystem); - } - - Future resolveUri(Uri? uri) async { - if (uri == null) return null; - var fileSystemEntity = compilerOptions.fileSystem.entityForUri(uri); - if (fileSystemEntity is MultiRootFileSystemEntity) { - fileSystemEntity = await fileSystemEntity.delegate; - } - return fileSystemEntity.uri; - } if (options.platformPath != null) { compilerOptions.sdkSummary = options.platformPath; @@ -191,10 +263,8 @@ Future compileToModule( compilerOptions.compileSdk = true; } - final dynamicMainModuleUri = await resolveUri(options.dynamicMainModuleUri); - final dynamicInterfaceUri = await resolveUri(options.dynamicInterfaceUri); - final isDynamicMainModule = - options.dynamicModuleType == DynamicModuleType.main; + final dynamicMainModuleUri = + await _resolveUri(fileSystem, options.dynamicMainModuleUri); final isDynamicSubmodule = options.dynamicModuleType == DynamicModuleType.submodule; if (isDynamicSubmodule) { @@ -227,19 +297,72 @@ Future compileToModule( if (hadCompileTimeError) { return CFECompileTimeErrors(compilerResult?.component); } - assert(compilerResult != null); - - Component component = compilerResult!.component!; - CoreTypes coreTypes = compilerResult.coreTypes!; - - ClosedWorldClassHierarchy classHierarchy = - ClassHierarchy(component, coreTypes) as ClosedWorldClassHierarchy; - LibraryIndex libraryIndex = LibraryIndex(component, _librariesToIndex); + final component = compilerResult!.component!; if (options.dumpKernelAfterCfe != null && writeFile != null) { writeFile(options.dumpKernelAfterCfe!, writeComponentToString(component)); } + return CfeResult(component, compilerResult.coreTypes!); +} + +Future _loadTfaResult(compiler.WasmCompilerOptions options, + WasmTarget target, FileSystem fileSystem) async { + final component = createEmptyComponent(); + final recordClassesRepository = _RecordClassesRepository(); + final interopMethodsRepository = _InteropMethodsRepository(); + component.addMetadataRepository(recordClassesRepository); + component.addMetadataRepository(interopMethodsRepository); + + BinaryBuilderWithMetadata(await File.fromUri(options.mainUri).readAsBytes()) + .readComponent(component); + final coreTypes = CoreTypes(component); + final libraryIndex = LibraryIndex(component, _librariesToIndex); + final classHierarchy = ClassHierarchy(component, coreTypes); + final dynamicMainModuleUri = + await _resolveUri(fileSystem, options.dynamicMainModuleUri); + final dynamicInterfaceUri = + await _resolveUri(fileSystem, options.dynamicInterfaceUri); + + final moduleStrategy = _createModuleStrategy(options, component, coreTypes, + target, classHierarchy, dynamicMainModuleUri, dynamicInterfaceUri); + + final recordClasses = {}; + recordClassesRepository.mapping.forEach((cls, shape) { + recordClasses[shape] = cls; + }); + + final isDynamicMainModule = + options.dynamicModuleType == DynamicModuleType.main; + final isDynamicSubmodule = + options.dynamicModuleType == DynamicModuleType.submodule; + MainModuleMetadata mainModuleMetadata = + MainModuleMetadata.empty(options.translatorOptions, options.environment); + + if (isDynamicSubmodule) { + mainModuleMetadata = + await deserializeMainModuleMetadata(component, options); + mainModuleMetadata.verifyDynamicSubmoduleOptions(options); + } else if (isDynamicMainModule) { + MainModuleMetadata.verifyMainModuleOptions(options); + } + + return TfaResult(component, coreTypes, libraryIndex, moduleStrategy, + mainModuleMetadata, interopMethodsRepository.mapping, recordClasses); +} + +Future _runTfaPhase( + CfeResult cfeResult, + compiler.WasmCompilerOptions options, + WasmTarget target, + FileSystem fileSystem, + {void Function(String, String)? writeFile}) async { + var CfeResult(:component, :coreTypes) = cfeResult; + + ClosedWorldClassHierarchy classHierarchy = + ClassHierarchy(component, coreTypes) as ClosedWorldClassHierarchy; + LibraryIndex libraryIndex = LibraryIndex(component, _librariesToIndex); + if (options.deleteToStringPackageUri.isNotEmpty) { to_string_transformer.transformComponent( component, options.deleteToStringPackageUri); @@ -250,6 +373,15 @@ Future compileToModule( coreTypes, classHierarchy); + final dynamicMainModuleUri = + await _resolveUri(fileSystem, options.dynamicMainModuleUri); + final dynamicInterfaceUri = + await _resolveUri(fileSystem, options.dynamicInterfaceUri); + final isDynamicMainModule = + options.dynamicModuleType == DynamicModuleType.main; + final isDynamicSubmodule = + options.dynamicModuleType == DynamicModuleType.submodule; + if (isDynamicSubmodule) { // Join the submodule libraries with the TFAed component from the main // module compilation. JS interop transformer must be run before this since @@ -280,26 +412,8 @@ Future compileToModule( writeFile(options.dumpKernelBeforeTfa!, writeComponentToString(component)); } - ModuleStrategy moduleStrategy; - if (options.translatorOptions.enableDeferredLoading) { - moduleStrategy = - DeferredLoadingModuleStrategy(component, options, target, coreTypes); - } else if (options.translatorOptions.enableMultiModuleStressTestMode) { - moduleStrategy = StressTestModuleStrategy( - component, coreTypes, options, target, classHierarchy); - } else if (isDynamicMainModule) { - moduleStrategy = DynamicMainModuleStrategy( - component, - coreTypes, - options, - File.fromUri(dynamicInterfaceUri!).readAsStringSync(), - options.dynamicInterfaceUri!); - } else if (isDynamicSubmodule) { - moduleStrategy = DynamicSubmoduleStrategy( - component, options, target, coreTypes, dynamicMainModuleUri!); - } else { - moduleStrategy = DefaultModuleStrategy(component, options); - } + final moduleStrategy = _createModuleStrategy(options, component, coreTypes, + target, classHierarchy, dynamicMainModuleUri, dynamicInterfaceUri); // DynamicMainModuleStrategy.prepareComponent() includes // dynamic_interface_annotator transformation which annotates AST nodes with @@ -337,9 +451,19 @@ Future compileToModule( useRapidTypeAnalysis: false); } - if (options.dumpKernelAfterTfa != null) { - writeComponentToText(component, - path: options.dumpKernelAfterTfa!, showMetadata: true); + if (options.phases.last == compiler.CompilerPhase.tfa) { + // Store metadata needed for codegen so that it can be serialized. + final recordClassesRepo = _RecordClassesRepository(); + recordClasses.forEach((shape, cls) { + recordClassesRepo.mapping[cls] = shape; + }); + component.addMetadataRepository(recordClassesRepo); + + final interopMethodsRepo = _InteropMethodsRepository(); + jsInteropMethods.forEach((method, info) { + interopMethodsRepo.mapping[method] = info; + }); + component.addMetadataRepository(interopMethodsRepo); } assert(() { @@ -348,6 +472,29 @@ Future compileToModule( return true; }()); + if (options.dumpKernelAfterTfa != null) { + writeComponentToText(component, + path: options.dumpKernelAfterTfa!, showMetadata: true); + } + + return TfaResult(component, coreTypes, libraryIndex, moduleStrategy, + mainModuleMetadata, jsInteropMethods, recordClasses); +} + +Future _runCodegenPhase( + TfaResult tfaSuccess, + compiler.WasmCompilerOptions options, + FileSystem fileSystem, + Uri Function(String moduleName)? sourceMapUrlGenerator) async { + final TfaResult( + :component, + :coreTypes, + :moduleStrategy, + :libraryIndex, + :recordClasses, + :mainModuleMetadata, + :jsInteropMethods + ) = tfaSuccess; await moduleStrategy.processComponentAfterTfa(); final moduleOutputData = moduleStrategy.buildModuleOutputData(); @@ -359,8 +506,8 @@ Future compileToModule( String? depFile = options.depFile; if (depFile != null) { - writeDepfile(compilerOptions.fileSystem, component.uriToSource.keys, - options.outputFile, depFile); + writeDepfile( + fileSystem, component.uriToSource.keys, options.outputFile, depFile); } final generateSourceMaps = options.translatorOptions.generateSourceMaps; @@ -380,6 +527,13 @@ Future compileToModule( final jsRuntimeFinalizer = js.RuntimeFinalizer(jsInteropMethods); + final dynamicMainModuleUri = + await _resolveUri(fileSystem, options.dynamicMainModuleUri); + final isDynamicMainModule = + options.dynamicModuleType == DynamicModuleType.main; + final isDynamicSubmodule = + options.dynamicModuleType == DynamicModuleType.submodule; + final jsRuntime = isDynamicSubmodule ? jsRuntimeFinalizer.generateDynamicSubmodule( translator.functions.translatedProcedures, @@ -400,7 +554,38 @@ Future compileToModule( optimized: true); } - return CompilationSuccess(wasmModules, jsRuntime, supportJs); + return CodegenResult(wasmModules, jsRuntime, supportJs); +} + +ModuleStrategy _createModuleStrategy( + compiler.WasmCompilerOptions options, + Component component, + CoreTypes coreTypes, + WasmTarget target, + ClassHierarchy classHierarchy, + Uri? dynamicMainModuleUri, + Uri? dynamicInterfaceUri) { + final isDynamicMainModule = + options.dynamicModuleType == DynamicModuleType.main; + final isDynamicSubmodule = + options.dynamicModuleType == DynamicModuleType.submodule; + if (options.translatorOptions.enableDeferredLoading) { + return DeferredLoadingModuleStrategy(component, options, target, coreTypes); + } else if (options.translatorOptions.enableMultiModuleStressTestMode) { + return StressTestModuleStrategy( + component, coreTypes, options, target, classHierarchy); + } else if (isDynamicMainModule) { + return DynamicMainModuleStrategy( + component, + coreTypes, + options, + File.fromUri(dynamicInterfaceUri!).readAsStringSync(), + options.dynamicInterfaceUri!); + } else if (isDynamicSubmodule) { + return DynamicSubmoduleStrategy( + component, options, target, coreTypes, dynamicMainModuleUri!); + } + return DefaultModuleStrategy(component, options); } // Patches `dart:_internal`s `mainTearOff{0,1,2}` getters. @@ -434,6 +619,69 @@ void _patchMainTearOffs(CoreTypes coreTypes, Component component) { if (mainHasType(mainArg0Type)) return patchToReturnMainTearOff(mainTearOff0); } +Future _resolveUri(FileSystem fileSystem, Uri? uri) async { + if (uri == null) return null; + var fileSystemEntity = fileSystem.entityForUri(uri); + if (fileSystemEntity is MultiRootFileSystemEntity) { + fileSystemEntity = await fileSystemEntity.delegate; + } + return fileSystemEntity.uri; +} + +class _RecordClassesRepository extends MetadataRepository { + static const String _tag = 'dart2wasm.recordClasses'; + @override + final Map mapping = {}; + + @override + RecordShape readFromBinary(Node node, BinarySource source) { + final positionals = source.readUInt30(); + final namesLength = source.readUInt30(); + final names = namesLength == 0 ? const [] : []; + for (int i = 0; i < namesLength; i++) { + names.add(source.readStringReference()); + } + return RecordShape(positionals, names); + } + + @override + String get tag => _tag; + + @override + void writeToBinary(RecordShape metadata, Node node, BinarySink sink) { + sink.writeUInt30(metadata.positionals); + sink.writeUInt30(metadata.names.length); + for (final name in metadata.names) { + sink.writeStringReference(name); + } + } +} + +class _InteropMethodsRepository + extends MetadataRepository<({String importName, String jsCode})> { + static const String _tag = 'dart2wasm.interopMethods'; + @override + final Map mapping = {}; + + @override + ({String importName, String jsCode}) readFromBinary( + Node node, BinarySource source) { + final importName = source.readStringReference(); + final jsCode = source.readStringReference(); + return (importName: importName, jsCode: jsCode); + } + + @override + String get tag => _tag; + + @override + void writeToBinary(({String importName, String jsCode}) metadata, Node node, + BinarySink sink) { + sink.writeStringReference(metadata.importName); + sink.writeStringReference(metadata.jsCode); + } +} + String _generateSupportJs(TranslatorOptions options) { // Copied from // https://github.com/GoogleChromeLabs/wasm-feature-detect/blob/main/src/detectors/gc/index.js diff --git a/pkg/dart2wasm/lib/compiler_options.dart b/pkg/dart2wasm/lib/compiler_options.dart index e36ff6caa2d..dcb81c77529 100644 --- a/pkg/dart2wasm/lib/compiler_options.dart +++ b/pkg/dart2wasm/lib/compiler_options.dart @@ -3,10 +3,33 @@ // BSD-style license that can be found in the LICENSE file. import 'package:front_end/src/api_unstable/vm.dart' as fe; +import 'package:path/path.dart' as path; import 'dynamic_modules.dart' show DynamicModuleType; import 'translator.dart'; +/// Represents a discrete phase of dart2wasm's compilation process. +/// +/// cfe: Runs the common frontend and applies any modular transforms as part of +/// that process. +/// +/// tfa: Runs global transforms on kernel including, but not limited to, TFA. +/// +/// codegen: Runs the main dart2wasm translation process converting the kernel +/// into WASM modules. +enum CompilerPhase { + cfe, + tfa, + codegen; + + static CompilerPhase parse(String name) { + for (final phase in values) { + if (phase.name == name) return phase; + } + throw ArgumentError('Invalid compiler phase name: $name'); + } +} + class WasmCompilerOptions { final TranslatorOptions translatorOptions = TranslatorOptions(); @@ -31,6 +54,11 @@ class WasmCompilerOptions { String? dumpKernelBeforeTfa; String? dumpKernelAfterTfa; bool dryRun = false; + List phases = const [ + CompilerPhase.cfe, + CompilerPhase.tfa, + CompilerPhase.codegen + ]; factory WasmCompilerOptions.defaultOptions() => WasmCompilerOptions(mainUri: Uri(), outputFile: ''); @@ -64,5 +92,57 @@ class WasmCompilerOptions { "compiling dynamic modules."); } } + + _validatePhases(); + } + + void _validatePhases() { + if (phases.isEmpty) { + throw ArgumentError('--phases must contain at least one phase.'); + } + + CompilerPhase? previousPhase; + for (final phase in phases) { + // Ensure phases are consecutive + if (previousPhase != null && previousPhase.index != phase.index - 1) { + throw ArgumentError('--phases must contain consecutive phases.'); + } + previousPhase = phase; + } + + // Ensure correct input file type + final inputExtension = path.extension(mainUri.path); + switch (phases.first) { + case CompilerPhase.cfe: + if (inputExtension != '.dart') { + throw ArgumentError('Input to cfe phase must be a .dart file.'); + } + case CompilerPhase.tfa: + if (inputExtension != '.dill') { + throw ArgumentError('Input to tfa phase must be a .dill file.'); + } + case CompilerPhase.codegen: + if (inputExtension != '.dill') { + throw ArgumentError('Input to codegen phase must be a .dill file.'); + } + } + + // Ensure correct output file type + final outputExtension = path.extension(outputFile); + switch (phases.last) { + case CompilerPhase.cfe: + if (outputExtension != '.dill') { + throw ArgumentError('Output from cfe phase must be a .dill file.'); + } + case CompilerPhase.tfa: + if (outputExtension != '.dill') { + throw ArgumentError('Output from tfa phase must be a .dill file.'); + } + case CompilerPhase.codegen: + if (outputExtension != '.wasm') { + throw ArgumentError( + 'Output from codegen phase must be a .wasm file.'); + } + } } } diff --git a/pkg/dart2wasm/lib/dart2wasm.dart b/pkg/dart2wasm/lib/dart2wasm.dart index 9c49b4e6c87..8050c1f3f62 100644 --- a/pkg/dart2wasm/lib/dart2wasm.dart +++ b/pkg/dart2wasm/lib/dart2wasm.dart @@ -27,6 +27,10 @@ final List