[dart2wasm] Add deferred loading support to dart2wasm (11/X).

This is the final CL for deferred loading. It wires up the library-module analysis logic to the compiler. With all the Translator module predicates implemented, code should now be generated in separate modules (assuming the flag is enabled).

This also handles the module naming scheme. For an invocation of dart2wasm like `dart2wasm main.dart out.wasm` this will produce files like `out.mjs, out.wasm, out_module1.wasm, out_module2.wasm, ...`. `out.wasm` is the main module that gets loaded on initialization. When the flag is disabled this will always be the only output.

If the flag is disabled then the `_importMapping` in `deferred.dart` will be empty and we will default to the same behavior as today which will be to just return an empty `Future`. When enabled, `loadLibrary` will fetch and instantiate the new module(s) before proceeding.

Change-Id: I0dd136c0af61b916be2a24b3d79052ff1b786b52
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/380440
Reviewed-by: Martin Kustermann <kustermann@google.com>

pkg/dart2wasm/lib/code_generator.dart

@@ -2835,20 +2835,15 @@ abstract class AstCodeGenerator
 
   @override
   w.ValueType visitLoadLibrary(LoadLibrary node, w.ValueType expectedType) {
-    LibraryDependency import = node.import;
-    _emitString(import.enclosingLibrary.importUri.toString());
-    _emitString(import.name!);
-    return translator.outputOrVoid(call(translator.loadLibrary.reference));
+    throw UnsupportedError(
+        'LoadLibrary should be lowered by modular transformer.');
   }
 
   @override
   w.ValueType visitCheckLibraryIsLoaded(
       CheckLibraryIsLoaded node, w.ValueType expectedType) {
-    LibraryDependency import = node.import;
-    _emitString(import.enclosingLibrary.importUri.toString());
-    _emitString(import.name!);
-    return translator
-        .outputOrVoid(call(translator.checkLibraryIsLoaded.reference));
+    throw UnsupportedError(
+        'CheckLibraryIsLoaded should be lowered by modular transformer.');
   }
 
   /// Pushes the `_Type` object for a function or class type parameter to the

pkg/dart2wasm/lib/compile.dart

@@ -37,6 +37,7 @@ import 'package:wasm_builder/wasm_builder.dart' show Serializer;
 
 import 'compiler_options.dart' as compiler;
 import 'constant_evaluator.dart';
+import 'deferred_loading.dart';
 import 'js/runtime_generator.dart' as js;
 import 'record_class_generator.dart';
 import 'records.dart';
@@ -45,11 +46,10 @@ import 'target.dart' hide Mode;
 import 'translator.dart';
 
 class CompilerOutput {
-  final Uint8List wasmModule;
+  final Map<String, ({Uint8List moduleBytes, String? sourceMap})> wasmModules;
   final String jsRuntime;
-  final String? sourceMap;
 
-  CompilerOutput(this.wasmModule, this.jsRuntime, this.sourceMap);
+  CompilerOutput(this.wasmModules, this.jsRuntime);
 }
 
 /// Compile a Dart file into a Wasm module.
@@ -58,13 +58,14 @@ class CompilerOutput {
 /// [handleDiagnosticMessage] callback will have received an error message
 /// describing the error.
 ///
-/// When generating a source map, `sourceMapUrl` argument should be provided
-/// with the URL of the source map. This value will be added to the Wasm module
-/// in `sourceMappingURL` section. When this argument is null the code
-/// generator does not generate source mappings.
+/// When generating source maps, `sourceMapUrlGenerator` argument should be
+/// provided which takes the module name and gives the URL of the source map.
+/// This value will be added to the Wasm module in `sourceMappingURL` section.
+/// When this argument is null the code generator does not generate source
+/// mappings.
 Future<CompilerOutput?> compileToModule(
     compiler.WasmCompilerOptions options,
-    Uri? sourceMapUrl,
+    Uri Function(String moduleName)? sourceMapUrlGenerator,
     void Function(DiagnosticMessage) handleDiagnosticMessage) async {
   var succeeded = true;
   void diagnosticMessageHandler(DiagnosticMessage message) {
@@ -194,8 +195,11 @@ Future<CompilerOutput?> compileToModule(
     return true;
   }());
 
+  final moduleOutputData =
+      modulesForComponent(component, options, target, coreTypes);
+
   var translator = Translator(component, coreTypes, libraryIndex, recordClasses,
-      options.translatorOptions);
+      moduleOutputData, options.translatorOptions);
 
   String? depFile = options.depFile;
   if (depFile != null) {
@@ -209,18 +213,23 @@ Future<CompilerOutput?> compileToModule(
   }
 
   final generateSourceMaps = options.translatorOptions.generateSourceMaps;
-  final wasmModule = translator.translate(sourceMapUrl);
-  final serializer = Serializer();
-  wasmModule.serialize(serializer);
-  final wasmModuleSerialized = serializer.data;
+  final modules = translator.translate(sourceMapUrlGenerator);
+  final wasmModules = <String, ({Uint8List moduleBytes, String? sourceMap})>{};
+  modules.forEach((moduleOutput, module) {
+    final serializer = Serializer();
+    module.serialize(serializer);
+    final wasmModuleSerialized = serializer.data;
 
-  final sourceMap =
-      generateSourceMaps ? serializer.sourceMapSerializer.serialize() : null;
+    final sourceMap =
+        generateSourceMaps ? serializer.sourceMapSerializer.serialize() : null;
+    wasmModules[moduleOutput.moduleName] =
+        (moduleBytes: wasmModuleSerialized, sourceMap: sourceMap);
+  });
 
   String jsRuntime = jsRuntimeFinalizer.generate(
       translator.functions.translatedProcedures,
       translator.internalizedStringsForJSRuntime,
       mode);
 
-  return CompilerOutput(wasmModuleSerialized, jsRuntime, sourceMap);
+  return CompilerOutput(wasmModules, jsRuntime);
 }

pkg/dart2wasm/lib/compiler_options.dart

@@ -3,7 +3,6 @@
 // BSD-style license that can be found in the LICENSE file.
 
 import 'package:front_end/src/api_unstable/vm.dart' as fe;
-import 'package:path/path.dart' as path;
 
 import 'translator.dart';
 
@@ -31,10 +30,6 @@ class WasmCompilerOptions {
 
   WasmCompilerOptions({required this.mainUri, required this.outputFile});
 
-  String get outputFileDir => path.dirname(outputFile);
-
-  String get outputFileName => path.basename(outputFile);
-
   void validate() {
     if (translatorOptions.importSharedMemory &&
         translatorOptions.sharedMemoryMaxPages == null) {

pkg/dart2wasm/lib/deferred_loading.dart

@@ -43,10 +43,10 @@ class ModuleOutput {
   bool get isMain => _id == _mainModuleId;
 
   /// The name used to import and export this module.
-  String get moduleName => 'module$_id';
+  String get moduleImportName => 'module$_id';
 
   /// The name added to the wasm output file for this module.
-  String get moduleFileName => isMain ? '' : moduleName;
+  String get moduleName => isMain ? '' : moduleImportName;
 
   ModuleOutput._(this._id);
 
@@ -58,7 +58,7 @@ class ModuleOutput {
   }
 
   @override
-  String toString() => '$moduleName($_libraries)';
+  String toString() => '$moduleImportName($_libraries)';
 }
 
 /// The root of a deferred import subgraph.
@@ -288,7 +288,8 @@ class ModuleOutputData {
     _importMap.forEach((lib, importMapping) {
       final nameMapping = <String, List<String>>{};
       importMapping.forEach((importName, modules) {
-        nameMapping[importName] = modules.map((o) => o.moduleName).toList();
+        nameMapping[importName] =
+            modules.map((o) => o.moduleImportName).toList();
       });
       result[lib.importUri.toString()] = nameMapping;
     });

pkg/dart2wasm/lib/generate_wasm.dart

@@ -27,29 +27,52 @@ Future<int> generateWasm(WasmCompilerOptions options,
         '  - generate source maps = ${options.translatorOptions.generateSourceMaps}');
   }
 
-  final relativeSourceMapUrl = options.translatorOptions.generateSourceMaps
-      ? Uri.file('${path.basename(options.outputFile)}.map')
-      : null;
+  String moduleNameToWasmOutputFile(String moduleName) {
+    final outputFile = options.outputFile;
+    if (moduleName.isEmpty) return outputFile;
+    final extension = path.extension(outputFile);
+    return path.setExtension(outputFile, '_$moduleName$extension');
+  }
 
-  CompilerOutput? output = await compileToModule(options, relativeSourceMapUrl,
+  String moduleNameToSourceMapFile(String moduleName) {
+    return '${moduleNameToWasmOutputFile(moduleName)}.map';
+  }
+
+  Uri moduleNameToRelativeSourceMapUri(String moduleName) {
+    return Uri.file(path.basename(moduleNameToSourceMapFile(moduleName)));
+  }
+
+  final relativeSourceMapUrlMapper =
+      options.translatorOptions.generateSourceMaps
+          ? moduleNameToRelativeSourceMapUri
+          : null;
+
+  CompilerOutput? output = await compileToModule(
+      options,
+      relativeSourceMapUrlMapper,
       (message) => printDiagnosticMessage(message, errorPrinter));
 
   if (output == null) {
     return 1;
   }
 
-  final File outFile = File(options.outputFile);
-  outFile.parent.createSync(recursive: true);
-  await outFile.writeAsBytes(output.wasmModule);
+  final writeFutures = <Future>[];
+  output.wasmModules.forEach((moduleName, moduleInfo) {
+    final (:moduleBytes, :sourceMap) = moduleInfo;
+    final File outFile = File(moduleNameToWasmOutputFile(moduleName));
+    outFile.parent.createSync(recursive: true);
+    writeFutures.add(outFile.writeAsBytes(moduleBytes));
+
+    if (sourceMap != null) {
+      writeFutures.add(
+          File(moduleNameToSourceMapFile(moduleName)).writeAsString(sourceMap));
+    }
+  });
+  await Future.wait(writeFutures);
 
   final jsFile = options.outputJSRuntimeFile ??
       path.setExtension(options.outputFile, '.mjs');
   await File(jsFile).writeAsString(output.jsRuntime);
 
-  final sourceMap = output.sourceMap;
-  if (sourceMap != null) {
-    await File('${options.outputFile}.map').writeAsString(sourceMap);
-  }
-
   return 0;
 }

pkg/dart2wasm/lib/kernel_nodes.dart

@@ -188,6 +188,8 @@ mixin KernelNodes {
       index.getTopLevelProcedure("dart:_internal", "loadLibrary");
   late final Procedure checkLibraryIsLoaded =
       index.getTopLevelProcedure("dart:_internal", "checkLibraryIsLoaded");
+  late final Procedure loadLibraryImportMap =
+      index.getTopLevelProcedure("dart:_internal", "get:_importMapping");
 
   // dart:_js_helper procedures
   late final Procedure getInternalizedString =

pkg/dart2wasm/lib/transformers.dart

@@ -56,6 +56,9 @@ class _WasmTransformer extends Transformer {
   final Procedure _trySetStackTraceForwarder;
   final Procedure _trySetStackTrace;
 
+  final Procedure _loadLibrary;
+  final Procedure _checkLibraryIsLoaded;
+
   final List<_AsyncStarFrame> _asyncStarFrames = [];
   bool _enclosingIsAsyncStar = false;
 
@@ -111,6 +114,10 @@ class _WasmTransformer extends Transformer {
             .getTopLevelProcedure('dart:async', '_trySetStackTrace'),
         _trySetStackTrace = coreTypes.index
             .getProcedure('dart:core', 'Error', '_trySetStackTrace'),
+        _loadLibrary = coreTypes.index
+            .getTopLevelProcedure("dart:_internal", "loadLibrary"),
+        _checkLibraryIsLoaded = coreTypes.index
+            .getTopLevelProcedure("dart:_internal", "checkLibraryIsLoaded"),
         _listFactorySpecializer = ListFactorySpecializer(coreTypes),
         _pushPopWasmArrayTransformer = PushPopWasmArrayTransformer(coreTypes);
 
@@ -738,6 +745,30 @@ class _WasmTransformer extends Transformer {
     node.transformChildren(this);
     return node.receiver;
   }
+
+  @override
+  TreeNode visitLoadLibrary(LoadLibrary node) {
+    node.transformChildren(this);
+    final import = node.import;
+    return StaticInvocation(
+        _loadLibrary,
+        Arguments([
+          StringLiteral('${import.enclosingLibrary.importUri}'),
+          StringLiteral(import.name!)
+        ]));
+  }
+
+  @override
+  TreeNode visitCheckLibraryIsLoaded(CheckLibraryIsLoaded node) {
+    node.transformChildren(this);
+    final import = node.import;
+    return StaticInvocation(
+        _checkLibraryIsLoaded,
+        Arguments([
+          StringLiteral('${import.enclosingLibrary.importUri}'),
+          StringLiteral(import.name!)
+        ]));
+  }
 }
 
 class _AsyncStarFrame {

pkg/dart2wasm/lib/translator.dart

@@ -18,6 +18,7 @@ import 'class_info.dart';
 import 'closures.dart';
 import 'code_generator.dart';
 import 'constants.dart';
+import 'deferred_loading.dart';
 import 'dispatch_table.dart';
 import 'dynamic_forwarders.dart';
 import 'functions.dart';
@@ -297,20 +298,25 @@ class Translator with KernelNodes {
   ]);
 
   // Module predicates and helpers
-  // TODO(natebiggs): Implement these with real module data.
-  Iterable<w.ModuleBuilder> get modules => [mainModule];
-  late final w.ModuleBuilder mainModule;
+  final ModuleOutputData _moduleOutputData;
+  Iterable<w.ModuleBuilder> get modules => _builderToOutput.keys;
+  w.ModuleBuilder get mainModule =>
+      _outputToBuilder[_moduleOutputData.mainModule]!;
   w.TypesBuilder get typesBuilder => mainModule.types;
-  bool get hasMultipleModules => false;
+  final Map<ModuleOutput, w.ModuleBuilder> _outputToBuilder = {};
+  final Map<w.ModuleBuilder, ModuleOutput> _builderToOutput = {};
+  bool get hasMultipleModules => _moduleOutputData.hasMultipleModules;
 
-  w.ModuleBuilder moduleForReference(Reference reference) => mainModule;
+  w.ModuleBuilder moduleForReference(Reference reference) =>
+      _outputToBuilder[_moduleOutputData.moduleForReference(reference)]!;
 
-  bool isMainModule(w.ModuleBuilder module) => true;
+  String nameForModule(w.ModuleBuilder module) =>
+      _builderToOutput[module]!.moduleImportName;
 
-  String nameForModule(w.ModuleBuilder module) => 'main';
+  bool isMainModule(w.ModuleBuilder module) => _builderToOutput[module]!.isMain;
 
   Translator(this.component, this.coreTypes, this.index, this.recordClasses,
-      this.options)
+      this._moduleOutputData, this.options)
       : libraries = component.libraries,
         hierarchy =
             ClassHierarchy(component, coreTypes) as ClosedWorldClassHierarchy {
@@ -326,9 +332,44 @@ class Translator with KernelNodes {
     exceptionTag = ExceptionTag(this);
   }
 
-  w.Module translate(Uri? sourceMapUrl) {
-    mainModule =
-        w.ModuleBuilder(sourceMapUrl, watchPoints: options.watchPoints);
+  void _initLoadLibraryImportMap() {
+    final mapEntries = <MapLiteralEntry>[];
+    _moduleOutputData.generateModuleImportMap().forEach((libName, importMap) {
+      final subMapEntries = <MapLiteralEntry>[];
+      importMap.forEach((importName, moduleNames) {
+        subMapEntries.add(MapLiteralEntry(StringLiteral(importName),
+            ListLiteral([...moduleNames.map(StringLiteral.new)])));
+      });
+      mapEntries.add(
+          MapLiteralEntry(StringLiteral(libName), MapLiteral(subMapEntries)));
+    });
+    final stringClass =
+        options.jsCompatibility ? jsStringClass : stringBaseClass;
+    loadLibraryImportMap.function.body = ReturnStatement(MapLiteral(mapEntries,
+        keyType: InterfaceType(stringClass, Nullability.nonNullable),
+        valueType: InterfaceType(coreTypes.mapNonNullableRawType.classNode,
+            Nullability.nonNullable, [
+          InterfaceType(stringClass, Nullability.nonNullable),
+          InterfaceType(stringClass, Nullability.nonNullable)
+        ])));
+    loadLibraryImportMap.isExternal = false;
+  }
+
+  void _initModules(Uri Function(String moduleName)? sourceMapUrlGenerator) {
+    for (final outputModule in _moduleOutputData.modules) {
+      final builder = w.ModuleBuilder(
+          sourceMapUrlGenerator?.call(outputModule.moduleName),
+          parent: outputModule.isMain ? null : mainModule,
+          watchPoints: options.watchPoints);
+      _outputToBuilder[outputModule] = builder;
+      _builderToOutput[builder] = outputModule;
+    }
+  }
+
+  Map<ModuleOutput, w.Module> translate(
+      Uri Function(String moduleName)? sourceMapUrlGenerator) {
+    _initLoadLibraryImportMap();
+    _initModules(sourceMapUrlGenerator);
     voidMarker = w.RefType.def(w.StructType("void"), nullable: true);
 
     closureLayouter.collect();
@@ -370,7 +411,11 @@ class Translator with KernelNodes {
     }
     _printFunction(initFunction, "init");
 
-    return mainModule.build();
+    final result = <ModuleOutput, w.Module>{};
+    _outputToBuilder.forEach((outputModule, builder) {
+      result[outputModule] = builder.build();
+    });
+    return result;
   }
 
   void _printFunction(w.BaseFunction function, Object name) {

sdk/lib/_internal/wasm/lib/deferred.dart

@@ -2,12 +2,16 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-// TODO(joshualitt): This is just a stub so apps can run. We should replace it
-// with an actual implementation of deferred loading.
-
 part of "internal_patch.dart";
 
-Map<String, Set<String>> _loadedLibraries = {};
+final Map<String, Future> _loadingModules = {};
+final Set<String> _loadedModules = {};
+final Map<String, Set<String>> _loadedLibraries = {};
+
+external Map<String, Map<String, List<String>>> get _importMapping;
+
+@pragma("wasm:import", "deferredLibraryHelper.loadModule")
+external WasmExternRef _loadModule(WasmExternRef moduleName);
 
 class DeferredNotLoadedError extends Error implements NoSuchMethodError {
   final String libraryName;
@@ -20,16 +24,62 @@ class DeferredNotLoadedError extends Error implements NoSuchMethodError {
   }
 }
 
-@pragma("wasm:entry-point")
 Future<void> loadLibrary(String enclosingLibrary, String importPrefix) {
-  (_loadedLibraries[enclosingLibrary] ??= {}).add(importPrefix);
-  return Future<void>.value();
+  if (_importMapping.isEmpty) {
+    // Only contains one unit.
+    (_loadedLibraries[enclosingLibrary] ??= {}).add(importPrefix);
+    return Future.value();
+  }
+  final loadedImports = _loadedLibraries[enclosingLibrary];
+  if (loadedImports != null && loadedImports.contains(importPrefix)) {
+    // Import already loaded.
+    return Future.value();
+  }
+  final importNameMapping = _importMapping[enclosingLibrary]!;
+  final moduleNames = importNameMapping[importPrefix];
+
+  if (moduleNames == null) {
+    // Since loadLibrary calls get lowered to static invocations of this method,
+    // TFA will tree-shake libraries (and their associated imports) that are
+    // only referenced via a loadLibrary call. In this case, we won't have an
+    // import mapping for the lowered loadLibrary call.
+    return Future.value();
+  }
+
+  // Start loading modules
+  final List<Future> loadFutures = [];
+  for (final moduleName in moduleNames) {
+    if (_loadedModules.contains(moduleName)) {
+      // Already loaded module
+      continue;
+    }
+    final existingLoad = _loadingModules[moduleName];
+    if (existingLoad != null) {
+      // Already loading module
+      loadFutures.add(existingLoad);
+      continue;
+    }
+
+    // Start module load
+    final promise =
+        (_loadModule(moduleName.toJS.toExternRef!).toJS as JSPromise);
+    final future = promise.toDart.then((_) {
+      // Module loaded
+      _loadedModules.add(moduleName);
+    }, onError: (e) {
+      throw DeferredLoadException('Error loading module: $moduleName\n$e');
+    });
+    loadFutures.add(future);
+    _loadingModules[moduleName] = future;
+  }
+  return Future.wait(loadFutures).then((_) {
+    (_loadedLibraries[enclosingLibrary] ??= {}).add(importPrefix);
+  });
 }
 
-@pragma("wasm:entry-point")
 Object checkLibraryIsLoaded(String enclosingLibrary, String importPrefix) {
-  bool? isLoaded = _loadedLibraries[enclosingLibrary]?.contains(importPrefix);
-  if (isLoaded == null || isLoaded == false) {
+  final loadedImports = _loadedLibraries[enclosingLibrary];
+  if (loadedImports == null || !loadedImports.contains(importPrefix)) {
     throw DeferredNotLoadedError(enclosingLibrary, importPrefix);
   }
   return true;

sdk/lib/_internal/wasm/lib/internal_patch.dart

@@ -2,12 +2,22 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-import "dart:_js_helper" show JS, jsStringFromDartString, jsStringToDartString;
+import 'dart:async';
+import "dart:_js_helper"
+    show JS, JSAnyToExternRef, jsStringFromDartString, jsStringToDartString;
 import "dart:_js_types" show JSStringImpl;
 import 'dart:_string';
 import 'dart:js_interop'
-    show JSArray, JSString, JSArrayToList, JSStringToString;
+    show
+        JSArray,
+        JSString,
+        JSArrayToList,
+        JSStringToString,
+        JSPromise,
+        JSPromiseToFuture,
+        StringToJSString;
 import 'dart:_js_helper' show JSValue;
+import 'dart:_js_types';
 import 'dart:_wasm';
 import 'dart:typed_data' show Uint8List;
 

tests/language/language_dart2wasm.status

@@ -7,4 +7,8 @@
 inference_update_2/why_not_promoted_external_error_test: SkipByDesign # Non-JS-interop external members are not supported
 number/separators_test: SkipByDesign # Wasm has real integers.
 number/web_int_literals_test: SkipByDesign # Wasm has real integers.
-vm/*: SkipByDesign # Tests for the VM.
+vm/*: SkipByDesign # Tests for the VM.
+
+[ $compiler == dart2wasm && $runtime == d8 ]
+import/conditional_import_string_test: Skip # Timeout does not work with async module loading.
+import/conditional_import_test: Skip # Timeout does not work with async module loading.