Files
sdk/pkg/dart2wasm/lib/io_util.dart
Martin Kustermann e3cf529f87 [dart2wasm] More compact encoding of deferred load lists
Measured on size of e main module (baseline is we don't embed
it in application code):

* embedding before: +16.5% uncompressed / +9.1% compressed
* embedding with this CL: +4% uncompressed / +4.3% compressed

When embeddeding deferred load list information into the app
(as opposed to a separate json file) we now use a more compact
encoding.

Specifically: Instead of encoding it as an array of an array of
strings (which are module names), we encode it as an array of an
array of module ids and construct the module name from the id.

To make the array of module ids more compact we utilize the fact
that we can sort them and encode in delta encoding (i.e. instead
of absolute module ids, encode the diff between previous module
id in the list).

We put the encoded module id lists in a data section and create
`WasmArray<WasmI8>`s from them at startup. When we trigger a load
we then decode them into the list of module names.

There's more opportunity to optimize it, but it's good to do
this as a first step.

Change-Id: I293fb8879d992fc370786f6c9b258ccd27e1559b
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/508980
Reviewed-by: Srujan Gaddam <srujzs@google.com>
Commit-Queue: Martin Kustermann <kustermann@google.com>
2026-06-05 00:50:46 -07:00

250 lines
7.7 KiB
Dart

// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:convert';
import 'dart:io' show File, Directory, Process, ProcessResult;
import 'dart:typed_data';
import 'package:build_integration/file_system/multi_root.dart'
show MultiRootFileSystemEntity, MultiRootFileSystem;
import 'package:front_end/src/api_prototype/file_system.dart' show FileSystem;
import 'package:kernel/ast.dart' show Component;
import 'package:kernel/binary/ast_from_binary.dart'
show BinaryBuilderWithMetadata;
import 'package:kernel/kernel.dart'
show writeComponentToBinary, writeComponentToText;
import 'package:path/path.dart' as path;
import 'compiler_options.dart';
import 'source_map_utils.dart';
class CompilerPhaseInputOutputManager {
final FileSystem fileSystem;
final WasmCompilerOptions options;
CompilerPhaseInputOutputManager(FileSystem fileSystem, this.options)
: fileSystem = options.multiRootScheme != null
? MultiRootFileSystem(
options.multiRootScheme!,
options.multiRoots.isEmpty ? [Uri.base] : options.multiRoots,
fileSystem,
)
: fileSystem;
String _moduleNameToWasmFile(String prefix, String moduleName) {
return path.join(path.dirname(prefix), moduleName);
}
String _moduleNameToSourceMapFile(String prefix, String moduleName) {
return '${_moduleNameToWasmFile(prefix, moduleName)}.map';
}
Uri _moduleNameToRelativeSourceMapUri(String moduleName) {
return Uri.file(
path.basename(_moduleNameToSourceMapFile(options.outputFile, moduleName)),
);
}
Uri Function(String)? get sourceMapUrlGenerator =>
options.translatorOptions.generateSourceMaps
? _moduleNameToRelativeSourceMapUri
: null;
Future<String> readString(Uri uri) async {
return await File.fromUri((await resolveUri(uri))!).readAsString();
}
Future<List<int>> readBytes(Uri uri) async {
return await File.fromUri((await resolveUri(uri))!).readAsBytes();
}
Future<void> readComponent(Uri componentUri, Component component) async {
BinaryBuilderWithMetadata(
await File.fromUri((await resolveUri(componentUri))!).readAsBytes(),
).readComponent(component);
}
final Set<String> _createdDirectories = {};
void _createDirIfNecessary(String filePath) {
final dirPath = path.dirname(filePath);
// Exit early if we've already created/checked this directory. For I/O heavy
// compilations (e.g. deferred loading with many modules), this can avoid
// a lot of redundant directory checks.
if (_createdDirectories.contains(dirPath)) return;
final Directory dir = Directory(dirPath);
// Do this synchronously to make sure it happens before subsequent async
// operations.
if (!dir.existsSync()) {
dir.createSync(recursive: true);
}
_createdDirectories.add(dirPath);
}
Future<void> writeComponent(
Component component,
String path, {
bool includeSource = true,
}) {
_createDirIfNecessary(path);
return writeComponentToBinary(
component,
path,
includeSource: includeSource,
);
}
void writeComponentAsText(Component component, String path) {
_createDirIfNecessary(path);
writeComponentToText(component, path: path, showMetadata: true);
}
Future<void> writeWasmModule(Uint8List wasmModule, String moduleName) {
final wasmFileName = _moduleNameToWasmFile(options.outputFile, moduleName);
_createDirIfNecessary(wasmFileName);
return File(wasmFileName).writeAsBytes(wasmModule);
}
Future<void> writeWasmSourceMap(String sourceMap, String moduleName) {
final sourceMapFileName = _moduleNameToSourceMapFile(
options.outputFile,
moduleName,
);
_createDirIfNecessary(sourceMapFileName);
return File(sourceMapFileName).writeAsString(sourceMap);
}
Future<void> writeJsRuntime(String jsRuntime) {
final jsRuntimeFileName = path.setExtension(options.outputFile, '.mjs');
_createDirIfNecessary(jsRuntimeFileName);
return File(jsRuntimeFileName).writeAsString(jsRuntime);
}
Future<void> writeSupportJs(String supportJs) {
final supportJsFileName = path.setExtension(
options.outputFile,
'.support.js',
);
_createDirIfNecessary(supportJsFileName);
return File(supportJsFileName).writeAsString(supportJs);
}
Future<void> runWasmOpt(
String mainWasmModule,
int moduleId,
List<String> flags,
) async {
final inputModuleName = WasmCompilerOptions.moduleNameForId(
mainWasmModule,
moduleId,
);
final outputModuleName = WasmCompilerOptions.moduleNameForId(
options.outputFile,
moduleId,
);
final wasmOutName = _moduleNameToWasmFile(
options.outputFile,
outputModuleName,
);
final wasmInName = _moduleNameToWasmFile(mainWasmModule, inputModuleName);
final sourceMapInName = _moduleNameToSourceMapFile(
mainWasmModule,
inputModuleName,
);
final sourceMapOutName = _moduleNameToSourceMapFile(
options.outputFile,
outputModuleName,
);
// wasm-opt drops custom sections and reorders names section, read custom
// section for deobfuscating class names before wasm-opt, add them back
// after.
List<String?>? classNames;
if (options.translatorOptions.generateSourceMaps &&
moduleId == WasmCompilerOptions.mainModuleId) {
classNames = getMinifiedClassNames(
jsonDecode(await File(sourceMapInName).readAsString()),
);
}
final args = [
...flags,
wasmInName,
'-o',
wasmOutName,
if (options.translatorOptions.generateSourceMaps) ...[
'-ism',
sourceMapInName,
'-osm',
sourceMapOutName,
'-osu',
_moduleNameToRelativeSourceMapUri(outputModuleName).toString(),
],
if (!options.stripWasm) '-g',
];
if (options.translatorOptions.verbose) {
print('Running wasm-opt $args');
}
if (options.saveUnopt) {
await File(
wasmInName,
).copy(path.setExtension(wasmOutName, '.unopt.wasm'));
}
final wasmOptPath = options.wasmOptPath?.toFilePath() ?? 'wasm-opt';
final result = await _runProcess(wasmOptPath, args);
if (result.exitCode != 0) {
throw Exception(
'wasm-opt failed on module $inputModuleName with exit code ${result.exitCode}:'
'\n${result.stdout}\n${result.stderr}',
);
}
if (classNames != null) {
final Map<String, Object?> sourceMapJson = jsonDecode(
await File(sourceMapOutName).readAsString(),
);
addMinifiedClassNames(sourceMapJson, classNames);
await File(sourceMapOutName).writeAsString(jsonEncode(sourceMapJson));
}
}
Future<ProcessResult> _runProcess(
String executable,
List<String> args,
) async {
return await Process.run(executable, args);
}
Future<Set<int>> getModuleIds(String mainWasmFilePath) async {
final files = (await Directory(
path.dirname(mainWasmFilePath),
).list().toList());
final mainWasmFilename = path.basename(mainWasmFilePath);
final moduleIds = <int>{};
for (final file in files) {
if (file is! File) continue;
final moduleId = WasmCompilerOptions.idForModuleName(
mainWasmFilename,
path.basename(file.path),
);
if (moduleId != null) {
moduleIds.add(moduleId);
}
}
return moduleIds;
}
Future<Uri?> resolveUri(Uri? uri) async {
if (uri == null) return null;
var fileSystemEntity = fileSystem.entityForUri(uri);
if (fileSystemEntity is MultiRootFileSystemEntity) {
fileSystemEntity = await fileSystemEntity.delegate;
}
return fileSystemEntity.uri;
}
}