[frontend_server] Add support for @Native assets

The VM can read native asset mappings from kernel.

Previous CLs already added support to embed native asset mappings for
one-shot compilation.
This CL adds support for adding native assets mappings to kernel files
created by the frontend_server with the incremental compiler.

The frontend_server accepts a `--native-assets=<uri>` at startup and
accepts a `native-assets <uri>` message on stdin as compilation
command.

The frontend_server caches the compiled native assets library.

When a `reset` command is sent to request a full dill from the
incremental compiler, the native assets mapping is taken from the
cache and added to the final dill file.

Split of DartSDK & flutter_tools prototype to land separately.

TEST=pkg/frontend_server/test/native_assets_test.dart

Bug: https://github.com/dart-lang/sdk/issues/49803
Bug: https://github.com/dart-lang/sdk/issues/50565
Change-Id: I6e15f177564b8a962e81261815e951e7c9525513
Cq-Include-Trybots: luci.dart.try:pkg-linux-debug-try,pkg-linux-release-try,pkg-mac-release-arm64-try,pkg-mac-release-try,pkg-win-release-try
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/282101
Reviewed-by: Jens Johansen <jensj@google.com>
Commit-Queue: Daco Harkes <dacoharkes@google.com>
This commit is contained in:
Daco Harkes
2023-02-21 07:29:29 +00:00
committed by Commit Queue
parent 9bc32f1914
commit 40787d84fc
7 changed files with 459 additions and 34 deletions
+100 -19
View File
@@ -99,6 +99,8 @@ ArgParser argParser = ArgParser(allowTrailingOptions: true)
..addMultiOption('source',
help: 'List additional source files to include into compilation.',
defaultsTo: const <String>[])
..addOption('native-assets',
help: 'Provide the native-assets mapping for @Native external functions.')
..addOption('target',
help: 'Target model that determines what core libraries are available',
allowed: <String>[
@@ -278,6 +280,9 @@ abstract class CompilerInterface {
IncrementalCompiler? generator,
});
/// Sets the native assets mapping to be embedded in the kernel.
Future<bool> setNativeAssets(String nativeAssets);
/// Assuming some Dart program was previously compiled, recompile it again
/// taking into account some changed(invalidated) sources.
Future<void> recompileDelta({String? entryPoint});
@@ -400,7 +405,15 @@ class FrontendCompiler implements CompilerInterface {
late bool _printIncrementalDependencies;
late ProcessedOptions _processedOptions;
/// Initialized in [writeJavaScriptBundle]
/// Initialized in [compile] from options, or (re)set in [setNativeAssets].
Uri? _nativeAssets;
/// Cached compilation of [_nativeAssets].
///
/// Managed by [_compileNativeAssets] and [setNativeAssets].
Library? _nativeAssetsLibrary;
/// Initialized in [writeJavaScriptBundle].
IncrementalJavaScriptBundler? _bundler;
/// Nullable fields
@@ -443,6 +456,8 @@ class FrontendCompiler implements CompilerInterface {
_mainSource = resolveInputUri(entryPoint);
_additionalSources =
(options['source'] as List<String>).map(resolveInputUri).toList();
final nativeAssets = options['native-assets'] as String?;
_nativeAssets = nativeAssets != null ? resolveInputUri(nativeAssets) : null;
_kernelBinaryFilenameFull = _options['output-dill'] ?? '$entryPoint.dill';
_kernelBinaryFilenameIncremental = _options['output-incremental-dill'] ??
(_options['output-dill'] != null
@@ -579,12 +594,16 @@ class FrontendCompiler implements CompilerInterface {
IncrementalCompilerResult compilerResult =
await _runWithPrintRedirection(() => _generator.compile());
Component component = compilerResult.component;
results = KernelCompilationResults(
component,
const {},
compilerResult.classHierarchy,
compilerResult.coreTypes,
component.uriToSource.keys);
await _compileNativeAssets();
results = KernelCompilationResults.named(
component: component,
nativeAssetsLibrary: _nativeAssetsLibrary,
classHierarchy: compilerResult.classHierarchy,
coreTypes: compilerResult.coreTypes,
compiledSources: component.uriToSource.keys,
);
incrementalSerializer = _generator.incrementalSerializer;
if (options['flutter-widget-cache']) {
@@ -601,6 +620,7 @@ class FrontendCompiler implements CompilerInterface {
results = await _runWithPrintRedirection(() => compileToKernel(
_mainSource, compilerOptions,
additionalSources: _additionalSources,
nativeAssets: _nativeAssets,
includePlatform: options['link-platform'],
deleteToStringPackageUris: options['delete-tostring-package-uri'],
aot: options['aot'],
@@ -622,9 +642,13 @@ class FrontendCompiler implements CompilerInterface {
options['filesystem-scheme'], options['dartdevc-module-format'],
fullComponent: true);
}
await writeDillFile(results, _kernelBinaryFilename,
filterExternal: importDill != null || options['minimal-kernel'],
incrementalSerializer: incrementalSerializer);
await writeDillFile(
results,
_kernelBinaryFilename,
filterExternal: importDill != null || options['minimal-kernel'],
incrementalSerializer: incrementalSerializer,
aot: options['aot'],
);
_outputStream.writeln(boundaryKey);
final compiledSources = results.compiledSources!;
@@ -645,6 +669,32 @@ class FrontendCompiler implements CompilerInterface {
return errors.isEmpty;
}
@override
Future<bool> setNativeAssets(String nativeAssets) async {
_nativeAssetsLibrary = null; // Purge compiled cache.
_nativeAssets = resolveInputUri(nativeAssets);
return true;
}
/// Compiles [_nativeAssets] into [_nativeAssetsLibrary].
///
/// [compile] and [recompileDelta] invoke this, and bundles the cached
/// [_nativeAssetsLibrary] in the dill file.
Future<void> _compileNativeAssets() async {
final nativeAssets = _nativeAssets;
if (nativeAssets == null || _nativeAssetsLibrary != null) {
return;
}
final results = await _runWithPrintRedirection(() => compileToKernel(
null,
_compilerOptions,
nativeAssets: _nativeAssets,
environmentDefines: {},
));
_nativeAssetsLibrary = results.nativeAssetsLibrary;
}
Future<void> _outputDependenciesDelta(Iterable<Uri> compiledSources) async {
if (!_printIncrementalDependencies) {
return;
@@ -744,11 +794,26 @@ class FrontendCompiler implements CompilerInterface {
]);
}
writeDillFile(KernelCompilationResults results, String filename,
{bool filterExternal = false,
IncrementalSerializer? incrementalSerializer}) async {
writeDillFile(
KernelCompilationResults results,
String filename, {
bool filterExternal = false,
IncrementalSerializer? incrementalSerializer,
bool aot = false,
}) async {
final Component component = results.component!;
final Library? nativeAssetsLibrary = results.nativeAssetsLibrary;
if (aot && nativeAssetsLibrary != null) {
// If Dart component in AOT, write the vm:native-assets library _inside_
// the Dart component.
// TODO(https://dartbug.com/50152): Support AOT dill concatenation.
component.libraries.add(nativeAssetsLibrary);
nativeAssetsLibrary.parent = component;
}
final IOSink sink = File(filename).openWrite();
final Set<Library> loadedLibraries = results.loadedLibraries;
final BinaryPrinter printer = filterExternal
? BinaryPrinter(sink,
@@ -766,6 +831,14 @@ class FrontendCompiler implements CompilerInterface {
}
printer.writeComponentFile(component);
if (nativeAssetsLibrary != null && !aot) {
final BinaryPrinter printer = BinaryPrinter(sink);
printer.writeComponentFile(Component(
libraries: [nativeAssetsLibrary],
mode: nativeAssetsLibrary.nonNullableByDefaultCompiledMode,
));
}
await sink.close();
if (_options['split-output-by-packages']) {
@@ -856,12 +929,15 @@ class FrontendCompiler implements CompilerInterface {
Component deltaProgram = deltaProgramResult.component;
transformer?.transform(deltaProgram);
KernelCompilationResults results = KernelCompilationResults(
deltaProgram,
const {},
deltaProgramResult.classHierarchy,
deltaProgramResult.coreTypes,
deltaProgram.uriToSource.keys);
await _compileNativeAssets();
KernelCompilationResults results = KernelCompilationResults.named(
component: deltaProgram,
classHierarchy: deltaProgramResult.classHierarchy,
coreTypes: deltaProgramResult.coreTypes,
compiledSources: deltaProgram.uriToSource.keys,
nativeAssetsLibrary: _nativeAssetsLibrary,
);
if (_compilerOptions.target!.name == 'dartdevc') {
await writeJavaScriptBundle(results, _kernelBinaryFilename,
@@ -1221,6 +1297,7 @@ StreamSubscription<String> listenAndCompile(CompilerInterface compiler,
case _State.READY_FOR_INSTRUCTION:
const String COMPILE_INSTRUCTION_SPACE = 'compile ';
const String RECOMPILE_INSTRUCTION_SPACE = 'recompile ';
const String NATIVE_ASSETS_INSTRUCTION_SPACE = 'native-assets ';
const String COMPILE_EXPRESSION_INSTRUCTION_SPACE =
'compile-expression ';
const String COMPILE_EXPRESSION_TO_JS_INSTRUCTION_SPACE =
@@ -1242,6 +1319,10 @@ StreamSubscription<String> listenAndCompile(CompilerInterface compiler,
boundaryKey = remainder;
}
state = _State.RECOMPILE_LIST;
} else if (string.startsWith(NATIVE_ASSETS_INSTRUCTION_SPACE)) {
final String nativeAssets =
string.substring(NATIVE_ASSETS_INSTRUCTION_SPACE.length);
await compiler.setNativeAssets(nativeAssets);
} else if (string
.startsWith(COMPILE_EXPRESSION_TO_JS_INSTRUCTION_SPACE)) {
// 'compile-expression-to-js <boundarykey>
@@ -3254,6 +3254,12 @@ class FrontendServer {
.codeUnits);
}
/// Sets the native assets yaml [uri].
void setNativeAssets({required Uri uri}) {
outputParser.expectSources = true;
inputStreamController.add('native-assets $uri\n'.codeUnits);
}
/// Compiles the [expression] as if it occurs in [library].
///
/// If [className] is provided, [expression] is compiled as if it occurs in
@@ -0,0 +1,325 @@
// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE.md file.
import 'dart:async';
import 'dart:convert';
import 'dart:ffi';
import 'dart:io';
import 'package:kernel/kernel.dart';
import 'package:test/test.dart';
// Reuse some test infrastructure.
import 'frontend_server_test.dart';
void main() async {
group('full compiler tests', () {
final platformKernel =
computePlatformBinariesLocation().resolve('vm_platform_strong.dill');
final sdkRoot = computePlatformBinariesLocation();
late Directory tempDir;
late File mainFile;
late File packageConfigFile;
late File nativeAssetsYamlFile;
late File dillFile;
setUp(() async {
tempDir = await Directory.systemTemp.createTemp('frontendServerTest');
mainFile = File('${tempDir.path}/a.dart');
await mainFile.create(recursive: true);
await mainFile.writeAsString('''
void main() {
print(42);
}
''');
packageConfigFile =
File('${tempDir.path}/.dart_tool/package_config.json');
await packageConfigFile.create(recursive: true);
await packageConfigFile.writeAsString(jsonEncode({
"configVersion": 2,
"packages": [],
}));
nativeAssetsYamlFile =
File('${tempDir.path}/.dart_tool/native_assets.yaml');
await nativeAssetsYamlFile.create(recursive: true);
await nativeAssetsYamlFile.writeAsString(jsonEncode({
'format-version': [1, 0, 0],
'native-assets': {
Abi.current().toString(): {
mainFile.uri.toString(): ['executable'],
},
},
}));
// Other setup.
dillFile = File('${tempDir.path}/app.dill');
});
tearDown(() async {
return await tempDir.delete(recursive: true);
});
group('--incremental', () {
testPassInNativeAssetsAtStartup({
List<String> additionalStartupArguments = const [],
}) async {
final frontendServer = FrontendServer();
Future<int> result = frontendServer.open(<String>[
'--sdk-root=${sdkRoot.toFilePath()}',
'--incremental',
'--platform=${platformKernel.path}',
'--output-dill=${dillFile.path}',
'--native-assets=${nativeAssetsYamlFile.path}',
]);
frontendServer.compile(mainFile.path);
int count = 0;
frontendServer.listen((Result compiledResult) async {
CompilationResult result =
CompilationResult.parse(compiledResult.status);
switch (count) {
case 0:
expect(await dillFile.exists(), equals(true));
expect(result.filename, dillFile.path);
expect(result.errorsCount, 0);
count += 1;
frontendServer.accept();
frontendServer.reset();
final component = loadComponentFromBinary(dillFile.path);
final nativeAssetsLibrary = _findNativeAssetsLibrary(component);
expect(nativeAssetsLibrary, isNotNull);
final firstLib = component.libraries.first;
expect(firstLib.importUri != _nativeAssetsLibraryUri, true);
expect(nativeAssetsLibrary!.isNonNullableByDefault,
firstLib.isNonNullableByDefault);
expect(nativeAssetsLibrary.nonNullable, firstLib.nonNullable);
expect(nativeAssetsLibrary.nonNullableByDefaultCompiledMode,
firstLib.nonNullableByDefaultCompiledMode);
await mainFile.writeAsString('''
void main() {
print(1337);
}
''');
frontendServer.recompile(mainFile.uri);
break;
case 1:
expect(await dillFile.exists(), equals(true));
expect(result.filename, dillFile.path);
expect(result.errorsCount, 0);
frontendServer.accept();
frontendServer.quit();
final component = loadComponentFromBinary(dillFile.path);
final nativeAssetsLibrary = _findNativeAssetsLibrary(component);
expect(nativeAssetsLibrary, isNotNull);
break;
}
});
expect(await result, 0);
frontendServer.close();
}
test('pass in native assets at startup', () async {
await testPassInNativeAssetsAtStartup();
});
test('--no-sound-null-safety', () async {
await testPassInNativeAssetsAtStartup(additionalStartupArguments: [
'--no-sound-null-safety',
]);
});
test('--incremental-serialization', () async {
await testPassInNativeAssetsAtStartup(additionalStartupArguments: [
'--incremental-serialization',
]);
});
test('set native assets later', () async {
final frontendServer = FrontendServer();
Future<int> result = frontendServer.open(<String>[
'--sdk-root=${sdkRoot.toFilePath()}',
'--incremental',
'--platform=${platformKernel.path}',
'--output-dill=${dillFile.path}',
]);
frontendServer.compile(mainFile.path);
int count = 0;
frontendServer.listen((Result compiledResult) async {
CompilationResult result =
CompilationResult.parse(compiledResult.status);
switch (count) {
case 0:
expect(await dillFile.exists(), equals(true));
expect(result.filename, dillFile.path);
expect(result.errorsCount, 0);
count += 1;
frontendServer.accept();
frontendServer.reset();
final component = loadComponentFromBinary(dillFile.path);
final nativeAssetsLibrary = _findNativeAssetsLibrary(component);
expect(nativeAssetsLibrary, isNull);
frontendServer.setNativeAssets(uri: nativeAssetsYamlFile.uri);
frontendServer.recompile(mainFile.uri);
break;
case 1:
expect(await dillFile.exists(), equals(true));
expect(result.filename, dillFile.path);
expect(result.errorsCount, 0);
frontendServer.accept();
frontendServer.quit();
final component = loadComponentFromBinary(dillFile.path);
final nativeAssetsLibrary = _findNativeAssetsLibrary(component);
expect(nativeAssetsLibrary, isNotNull);
break;
}
});
expect(await result, 0);
frontendServer.close();
});
testInitializeFromDill({
bool passNativeAssetsOnFirstStartup = false,
bool passNativeAssetsOnSecondStartup = false,
}) async {
{
final frontendServer = FrontendServer();
Future<int> frontendServerResult = frontendServer.open(<String>[
'--sdk-root=${sdkRoot.toFilePath()}',
'--incremental',
'--platform=${platformKernel.path}',
'--output-dill=${dillFile.path}',
if (passNativeAssetsOnFirstStartup)
'--native-assets=${nativeAssetsYamlFile.path}',
]);
frontendServer.compile(mainFile.path);
final compiledResult =
await frontendServer.receivedResults.stream.first;
CompilationResult result =
CompilationResult.parse(compiledResult.status);
expect(await dillFile.exists(), equals(true));
expect(result.filename, dillFile.path);
expect(result.errorsCount, 0);
frontendServer.accept();
frontendServer.quit();
final component = loadComponentFromBinary(dillFile.path);
final nativeAssetsLibrary = _findNativeAssetsLibrary(component);
expect(nativeAssetsLibrary,
passNativeAssetsOnFirstStartup ? isNotNull : isNull);
expect(await frontendServerResult, 0);
frontendServer.close();
}
{
final frontendServer = FrontendServer();
Future<int> frontendServerResult = frontendServer.open(<String>[
'--sdk-root=${sdkRoot.toFilePath()}',
'--incremental',
'--platform=${platformKernel.path}',
'--output-dill=${dillFile.path}',
'--initialize-from-dill=${dillFile.path}',
if (passNativeAssetsOnSecondStartup)
'--native-assets=${nativeAssetsYamlFile.path}',
]);
frontendServer.compile(mainFile.path);
final compiledResult =
await frontendServer.receivedResults.stream.first;
CompilationResult result =
CompilationResult.parse(compiledResult.status);
expect(await dillFile.exists(), equals(true));
expect(result.filename, dillFile.path);
expect(result.errorsCount, 0);
frontendServer.accept();
frontendServer.quit();
final component = loadComponentFromBinary(dillFile.path);
final nativeAssetsLibrary = _findNativeAssetsLibrary(component);
expect(nativeAssetsLibrary,
passNativeAssetsOnSecondStartup ? isNotNull : isNull);
expect(await frontendServerResult, 0);
frontendServer.close();
}
}
test('--initialize-from-dill embed native-assets', () async {
// This should forget the native assets, the incremental compiler
// should be seen as an optimization, _not_ as a thing that keeps
// state intentionally.
await testInitializeFromDill(passNativeAssetsOnFirstStartup: true);
});
test('--initialize-from-dill second start with --native-assets',
() async {
await testInitializeFromDill(passNativeAssetsOnSecondStartup: true);
});
test('--initialize-from-dill replace --native-assets', () async {
// This should forget the native assets from the first invocation.
await testInitializeFromDill(
passNativeAssetsOnFirstStartup: true,
passNativeAssetsOnSecondStartup: true,
);
});
});
group('--aot --tfa', () {
test('pass in native assets at startup', () async {
final frontendServer = FrontendServer();
Future<int> frontendServerResult = frontendServer.open(<String>[
'--sdk-root=${sdkRoot.toFilePath()}',
'--platform=${platformKernel.path}',
'--aot',
'--tfa',
'--output-dill=${dillFile.path}',
'--native-assets=${nativeAssetsYamlFile.path}',
mainFile.path
]);
expect(await frontendServerResult, 0);
frontendServer.close();
expect(await dillFile.exists(), equals(true));
final component = loadComponentFromBinary(dillFile.path);
final nativeAssetsLibrary = _findNativeAssetsLibrary(component);
expect(nativeAssetsLibrary, isNotNull);
});
});
});
}
final _nativeAssetsLibraryUri = Uri.parse('vm:ffi:native-assets');
Library? _findNativeAssetsLibrary(Component component) {
for (final library in component.libraries) {
if (library.importUri == _nativeAssetsLibraryUri) {
return library;
}
}
return null;
}
+1 -1
View File
@@ -835,7 +835,7 @@ Future _processLoadRequest(request) async {
} else {
await wrapper.reject();
}
} catch(e, st) {
} catch (e, st) {
port.send(CompilationResult.crash(e, st).toResponse());
return;
}
+22 -12
View File
@@ -318,28 +318,32 @@ Future<int> runCompiler(ArgResults options, String usage) async {
final Component? component = results.component;
final Library? nativeAssetsLibrary = results.nativeAssetsLibrary;
if (errorDetector.hasCompilationErrors || (component == null)) {
if (errorDetector.hasCompilationErrors ||
(component == null && nativeAssetsLibrary == null)) {
return compileTimeErrorExitCode;
}
final IOSink sink = new File(outputFileName).openWrite();
final BinaryPrinter printer = new BinaryPrinter(sink,
libraryFilter: (lib) => !results.loadedLibraries.contains(lib));
if (aot && nativeAssetsLibrary != null && aot) {
// If Dart component in AOT, write the vm:native-assets library _inside_
// the Dart component.
component.libraries.add(nativeAssetsLibrary);
nativeAssetsLibrary.parent = component;
if (component != null) {
final BinaryPrinter printer = new BinaryPrinter(sink,
libraryFilter: (lib) => !results.loadedLibraries.contains(lib));
if (aot && nativeAssetsLibrary != null) {
// If Dart component in AOT, write the vm:native-assets library _inside_
// the Dart component.
// TODO(https://dartbug.com/50152): Support AOT dill concatenation.
component.libraries.add(nativeAssetsLibrary);
nativeAssetsLibrary.parent = component;
}
printer.writeComponentFile(component);
}
printer.writeComponentFile(component);
if (nativeAssetsLibrary != null && !aot) {
if ((nativeAssetsLibrary != null && (!aot || component == null))) {
// If no Dart component, write as separate dill.
// If Dart component in JIT, write as concatenated dill, to not mess with
// the incremental compiler.
final BinaryPrinter printer = new BinaryPrinter(sink);
printer.writeComponentFile(Component(
libraries: [nativeAssetsLibrary],
mode: NonNullableByDefaultCompiledMode.Strong,
mode: nativeAssetsLibrary.nonNullableByDefaultCompiledMode,
));
}
await sink.close();
@@ -398,8 +402,9 @@ class KernelCompilationResults {
///
/// VM-specific replacement of [kernelForProgram].
///
/// Either [source], or [nativeAssets], or both must be non-null.
Future<KernelCompilationResults> compileToKernel(
Uri source,
Uri? source,
CompilerOptions options, {
List<Uri> additionalSources = const <Uri>[],
Uri? nativeAssets,
@@ -429,6 +434,11 @@ Future<KernelCompilationResults> compileToKernel(
? NonNullableByDefaultCompiledMode.Strong
: NonNullableByDefaultCompiledMode.Weak,
);
if (source == null) {
return KernelCompilationResults.named(
nativeAssetsLibrary: nativeAssetsLibrary,
);
}
final target = options.target!;
options.environmentDefines =
+4 -1
View File
@@ -72,7 +72,10 @@ class NativeAssetsSynthesizer {
pragmaOptions.fieldReference: nativeAssetsConstant,
}))
],
)..nonNullableByDefaultCompiledMode = nonNullableByDefaultCompiledMode;
)
..nonNullableByDefaultCompiledMode = nonNullableByDefaultCompiledMode
..isNonNullableByDefault = nonNullableByDefaultCompiledMode ==
NonNullableByDefaultCompiledMode.Strong;
}
/// Loads [nativeAssetsYamlString], validates the contents, and synthesizes
@@ -30,7 +30,7 @@ native-assets:
);
final libraryToString = kernelLibraryToString(component.libraries.single);
final expectedKernel = '''@#C9
library;
library /*isNonNullableByDefault*/;
import self as self;
constants {