From 81af7ebbf400b8be32dab8f0b2d5f62b059603dd Mon Sep 17 00:00:00 2001 From: Daco Harkes Date: Fri, 19 Apr 2024 14:37:26 +0000 Subject: [PATCH] [frontend_server] `native-assets-only` compilation This CL adds a `--native-assets-only` CLI option to the frontend_server startup for single shot compilation. This CL adds a `native-assets-only` instruction to the frontend_server protocol. TEST=pkg/frontend_server/test/native_assets_test.dart Unit test producing kernel file. Closes: https://github.com/dart-lang/sdk/issues/55503 Change-Id: Ice6281162460032e669d2dda2a128b357e81bc50 Cq-Include-Trybots: dart/try:pkg-linux-debug-try,pkg-linux-release-arm64-try,pkg-mac-release-try,pkg-mac-release-arm64-try,pkg-win-release-try,pkg-win-release-arm64-try Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/363567 Reviewed-by: Jens Johansen Commit-Queue: Daco Harkes --- pkg/frontend_server/lib/frontend_server.dart | 80 ++++++++++++++++++- pkg/frontend_server/lib/starter.dart | 8 ++ .../test/frontend_server_test.dart | 6 ++ .../test/native_assets_test.dart | 38 ++++++++- 4 files changed, 130 insertions(+), 2 deletions(-) diff --git a/pkg/frontend_server/lib/frontend_server.dart b/pkg/frontend_server/lib/frontend_server.dart index 360683ea394..2989a53df61 100644 --- a/pkg/frontend_server/lib/frontend_server.dart +++ b/pkg/frontend_server/lib/frontend_server.dart @@ -100,6 +100,9 @@ ArgParser argParser = new ArgParser(allowTrailingOptions: true) defaultsTo: const []) ..addOption('native-assets', help: 'Provide the native-assets mapping for @Native external functions.') + ..addFlag('native-assets-only', + help: "Only compile the native-assets mapping. " + "Don't compile the dart program.") ..addOption('target', help: 'Target model that determines what core libraries are available', allowed: [ @@ -291,6 +294,14 @@ abstract class CompilerInterface { IncrementalCompiler? generator, }); + /// Compiles the native_assets.yaml into a dill file. + /// + /// Returns [true] if compilation was successful and produced no errors. + Future compileNativeAssetsOnly( + ArgResults options, { + IncrementalCompiler? generator, + }); + /// Sets the native assets mapping to be embedded in the kernel. Future setNativeAssets(String nativeAssets); @@ -480,7 +491,9 @@ class FrontendCompiler implements CompilerInterface { _additionalSources = (options['source'] as List).map(resolveInputUri).toList(); final String? nativeAssets = options['native-assets'] as String?; - _nativeAssets = nativeAssets != null ? resolveInputUri(nativeAssets) : null; + if (_nativeAssets == null && nativeAssets != null) { + _nativeAssets = resolveInputUri(nativeAssets); + } _kernelBinaryFilenameFull = _options['output-dill'] ?? '$entryPoint.dill'; _kernelBinaryFilenameIncremental = _options['output-incremental-dill'] ?? (_options['output-dill'] != null @@ -694,6 +707,55 @@ class FrontendCompiler implements CompilerInterface { return errors.isEmpty; } + @override + Future compileNativeAssetsOnly( + ArgResults options, { + IncrementalCompiler? generator, + }) async { + _fileSystem = createFrontEndFileSystem( + options['filesystem-scheme'], + options['filesystem-root'], + allowHttp: options['enable-http-uris'], + ); + _options = options; + final String? nativeAssets = options['native-assets'] as String?; + if (_nativeAssets == null && nativeAssets != null) { + _nativeAssets = resolveInputUri(nativeAssets); + } + if (_nativeAssets == null) { + print( + 'Error: When --native-assets-only is specified it is required to' + ' specify --native-assets option that points to physical file system' + ' location of a source native_assets.yaml file.', + ); + return false; + } + if (_options['output-dill'] == null) { + print( + 'Error: When --native-assets-only is specified it is required to' + ' specify --output-dill option that points to physical file system' + ' location of a target dill file.', + ); + return false; + } + _kernelBinaryFilename = _options['output-dill']; + final CompilerOptions compilerOptions = new CompilerOptions(); + _compilerOptions = compilerOptions; + + final String boundaryKey = generateV4UUID(); + _outputStream.writeln('result $boundaryKey'); + await _compileNativeAssets(); + await writeDillFileNativeAssets( + _nativeAssetsLibrary!, + _kernelBinaryFilename, + ); + _outputStream.writeln(boundaryKey); + _outputStream.writeln('+${await asFileUri(_fileSystem, _nativeAssets!)}'); + _outputStream + .writeln('$boundaryKey $_kernelBinaryFilename ${errors.length}'); + return true; + } + @override Future setNativeAssets(String nativeAssets) async { _nativeAssetsLibrary = null; // Purge compiled cache. @@ -875,6 +937,19 @@ class FrontendCompiler implements CompilerInterface { } } + Future writeDillFileNativeAssets( + Library nativeAssetsLibrary, + String filename, + ) async { + final IOSink sink = new File(filename).openWrite(); + final BinaryPrinter printer = new BinaryPrinter(sink); + printer.writeComponentFile(new Component( + libraries: [nativeAssetsLibrary], + mode: nativeAssetsLibrary.nonNullableByDefaultCompiledMode, + )); + await sink.close(); + } + Future invalidateIfInitializingFromDill() async { if (_assumeInitializeFromDillUpToDate) return; if (_kernelBinaryFilename != _kernelBinaryFilenameFull) return; @@ -1297,6 +1372,7 @@ StreamSubscription listenAndCompile(CompilerInterface compiler, const String COMPILE_INSTRUCTION_SPACE = 'compile '; const String RECOMPILE_INSTRUCTION_SPACE = 'recompile '; const String NATIVE_ASSETS_INSTRUCTION_SPACE = 'native-assets '; + const String NATIVE_ASSETS_ONLY_INSTRUCTION = 'native-assets-only'; const String COMPILE_EXPRESSION_INSTRUCTION_SPACE = 'compile-expression '; const String COMPILE_EXPRESSION_TO_JS_INSTRUCTION_SPACE = @@ -1305,6 +1381,8 @@ StreamSubscription listenAndCompile(CompilerInterface compiler, final String entryPoint = string.substring(COMPILE_INSTRUCTION_SPACE.length); await compiler.compile(entryPoint, options, generator: generator); + } else if (string == NATIVE_ASSETS_ONLY_INSTRUCTION) { + await compiler.compileNativeAssetsOnly(options, generator: generator); } else if (string.startsWith(RECOMPILE_INSTRUCTION_SPACE)) { // 'recompile [] ' // where can't have spaces diff --git a/pkg/frontend_server/lib/starter.dart b/pkg/frontend_server/lib/starter.dart index 2022e901ca5..51714910425 100644 --- a/pkg/frontend_server/lib/starter.dart +++ b/pkg/frontend_server/lib/starter.dart @@ -97,6 +97,14 @@ Future starter( canaryFeatures: options['dartdevc-canary'], ); + if (options['native-assets-only']) { + final bool compileResult = await compiler.compileNativeAssetsOnly( + options, + generator: generator, + ); + return compileResult ? 0 : 254; + } + if (options.rest.isNotEmpty) { return await compiler.compile(options.rest[0], options, generator: generator) diff --git a/pkg/frontend_server/test/frontend_server_test.dart b/pkg/frontend_server/test/frontend_server_test.dart index b2c805d8bb9..d4fa7bdefcc 100644 --- a/pkg/frontend_server/test/frontend_server_test.dart +++ b/pkg/frontend_server/test/frontend_server_test.dart @@ -3328,6 +3328,12 @@ class FrontendServer { .codeUnits); } + /// Compiles the native assets in isolation. + void compileNativeAssetsOnly() { + outputParser.expectSources = true; + inputStreamController.add('native-assets-only\n'.codeUnits); + } + /// Sets the native assets yaml [uri]. void setNativeAssets({required Uri uri}) { outputParser.expectSources = true; diff --git a/pkg/frontend_server/test/native_assets_test.dart b/pkg/frontend_server/test/native_assets_test.dart index a13631dc177..6f2a57a8e06 100644 --- a/pkg/frontend_server/test/native_assets_test.dart +++ b/pkg/frontend_server/test/native_assets_test.dart @@ -187,8 +187,8 @@ void main() { expect(await dillFile.exists(), equals(true)); expect(result.filename, dillFile.path); expect(result.errorsCount, 0); + count += 1; frontendServer.accept(); - frontendServer.quit(); final Component component = loadComponentFromBinary(dillFile.path); @@ -196,7 +196,21 @@ void main() { _findNativeAssetsLibrary(component); expect(nativeAssetsLibrary, isNotNull); + frontendServer.compileNativeAssetsOnly(); break; + case 2: + expect(await dillFile.exists(), equals(true)); + expect(result.filename, dillFile.path); + expect(result.errorsCount, 0); + count += 1; + + final Component component = + loadComponentFromBinary(dillFile.path); + final Library? nativeAssetsLibrary = + _findNativeAssetsLibrary(component); + expect(nativeAssetsLibrary, isNotNull); + + frontendServer.quit(); } }); expect(await result, 0); @@ -320,6 +334,28 @@ void main() { _findNativeAssetsLibrary(component); expect(nativeAssetsLibrary, isNotNull); }); + + test('pass in native assets only at startup', () async { + final FrontendServer frontendServer = new FrontendServer(); + Future frontendServerResult = frontendServer.open([ + '--sdk-root=${sdkRoot.toFilePath()}', + '--platform=${platformKernel.path}', + '--aot', + '--tfa', + '--output-dill=${dillFile.path}', + '--native-assets=${nativeAssetsYamlFile.path}', + '--native-assets-only', + ]); + + expect(await frontendServerResult, 0); + frontendServer.close(); + + expect(await dillFile.exists(), equals(true)); + final Component component = loadComponentFromBinary(dillFile.path); + final Library? nativeAssetsLibrary = + _findNativeAssetsLibrary(component); + expect(nativeAssetsLibrary, isNotNull); + }); }); }); }