diff --git a/pkg/frontend_server_client/CHANGELOG.md b/pkg/frontend_server_client/CHANGELOG.md new file mode 100644 index 00000000000..0530b11b4e6 --- /dev/null +++ b/pkg/frontend_server_client/CHANGELOG.md @@ -0,0 +1,57 @@ +## 4.0.0 + +- Update Dart SDK constraint to `^3.0.0`. +- Support changes in the SDK layout for Dart 3.0. +- By default, start the frontend server from the AOT snapshot shipped in the + Dart SDK. +- Throw an `ArgumentError` when `FrontendServerClient.start` is called with the + `frontendServerPath` argument omitted and the `debug` argument set to true. +- Update `package:vm_service` constraint to `^14.0.0`. + +## 3.2.0 + +- Add `nativeAssets` parameter to `FrontendServerClient`, for passing + additional `--native-assets` to the kernel compiler. + +## 3.1.0 + +- Add `additionalSources` parameter to `FrontendServerClient`, for passing + additional `--source`s to the kernel compiler. + +## 3.0.0 + +- Update the `compile` api to return a non-null `CompileResult`, and instead + make the `dillOutput` field nullable. This allows you to still get compiler + output if no dill file was produced. + +## 2.1.3 + +- Update `package:vm_service` to version `^8.0.0` + +## 2.1.2 + +- Force kill the frontend server after one second when calling shutdown. It + appears to hang on windows sometimes. + +## 2.1.1 + +- Fix a bug where spaces in the output dill path would cause a parse error when + reading the error count output. + +## 2.1.0 + +- Support enabling experiments when starting the compiler. + +## 2.0.1 + +- Widen the upper bound sdk constraint to `<3.0.0`. The frontend server api is + now considered quite stable and this package is now depended on by + package:test, so a tight constraint would cause unnecessary headaches. + +## 2.0.0 + +- Support null safety. + +## 1.0.0 + +- Initial version diff --git a/pkg/frontend_server_client/LICENSE b/pkg/frontend_server_client/LICENSE new file mode 100644 index 00000000000..be5c7b45c38 --- /dev/null +++ b/pkg/frontend_server_client/LICENSE @@ -0,0 +1,26 @@ +Copyright 2012, the Dart project authors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google LLC nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pkg/frontend_server_client/OWNERS b/pkg/frontend_server_client/OWNERS new file mode 100644 index 00000000000..0de49096064 --- /dev/null +++ b/pkg/frontend_server_client/OWNERS @@ -0,0 +1,7 @@ +set noparent +# Dart Development Infrastructure Team +file:/tools/OWNERS_DEV_INFRA +# VM Team +file:/tools/OWNERS_VM +# Global owners. +file:/OWNERS diff --git a/pkg/frontend_server_client/README.md b/pkg/frontend_server_client/README.md new file mode 100644 index 00000000000..e18c0979e9e --- /dev/null +++ b/pkg/frontend_server_client/README.md @@ -0,0 +1,37 @@ +[![pub package](https://img.shields.io/pub/v/frontend_server_client.svg)](https://pub.dev/packages/frontend_server_client) +[![package publisher](https://img.shields.io/pub/publisher/frontend_server_client.svg)](https://pub.dev/packages/frontend_server_client/publisher) + +This package provides a client interface around the frontend_server compiler +which ships with the Dart SDK. + +## SDK Versioning Policy + +This package keeps a relatively tight version constraint on the SDK to allow for +breaking changes in the frontend_server binary itself. + +Specifically, releases of this package will have an upper bound of less than the +next _minor_ (middle) version number of the latest stable SDK. There are no +requirements for the lower bound (other than the package must pass tests on that +SDK). + +The effect of this policy is that breaking changes will be allowed to the +frontend_server binary, but only in _minor_ SDK version releases. + +**Note**: This also means that when a new stable SDK is released, this package +will also need a new version published on pub before users can get a valid +version solve. + +### Working with dev SDK releases + +By default, when you do a pub get/upgrade, a constraint like `<2.9.0` will +actually allow dev releases of `2.9.0` if the current SDK is a dev release. It +emits a warning when it does this, but will happily allow it. + +- This means that we don't have to publish versions that explicitly allow dev + releases. We will be notified of breakages by our bots if a dev release does + break us, and can release a patch. + +If we need to depend on some new feature from a dev release, the min sdk +constraint should be bumped to that version but the max sdk constraint should +_not_ be changed. So for example we will have constraints like +`>=2.9.0-dev.1 <2.9.0`. diff --git a/pkg/frontend_server_client/analysis_options.yaml b/pkg/frontend_server_client/analysis_options.yaml new file mode 100644 index 00000000000..d978f811cce --- /dev/null +++ b/pkg/frontend_server_client/analysis_options.yaml @@ -0,0 +1 @@ +include: package:dart_flutter_team_lints/analysis_options.yaml diff --git a/pkg/frontend_server_client/example/app/index.html b/pkg/frontend_server_client/example/app/index.html new file mode 100644 index 00000000000..271b5b4f8fb --- /dev/null +++ b/pkg/frontend_server_client/example/app/index.html @@ -0,0 +1,6 @@ + + + + + + diff --git a/pkg/frontend_server_client/example/app/main.dart b/pkg/frontend_server_client/example/app/main.dart new file mode 100644 index 00000000000..6ee01dd236d --- /dev/null +++ b/pkg/frontend_server_client/example/app/main.dart @@ -0,0 +1,16 @@ +// Copyright 2020 The Dart Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:path/path.dart' as p; + +Future main() async { + print(message); + while (!message.contains('goodbye')) { + print('waiting for hot reload to change message'); + await Future.delayed(const Duration(seconds: 1)); + } + print(message); +} + +String get message => p.join('hello', 'world'); diff --git a/pkg/frontend_server_client/example/vm_client.dart b/pkg/frontend_server_client/example/vm_client.dart new file mode 100644 index 00000000000..1f73c095a5c --- /dev/null +++ b/pkg/frontend_server_client/example/vm_client.dart @@ -0,0 +1,117 @@ +// Copyright 2020 The Dart Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:frontend_server_client/frontend_server_client.dart'; +import 'package:path/path.dart' as p; +import 'package:vm_service/vm_service.dart'; +import 'package:vm_service/vm_service_io.dart'; + +void main(List args) async { + // Change to package root so relative paths work in CI + final scriptDir = p.dirname(p.fromUri(Platform.script)); + final packageRoot = p.dirname(scriptDir); + Directory.current = packageRoot; + + try { + watch.start(); + if (args.isNotEmpty) { + throw ArgumentError('No command line args are supported'); + } + + final packagesPath = findNearestPackageConfigPath(); + final client = await FrontendServerClient.start( + 'org-dartlang-root:///$app', + outputDill, + p.join(sdkDir, 'lib', '_internal', 'vm_platform_strong.dill'), + packagesJson: packagesPath ?? '.dart_tool/package_config.json', + target: 'vm', + // Use an absolute filesystem root so org-dartlang-root:/// URIs resolve reliably in CI + fileSystemRoots: [Directory.current.path], + fileSystemScheme: 'org-dartlang-root', + verbose: true, + ); + _print('compiling $app'); + var result = await client.compile(); + client.accept(); + _print('done compiling $app'); + + Process appProcess; + final vmServiceCompleter = Completer(); + appProcess = await Process.start(Platform.resolvedExecutable, [ + '--enable-vm-service', + result.dillOutput!, + ]); + final sawHelloWorld = Completer(); + appProcess.stdout + .transform(utf8.decoder) + .transform(const LineSplitter()) + .listen((line) { + stdout.writeln('APP -> $line'); + if (line == 'hello/world') { + sawHelloWorld.complete(); + } + if (line.startsWith( + 'The Dart DevTools debugger and profiler is available at:', + )) { + final observatoryUri = + '${line.split(' ').last.replaceFirst('http', 'ws')}ws'; + if (!vmServiceCompleter.isCompleted) { + vmServiceCompleter.complete(vmServiceConnectUri(observatoryUri)); + } + } + }); + appProcess.stderr + .transform(utf8.decoder) + .transform(const LineSplitter()) + .listen((line) { + stderr.writeln('APP -> $line'); + }); + + final vmService = await vmServiceCompleter.future; + await sawHelloWorld.future; + + _print('editing $app'); + final appFile = File(app); + final originalContent = await appFile.readAsString(); + final newContent = originalContent.replaceFirst('hello', 'goodbye'); + await appFile.writeAsString(newContent); + + _print('recompiling $app with edits'); + result = await client.compile([Uri.parse('org-dartlang-root:///$app')]); + client.accept(); + _print('done recompiling $app'); + _print('reloading $app'); + final vm = await vmService.getVM(); + await vmService.reloadSources( + vm.isolates!.first.id!, + rootLibUri: result.dillOutput!, + ); + + _print('restoring $app to original contents'); + await appFile.writeAsString(originalContent); + _print('exiting'); + await client.shutdown().timeout( + const Duration(seconds: 1), + onTimeout: () { + client.kill(); + return 1; + }, + ); + } finally { + Directory(p.join('.dart_tool', 'out')).deleteSync(recursive: true); + } +} + +void _print(String message) { + print('${watch.elapsed}: $message'); +} + +final app = 'example/app/main.dart'; +final outputDill = p.join('.dart_tool', 'out', 'example_app.dill'); +final sdkDir = p.dirname(p.dirname(Platform.resolvedExecutable)); +final watch = Stopwatch(); diff --git a/pkg/frontend_server_client/example/web_client.dart b/pkg/frontend_server_client/example/web_client.dart new file mode 100644 index 00000000000..6d971acc823 --- /dev/null +++ b/pkg/frontend_server_client/example/web_client.dart @@ -0,0 +1,185 @@ +// Copyright 2020 The Dart Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; + +import 'package:async/async.dart'; +import 'package:frontend_server_client/frontend_server_client.dart'; +import 'package:path/path.dart' as p; +import 'package:shelf/shelf.dart'; +import 'package:shelf/shelf_io.dart' as shelf_io; +import 'package:shelf_packages_handler/shelf_packages_handler.dart'; +import 'package:shelf_static/shelf_static.dart'; + +void main(List args) async { + // Change to package root so relative paths work in CI + final scriptDir = p.dirname(p.fromUri(Platform.script)); + final packageRoot = p.dirname(scriptDir); + Directory.current = packageRoot; + + try { + watch.start(); + if (args.isNotEmpty) { + throw ArgumentError('No command line args are supported'); + } + + _print('compiling the dart sdk'); + final sdkCompileResult = await Process.run(Platform.resolvedExecutable, [ + 'compile', + 'js-dev', + '--multi-root-scheme=org-dartlang-sdk', + '--modules=amd', + '--module-name=dart_sdk', + '-o', + dartSdkJs, + p.url.join(sdkDir, sdkKernelPath), + ]); + if (sdkCompileResult.exitCode != 0) { + _print( + 'Failed to compile the dart sdk to JS:\n' + '${sdkCompileResult.stdout}\n' + '${sdkCompileResult.stderr}', + ); + exit(sdkCompileResult.exitCode); + } + + _print('starting frontend server'); + final packagesPath = findNearestPackageConfigPath(); + final client = await DartDevcFrontendServerClient.start( + 'org-dartlang-root:///$app', + outputDill, + // Use an absolute filesystem root so org-dartlang-root:/// URIs resolve reliably in CI + fileSystemRoots: [Directory.current.path], + fileSystemScheme: 'org-dartlang-root', + platformKernel: p.toUri(sdkKernelPath).toString(), + packagesJson: packagesPath ?? '.dart_tool/package_config.json', + verbose: true, + ); + + _print('compiling $app'); + await client.compile([]); + client.accept(); + _print('done compiling $app'); + + _print('starting shelf server'); + final cascade = Cascade() + .add(_clientHandler(client)) + .add(createStaticHandler(Directory.current.path)) + .add(createFileHandler(dartSdkJs, url: 'example/app/dart_sdk.js')) + .add( + createFileHandler( + p.join( + sdkDir, + 'lib', + 'dev_compiler', + 'web', + 'dart_stack_trace_mapper.js', + ), + url: 'example/app/dart_stack_trace_mapper.js', + ), + ) + .add( + createFileHandler( + p.join(sdkDir, 'lib', 'dev_compiler', 'amd', 'require.js'), + url: 'example/app/require.js', + ), + ) + .add(packagesDirHandler()); + final server = await shelf_io.serve(cascade.handler, 'localhost', 8080); + _print('server ready'); + + // The file we will be editing in the repl + final appFile = File(app); + final originalContent = await appFile.readAsString(); + final appLines = const LineSplitter().convert(originalContent); + final getterText = 'String get message =>'; + final messageLine = appLines.indexWhere( + (line) => line.startsWith(getterText), + ); + + final stdinQueue = StreamQueue( + stdin.transform(utf8.decoder).transform(const LineSplitter()), + ); + _prompt(); + while (await stdinQueue.hasNext) { + final newMessage = await stdinQueue.next; + if (newMessage == 'quit') { + await server.close(); + await stdinQueue.cancel(); + break; + } else if (newMessage == 'reset') { + print('resetting'); + client.reset(); + _print('restoring $app'); + await appFile.writeAsString(originalContent); + } else { + _print('editing $app'); + appLines[messageLine] = '$getterText "$newMessage";'; + final newContent = appLines.join('\n'); + await appFile.writeAsString(newContent); + + _print('recompiling $app with edits'); + final result = await client.compile([ + Uri.parse('org-dartlang-root:///$app'), + ]); + if (result.errorCount > 0) { + print('Compile errors: \n${result.compilerOutputLines.join('\n')}'); + await client.reject(); + } else { + _print('Recompile succeeded for $app'); + client.accept(); + // TODO: support hot restart + print('reload app to see the new message'); + } + } + + _prompt(); + } + + _print('restoring $app'); + await appFile.writeAsString(originalContent); + _print('exiting'); + await client.shutdown(); + } finally { + Directory(p.join('.dart_tool', 'out')).deleteSync(recursive: true); + } +} + +Handler _clientHandler(DartDevcFrontendServerClient client) { + return (Request request) { + var path = request.url.path; + final packagesIndex = path.indexOf('/packages/'); + if (packagesIndex > 0) { + path = request.url.path.substring(packagesIndex); + } else { + path = request.url.path; + } + if (!path.startsWith('/')) path = '/$path'; + if (path.endsWith('.dart.js') && path != '/example/app/main.dart.js') { + path = path.replaceFirst('.dart.js', '.dart.lib.js', path.length - 8); + } + final assetBytes = client.assetBytes(path); + if (assetBytes == null) return Response.notFound('path not found'); + return Response.ok( + assetBytes, + headers: {HttpHeaders.contentTypeHeader: 'application/javascript'}, + ); + }; +} + +void _print(String message) { + print('${watch.elapsed}: $message'); +} + +void _prompt() => stdout.write( + 'Enter a new message to print and recompile, or type `quit` to exit:', + ); + +final app = 'example/app/main.dart'; +final dartSdkJs = p.join('.dart_tool', 'out', 'dart_sdk.js'); +final outputDill = p.join('.dart_tool', 'out', 'example_app.dill'); +final sdkDir = p.dirname(p.dirname(Platform.resolvedExecutable)); +final sdkKernelPath = p.join(sdkDir, 'lib', '_internal', 'ddc_platform.dill'); +final watch = Stopwatch(); diff --git a/pkg/frontend_server_client/lib/frontend_server_client.dart b/pkg/frontend_server_client/lib/frontend_server_client.dart new file mode 100644 index 00000000000..06fba641b1a --- /dev/null +++ b/pkg/frontend_server_client/lib/frontend_server_client.dart @@ -0,0 +1,9 @@ +// Copyright 2020 The Dart Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +export 'src/dartdevc_frontend_server_client.dart' + show DartDevcFrontendServerClient; +export 'src/frontend_server_client.dart' + show CompileResult, FrontendServerClient; +export 'src/package_config_utils.dart' show findNearestPackageConfigPath; diff --git a/pkg/frontend_server_client/lib/src/dartdevc_bootstrap_amd.dart b/pkg/frontend_server_client/lib/src/dartdevc_bootstrap_amd.dart new file mode 100644 index 00000000000..50ed7d9f096 --- /dev/null +++ b/pkg/frontend_server_client/lib/src/dartdevc_bootstrap_amd.dart @@ -0,0 +1,61 @@ +// Copyright 2020 The Dart Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +/// The JavaScript bootstrap script to support in-browser hot restart. +/// +/// The [requireUrl] loads our cached RequireJS script file. The [mapperUrl] +/// loads the special Dart stack trace mapper. +/// +/// This file is served when the browser requests `$entrypoint.js` in debug +/// mode, and is responsible for bootstrapping the RequireJS modules and +/// attaching the hot reload hooks. +String generateAmdBootstrapScript({ + required String requireUrl, + required String mapperUrl, + required String entrypoint, +}) { + return ''' +"use strict"; + +// Attach source mapping. +var mapperEl = document.createElement("script"); +mapperEl.defer = true; +mapperEl.async = false; +mapperEl.src = "$mapperUrl"; +document.head.appendChild(mapperEl); + +// Attach require JS. +var requireEl = document.createElement("script"); +requireEl.defer = true; +requireEl.async = false; +requireEl.src = "$requireUrl"; + +// This attribute tells require JS what to load as main (defined below). +requireEl.setAttribute("data-main", "$entrypoint.bootstrap"); +document.head.appendChild(requireEl); +'''; +} + +/// Generate a synthetic main module which captures the application's main +/// method. +/// +/// RE: Object.keys usage in app.main: +/// This attaches the main entrypoint and hot reload functionality to the +/// window. The app module will have a single property which contains the +/// actual application code. The property name is based off of the entrypoint +/// that is generated, for example the file `foo/bar/baz.dart` will generate +/// a property named approximately `foo__bar__baz`. Rather than attempt to +/// guess, we assume the first property of this object is the module. +String generateAmdMainModule({required String entrypoint}) { + return '''/* ENTRYPOINT_EXTENTION_MARKER */ +// Create the main module loaded below. +require(["$entrypoint.lib.js", "dart_sdk"], function(app, dart_sdk) { + dart_sdk.dart.setStartAsyncSynchronously(true); + dart_sdk._debugger.registerDevtoolsFormatter(); + + // See the generateMainModule doc comment. + app[Object.keys(app)[0]].main(); +}); +'''; +} diff --git a/pkg/frontend_server_client/lib/src/dartdevc_frontend_server_client.dart b/pkg/frontend_server_client/lib/src/dartdevc_frontend_server_client.dart new file mode 100644 index 00000000000..15e78cd02e9 --- /dev/null +++ b/pkg/frontend_server_client/lib/src/dartdevc_frontend_server_client.dart @@ -0,0 +1,211 @@ +// Copyright 2020 The Dart Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:path/path.dart' as p; + +import 'dartdevc_bootstrap_amd.dart'; +import 'frontend_server_client.dart'; +import 'shared.dart'; + +/// Wraps a [FrontendServerClient] with opinionated web specific functionality, +/// and provides some typical defaults. +/// +/// Loads into memory the [CompileResult]s from each [compile] call, and +/// provides access to the up to date sources and source maps. +/// +/// Also has the ability to create a bootstrap file for the current entrypoint. +class DartDevcFrontendServerClient implements FrontendServerClient { + final FrontendServerClient _frontendServerClient; + + final _assets = {}; + final String _entrypoint; + + /// The last compile, or `null` once it has been accepted or rejected. + CompileResult? _lastResult; + + /// The bootstrap js contents, provided in [_assets] at + /// the path `${_entrypointModule}.js`. + /// + /// This is `null` if the module format is not supported for bootstrapping. + final String? _bootstrapJs; + + /// The generated main module js contents, provided in [_assets] at + /// the path `${_entrypointModule}.bootstrap.js`. + /// + /// This is `null` if the module format is not supported for bootstrapping. + final String? _mainModuleJs; + + DartDevcFrontendServerClient._( + this._frontendServerClient, this._entrypoint, String moduleFormat) + : _bootstrapJs = moduleFormat == 'amd' + ? generateAmdBootstrapScript( + requireUrl: 'require.js', + mapperUrl: 'dart_stack_trace_mapper.js', + entrypoint: _entrypoint) + : null, + _mainModuleJs = moduleFormat == 'amd' + ? generateAmdMainModule(entrypoint: _entrypoint) + : null { + _resetAssets(); + } + + static Future start( + String entrypoint, + String outputDillPath, { + String dartdevcModuleFormat = 'amd', + bool debug = false, + bool enableHttpUris = false, + List fileSystemRoots = const [], // For `fileSystemScheme` uris, + String fileSystemScheme = + 'org-dartlang-root', // Custom scheme for virtual `fileSystemRoots`. + String? frontendServerPath, // Defaults to the snapshot in the sdk. + String packagesJson = '.dart_tool/package_config.json', + String? platformKernel, // Defaults to the dartdevc platform from the sdk. + String? sdkRoot, // Defaults to the current SDK root. + bool verbose = false, + bool printIncrementalDependencies = true, + }) async { + final feServer = await FrontendServerClient.start( + entrypoint, + outputDillPath, + platformKernel ?? _dartdevcPlatformKernel, + dartdevcModuleFormat: dartdevcModuleFormat, + debug: debug, + enableHttpUris: enableHttpUris, + fileSystemRoots: fileSystemRoots, + fileSystemScheme: fileSystemScheme, + frontendServerPath: frontendServerPath, + packagesJson: packagesJson, + sdkRoot: sdkRoot, + target: 'dartdevc', + verbose: verbose, + ); + return DartDevcFrontendServerClient._( + feServer, Uri.parse(entrypoint).path, dartdevcModuleFormat); + } + + /// Returns the current bytes for the asset at [path]. + /// + /// The [path] should be exactly as it appears in the + /// [CompileResult.jsManifestOutput] file. + /// + /// **Note**: Assets are not updated until `accept` is called after a + /// successful compile. They are not updated if `reject` is called. + /// + /// Returns `null` if no asset exists at [path]. + /// + /// In addition to any DDC compiled assets, this serves + Uint8List? assetBytes(String path) => _assets[path]; + + /// The contents of a JS file capable of bootstrapping the current app. + /// + /// TODO: implement + String bootstrapJs() => throw UnimplementedError(); + + /// Updates [_assets] for [result]. + void _updateAssets(CompileResult result) { + if (result.dillOutput == null) { + return; + } + final manifest = + jsonDecode(File(result.jsManifestOutput!).readAsStringSync()) + as Map; + final sourceBytes = File(result.jsSourcesOutput!).readAsBytesSync(); + final sourceMapBytes = File(result.jsSourceMapsOutput!).readAsBytesSync(); + + for (final entry in manifest.entries) { + final metadata = entry.value as Map; + final sourceOffsets = metadata['code'] as List; + _assets[entry.key] = + sourceBytes.sublist(sourceOffsets[0] as int, sourceOffsets[1] as int); + final sourceMapOffsets = metadata['sourcemap'] as List; + _assets['${entry.key}.map'] = sourceMapBytes.sublist( + sourceMapOffsets[0] as int, sourceMapOffsets[1] as int); + } + } + + @override + Future compile([List? invalidatedUris]) async { + return _lastResult = await _frontendServerClient.compile(invalidatedUris); + } + + @override + Future compileExpression({ + required String expression, + required List definitions, + required bool isStatic, + required String klass, + required String libraryUri, + required List typeDefinitions, + }) => + throw UnsupportedError( + 'Use `compileExpressionToJs` for dartdevc based clients'); + + @override + Future compileExpressionToJs({ + required String expression, + required int column, + required Map jsFrameValues, + required Map jsModules, + required String libraryUri, + required int line, + required String moduleName, + }) => + _frontendServerClient.compileExpressionToJs( + expression: expression, + column: column, + jsFrameValues: jsFrameValues, + jsModules: jsModules, + libraryUri: libraryUri, + line: line, + moduleName: moduleName); + + @override + void accept() { + _frontendServerClient.accept(); + if (_lastResult != null) _updateAssets(_lastResult!); + _lastResult = null; + } + + @override + Future reject() async { + await _frontendServerClient.reject(); + _lastResult = null; + } + + @override + void reset() { + _frontendServerClient.reset(); + _resetAssets(); + } + + @override + bool kill({ProcessSignal processSignal = ProcessSignal.sigterm}) => + _frontendServerClient.kill(); + + @override + Future shutdown() => _frontendServerClient.shutdown(); + + /// Clears any previously compiled assets and adds the bootstrap modules as + /// assets if available. + void _resetAssets() { + _assets.clear(); + final bootstrapJs = _bootstrapJs; + if (bootstrapJs != null) { + _assets['$_entrypoint.js'] = Uint8List.fromList(utf8.encode(bootstrapJs)); + } + final mainModuleJs = _mainModuleJs; + if (mainModuleJs != null) { + _assets['$_entrypoint.bootstrap.js'] = + Uint8List.fromList(utf8.encode(mainModuleJs)); + } + } +} + +final _dartdevcPlatformKernel = + p.toUri(p.join(sdkDir, 'lib', '_internal', 'ddc_sdk.dill')).toString(); diff --git a/pkg/frontend_server_client/lib/src/frontend_server_client.dart b/pkg/frontend_server_client/lib/src/frontend_server_client.dart new file mode 100644 index 00000000000..d8d8d746d7c --- /dev/null +++ b/pkg/frontend_server_client/lib/src/frontend_server_client.dart @@ -0,0 +1,455 @@ +// Copyright 2020 The Dart Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:async/async.dart'; +import 'package:path/path.dart' as p; + +import 'shared.dart'; + +/// Wrapper around the incremental frontend server compiler. +class FrontendServerClient { + final String _entrypoint; + final Process _feServer; + final StreamQueue _feServerStdoutLines; + final bool _verbose; + + _ClientState _state; + + FrontendServerClient._( + this._entrypoint, this._feServer, this._feServerStdoutLines, + {bool? verbose}) + : _verbose = verbose ?? false, + _state = _ClientState.waitingForFirstCompile { + _feServer.stderr.transform(utf8.decoder).listen(stderr.write); + } + + /// Starts the frontend server. + /// + /// Most arguments directly mirror the command line arguments for the + /// frontend_server (see `pkg/frontend_server/lib/frontend_server.dart` in + /// the sdk). Options are exposed on an as-needed basis. + /// + /// The [entrypoint] and [packagesJson] may be a relative path or any uri + /// supported by the frontend server. + /// + /// The [outputDillPath] determines where the primary output should be, and + /// some targets may output additional files based on that file name (by + /// adding file extensions for instance). + /// + /// When the [frontendServerPath] argument is provided, the frontend server + /// will be started from the specified file. The specified file can either be + /// a Dart source file or an AppJIT snapshot. + /// + /// When the [frontendServerPath] argument is provided, setting [debug] to + /// true permits debuggers to attach to the frontend server. When the + /// [frontendServerPath] argument is omitted, setting [debug] to true will + /// cause an [ArgumentError] to be thrown. + static Future start( + String entrypoint, + String outputDillPath, + String platformKernel, { + String dartdevcModuleFormat = 'amd', + bool debug = false, + List? enabledExperiments, + bool enableHttpUris = false, + List fileSystemRoots = const [], // For `fileSystemScheme` uris, + String fileSystemScheme = + 'org-dartlang-root', // Custom scheme for virtual `fileSystemRoots`. + String? frontendServerPath, // Defaults to the snapshot in the sdk. + String packagesJson = '.dart_tool/package_config.json', + String? sdkRoot, // Defaults to the current SDK root. + String target = 'vm', // The kernel target type. + bool verbose = false, // Verbose logs, including server/client messages + bool printIncrementalDependencies = true, + List additionalSources = const [], + String? nativeAssets, + }) async { + final commonArguments = [ + '--sdk-root', + sdkRoot ?? sdkDir, + '--platform=$platformKernel', + '--target=$target', + if (target == 'dartdevc') + '--dartdevc-module-format=$dartdevcModuleFormat', + for (final root in fileSystemRoots) '--filesystem-root=$root', + '--filesystem-scheme', + fileSystemScheme, + '--output-dill', + outputDillPath, + '--packages=$packagesJson', + if (enableHttpUris) '--enable-http-uris', + '--incremental', + if (verbose) '--verbose', + if (!printIncrementalDependencies) '--no-print-incremental-dependencies', + if (enabledExperiments != null) + for (final experiment in enabledExperiments) + '--enable-experiment=$experiment', + for (final source in additionalSources) ...[ + '--source', + source, + ], + if (nativeAssets != null) ...[ + '--native-assets', + nativeAssets, + ], + ]; + late final Process feServer; + if (frontendServerPath != null) { + feServer = await Process.start( + Platform.resolvedExecutable, + [ + if (debug) '--observe', + frontendServerPath, + ...commonArguments, + ], + ); + } else if (File(_feServerAotSnapshotPath).existsSync()) { + if (debug) { + throw ArgumentError('The debug argument cannot be set to true when the ' + 'frontendServerPath argument is omitted.'); + } + feServer = await Process.start( + _dartAotRuntimePath, + [_feServerAotSnapshotPath, ...commonArguments], + ); + } else { + // AOT snapshots cannot be generated on IA32, so we need this fallback + // branch until support for IA32 is dropped (https://dartbug.com/49969). + feServer = await Process.start( + Platform.resolvedExecutable, + [ + if (debug) '--observe', + _feServerAppJitSnapshotPath, + ...commonArguments, + ], + ); + } + final feServerStdoutLines = StreamQueue(feServer.stdout + .transform(utf8.decoder) + .transform(const LineSplitter())); + + // The frontend_server doesn't appear to recursively create files, so we + // need to make sure the output dir already exists. + final outputDir = Directory(p.dirname(outputDillPath)); + if (!await outputDir.exists()) await outputDir.create(); + + return FrontendServerClient._( + entrypoint, + feServer, + feServerStdoutLines, + verbose: verbose, + ); + } + + /// Compiles [_entrypoint], using an incremental recompile if possible. + /// + /// [invalidatedUris] must not be null for all but the very first compile. + /// + /// The frontend server _does not_ do any of its own invalidation. + Future compile([List? invalidatedUris]) async { + String action; + switch (_state) { + case _ClientState.waitingForFirstCompile: + action = 'compile'; + break; + case _ClientState.waitingForRecompile: + action = 'recompile'; + break; + case _ClientState.waitingForAcceptOrReject: + throw StateError( + 'Previous `CompileResult` must be accepted or rejected by ' + 'calling `accept` or `reject`.'); + case _ClientState.compiling: + throw StateError( + 'App is already being compiled, you must wait for that to ' + 'complete and `accept` or `reject` the result before compiling ' + 'again.'); + case _ClientState.rejecting: + throw StateError('Still waiting for previous `reject` call to finish. ' + 'You must await that before compiling again.'); + } + _state = _ClientState.compiling; + + try { + final command = StringBuffer('$action $_entrypoint'); + if (action == 'recompile') { + if (invalidatedUris == null || invalidatedUris.isEmpty) { + throw StateError( + 'Subsequent compile invocations must provide a non-empty list ' + 'of invalidated uris.'); + } + final boundaryKey = generateUuidV4(); + command.writeln(' $boundaryKey'); + for (final uri in invalidatedUris) { + command.writeln('$uri'); + } + command.write(boundaryKey); + } + + _sendCommand(command.toString()); + var state = _CompileState.started; + late String feBoundaryKey; + final newSources = {}; + final removedSources = {}; + final compilerOutputLines = []; + var errorCount = 0; + String? outputDillPath; + while ( + state != _CompileState.done && await _feServerStdoutLines.hasNext) { + final line = await _nextInputLine(); + switch (state) { + case _CompileState.started: + assert(line.startsWith('result')); + feBoundaryKey = line.substring(line.indexOf(' ') + 1); + state = _CompileState.waitingForKey; + continue; + case _CompileState.waitingForKey: + if (line == feBoundaryKey) { + state = _CompileState.gettingSourceDiffs; + } else { + compilerOutputLines.add(line); + } + continue; + case _CompileState.gettingSourceDiffs: + if (line.startsWith(feBoundaryKey)) { + state = _CompileState.done; + final parts = line.split(' '); + outputDillPath = parts.getRange(1, parts.length - 1).join(' '); + errorCount = int.parse(parts.last); + continue; + } + final diffUri = Uri.parse(line.substring(1)); + if (line.startsWith('+')) { + newSources.add(diffUri); + } else if (line.startsWith('-')) { + removedSources.add(diffUri); + } else { + throw StateError( + 'unrecognized diff line, should start with a + or - ' + 'but got: $line'); + } + continue; + case _CompileState.done: + throw StateError('Unreachable'); + } + } + + return CompileResult._( + dillOutput: outputDillPath, + errorCount: errorCount, + newSources: newSources, + removedSources: removedSources, + compilerOutputLines: compilerOutputLines); + } finally { + _state = _ClientState.waitingForAcceptOrReject; + } + } + + /// TODO: Document + Future compileExpression({ + required String expression, + required List definitions, + required bool isStatic, + required String klass, + required String libraryUri, + required List typeDefinitions, + }) => + throw UnimplementedError(); + + /// TODO: Document + Future compileExpressionToJs({ + required String expression, + required int column, + required Map jsFrameValues, + required Map jsModules, + required String libraryUri, + required int line, + required String moduleName, + }) => + throw UnimplementedError(); + + /// Should be invoked when results of compilation are accepted by the client. + /// + /// Either [accept] or [reject] should be called after every [compile] call. + void accept() { + if (_state != _ClientState.waitingForAcceptOrReject) { + throw StateError( + 'Called `accept` but there was no previous compile to accept.'); + } + _sendCommand('accept'); + _state = _ClientState.waitingForRecompile; + } + + /// Should be invoked when results of compilation are rejected by the client. + /// + /// Either [accept] or [reject] should be called after every [compile] call. + /// + /// The result of this call must be awaited before a new [compile] can be + /// done. + Future reject() async { + if (_state != _ClientState.waitingForAcceptOrReject) { + throw StateError( + 'Called `reject` but there was no previous compile to reject.'); + } + _state = _ClientState.rejecting; + _sendCommand('reject'); + late String boundaryKey; + var rejectState = _RejectState.started; + while (rejectState != _RejectState.done && + await _feServerStdoutLines.hasNext) { + final line = await _nextInputLine(); + switch (rejectState) { + case _RejectState.started: + if (!line.startsWith('result')) { + throw StateError( + 'Expected a line like `result ` after a `reject` ' + 'command, but got:\n$line'); + } + boundaryKey = line.split(' ').last; + rejectState = _RejectState.waitingForKey; + continue; + case _RejectState.waitingForKey: + if (line != boundaryKey) { + throw StateError('Expected exactly `$boundaryKey` but got:\n$line'); + } + rejectState = _RejectState.done; + continue; + case _RejectState.done: + throw StateError('Unreachable'); + } + } + _state = _ClientState.waitingForRecompile; + } + + /// Should be invoked when frontend server compiler should forget what was + /// accepted previously so that next call to [compile] produces complete + /// kernel file. + void reset() { + if (_state == _ClientState.compiling) { + throw StateError( + 'Called `reset` during an active compile, you must wait for that to ' + 'complete first.'); + } + _sendCommand('reset'); + _state = _ClientState.waitingForRecompile; + } + + /// Stop the service gracefully (using the shutdown command) + Future shutdown() async { + _sendCommand('quit'); + final timer = Timer(const Duration(seconds: 1), _feServer.kill); + final exitCode = await _feServer.exitCode; + timer.cancel(); + await _feServerStdoutLines.cancel(); + return exitCode; + } + + /// Kills the server forcefully by calling `kill` on the process, and + /// returns the result. + bool kill({ProcessSignal processSignal = ProcessSignal.sigterm}) { + _feServerStdoutLines.cancel(); + return _feServer.kill(processSignal); + } + + /// Sends [command] to the [_feServer] via stdin, and logs it if [_verbose]. + void _sendCommand(String command) { + if (_verbose) { + final lines = const LineSplitter().convert(command); + for (final line in lines) { + print('>> $line'); + } + } + _feServer.stdin.writeln(command); + } + + /// Reads a line from [_feServerStdoutLines] and logs it if [_verbose]. + Future _nextInputLine() async { + final line = await _feServerStdoutLines.next; + if (_verbose) print('<< $line'); + return line; + } +} + +/// The result of a compile call. +class CompileResult { + const CompileResult._( + {required this.dillOutput, + required this.compilerOutputLines, + required this.errorCount, + required this.newSources, + required this.removedSources}); + + /// The produced dill output file, this will either be a full dill file, an + /// incremental dill file, or `null` if no file was produced. + final String? dillOutput; + + /// All output from the compiler, typically this would contain errors or + /// warnings. + final Iterable compilerOutputLines; + + /// The total count of errors, details should appear in + /// [compilerOutputLines]. + final int errorCount; + + /// A single file containing all source maps for all JS outputs. + /// + /// Read [jsManifestOutput] for file offsets for each sourcemap. + String? get jsSourceMapsOutput => + dillOutput == null ? null : '$dillOutput.map'; + + /// A single file containing all JS outputs. + /// + /// Read [jsManifestOutput] for file offsets for each source. + String? get jsSourcesOutput => + dillOutput == null ? null : '$dillOutput.sources'; + + /// A JSON manifest containing offsets for the sources and source maps in + /// the [jsSourcesOutput] and [jsSourceMapsOutput] files. + String? get jsManifestOutput => + dillOutput == null ? null : '$dillOutput.json'; + + /// All the transitive source dependencies that were added as a part of this + /// compile. + final Iterable newSources; + + /// All the transitive source dependencies that were removed as a part of + /// this compile. + final Iterable removedSources; +} + +/// Internal states for the client. +enum _ClientState { + compiling, + rejecting, + waitingForAcceptOrReject, + waitingForFirstCompile, + waitingForRecompile, +} + +/// Frontend server interaction states for a `compile` call. +enum _CompileState { + started, + waitingForKey, + gettingSourceDiffs, + done, +} + +/// Frontend server interaction states for a `reject` call. +enum _RejectState { + started, + waitingForKey, + done, +} + +final _dartAotRuntimePath = p.join(sdkDir, 'bin', 'dartaotruntime'); + +final _feServerAppJitSnapshotPath = + p.join(sdkDir, 'bin', 'snapshots', 'frontend_server.dart.snapshot'); + +final _feServerAotSnapshotPath = + p.join(sdkDir, 'bin', 'snapshots', 'frontend_server_aot.dart.snapshot'); diff --git a/pkg/frontend_server_client/lib/src/package_config_utils.dart b/pkg/frontend_server_client/lib/src/package_config_utils.dart new file mode 100644 index 00000000000..b3eaaa4e8dd --- /dev/null +++ b/pkg/frontend_server_client/lib/src/package_config_utils.dart @@ -0,0 +1,46 @@ +// Utility functions to locate a package_config.json for pub/workspace setups. + +import 'dart:io'; + +import 'package:package_config/package_config.dart'; +import 'package:path/path.dart' as p; + +/// Walks up from [start] (or the current directory if omitted) to find the +/// nearest `.dart_tool/package_config.json`. +/// +/// Returns the absolute file path, or `null` if none is found. +String? findNearestPackageConfigPath([Directory? start]) { + var dir = (start ?? Directory.current).absolute; + while (true) { + final file = File(p.join(dir.path, '.dart_tool', 'package_config.json')); + if (file.existsSync()) return file.path; + final parent = dir.parent; + if (parent.path == dir.path) return null; + dir = parent; + } +} + +/// Returns an absolute path under the given [packageName]'s root directory, +/// resolving using the nearest workspace `.dart_tool/package_config.json`. +/// +/// This is robust for pub workspace monorepos where the nearest package +/// config lives at the repo root and contains individual entries for each +/// package with its own root. +Future pathFromNearestPackageConfig( + String relativePath, { + String packageName = 'frontend_server_client', +}) async { + final configPath = findNearestPackageConfigPath(); + if (configPath == null) { + throw StateError('Could not locate .dart_tool/package_config.json'); + } + final config = await loadPackageConfigUri(Uri.file(configPath)); + final pkg = config.packages.firstWhere( + (p0) => p0.name == packageName, + orElse: () => throw StateError( + 'Package $packageName not found in package config at $configPath', + ), + ); + final packageRootDir = p.fromUri(pkg.root); + return p.normalize(p.join(packageRootDir, relativePath)); +} diff --git a/pkg/frontend_server_client/lib/src/shared.dart b/pkg/frontend_server_client/lib/src/shared.dart new file mode 100644 index 00000000000..5e6faa62af0 --- /dev/null +++ b/pkg/frontend_server_client/lib/src/shared.dart @@ -0,0 +1,38 @@ +// Copyright 2020 The Dart Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io'; +import 'dart:math' show Random; + +import 'package:path/path.dart' as p; + +final sdkDir = p.dirname(p.dirname(Platform.resolvedExecutable)); +final sdkUri = p.toUri(sdkDir).toString(); + +/// Returns a unique ID in the format: +/// +/// f47ac10b-58cc-4372-a567-0e02b2c3d479 +/// +/// The generated uuids are 128 bit numbers encoded in a specific string format. +/// For more information, see +/// [en.wikipedia.org/wiki/Universally_unique_identifier](http://en.wikipedia.org/wiki/Universally_unique_identifier). +String generateUuidV4() { + final random = Random(); + + int generateBits(int bitCount) => random.nextInt(1 << bitCount); + + String printDigits(int value, int count) => + value.toRadixString(16).padLeft(count, '0'); + String bitsDigits(int bitCount, int digitCount) => + printDigits(generateBits(bitCount), digitCount); + + // Generate xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx / 8-4-4-4-12. + final special = 8 + random.nextInt(4); + + return '${bitsDigits(16, 4)}${bitsDigits(16, 4)}-' + '${bitsDigits(16, 4)}-' + '4${bitsDigits(12, 3)}-' + '${printDigits(special, 1)}${bitsDigits(12, 3)}-' + '${bitsDigits(16, 4)}${bitsDigits(16, 4)}${bitsDigits(16, 4)}'; +} diff --git a/pkg/frontend_server_client/pubspec.yaml b/pkg/frontend_server_client/pubspec.yaml new file mode 100644 index 00000000000..5949ffdd60b --- /dev/null +++ b/pkg/frontend_server_client/pubspec.yaml @@ -0,0 +1,26 @@ +name: frontend_server_client +version: 4.0.0 +description: >- + Client code to start and interact with the frontend_server compiler from the + Dart SDK. +repository: https://github.com/dart-lang/webdev/tree/main/frontend_server_client + +resolution: workspace + +environment: + sdk: ^3.5.0 + +dependencies: + async: ^2.5.0 + package_config: ^2.1.0 + path: ^1.8.0 + +dev_dependencies: + dart_flutter_team_lints: any + shelf: any + shelf_packages_handler: any + shelf_static: any + test: any + test_descriptor: any + test_process: any + vm_service: any diff --git a/pkg/frontend_server_client/test/example/vm_client_test.dart b/pkg/frontend_server_client/test/example/vm_client_test.dart new file mode 100644 index 00000000000..846f6246bc3 --- /dev/null +++ b/pkg/frontend_server_client/test/example/vm_client_test.dart @@ -0,0 +1,33 @@ +// Copyright 2022 The Dart Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// TODO: The examples don't work on windows +@TestOn('!windows') +library; + +import 'dart:io'; + +import 'package:frontend_server_client/src/package_config_utils.dart'; +import 'package:test/test.dart'; +import 'package:test_process/test_process.dart'; + +void main() { + test('vm client example can build and rebuild an app', () async { + // Resolve the example script path based on the package root. + final exampleFilePath = await pathFromNearestPackageConfig( + 'example/vm_client.dart', + ); + final process = await TestProcess.start( + Platform.resolvedExecutable, ['run', exampleFilePath]); + await expectLater(process.stdout, + emitsThrough(contains('done compiling example/app/main.dart'))); + await expectLater( + process.stdout, emitsThrough(contains('APP -> hello/world'))); + await expectLater(process.stdout, + emitsThrough(contains('done recompiling example/app/main.dart'))); + await expectLater( + process.stdout, emitsThrough(contains('APP -> goodbye/world'))); + expect(await process.exitCode, 0); + }); +} diff --git a/pkg/frontend_server_client/test/example/web_client_test.dart b/pkg/frontend_server_client/test/example/web_client_test.dart new file mode 100644 index 00000000000..959ffea1099 --- /dev/null +++ b/pkg/frontend_server_client/test/example/web_client_test.dart @@ -0,0 +1,33 @@ +// Copyright 2022 The Dart Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// TODO: The examples don't work on windows +@TestOn('!windows') +library; + +import 'dart:io'; + +import 'package:frontend_server_client/src/package_config_utils.dart'; +import 'package:test/test.dart'; +import 'package:test_process/test_process.dart'; + +void main() { + test('web client example can build and rebuild an app', () async { + // Resolve the example script path based on the package root. + final exampleFilePath = await pathFromNearestPackageConfig( + 'example/web_client.dart', + ); + final process = await TestProcess.start( + Platform.resolvedExecutable, ['run', exampleFilePath]); + await expectLater(process.stdout, + emitsThrough(contains('done compiling example/app/main.dart'))); + process.stdin.writeln('new message'); + await expectLater( + process.stdout, + emitsThrough( + contains('Recompile succeeded for example/app/main.dart'))); + process.stdin.writeln('quit'); + expect(await process.exitCode, 0); + }); +} diff --git a/pkg/frontend_server_client/test/frontend_server_client_test.dart b/pkg/frontend_server_client/test/frontend_server_client_test.dart new file mode 100644 index 00000000000..b779da5ed7c --- /dev/null +++ b/pkg/frontend_server_client/test/frontend_server_client_test.dart @@ -0,0 +1,364 @@ +// Copyright 2020 The Dart Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +@TestOn('vm') +library; + +import 'dart:convert'; +import 'dart:io'; + +import 'package:async/async.dart'; +import 'package:frontend_server_client/frontend_server_client.dart'; +import 'package:package_config/package_config.dart'; +import 'package:path/path.dart' as p; +import 'package:test/test.dart'; +import 'package:test_descriptor/test_descriptor.dart' as d; +import 'package:vm_service/vm_service.dart'; +import 'package:vm_service/vm_service_io.dart'; + +void main() async { + FrontendServerClient? client; + late PackageConfig packageConfig; + late String packageRoot; + late String packagesJsonPath; + + setUp(() async { + await d.dir('a', [ + d.file('pubspec.yaml', ''' +name: a +dependencies: + path: ^1.0.0 + +environment: + sdk: ^3.0.0 + '''), + d.dir('bin', [ + d.file('main.dart', ''' +import 'package:path/path.dart' as p; + +void main() async { + print(message); + /// Runs in a loop until it is hot reloaded with a new message. + while (!message.contains('goodbye')) { + await Future.delayed(const Duration(seconds: 1)); + } + print(message); +} + +String get message => p.join('hello', 'world'); + +'''), + ]), + ]).create(); + packageRoot = p.join(d.sandbox, 'a'); + await Process.run( + Platform.resolvedExecutable, + [ + 'pub', + 'get', + ], + workingDirectory: packageRoot); + packageConfig = (await findPackageConfig(Directory(packageRoot)))!; + packagesJsonPath = findNearestPackageConfigPath(Directory(packageRoot)) ?? + p.join(packageRoot, '.dart_tool', 'package_config.json'); + }); + + tearDown(() async { + await client?.shutdown(); + }); + + test('can compile, recompile, and hot reload a vm app', () async { + final entrypoint = p.join(packageRoot, 'bin', 'main.dart'); + client = await FrontendServerClient.start( + entrypoint, + p.join(packageRoot, 'out.dill'), + vmPlatformDill, + packagesJson: packagesJsonPath, + ); + var result = await client!.compile(); + client!.accept(); + expect(result.compilerOutputLines, isEmpty); + expect(result.errorCount, 0); + expect( + result.newSources, + containsAll([ + File(entrypoint).uri, + packageConfig.resolve(Uri.parse('package:path/path.dart')), + ]), + ); + expect(result.removedSources, isEmpty); + expect(result.dillOutput, isNotNull); + expect(File(result.dillOutput!).existsSync(), true); + final process = await Process.start(Platform.resolvedExecutable, [ + '--observe', + '--no-pause-isolates-on-exit', + '--pause-isolates-on-start', + result.dillOutput!, + ]); + addTearDown(process.kill); + final stdoutLines = StreamQueue( + process.stdout.transform(utf8.decoder).transform(const LineSplitter()), + ); + addTearDown(stdoutLines.cancel); + + final observatoryLine = await stdoutLines.next; + final observatoryUri = + '${observatoryLine.split(' ').last.replaceFirst('http', 'ws')}ws'; + final vmService = await vmServiceConnectUri(observatoryUri); + addTearDown(vmService.dispose); + final isolate = await waitForIsolatesAndResume(vmService); + + await expectLater(stdoutLines, emitsThrough(p.join('hello', 'world'))); + + final appFile = File(entrypoint); + final originalContent = await appFile.readAsString(); + final newContent = originalContent.replaceFirst('hello', 'goodbye'); + await appFile.writeAsString(newContent); + + result = await client!.compile([File(entrypoint).uri]); + + client!.accept(); + expect(result.newSources, isEmpty); + expect(result.removedSources, isEmpty); + expect(result.compilerOutputLines, isEmpty); + expect(result.errorCount, 0); + expect(result.dillOutput, endsWith('.incremental.dill')); + + await vmService.reloadSources(isolate.id!, rootLibUri: result.dillOutput); + + expect(await stdoutLines.next, p.join('goodbye', 'world')); + expect(await process.exitCode, 0); + }); + + test('can handle compile errors and reload fixes', () async { + final entrypoint = p.join(packageRoot, 'bin', 'main.dart'); + final entrypointFile = File(entrypoint); + final originalContent = await entrypointFile.readAsString(); + // append two compile errors to the bottom + await entrypointFile.writeAsString( + '$originalContent\nint foo = 1.0;\nString bar = 4;', + ); + + client = await FrontendServerClient.start( + entrypoint, + p.join(packageRoot, 'out.dill'), + vmPlatformDill, + packagesJson: packagesJsonPath, + ); + var result = await client!.compile(); + + client!.accept(); + expect(result.errorCount, 2); + expect( + result.compilerOutputLines, + allOf(contains('int foo = 1.0;'), contains('String bar = 4;')), + ); + expect( + result.newSources, + containsAll([ + File(entrypoint).uri, + packageConfig.resolve(Uri.parse('package:path/path.dart')), + ]), + ); + expect(result.removedSources, isEmpty); + expect(result.dillOutput, isNotNull); + expect(File(result.dillOutput!).existsSync(), true); + + final process = await Process.start(Platform.resolvedExecutable, [ + '--observe', + '--no-pause-isolates-on-exit', + '--pause-isolates-on-start', + result.dillOutput!, + ]); + addTearDown(process.kill); + final stdoutLines = StreamQueue( + process.stdout.transform(utf8.decoder).transform(const LineSplitter()), + ); + addTearDown(stdoutLines.cancel); + + final observatoryLine = await stdoutLines.next; + final observatoryUri = + '${observatoryLine.split(' ').last.replaceFirst('http', 'ws')}ws'; + final vmService = await vmServiceConnectUri(observatoryUri); + addTearDown(vmService.dispose); + final isolate = await waitForIsolatesAndResume(vmService); + + // The program actually runs regardless of the errors, as they don't affect + // the runtime behavior. + await expectLater(stdoutLines, emitsThrough(p.join('hello', 'world'))); + + await entrypointFile.writeAsString( + originalContent.replaceFirst('hello', 'goodbye'), + ); + result = await client!.compile([entrypointFile.uri]); + client!.accept(); + expect(result.errorCount, 0); + expect(result.compilerOutputLines, isEmpty); + expect(result.newSources, isEmpty); + expect(result.removedSources, isEmpty); + expect(result.dillOutput, isNotNull); + expect(File(result.dillOutput!).existsSync(), true); + + await vmService.reloadSources(isolate.id!, rootLibUri: result.dillOutput); + + expect(await stdoutLines.next, p.join('goodbye', 'world')); + expect(await process.exitCode, 0); + }); + + test('can compile and recompile a dartdevc app', () async { + final entrypoint = + p.toUri(p.join(packageRoot, 'bin', 'main.dart')).toString(); + final dartDevcClient = client = await DartDevcFrontendServerClient.start( + entrypoint, + p.join(packageRoot, 'out.dill'), + platformKernel: p + .toUri(p.join(sdkDir, 'lib', '_internal', 'ddc_platform.dill')) + .toString(), + packagesJson: packagesJsonPath, + ); + var result = await client!.compile(); + client!.accept(); + + expect(result.compilerOutputLines, isEmpty); + expect(result.errorCount, 0); + expect( + result.newSources, + containsAll([ + Uri.parse(entrypoint), + packageConfig.resolve(Uri.parse('package:path/path.dart')), + ]), + ); + expect(result.removedSources, isEmpty); + + expect(result.dillOutput, isNotNull); + expect(File(result.jsManifestOutput!).existsSync(), true); + expect(File(result.jsSourcesOutput!).existsSync(), true); + expect(File(result.jsSourceMapsOutput!).existsSync(), true); + + final entrypointUri = Uri.parse(entrypoint); + expect( + utf8.decode(dartDevcClient.assetBytes('${entrypointUri.path}.lib.js')!), + contains('hello'), + ); + + final appFile = File(entrypointUri.toFilePath()); + final originalContent = await appFile.readAsString(); + final newContent = originalContent.replaceFirst('hello', 'goodbye'); + await appFile.writeAsString(newContent); + + result = await client!.compile([entrypointUri]); + client!.accept(); + expect(result.newSources, isEmpty); + expect(result.removedSources, isEmpty); + expect(result.compilerOutputLines, isEmpty); + expect(result.errorCount, 0); + expect(result.jsManifestOutput, endsWith('.incremental.dill.json')); + + expect( + utf8.decode(dartDevcClient.assetBytes('${entrypointUri.path}.lib.js')!), + contains('goodbye'), + ); + }); + + test('can enable experiments', () async { + await d.dir('a', [ + d.dir('bin', [ + d.file('nnbd.dart', ''' + +// Compile time error if nnbd is enabled +int x; + +void main() { + print(x); +} +'''), + ]), + ]).create(); + final entrypoint = p.join(packageRoot, 'bin', 'nnbd.dart'); + client = await FrontendServerClient.start( + entrypoint, + p.join(packageRoot, 'out.dill'), + vmPlatformDill, + enabledExperiments: ['non-nullable'], + packagesJson: packagesJsonPath, + ); + final result = await client!.compile(); + client!.accept(); + expect(result.errorCount, 1); + expect(result.compilerOutputLines, contains(contains('int x;'))); + }); + + test('can compile and recompile filenames with spaces', () async { + await d.dir('a', [ + d.dir('bin', [ + d.file('main with spaces.dart', ''' +void main() { + print('hello world'); +} +'''), + ]), + ]).create(); + + final entrypoint = p.join(packageRoot, 'bin', 'main with spaces.dart'); + client = await FrontendServerClient.start( + entrypoint, + p.join(packageRoot, 'out with spaces.dill'), + vmPlatformDill, + packagesJson: packagesJsonPath, + ); + var result = await client!.compile(); + client!.accept(); + expect(result.compilerOutputLines, isEmpty); + expect(result.errorCount, 0); + expect(result.newSources, containsAll([File(entrypoint).uri])); + expect(result.removedSources, isEmpty); + expect(result.dillOutput, isNotNull); + expect(File(result.dillOutput!).existsSync(), true); + var processResult = await Process.run(Platform.resolvedExecutable, [ + result.dillOutput!, + ]); + + expect(processResult.stdout, startsWith('hello world')); + expect(processResult.exitCode, 0); + + final appFile = File(entrypoint); + final originalContent = await appFile.readAsString(); + final newContent = originalContent.replaceFirst('hello', 'goodbye'); + await appFile.writeAsString(newContent); + result = await client!.compile([appFile.uri]); + expect(result.compilerOutputLines, isEmpty); + expect(result.errorCount, 0); + expect(result.newSources, isEmpty); + expect(result.removedSources, isEmpty); + + processResult = await Process.run(Platform.resolvedExecutable, [ + result.dillOutput!, + ]); + expect(processResult.stdout, startsWith('goodbye world')); + expect(processResult.exitCode, 0); + }); +} + +Future waitForIsolatesAndResume(VmService vmService) async { + var vm = await vmService.getVM(); + var isolates = vm.isolates; + while (isolates == null || isolates.isEmpty) { + await Future.delayed(const Duration(milliseconds: 100)); + vm = await vmService.getVM(); + isolates = vm.isolates; + } + final isolateRef = isolates.first; + var isolate = await vmService.getIsolate(isolateRef.id!); + while (isolate.pauseEvent?.kind != EventKind.kPauseStart) { + await Future.delayed(const Duration(milliseconds: 100)); + isolate = await vmService.getIsolate(isolateRef.id!); + } + await vmService.resume(isolate.id!); + return isolate; +} + +final vmPlatformDill = p + .toUri(p.join(sdkDir, 'lib', '_internal', 'vm_platform_strong.dill')) + .toString(); +final sdkDir = p.dirname(p.dirname(Platform.resolvedExecutable)); diff --git a/pkg/pkg.status b/pkg/pkg.status index f5a2695def2..b6c03ef3e22 100644 --- a/pkg/pkg.status +++ b/pkg/pkg.status @@ -256,6 +256,8 @@ vm_snapshot_analysis/test/*: SkipByDesign # Only meant to run on vm [ $system == windows ] front_end/test/bootstrap_test: Skip # Issue 31902 front_end/test/incremental_dart2js_load_from_dill_test: Pass, Slow +frontend_server_client/test/example/vm_client_test: SkipByDesign # Test is incompatible with Windows platform. +frontend_server_client/test/example/web_client_test: SkipByDesign # Test is incompatible with Windows platform. vm_service/test/private_rpcs/dev_fs_http_put_test: Skip # Windows disallows "?" in paths vm_service/test/private_rpcs/dev_fs_http_put_weird_char_test: Skip # Windows disallows "\r" in paths vm_service/test/private_rpcs/dev_fs_weird_char_test: Skip # Windows disallows "\r" in paths diff --git a/pubspec.yaml b/pubspec.yaml index 26ab9d72586..b68b4065060 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -48,6 +48,7 @@ workspace: - pkg/_fe_analyzer_shared - pkg/front_end - pkg/frontend_server + - pkg/frontend_server_client - pkg/heap_snapshot - pkg/js - pkg/js_ast @@ -96,8 +97,6 @@ workspace: # All third_party packages are retrieved via the DEPS-file and overridden here. dependency_overrides: - analysis_config: - path: third_party/pkg/webdev/_analysis_config args: path: third_party/pkg/core/pkgs/args async: @@ -156,8 +155,6 @@ dependency_overrides: path: third_party/pkg/tools/pkgs/file_testing fixnum: path: third_party/pkg/core/pkgs/fixnum - frontend_server_client: - path: third_party/pkg/webdev/frontend_server_client glob: path: third_party/pkg/tools/pkgs/glob graphs: diff --git a/tools/OWNERS_DEV_INFRA b/tools/OWNERS_DEV_INFRA new file mode 100644 index 00000000000..1aaa395db49 --- /dev/null +++ b/tools/OWNERS_DEV_INFRA @@ -0,0 +1,5 @@ +bkonyi@google.com +yjessy@google.com +nshahan@google.com +markzipan@google.com +natebiggs@google.com \ No newline at end of file