diff --git a/tools/testing/dart/compiler_configuration.dart b/tools/testing/dart/compiler_configuration.dart index 3cf7fffbcdd..b225c15b7b5 100644 --- a/tools/testing/dart/compiler_configuration.dart +++ b/tools/testing/dart/compiler_configuration.dart @@ -69,8 +69,6 @@ abstract class CompilerConfiguration { bool useFastStartup = configuration['fast_startup']; bool useKernelInDart2js = configuration['dart2js_with_kernel']; bool verifyKernel = configuration['verify-ir']; - bool useDFE = !configuration['noDFE']; - bool useFasta = !configuration['noUseFasta']; bool treeShake = !configuration['no-tree-shake']; switch (compiler) { @@ -104,16 +102,6 @@ abstract class CompilerConfiguration { useBlobs: useBlobs, isAndroid: configuration['system'] == 'android'); case 'dartk': - if (!useDFE) { - return ComposedCompilerConfiguration.createDartKConfiguration( - isChecked: isChecked, - isHostChecked: isHostChecked, - useSdk: useSdk, - verify: verifyKernel, - strong: isStrong, - treeShake: treeShake); - } - return new NoneCompilerConfiguration( isDebug: isDebug, isChecked: isChecked, @@ -121,29 +109,15 @@ abstract class CompilerConfiguration { useSdk: useSdk, hotReload: hotReload, hotReloadRollback: hotReloadRollback, - dfeMode: useFasta ? DFEMode.Fasta : DFEMode.DartK); - + useDFE: true); case 'dartkp': - if (!useDFE) { - return ComposedCompilerConfiguration.createDartKPConfiguration( - isChecked: isChecked, - isHostChecked: isHostChecked, - arch: configuration['arch'], - useBlobs: useBlobs, - isAndroid: configuration['system'] == 'android', - useSdk: useSdk, - verify: verifyKernel, - strong: isStrong, - treeShake: treeShake); - } return new PrecompilerCompilerConfiguration( isDebug: isDebug, isChecked: isChecked, arch: configuration['arch'], useBlobs: useBlobs, isAndroid: configuration['system'] == 'android', - dfeMode: useFasta ? DFEMode.Fasta : DFEMode.DartK); - + useDFE: true); case 'none': return new NoneCompilerConfiguration( isDebug: isDebug, @@ -210,23 +184,17 @@ abstract class CompilerConfiguration { } } -enum DFEMode { - None, - DartK, - Fasta -} - /// The "none" compiler. class NoneCompilerConfiguration extends CompilerConfiguration { final bool hotReload; final bool hotReloadRollback; - final DFEMode dfeMode; + final bool useDFE; NoneCompilerConfiguration( {bool isDebug, bool isChecked, bool isHostChecked, bool useSdk, bool this.hotReload, bool this.hotReloadRollback, - DFEMode this.dfeMode: DFEMode.None}) + this.useDFE: false}) : super._subclass( isDebug: isDebug, isChecked: isChecked, @@ -244,12 +212,9 @@ class NoneCompilerConfiguration extends CompilerConfiguration { List originalArguments, CommandArtifact artifact) { List args = []; - if (dfeMode != DFEMode.None) { + if (useDFE) { args.add('--dfe=utils/kernel-service/kernel-service.dart'); } - if (dfeMode == DFEMode.Fasta) { - args.add('-DDFE_USE_FASTA=true'); - } if (isChecked) { args.add('--enable_asserts'); args.add('--enable_type_checks'); @@ -613,10 +578,10 @@ class PrecompilerCompilerConfiguration extends CompilerConfiguration { final String arch; final bool useBlobs; final bool isAndroid; - final DFEMode dfeMode; + final bool useDFE; PrecompilerCompilerConfiguration({bool isDebug, bool isChecked, - this.arch, this.useBlobs, this.isAndroid, this.dfeMode: DFEMode.None}) + this.arch, this.useBlobs, this.isAndroid, this.useDFE: false}) : super._subclass(isDebug: isDebug, isChecked: isChecked); int computeTimeoutMultiplier() { @@ -657,12 +622,9 @@ class PrecompilerCompilerConfiguration extends CompilerConfiguration { exec = "$buildDir/dart_bootstrap"; } var args = new List(); - if (dfeMode != DFEMode.None) { + if (useDFE) { args.add('--dfe=utils/kernel-service/kernel-service.dart'); } - if (dfeMode == DFEMode.Fasta) { - args.add('-DDFE_USE_FASTA=true'); - } args.add("--snapshot-kind=app-aot"); if (useBlobs) { args.add("--snapshot=$tempDir/out.aotsnapshot"); diff --git a/tools/testing/dart/runtime_configuration.dart b/tools/testing/dart/runtime_configuration.dart index 94a5dd59658..c077e7d9690 100644 --- a/tools/testing/dart/runtime_configuration.dart +++ b/tools/testing/dart/runtime_configuration.dart @@ -227,9 +227,6 @@ class StandaloneDartRuntimeConfiguration extends DartVmRuntimeConfiguration { CommandArtifact artifact, List arguments, Map environmentOverrides) { - final bool needsDFERunner = suite.configuration['compiler'] == 'dartk' && - suite.configuration['noUseFasta'] && - !suite.configuration['noBatch']; String script = artifact.filename; String type = artifact.mimeType; if (script != null && @@ -239,7 +236,7 @@ class StandaloneDartRuntimeConfiguration extends DartVmRuntimeConfiguration { } String executable = suite.dartVmBinaryFileName; return [ - commandBuilder.getVmCommand(executable, arguments, environmentOverrides, needsDFERunner: needsDFERunner) + commandBuilder.getVmCommand(executable, arguments, environmentOverrides) ]; } } diff --git a/tools/testing/dart/test_options.dart b/tools/testing/dart/test_options.dart index bb7d7fbd633..d15680179c2 100644 --- a/tools/testing/dart/test_options.dart +++ b/tools/testing/dart/test_options.dart @@ -300,12 +300,6 @@ class TestOptionsParser { new _TestOptionSpecification( 'no-tree-shake', 'Disable kernel IR tree shaking', ['--no-tree-shake'], [], false, type: 'bool'), - new _TestOptionSpecification( - 'noDFE', 'Do not use Kernel Isolate', ['--no-dfe'], [], false, - type: 'bool'), - new _TestOptionSpecification( - 'noUseFasta', 'Use Fasta in Kernel Isolate', ['--no-use-fasta'], [], false, - type: 'bool'), new _TestOptionSpecification( 'list', 'List tests only, do not run them', ['--list'], [], false, type: 'bool'), diff --git a/tools/testing/dart/test_runner.dart b/tools/testing/dart/test_runner.dart index 51bcde69415..9df2b2565a7 100644 --- a/tools/testing/dart/test_runner.dart +++ b/tools/testing/dart/test_runner.dart @@ -364,25 +364,14 @@ class AnalysisCommand extends ProcessCommand { } class VmCommand extends ProcessCommand { - final bool needsDFERunner; VmCommand._(String executable, List arguments, - Map environmentOverrides, - bool this.needsDFERunner) + Map environmentOverrides) : super._("vm", executable, arguments, environmentOverrides); - - void _buildHashCode(HashCodeBuilder builder) { - super._buildHashCode(builder); - builder.add(needsDFERunner); - } - - bool _equal(VmCommand other) => - super._equal(other) && needsDFERunner == other.needsDFERunner; } class VmBatchCommand extends ProcessCommand implements VmCommand { final String dartFile; final bool checked; - final needsDFERunner = false; VmBatchCommand._(String executable, String dartFile, List arguments, Map environmentOverrides, {this.checked: true}) @@ -651,8 +640,8 @@ class CommandBuilder { } VmCommand getVmCommand(String executable, List arguments, - Map environmentOverrides, {bool needsDFERunner: false}) { - var command = new VmCommand._(executable, arguments, environmentOverrides, needsDFERunner); + Map environmentOverrides) { + var command = new VmCommand._(executable, arguments, environmentOverrides); return _getUniqueCommand(command); } @@ -1901,12 +1890,10 @@ class RunningProcess { bool compilationSkipped = false; Completer completer; Map configuration; - List preArguments; RunningProcess(this.command, this.timeout, - {this.configuration, - this.preArguments}); + {this.configuration}); Future run() { completer = new Completer(); @@ -1923,9 +1910,6 @@ class RunningProcess { } else { var processEnvironment = _createProcessEnvironment(); var args = command.arguments; - if (preArguments != null) { - args = []..addAll(preArguments)..addAll(args); - } Future processFuture = io.Process.start( command.executable, args, environment: processEnvironment, @@ -2333,102 +2317,6 @@ class BatchRunnerProcess { } } -class BatchDFEProcess { - io.Process _process; - int _port = -1; - Function _processExitHandler; - - Completer terminating = null; - - bool locked = false; - - Future acquire() async { - try { - assert(!locked); - locked = true; - if (_process == null) { - await _startProcess(); - } - return _port; - } catch(e) { - locked = false; - rethrow; - } - } - - void release() { - locked = false; - } - - Future terminate() { - locked = true; - if (_process == null) { - return new Future.value(true); - } - if (terminating == null) { - terminating = new Completer(); - _process.kill(); - } - return terminating.future; - } - - _onExit(exitCode) { - if (terminating != null) { - terminating.complete(); - return; - } - - _process = null; - locked = false; - _port = -1; - } - - static Future _firstLine(stream) { - var completer = new Completer(); - stream.transform(UTF8.decoder) - .transform(new LineSplitter()) - .listen((line) { - if (!completer.isCompleted) { - completer.complete(line); - } - // We need to drain a pipe continuously. - }, onDone: () { - if (!completer.isCompleted) { - completer.completeError( - "DFE kernel compiler server did not sucessfully start up"); - } - }); - return completer.future; - } - - Future _startProcess() async { - final executable = io.Platform.executable; - final arguments = ['utils/kernel-service/kernel-service.dart', '--batch']; - - try { - _port = -1; - _process = await io.Process.start(executable, arguments); - _process.exitCode.then(_onExit); - _process.stderr.transform(UTF8.decoder).listen(DebugLogger.error); - - final readyMsg = await _firstLine(_process.stdout); - final data = readyMsg.split(' '); - assert(data[0] == 'READY'); - - _port = int.parse(data[1]); - } catch (e) { - print("Process error:"); - print(" Command: $executable ${arguments.join(' ')}"); - print(" Error: $e"); - // If there is an error starting a batch process, chances are that - // it will always fail. So rather than re-trying a 1000+ times, we - // exit. - io.exit(1); - return true; - } - } -} - /** * [TestCaseEnqueuer] takes a list of TestSuites, generates TestCases and * builds a dependency graph of all commands in every TestSuite. @@ -2737,8 +2625,6 @@ class CommandExecutorImpl implements CommandExecutor { // We keep a BrowserTestRunner for every configuration. final _browserTestRunners = new Map(); - List _dfeProcesses = null; - bool _finishing = false; CommandExecutorImpl( @@ -2763,13 +2649,9 @@ class CommandExecutorImpl implements CommandExecutor { return Future.wait(futures); } - Future _terminateDFEWorkers() => - Future.wait((_dfeProcesses ?? []).map((p) => p.terminate())); - return Future.wait([ _terminateBatchRunners(), _terminateBrowserRunners(), - _terminateDFEWorkers() ]); } @@ -2818,13 +2700,6 @@ class CommandExecutorImpl implements CommandExecutor { adbDevicePool.releaseDevice(device); }); }); - } else if (command is VmCommand && command.needsDFERunner) { - final runner = _getDFEProcess(); - return runner.acquire().then((port) { - return new RunningProcess(command, timeout, - configuration: globalConfiguration, - preArguments: ['-DDFE_WORKER_PORT=${port}']).run(); - }).whenComplete(() => runner.release()); } else if (command is VmBatchCommand) { var name = command.displayName; return _getBatchRunner(command.displayName + command.dartFile) @@ -2928,12 +2803,6 @@ class CommandExecutorImpl implements CommandExecutor { throw new Exception('Unable to find inactive batch runner.'); } - BatchDFEProcess _getDFEProcess() { - _dfeProcesses ??= new List.generate(maxProcesses, - (_) => new BatchDFEProcess()); - return _dfeProcesses.firstWhere((runner) => !runner.locked); - } - Future _startBrowserControllerTest( BrowserTestCommand browserCommand, int timeout) { var completer = new Completer(); diff --git a/utils/kernel-service/kernel-service.dart b/utils/kernel-service/kernel-service.dart index cef18297350..d8fdbd08856 100644 --- a/utils/kernel-service/kernel-service.dart +++ b/utils/kernel-service/kernel-service.dart @@ -10,30 +10,13 @@ /// This is either invoked as the root script of the Kernel isolate when used /// as a part of /// -/// dart --dfe=utils/kernel-service/kernel-service.dart ... +/// dart --dfe=utils/kernel-service/kernel-service.dart ... /// -/// invocation or it is invoked as a standalone script to perform batch mode -/// compilation requested via an HTTP interface +/// invocation or it is invoked as a standalone script to perform training for +/// the app-jit snapshot /// -/// dart utils/kernel-service/kernel-service.dart --batch +/// dart utils/kernel-service/kernel-service.dart --train /// -/// The port for the batch mode worker is controlled by DFE_WORKER_PORT -/// environment declarations (set by -DDFE_WORKER_PORT=... command line flag). -/// When not set (or set to 0) an ephemeral port returned by the OS is used -/// instead. -/// -/// When this script is used as a Kernel isolate root script and DFE_WORKER_PORT -/// is set to non-zero value then Kernel isolate will forward all compilation -/// requests it receives to the batch worker on the given port. -/// -/// Set DFE_USE_FASTA environment declaration to true to use fasta front-end -/// instead of dartk. Note: we expect patched_sdk folder to contain -/// platform.dill file that contains patched SDK in the Kernel binary form. -/// This file can be created using the following command line: -/// -/// export DART_AOT_SDK= -/// dart pkg/front_end/lib/src/fasta/bin/compile_platform.dart \ -/// ${DART_AOT_SDK}/platform.dill /// library runtime.tools.kernel_service; @@ -43,7 +26,6 @@ import 'dart:io'; import 'dart:isolate'; import 'dart:typed_data'; -import 'package:kernel/analyzer/loader.dart'; import 'package:kernel/binary/ast_to_binary.dart'; import 'package:kernel/kernel.dart'; import 'package:kernel/target/targets.dart'; @@ -57,8 +39,6 @@ import 'package:front_end/src/fasta/ast_kind.dart' show AstKind; import 'package:front_end/src/fasta/errors.dart' show InputError; const bool verbose = const bool.fromEnvironment('DFE_VERBOSE'); -const int workerPort = const int.fromEnvironment('DFE_WORKER_PORT') ?? 0; -const bool useFasta = const bool.fromEnvironment('DFE_USE_FASTA'); class DataSink implements Sink> { final BytesBuilder builder = new BytesBuilder(); @@ -93,19 +73,6 @@ class CompilationOk extends CompilationResult { abstract class CompilationFail extends CompilationResult { String get errorString; - - Map toJson(); - - static CompilationFail fromJson(Map m) { - switch (m['status']) { - case STATUS_ERROR: - return new CompilationError(m['errors']); - case STATUS_CRASH: - return new CompilationCrash(m['exception'], m['stack']); - default: - throw "Can't deserialize CompilationFail from ${m}."; - } - } } class CompilationError extends CompilationFail { @@ -115,11 +82,6 @@ class CompilationError extends CompilationFail { List toResponse() => [STATUS_ERROR, errorString]; - Map toJson() => { - "status": STATUS_ERROR, - "errors": errors, - }; - String get errorString => errors.take(10).join('\n'); String toString() => "CompilationError(${errorString})"; @@ -133,18 +95,12 @@ class CompilationCrash extends CompilationFail { List toResponse() => [STATUS_CRASH, errorString]; - Map toJson() => { - "status": STATUS_CRASH, - "exception": exception, - "stack": stack, - }; - String get errorString => "${exception}\n${stack}"; String toString() => "CompilationCrash(${errorString})"; } -Future parseScriptImpl(DartLoaderBatch batch_loader, +Future parseScriptImpl( Uri fileName, String packageConfig, String sdkPath) async { if (!FileSystemEntity.isFileSync(fileName.path)) { throw "Input file '${fileName.path}' does not exist."; @@ -157,42 +113,24 @@ Future parseScriptImpl(DartLoaderBatch batch_loader, Target target = getTarget("vm", new TargetFlags(strongMode: false)); Program program; - if (useFasta) { - final uriTranslator = await TranslateUri.parse(new Uri.file(packageConfig)); - final Ticker ticker = new Ticker(isVerbose: verbose); - final DillTarget dillTarget = new DillTarget(ticker, uriTranslator); - dillTarget.read(new Uri.directory(sdkPath).resolve('platform.dill')); - final KernelTarget kernelTarget = - new KernelTarget(dillTarget, uriTranslator); - try { - kernelTarget.read(fileName); - await dillTarget.writeOutline(null); - program = await kernelTarget.writeOutline(null); - program = await kernelTarget.writeProgram(null, AstKind.Kernel); - if (kernelTarget.errors.isNotEmpty) { - return new CompilationError(kernelTarget.errors - .map((err) => err.toString()) - .toList(growable: false)); - } - } on InputError catch (e) { - return new CompilationError([e.format()]); - } - } else { - DartOptions dartOptions = new DartOptions( - strongMode: false, - strongModeSdk: false, - sdk: sdkPath, - packagePath: packageConfig, - customUriMappings: const {}, - declaredVariables: const {}); - program = new Program(); - DartLoader loader = - await batch_loader.getLoader(program, dartOptions); - loader.loadProgram(fileName, target: target); - - if (loader.errors.isNotEmpty) { - return new CompilationError(loader.errors.toList(growable: false)); + final uriTranslator = await TranslateUri.parse(new Uri.file(packageConfig)); + final Ticker ticker = new Ticker(isVerbose: verbose); + final DillTarget dillTarget = new DillTarget(ticker, uriTranslator); + dillTarget.read(new Uri.directory(sdkPath).resolve('platform.dill')); + final KernelTarget kernelTarget = + new KernelTarget(dillTarget, uriTranslator); + try { + kernelTarget.read(fileName); + await dillTarget.writeOutline(null); + program = await kernelTarget.writeOutline(null); + program = await kernelTarget.writeProgram(null, AstKind.Kernel); + if (kernelTarget.errors.isNotEmpty) { + return new CompilationError(kernelTarget.errors + .map((err) => err.toString()) + .toList(growable: false)); } + } on InputError catch (e) { + return new CompilationError([e.format()]); } // Perform target-specific transformations. @@ -205,10 +143,10 @@ Future parseScriptImpl(DartLoaderBatch batch_loader, return new CompilationOk(sink.builder.takeBytes()); } -Future parseScript(DartLoaderBatch loader, Uri fileName, - String packageConfig, String sdkPath) async { +Future parseScript( + Uri fileName, String packageConfig, String sdkPath) async { try { - return await parseScriptImpl(loader, fileName, packageConfig, sdkPath); + return await parseScriptImpl(fileName, packageConfig, sdkPath); } catch (err, stack) { return new CompilationCrash(err.toString(), stack.toString()); } @@ -249,12 +187,7 @@ Future _processLoadRequestImpl(String inputFileUrl) async { }"""); } - if (workerPort != 0) { - return await requestParse(scriptUri, packagesUri, patchedSdk); - } else { - return await parseScript( - new DartLoaderBatch(), scriptUri, packagesUri.path, patchedSdk.path); - } + return await parseScript(scriptUri, packagesUri.path, patchedSdk.path); } // Process a request from the runtime. See KernelIsolate::CompileToKernel in @@ -296,66 +229,6 @@ Future _processLoadRequest(request) async { } } -Future requestParse( - Uri scriptUri, Uri packagesUri, Uri patchedSdk) async { - if (verbose) { - print( - "DFE: forwarding request to worker at http://localhost:${workerPort}/"); - } - - HttpClient client = new HttpClient(); - final rq = await client - .postUrl(new Uri(host: 'localhost', port: workerPort, scheme: 'http')); - rq.headers.contentType = ContentType.JSON; - rq.write(JSON.encode({ - "inputFileUrl": scriptUri.toString(), - "packagesUri": packagesUri.toString(), - "patchedSdk": patchedSdk.toString(), - })); - final rs = await rq.close(); - try { - if (rs.statusCode == HttpStatus.OK) { - final BytesBuilder bb = new BytesBuilder(); - await rs.forEach(bb.add); - return new CompilationOk(bb.takeBytes()); - } else { - return CompilationFail.fromJson(JSON.decode(await UTF8.decodeStream(rs))); - } - } finally { - await client.close(); - } -} - -void startBatchServer() { - final loader = new DartLoaderBatch(); - HttpServer.bind('localhost', workerPort).then((server) { - print('READY ${server.port}'); - server.listen((HttpRequest request) async { - final rq = JSON.decode(await UTF8.decodeStream(request)); - - final Uri scriptUri = Uri.parse(rq['inputFileUrl']); - final Uri packagesUri = Uri.parse(rq['packagesUri']); - final Uri patchedSdk = Uri.parse(rq['patchedSdk']); - - final CompilationResult result = await parseScript( - loader, scriptUri, packagesUri.path, patchedSdk.path); - - if (result is CompilationOk) { - request.response.statusCode = HttpStatus.OK; - request.response.headers.contentType = ContentType.BINARY; - request.response.add(result.binary); - request.response.close(); - } else { - request.response.statusCode = HttpStatus.INTERNAL_SERVER_ERROR; - request.response.headers.contentType = ContentType.TEXT; - request.response.write(JSON.encode(result)); - request.response.close(); - } - }); - ProcessSignal.SIGTERM.watch().first.then((_) => server.close()); - }); -} - train(String scriptUri) { // TODO(28532): Enable on Windows. if (Platform.isWindows) return; @@ -378,9 +251,7 @@ train(String scriptUri) { } main([args]) { - if (args?.length == 1 && args[0] == '--batch') { - startBatchServer(); - } else if (args?.length == 2 && args[0] == '--train') { + if (args?.length == 2 && args[0] == '--train') { // This entry point is used when creating an app snapshot. The argument // provides a script to compile to warm-up generated code. train(args[1]);