diff --git a/pkg/dynamic_modules/test/runner/aot.dart b/pkg/dynamic_modules/test/runner/aot.dart index 76fd0053b7f..18ffb1a94fa 100644 --- a/pkg/dynamic_modules/test/runner/aot.dart +++ b/pkg/dynamic_modules/test/runner/aot.dart @@ -99,14 +99,12 @@ class AotExecutor implements TargetExecutor { var testDir = _tmp.uri.resolve(test.name).toFilePath(); var args = [ - 'compile', - 'exe', + '--snapshot-kind=app-aot-elf', + '--elf=${test.main}.snapshot', '${test.main}_aot.dill', - '--output', - '${test.main}.exe', ]; - await runProcess(Platform.resolvedExecutable, args, testDir, _logger, - 'compile exe ${test.name}/${test.main}'); + await runProcess(genSnapshotBin.toFilePath(), args, testDir, _logger, + 'aot snapshot ${test.name}/${test.main}'); } @override @@ -151,8 +149,10 @@ class AotExecutor implements TargetExecutor { // and finally launches the app. var testDir = _tmp.uri.resolve('${test.name}/'); var result = await runProcess( - testDir.resolve('${test.main}.exe').toFilePath(), - [], + aotRuntimeBin.toFilePath(), + [ + '${test.main}.snapshot', + ], testDir.toFilePath(), _logger, 'executable test ${test.main}.exe'); diff --git a/pkg/dynamic_modules/test/runner/main.dart b/pkg/dynamic_modules/test/runner/main.dart index 3c42f13016d..7a779e58b3e 100644 --- a/pkg/dynamic_modules/test/runner/main.dart +++ b/pkg/dynamic_modules/test/runner/main.dart @@ -5,6 +5,8 @@ /// Entrypoint to run dynamic module tests. library; +import 'dart:io'; + import 'package:args/args.dart'; import 'aot.dart'; @@ -29,7 +31,9 @@ void main(List args) async { defaultsTo: Target.ddc.name, abbr: 'r') ..addOption('configuration', - help: 'Configuration to use for reporting test results', abbr: 'c') + help: 'Configuration to use for reporting test results', abbr: 'n') + ..addOption('output-directory', + help: 'location where to emit the json-l result and log files') ..addFlag('verbose', help: 'Show a lot of information', negatable: false, abbr: 'v'); final options = parser.parse(args); @@ -58,7 +62,13 @@ void main(List args) async { for (final t in tests) { results.add(await _runSingleTest(t, executor)); } - _reportResults(results); + final result = _reportResults(results, + writeLog: singleTest == null, + configuration: options['configuration'], + logDir: options['output-directory']); + if (result != 0) { + exitCode = result; + } } finally { executor.suiteComplete(); } @@ -68,35 +78,68 @@ void main(List args) async { /// on the target environment. Future _runSingleTest( DynamicModuleTest test, TargetExecutor target) async { + var timer = Stopwatch()..start(); try { await target.compileApplication(test); for (var name in test.dynamicModules.keys) { await target.compileDynamicModule(test, name); } } catch (e, st) { - return DynamicModuleTestResult.compileError(test, '$e\n$st'); + return DynamicModuleTestResult.compileError(test, '$e\n$st', timer.elapsed); } try { await target.executeApplication(test); } catch (e, st) { - return DynamicModuleTestResult.runtimeError(test, '$e\n$st'); + return DynamicModuleTestResult.runtimeError(test, '$e\n$st', timer.elapsed); } - return DynamicModuleTestResult.pass(test); + return DynamicModuleTestResult.pass(test, timer.elapsed); } /// Generates a report of the test results in the JSON format /// that is expected by our testing infrastructure. -void _reportResults(List results) { - // TODO(sigmund): replace this with proper infra reporting +int _reportResults( + List results, { + required bool writeLog, + String? configuration, + String? logDir, +}) { bool fail = false; print('Test results:'); for (var result in results) { print(' ${result.name}: ${result.status}'); if (result.status != Status.pass) fail = true; } - if (fail) throw "Some tests failed..."; + if (fail) print('Error: some tests failed'); + + if (writeLog) { + if (logDir == null) { + print('Error: no output directory provided, logs won\'t be emitted.'); + return 1; + } + if (configuration == null) { + print('Error: no configuration name provided, logs won\'t be emitted.'); + return 1; + } + + // Ensure the directory URI ends with a path separator. + var dirUri = Directory(logDir).uri; + File.fromUri(dirUri.resolve('results.json')).writeAsStringSync( + results.map((r) => '${r.toRecordJson(configuration)}\n').join(), + flush: true); + File.fromUri(dirUri.resolve('logs.json')).writeAsStringSync( + results + .where((r) => r.status != Status.pass) + .map((r) => '${r.toLogJson(configuration)}\n') + .join(), + flush: true); + + print('Success: log files emitted under $dirUri'); + } else if (fail) { + return 1; + } + return 0; } /// Placeholder until we implement all executors. diff --git a/pkg/dynamic_modules/test/runner/model.dart b/pkg/dynamic_modules/test/runner/model.dart index 3d1a2a626d2..e058e7e1317 100644 --- a/pkg/dynamic_modules/test/runner/model.dart +++ b/pkg/dynamic_modules/test/runner/model.dart @@ -2,6 +2,8 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'dart:convert'; + /// Data needed to build and run a single dynamic module test. class DynamicModuleTest { /// Name of the test. Matches the folder containing the test. @@ -30,19 +32,56 @@ class DynamicModuleTestResult { final String name; final Status status; final String details; + final Duration time; - DynamicModuleTestResult._(this.name, this.status, this.details); + DynamicModuleTestResult._(this.name, this.status, this.details, this.time); - factory DynamicModuleTestResult.pass(DynamicModuleTest test) => - DynamicModuleTestResult._(test.name, Status.pass, ''); + factory DynamicModuleTestResult.pass(DynamicModuleTest test, Duration time) => + DynamicModuleTestResult._(test.name, Status.pass, '', time); factory DynamicModuleTestResult.compileError( - DynamicModuleTest test, String details) => - DynamicModuleTestResult._(test.name, Status.compileTimeError, details); + DynamicModuleTest test, String details, Duration time) => + DynamicModuleTestResult._( + test.name, Status.compileTimeError, details, time); factory DynamicModuleTestResult.runtimeError( - DynamicModuleTest test, String details) => - DynamicModuleTestResult._(test.name, Status.runtimeError, details); + DynamicModuleTest test, String details, Duration time) => + DynamicModuleTestResult._(test.name, Status.runtimeError, details, time); + + /// Emit the result in the JSON format expected by the test infrastructure. + String toRecordJson(String configuration) { + final outcome = switch (status) { + Status.pass => 'Pass', + Status.compileTimeError => 'CompileTimeError', + Status.runtimeError => 'RuntimeError', + }; + return jsonEncode({ + 'name': 'dynamic_modules_suite/$name', + 'configuration': configuration, + 'suite': 'dynamic_modules_suite', + 'test_name': name, + 'time_ms': time.inMilliseconds, + 'expected': 'Pass', + 'result': outcome, + 'matches': status == Status.pass, + }); + } + + /// Emit the log entry with details of a failure in the JSON format expected + /// by the test infrastructure. + String toLogJson(String configuration) { + final outcome = switch (status) { + Status.pass => 'Pass', + Status.compileTimeError => 'CompileTimeError', + Status.runtimeError => 'RuntimeError', + }; + return jsonEncode({ + 'name': 'dynamic_modules_suite/$name', + 'configuration': configuration, + 'result': outcome, + 'log': details, + }); + } } enum Status { pass, compileTimeError, runtimeError } diff --git a/pkg/dynamic_modules/test/runner/util.dart b/pkg/dynamic_modules/test/runner/util.dart index 11ce7d70652..952bc4e708a 100644 --- a/pkg/dynamic_modules/test/runner/util.dart +++ b/pkg/dynamic_modules/test/runner/util.dart @@ -25,8 +25,29 @@ Uri repoRoot = (() { })(); String _outFolder = Platform.isMacOS ? 'xcodebuild' : 'out'; -String configuration = - Platform.environment['DART_CONFIGURATION'] ?? 'ReleaseX64'; + +String configuration = () { + var env = Platform.environment['DART_CONFIGURATION']; + if (env != null) return env; + var folderSegments = _dartBin.resolve('.').pathSegments; + for (int i = folderSegments.length - 1; i > 0; i--) { + if (folderSegments[i] == _outFolder) { + var candidate = folderSegments[i + 1]; + if (candidate.startsWith('Debug') || + candidate.startsWith('Release') || + candidate.startsWith('Product')) { + return candidate; + } + } + } + return 'ReleaseX64'; +}(); + +// See also utils/gen_kernel/BUILD.gn: +// dartaotruntime has dart_product_config applied to it and is built in product +// mode in both release and product builds. +bool get useProduct => !configuration.startsWith('Debug'); + String buildFolder = '$_outFolder/$configuration/'; String arch = Abi.current().toString().split('_')[1]; String _d8Path = (() { @@ -58,13 +79,15 @@ Uri ddcModuleLoaderJs = repoRoot.resolve('pkg/dev_compiler/lib/js/ddc/ddc_module_loader.js'); Uri genKernelSnapshot = - _dartBin.resolve('snapshots/gen_kernel_aot.dart.snapshot'); + buildRootUri.resolve('gen/gen_kernel_aot.dart.snapshot'); +Uri genSnapshotBin = + buildRootUri.resolve(useProduct ? 'gen_snapshot_product' : 'gen_snapshot'); Uri dart2bytecodeSnapshot = - _dartBin.resolve('snapshots/dart2bytecode.dart.snapshot'); -Uri aotRuntimeBin = - Uri.parse(Platform.resolvedExecutable).resolve('dartaotruntime'); -Uri vmPlatformDill = - _dartBin.resolve('../lib/_internal/vm_platform_strong_product.dill'); + buildRootUri.resolve('gen/dart2bytecode.dart.snapshot'); +Uri aotRuntimeBin = buildRootUri.resolve(useProduct + ? 'dart_precompiled_runtime_product' + : 'dart_precompiled_runtime'); +Uri vmPlatformDill = buildRootUri.resolve('vm_platform_strong.dill'); // Encodes test results in the format expected by Dart's CI infrastructure. class TestResultOutcome { diff --git a/tools/bots/test_matrix.json b/tools/bots/test_matrix.json index 05f6263d213..30201d52744 100644 --- a/tools/bots/test_matrix.json +++ b/tools/bots/test_matrix.json @@ -1331,7 +1331,8 @@ "arguments": [ "--dart-dynamic-modules", "runtime", - "runtime_precompiled" + "runtime_precompiled", + "utils/gen_kernel" ] }, { @@ -1343,6 +1344,17 @@ ], "fileset": "vm", "shards": 5 + }, + { + "name": "dynamic module tests", + "script": "out/DebugX64/dart", + "testRunner": true, + "arguments": [ + "pkg/dynamic_modules/test/runner/main.dart", + "-nvm-aot-dyn-${system}-${mode}-${arch}", + "-raot", + "--verbose" + ] } ] }, @@ -1878,6 +1890,17 @@ "--use-sdk" ] }, + { + "name": "ddc dynamic module tests", + "script": "out/ReleaseX64/dart-sdk/bin/dart", + "testRunner": true, + "arguments": [ + "pkg/dynamic_modules/test/runner/main.dart", + "-nddc-${system}-chrome", + "-rddc", + "--verbose" + ] + }, { "name": "ddc sourcemap tests", "script": "out/ReleaseX64/dart",