dart-lang/sdk
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[dynamic_modules] Run dynamic module tests in CQ

Browse Source 
* Adds support to emit log records for test outcomes and failure logs
* Adds steps to the test_matrix

Change-Id: Ibabf0410a0304aae446387a0d3ca147488f56df3
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/383929
Reviewed-by: Alexander Markov <alexmarkov@google.com>
Commit-Queue: Sigmund Cherem <sigmund@google.com>
This commit is contained in:
Sigmund Cherem
2024-09-12 15:36:40 +00:00
committed by Commit Queue
parent e2efffa005
commit fd3cb424e2
5 changed files with 160 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/dynamic_modules/test/runner/aot.dart
+8 -8
View File
@@ -99,14 +99,12 @@ class AotExecutor implements TargetExecutor {
var testDir = _tmp.uri.resolve(test.name).toFilePath();
var args = [
'compile',
'exe',
'--snapshot-kind=app-aot-elf',
'--elf=${test.main}.snapshot',
'${test.main}_aot.dill',
'--output',
'${test.main}.exe',
];
await runProcess(Platform.resolvedExecutable, args, testDir, _logger,
'compile exe ${test.name}/${test.main}');
await runProcess(genSnapshotBin.toFilePath(), args, testDir, _logger,
'aot snapshot ${test.name}/${test.main}');
}
@override
@@ -151,8 +149,10 @@ class AotExecutor implements TargetExecutor {
// and finally launches the app.
var testDir = _tmp.uri.resolve('${test.name}/');
var result = await runProcess(
testDir.resolve('${test.main}.exe').toFilePath(),
[],
aotRuntimeBin.toFilePath(),
[
'${test.main}.snapshot',
],
testDir.toFilePath(),
_logger,
'executable test ${test.main}.exe');
pkg/dynamic_modules/test/runner/main.dart
+51 -8
View File
@@ -5,6 +5,8 @@
/// Entrypoint to run dynamic module tests.
library;
import 'dart:io';
import 'package:args/args.dart';
import 'aot.dart';
@@ -29,7 +31,9 @@ void main(List<String> args) async {
defaultsTo: Target.ddc.name,
abbr: 'r')
..addOption('configuration',
help: 'Configuration to use for reporting test results', abbr: 'c')
help: 'Configuration to use for reporting test results', abbr: 'n')
..addOption('output-directory',
help: 'location where to emit the json-l result and log files')
..addFlag('verbose',
help: 'Show a lot of information', negatable: false, abbr: 'v');
final options = parser.parse(args);
@@ -58,7 +62,13 @@ void main(List<String> args) async {
for (final t in tests) {
results.add(await _runSingleTest(t, executor));
}
_reportResults(results);
final result = _reportResults(results,
writeLog: singleTest == null,
configuration: options['configuration'],
logDir: options['output-directory']);
if (result != 0) {
exitCode = result;
}
} finally {
executor.suiteComplete();
}
@@ -68,35 +78,68 @@ void main(List<String> args) async {
/// on the target environment.
Future<DynamicModuleTestResult> _runSingleTest(
DynamicModuleTest test, TargetExecutor target) async {
var timer = Stopwatch()..start();
try {
await target.compileApplication(test);
for (var name in test.dynamicModules.keys) {
await target.compileDynamicModule(test, name);
}
} catch (e, st) {
return DynamicModuleTestResult.compileError(test, '$e\n$st');
return DynamicModuleTestResult.compileError(test, '$e\n$st', timer.elapsed);
}
try {
await target.executeApplication(test);
} catch (e, st) {
return DynamicModuleTestResult.runtimeError(test, '$e\n$st');
return DynamicModuleTestResult.runtimeError(test, '$e\n$st', timer.elapsed);
}
return DynamicModuleTestResult.pass(test);
return DynamicModuleTestResult.pass(test, timer.elapsed);
}
/// Generates a report of the test results in the JSON format
/// that is expected by our testing infrastructure.
void _reportResults(List<DynamicModuleTestResult> results) {
// TODO(sigmund): replace this with proper infra reporting
int _reportResults(
List<DynamicModuleTestResult> results, {
required bool writeLog,
String? configuration,
String? logDir,
}) {
bool fail = false;
print('Test results:');
for (var result in results) {
print(' ${result.name}: ${result.status}');
if (result.status != Status.pass) fail = true;
}
if (fail) throw "Some tests failed...";
if (fail) print('Error: some tests failed');
if (writeLog) {
if (logDir == null) {
print('Error: no output directory provided, logs won\'t be emitted.');
return 1;
}
if (configuration == null) {
print('Error: no configuration name provided, logs won\'t be emitted.');
return 1;
}
// Ensure the directory URI ends with a path separator.
var dirUri = Directory(logDir).uri;
File.fromUri(dirUri.resolve('results.json')).writeAsStringSync(
results.map((r) => '${r.toRecordJson(configuration)}\n').join(),
flush: true);
File.fromUri(dirUri.resolve('logs.json')).writeAsStringSync(
results
.where((r) => r.status != Status.pass)
.map((r) => '${r.toLogJson(configuration)}\n')
.join(),
flush: true);
print('Success: log files emitted under $dirUri');
} else if (fail) {
return 1;
}
return 0;
}
/// Placeholder until we implement all executors.
pkg/dynamic_modules/test/runner/model.dart
+46 -7
View File
@@ -2,6 +2,8 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:convert';
/// Data needed to build and run a single dynamic module test.
class DynamicModuleTest {
/// Name of the test. Matches the folder containing the test.
@@ -30,19 +32,56 @@ class DynamicModuleTestResult {
final String name;
final Status status;
final String details;
final Duration time;
DynamicModuleTestResult._(this.name, this.status, this.details);
DynamicModuleTestResult._(this.name, this.status, this.details, this.time);
factory DynamicModuleTestResult.pass(DynamicModuleTest test) =>
DynamicModuleTestResult._(test.name, Status.pass, '');
factory DynamicModuleTestResult.pass(DynamicModuleTest test, Duration time) =>
DynamicModuleTestResult._(test.name, Status.pass, '', time);
factory DynamicModuleTestResult.compileError(
DynamicModuleTest test, String details) =>
DynamicModuleTestResult._(test.name, Status.compileTimeError, details);
DynamicModuleTest test, String details, Duration time) =>
DynamicModuleTestResult._(
test.name, Status.compileTimeError, details, time);
factory DynamicModuleTestResult.runtimeError(
DynamicModuleTest test, String details) =>
DynamicModuleTestResult._(test.name, Status.runtimeError, details);
DynamicModuleTest test, String details, Duration time) =>
DynamicModuleTestResult._(test.name, Status.runtimeError, details, time);
/// Emit the result in the JSON format expected by the test infrastructure.
String toRecordJson(String configuration) {
final outcome = switch (status) {
Status.pass => 'Pass',
Status.compileTimeError => 'CompileTimeError',
Status.runtimeError => 'RuntimeError',
};
return jsonEncode({
'name': 'dynamic_modules_suite/$name',
'configuration': configuration,
'suite': 'dynamic_modules_suite',
'test_name': name,
'time_ms': time.inMilliseconds,
'expected': 'Pass',
'result': outcome,
'matches': status == Status.pass,
});
}
/// Emit the log entry with details of a failure in the JSON format expected
/// by the test infrastructure.
String toLogJson(String configuration) {
final outcome = switch (status) {
Status.pass => 'Pass',
Status.compileTimeError => 'CompileTimeError',
Status.runtimeError => 'RuntimeError',
};
return jsonEncode({
'name': 'dynamic_modules_suite/$name',
'configuration': configuration,
'result': outcome,
'log': details,
});
}
}
enum Status { pass, compileTimeError, runtimeError }
pkg/dynamic_modules/test/runner/util.dart
+31 -8
View File
@@ -25,8 +25,29 @@ Uri repoRoot = (() {
})();
String _outFolder = Platform.isMacOS ? 'xcodebuild' : 'out';
String configuration =
Platform.environment['DART_CONFIGURATION'] ?? 'ReleaseX64';
String configuration = () {
var env = Platform.environment['DART_CONFIGURATION'];
if (env != null) return env;
var folderSegments = _dartBin.resolve('.').pathSegments;
for (int i = folderSegments.length - 1; i > 0; i--) {
if (folderSegments[i] == _outFolder) {
var candidate = folderSegments[i + 1];
if (candidate.startsWith('Debug') ||
candidate.startsWith('Release') ||
candidate.startsWith('Product')) {
return candidate;
}
}
}
return 'ReleaseX64';
}();
// See also utils/gen_kernel/BUILD.gn:
// dartaotruntime has dart_product_config applied to it and is built in product
// mode in both release and product builds.
bool get useProduct => !configuration.startsWith('Debug');
String buildFolder = '$_outFolder/$configuration/';
String arch = Abi.current().toString().split('_')[1];
String _d8Path = (() {
@@ -58,13 +79,15 @@ Uri ddcModuleLoaderJs =
repoRoot.resolve('pkg/dev_compiler/lib/js/ddc/ddc_module_loader.js');
Uri genKernelSnapshot =
_dartBin.resolve('snapshots/gen_kernel_aot.dart.snapshot');
buildRootUri.resolve('gen/gen_kernel_aot.dart.snapshot');
Uri genSnapshotBin =
buildRootUri.resolve(useProduct ? 'gen_snapshot_product' : 'gen_snapshot');
Uri dart2bytecodeSnapshot =
_dartBin.resolve('snapshots/dart2bytecode.dart.snapshot');
Uri aotRuntimeBin =
Uri.parse(Platform.resolvedExecutable).resolve('dartaotruntime');
Uri vmPlatformDill =
_dartBin.resolve('../lib/_internal/vm_platform_strong_product.dill');
buildRootUri.resolve('gen/dart2bytecode.dart.snapshot');
Uri aotRuntimeBin = buildRootUri.resolve(useProduct
? 'dart_precompiled_runtime_product'
: 'dart_precompiled_runtime');
Uri vmPlatformDill = buildRootUri.resolve('vm_platform_strong.dill');
// Encodes test results in the format expected by Dart's CI infrastructure.
class TestResultOutcome {
tools/bots/test_matrix.json
+24 -1
View File
@@ -1331,7 +1331,8 @@
"arguments": [
"--dart-dynamic-modules",
"runtime",
"runtime_precompiled"
"runtime_precompiled",
"utils/gen_kernel"
]
},
{
@@ -1343,6 +1344,17 @@
],
"fileset": "vm",
"shards": 5
},
{
"name": "dynamic module tests",
"script": "out/DebugX64/dart",
"testRunner": true,
"arguments": [
"pkg/dynamic_modules/test/runner/main.dart",
"-nvm-aot-dyn-${system}-${mode}-${arch}",
"-raot",
"--verbose"
]
}
]
},
@@ -1878,6 +1890,17 @@
"--use-sdk"
]
},
{
"name": "ddc dynamic module tests",
"script": "out/ReleaseX64/dart-sdk/bin/dart",
"testRunner": true,
"arguments": [
"pkg/dynamic_modules/test/runner/main.dart",
"-nddc-${system}-chrome",
"-rddc",
"--verbose"
]
},
{
"name": "ddc sourcemap tests",
"script": "out/ReleaseX64/dart",