dart-lang/sdk
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[testing] Format package testing

Browse Source 
Change-Id: Ia4503844bcb6ea76627ab050ff198de2c14cb813
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/498860
Reviewed-by: Johnni Winther <johnniwinther@google.com>
Commit-Queue: Jens Johansen <jensj@google.com>
This commit is contained in:
Jens Johansen
2026-04-28 05:59:53 -07:00
committed by dart-scoped@luci-project-accounts.iam.gserviceaccount.com
parent a585ead993
commit 37de43c52f
13 changed files with 519 additions and 255 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/testing/lib/src/analyze.dart
+98 -52
View File
@@ -28,21 +28,34 @@ class Analyze extends Suite {
final List<String>? gitGrepPatterns;
Analyze(this.analysisOptions, this.uris, this.exclude, this.gitGrepPathspecs,
this.gitGrepPatterns)
: super("analyze", "analyze", null);
Analyze(
this.analysisOptions,
this.uris,
this.exclude,
this.gitGrepPathspecs,
this.gitGrepPatterns,
) : super("analyze", "analyze", null);
Future<void> run(Uri packages, List<Uri>? extraUris) {
List<Uri> allUris = List<Uri>.from(uris);
if (extraUris != null) {
allUris.addAll(extraUris);
}
return analyzeUris(analysisOptions, packages, allUris, exclude,
gitGrepPathspecs, gitGrepPatterns);
return analyzeUris(
analysisOptions,
packages,
allUris,
exclude,
gitGrepPathspecs,
gitGrepPatterns,
);
}
static Future<Analyze> fromJsonMap(
Uri base, Map json, List<Suite> suites) async {
Uri base,
Map json,
List<Suite> suites,
) async {
String? optionsPath = json["options"];
Uri? optionsUri = optionsPath == null ? null : base.resolve(optionsPath);
@@ -51,8 +64,9 @@ class Analyze extends Suite {
return base.resolve(r);
}).toList();
List<RegExp> exclude =
json["exclude"].map<RegExp>((p) => RegExp(p)).toList();
List<RegExp> exclude = json["exclude"]
.map<RegExp>((p) => RegExp(p))
.toList();
Map? gitGrep = json["git grep"];
List<String>? gitGrepPathspecs;
@@ -67,7 +81,12 @@ class Analyze extends Suite {
}
return Analyze(
optionsUri, uris, exclude, gitGrepPathspecs, gitGrepPatterns);
optionsUri,
uris,
exclude,
gitGrepPathspecs,
gitGrepPatterns,
);
}
@override
@@ -95,20 +114,30 @@ class AnalyzerDiagnostic {
static final Pattern unescapePattern = RegExp(r"\\(.)");
AnalyzerDiagnostic(this.kind, this.detailedKind, this.code, this.uri,
this.line, this.startColumn, this.endColumn, this.message);
AnalyzerDiagnostic(
this.kind,
this.detailedKind,
this.code,
this.uri,
this.line,
this.startColumn,
this.endColumn,
this.message,
);
AnalyzerDiagnostic.malformed(String line)
: this(null, null, null, null, -1, -1, -1, line);
: this(null, null, null, null, -1, -1, -1, line);
factory AnalyzerDiagnostic.fromLine(String line) {
List<String> parts = <String>[];
int start = 0;
int index = line.indexOf(potentialSplitPattern);
void addPart() {
parts.add(line
.substring(start, index == -1 ? null : index)
.replaceAllMapped(unescapePattern, (Match m) => m[1]!));
parts.add(
line
.substring(start, index == -1 ? null : index)
.replaceAllMapped(unescapePattern, (Match m) => m[1]!),
);
}
while (index != -1) {
@@ -125,14 +154,15 @@ class AnalyzerDiagnostic {
return AnalyzerDiagnostic.malformed(line);
}
return AnalyzerDiagnostic(
parts[0],
parts[1],
parts[2],
Uri.base.resolveUri(Uri.file(parts[3])),
int.parse(parts[4]),
int.parse(parts[5]),
int.parse(parts[6]),
parts[7]);
parts[0],
parts[1],
parts[2],
Uri.base.resolveUri(Uri.file(parts[3])),
int.parse(parts[4]),
int.parse(parts[5]),
int.parse(parts[6]),
parts[7],
);
}
@override
@@ -140,15 +170,17 @@ class AnalyzerDiagnostic {
return kind == null
? "Malformed output from dartanalyzer:\n$message"
: "${uri!.toFilePath()}:$line:$startColumn: "
"${kind == 'INFO' ? 'warning: hint' : kind!.toLowerCase()}:\n"
"[$code] $message";
"${kind == 'INFO' ? 'warning: hint' : kind!.toLowerCase()}:\n"
"[$code] $message";
}
}
Stream<AnalyzerDiagnostic> parseAnalyzerOutput(
Stream<List<int>> stream) async* {
Stream<String> lines =
stream.transform(utf8.decoder).transform(LineSplitter());
Stream<List<int>> stream,
) async* {
Stream<String> lines = stream
.transform(utf8.decoder)
.transform(LineSplitter());
await for (String line in lines) {
if (line.startsWith(">>> ")) continue;
yield AnalyzerDiagnostic.fromLine(line);
@@ -157,18 +189,19 @@ Stream<AnalyzerDiagnostic> parseAnalyzerOutput(
/// Run dartanalyzer on all tests in [uris].
Future<void> analyzeUris(
Uri? analysisOptions,
Uri packages,
List<Uri> uris,
List<RegExp> exclude,
List<String>? gitGrepPathspecs,
List<String>? gitGrepPatterns) async {
Uri? analysisOptions,
Uri packages,
List<Uri> uris,
List<RegExp> exclude,
List<String>? gitGrepPathspecs,
List<String>? gitGrepPatterns,
) async {
if (uris.isEmpty) return;
String topLevel;
try {
topLevel = Uri.directory(
(await git("rev-parse", <String>["--show-toplevel"])).trimRight())
.toFilePath(windows: false);
(await git("rev-parse", <String>["--show-toplevel"])).trimRight(),
).toFilePath(windows: false);
} catch (e) {
topLevel = Uri.base.toFilePath(windows: false);
}
@@ -190,8 +223,9 @@ Future<void> analyzeUris(
for (Uri uri in uris) {
if (await Directory.fromUri(uri).exists()) {
await for (FileSystemEntity entity
in Directory.fromUri(uri).list(recursive: true, followLinks: false)) {
await for (FileSystemEntity entity in Directory.fromUri(
uri,
).list(recursive: true, followLinks: false)) {
if (entity is File && entity.path.endsWith(".dart")) {
filesToAnalyze.add(toFilePath(entity.uri));
}
@@ -206,11 +240,15 @@ Future<void> analyzeUris(
if (gitGrepPatterns != null) {
List<String> arguments = <String>["-l"];
arguments.addAll(
gitGrepPatterns.expand((String pattern) => <String>["-e", pattern]));
gitGrepPatterns.expand((String pattern) => <String>["-e", pattern]),
);
arguments.add("--");
arguments.addAll(gitGrepPathspecs!);
filesToAnalyze.addAll(splitLines(await git("grep", arguments))
.map((String line) => line.trimRight()));
filesToAnalyze.addAll(
splitLines(
await git("grep", arguments),
).map((String line) => line.trimRight()),
);
}
const String analyzerPath = "pkg/analyzer_cli/bin/analyzer.dart";
@@ -237,8 +275,9 @@ Future<void> analyzeUris(
print("Running dartanalyzer.");
}
Stopwatch sw = Stopwatch()..start();
Process process = await startDart(
analyzer, const <String>["--batch"], dartArguments..remove("-c"));
Process process = await startDart(analyzer, const <String>[
"--batch",
], dartArguments..remove("-c"));
process.stdin.writeln(arguments.join(" "));
await process.stdin.close();
@@ -246,7 +285,8 @@ Future<void> analyzeUris(
Set<String> seen = <String>{};
Future<void> processAnalyzerOutput(
Stream<AnalyzerDiagnostic> diagnostics) async {
Stream<AnalyzerDiagnostic> diagnostics,
) async {
await for (AnalyzerDiagnostic diagnostic in diagnostics) {
if (diagnostic.uri != null) {
String path = toFilePath(diagnostic.uri!);
@@ -260,10 +300,12 @@ Future<void> analyzeUris(
}
}
Future stderrFuture =
processAnalyzerOutput(parseAnalyzerOutput(process.stderr));
Future stdoutFuture =
processAnalyzerOutput(parseAnalyzerOutput(process.stdout));
Future stderrFuture = processAnalyzerOutput(
parseAnalyzerOutput(process.stderr),
);
Future stdoutFuture = processAnalyzerOutput(
parseAnalyzerOutput(process.stdout),
);
await process.exitCode;
await stdoutFuture;
await stderrFuture;
@@ -278,16 +320,20 @@ String _findSdkPath() {
var executableUri = Uri.file(Platform.executable);
if (File.fromUri(executableUri.resolve('../version')).existsSync()) {
return executableUri.resolve('..').toFilePath();
} else if (File.fromUri(executableUri.resolve('dart-sdk/version'))
.existsSync()) {
} else if (File.fromUri(
executableUri.resolve('dart-sdk/version'),
).existsSync()) {
return executableUri.resolve('dart-sdk').toFilePath();
} else {
throw StateError('Cannot find dart-sdk for $executableUri');
}
}
Future<String> git(String command, Iterable<String> arguments,
{String? workingDirectory}) async {
Future<String> git(
String command,
Iterable<String> arguments, {
String? workingDirectory,
}) async {
ProcessResult result = await Process.run(
Platform.isWindows ? "git.bat" : "git",
<String>[command, ...arguments],
pkg/testing/lib/src/chain.dart
+119 -46
View File
@@ -22,8 +22,8 @@ import 'log.dart' show Logger, StdoutLogger, splitLines;
import 'expectation.dart' show Expectation, ExpectationGroup, ExpectationSet;
typedef CreateContext = Future<ChainContext> Function(
Chain suite, Map<String, String> environment);
typedef CreateContext =
Future<ChainContext> Function(Chain suite, Map<String, String> environment);
/// A test suite for tool chains, for example, a compiler.
class Chain extends Suite {
@@ -39,9 +39,17 @@ class Chain extends Suite {
final List<RegExp> exclude;
Chain(String name, String kind, this.source, this.root, this.subRoots,
Uri statusFile, this.includeEndsWith, this.pattern, this.exclude)
: super(name, kind, statusFile);
Chain(
String name,
String kind,
this.source,
this.root,
this.subRoots,
Uri statusFile,
this.includeEndsWith,
this.pattern,
this.exclude,
) : super(name, kind, statusFile);
factory Chain.fromJsonMap(Uri base, Map json, String name, String kind) {
Uri source = base.resolve(json["source"]);
@@ -63,16 +71,26 @@ class Chain extends Suite {
subRoots.add(rootUri);
}
Uri statusFile = base.resolve(json["status"]);
List<String> includeEndsWith =
List<String>.from(json['includeEndsWith'] ?? const []);
List<String> includeEndsWith = List<String>.from(
json['includeEndsWith'] ?? const [],
);
List<RegExp> pattern = [
for (final p in json['pattern'] ?? const []) RegExp(p)
for (final p in json['pattern'] ?? const []) RegExp(p),
];
List<RegExp> exclude = [
for (final e in json['exclude'] ?? const []) RegExp(e)
for (final e in json['exclude'] ?? const []) RegExp(e),
];
return Chain(name, kind, source, rootUri, subRoots, statusFile,
includeEndsWith, pattern, exclude);
return Chain(
name,
kind,
source,
rootUri,
subRoots,
statusFile,
includeEndsWith,
pattern,
exclude,
);
}
void writeImportOn(StringSink sink) {
@@ -89,8 +107,10 @@ class Chain extends Suite {
sink.writeln(".createContext, {...environment}, selectors, r'''");
const String jsonExtraIndent = " ";
sink.write(jsonExtraIndent);
sink.writeAll(splitLines(JsonEncoder.withIndent(" ").convert(this)),
jsonExtraIndent);
sink.writeAll(
splitLines(JsonEncoder.withIndent(" ").convert(this)),
jsonExtraIndent,
);
sink.writeln("''');");
}
@@ -115,14 +135,19 @@ abstract class ChainContext {
ExpectationSet get expectationSet => ExpectationSet.defaultExpectations;
Future<void> run(Chain suite, Set<String> selectors,
{int shards = 1,
int shard = 0,
int? limitTo,
Logger logger = const StdoutLogger()}) async {
Future<void> run(
Chain suite,
Set<String> selectors, {
int shards = 1,
int shard = 0,
int? limitTo,
Logger logger = const StdoutLogger(),
}) async {
assert(shards >= 1, "Invalid shards count: $shards");
assert(0 <= shard && shard < shards,
"Invalid shard index: $shard, not in range [0,$shards[.");
assert(
0 <= shard && shard < shards,
"Invalid shard index: $shard, not in range [0,$shards[.",
);
List<String> tripleDotSelectors = selectors
.where((s) => s.endsWith('...'))
.map((s) => s.substring(0, s.length - 3))
@@ -131,8 +156,9 @@ abstract class ChainContext {
.where((s) => s.contains('*'))
.map((s) => _createRegExpForAsterisk(s))
.toList();
TestExpectations expectations = readTestExpectations(
<String>[suite.statusFile!.toFilePath()], expectationSet);
TestExpectations expectations = readTestExpectations(<String>[
suite.statusFile!.toFilePath(),
], expectationSet);
List<TestDescription> descriptions = await list(suite);
descriptions.sort();
@@ -168,7 +194,9 @@ abstract class ChainContext {
continue;
}
final Set<Expectation> expectedOutcomes = processExpectedOutcomes(
expectations.expectations(description.shortName), description);
expectations.expectations(description.shortName),
description,
);
bool shouldSkip = false;
for (Expectation expectation in expectedOutcomes) {
if (expectation.group == ExpectationGroup.skip) {
@@ -205,8 +233,14 @@ abstract class ChainContext {
Step step = iterator.current;
lastStepRun = step;
isAsync = step.isAsync;
logger.logStepStart(completed, unexpectedResults.length,
descriptions.length, suite, description, step);
logger.logStepStart(
completed,
unexpectedResults.length,
descriptions.length,
suite,
description,
step,
);
// TODO(ahe): It's important to share the zone error reporting zone
// between all the tasks. Otherwise, if a future completes with an
// error in one zone, and gets stored, it becomes an uncaught error
@@ -223,8 +257,14 @@ abstract class ChainContext {
}
future = future.then((currentResult) async {
if (currentResult != null) {
logger.logStepComplete(completed, unexpectedResults.length,
descriptions.length, suite, description, lastStepRun!);
logger.logStepComplete(
completed,
unexpectedResults.length,
descriptions.length,
suite,
description,
lastStepRun!,
);
result = currentResult;
if ((currentResult as Result).outcome == Expectation.pass) {
// The input to the next step is the output of this step.
@@ -238,15 +278,28 @@ abstract class ChainContext {
unexpectedResults[description] = result!;
unexpectedOutcomes[description] = expectedOutcomes;
logger.logUnexpectedResult(
suite, description, result!, expectedOutcomes);
suite,
description,
result!,
expectedOutcomes,
);
exitCode = 1;
} else {
logger.logExpectedResult(
suite, description, result!, expectedOutcomes);
suite,
description,
result!,
expectedOutcomes,
);
logger.logMessage(sb);
}
logger.logTestComplete(++completed, unexpectedResults.length,
descriptions.length, suite, description);
logger.logTestComplete(
++completed,
unexpectedResults.length,
descriptions.length,
suite,
description,
);
});
if (isAsync) {
futures.add(future);
@@ -256,8 +309,13 @@ abstract class ChainContext {
}
}
logger.logTestStart(completed, unexpectedResults.length,
descriptions.length, suite, description);
logger.logTestStart(
completed,
unexpectedResults.length,
descriptions.length,
suite,
description,
);
// The input of the first step is [description].
await doStep(description);
}
@@ -266,7 +324,11 @@ abstract class ChainContext {
if (unexpectedResults.isNotEmpty) {
unexpectedResults.forEach((TestDescription description, Result result) {
logger.logUnexpectedResult(
suite, description, result, unexpectedOutcomes[description]!);
suite,
description,
result,
unexpectedOutcomes[description]!,
);
});
print("${unexpectedResults.length} failed:");
unexpectedResults.forEach((TestDescription description, Result result) {
@@ -281,8 +343,10 @@ abstract class ChainContext {
for (Uri subRoot in suite.subRoots) {
Directory testRoot = Directory.fromUri(subRoot);
if (testRoot.existsSync()) {
for (FileSystemEntity entity
in testRoot.listSync(recursive: true, followLinks: false)) {
for (FileSystemEntity entity in testRoot.listSync(
recursive: true,
followLinks: false,
)) {
if (entity is! File) continue;
// Use `.uri.path` instead of just `.path` to ensure forward slashes.
String path = entity.uri.path;
@@ -308,7 +372,9 @@ abstract class ChainContext {
}
Set<Expectation> processExpectedOutcomes(
Set<Expectation> outcomes, TestDescription description) {
Set<Expectation> outcomes,
TestDescription description,
) {
return outcomes;
}
@@ -394,10 +460,10 @@ class Result<O> {
Result.pass(O output) : this(output, Expectation.pass, null);
Result.crash(Object? error, StackTrace trace)
: this(null, Expectation.crash, error, trace: trace);
: this(null, Expectation.crash, error, trace: trace);
Result.fail(O output, [error, StackTrace? trace])
: this(output, Expectation.fail, error, trace: trace);
: this(output, Expectation.fail, error, trace: trace);
bool get isPass => outcome == Expectation.pass;
@@ -408,17 +474,24 @@ class Result<O> {
}
Result<O2> copyWithOutput<O2>(O2 output) {
return Result<O2>(output, outcome, error,
trace: trace,
autoFixCommand: autoFixCommand,
canBeFixWithUpdateExpectations: canBeFixWithUpdateExpectations)
..logs.addAll(logs);
return Result<O2>(
output,
outcome,
error,
trace: trace,
autoFixCommand: autoFixCommand,
canBeFixWithUpdateExpectations: canBeFixWithUpdateExpectations,
)..logs.addAll(logs);
}
}
/// This is called from generated code.
Future<void> runChain(CreateContext f, Map<String, String> environment,
Set<String> selectors, String jsonText) {
Future<void> runChain(
CreateContext f,
Map<String, String> environment,
Set<String> selectors,
String jsonText,
) {
return withErrorHandling(() async {
Chain suite = Suite.fromJsonMap(Uri.base, json.decode(jsonText)) as Chain;
print("Running ${suite.name}");
pkg/testing/lib/src/discover.dart
+39 -23
View File
@@ -14,11 +14,15 @@ final Uri packageConfig = computePackageConfig();
/// Common arguments when running a dart program. Returns a copy that can
/// safely be modified by caller.
List<String> get dartArguments =>
<String>["-c", "--packages=${packageConfig.toFilePath()}"];
List<String> get dartArguments => <String>[
"-c",
"--packages=${packageConfig.toFilePath()}",
];
Stream<FileBasedTestDescription> listTests(List<Uri> testRoots,
{Pattern? pattern}) {
Stream<FileBasedTestDescription> listTests(
List<Uri> testRoots, {
Pattern? pattern,
}) {
StreamController<FileBasedTestDescription> controller =
StreamController<FileBasedTestDescription>();
Map<Uri, StreamSubscription?> subscriptions = <Uri, StreamSubscription>{};
@@ -27,23 +31,32 @@ Stream<FileBasedTestDescription> listTests(List<Uri> testRoots,
Directory testRoot = Directory.fromUri(testRootUri);
testRoot.exists().then((bool exists) {
if (exists) {
Stream<FileSystemEntity> stream =
testRoot.list(recursive: true, followLinks: false);
var subscription = stream.listen((FileSystemEntity entity) {
FileBasedTestDescription? description = FileBasedTestDescription.from(
testRootUri, entity,
pattern: pattern);
if (description != null) {
controller.add(description);
}
}, onError: (error, StackTrace trace) {
controller.addError(error, trace);
}, onDone: () {
subscriptions.remove(testRootUri);
if (subscriptions.isEmpty) {
controller.close(); // TODO(ahe): catchError???
}
});
Stream<FileSystemEntity> stream = testRoot.list(
recursive: true,
followLinks: false,
);
var subscription = stream.listen(
(FileSystemEntity entity) {
FileBasedTestDescription? description =
FileBasedTestDescription.from(
testRootUri,
entity,
pattern: pattern,
);
if (description != null) {
controller.add(description);
}
},
onError: (error, StackTrace trace) {
controller.addError(error, trace);
},
onDone: () {
subscriptions.remove(testRootUri);
if (subscriptions.isEmpty) {
controller.close(); // TODO(ahe): catchError???
}
},
);
subscriptions[testRootUri] = subscription;
} else {
controller.addError("$testRootUri isn't a directory");
@@ -63,8 +76,11 @@ Uri computePackageConfig() {
return Uri.base.resolve(".dart_tool/package_config.json");
}
Future<Process> startDart(Uri program,
[List<String>? arguments, List<String>? vmArguments]) {
Future<Process> startDart(
Uri program, [
List<String>? arguments,
List<String>? vmArguments,
]) {
List<String> allArguments = <String>[];
allArguments.addAll(vmArguments ?? dartArguments);
allArguments.add(program.toFilePath());
pkg/testing/lib/src/error_handling.dart
+4 -2
View File
@@ -12,8 +12,10 @@ import 'dart:isolate' show ReceivePort;
import 'log.dart';
Future<T?> withErrorHandling<T>(Future<T> Function() f,
{Logger? logger}) async {
Future<T?> withErrorHandling<T>(
Future<T> Function() f, {
Logger? logger,
}) async {
final ReceivePort port = ReceivePort();
try {
return await f();
pkg/testing/lib/src/expectation.dart
+25 -26
View File
@@ -17,8 +17,10 @@ class Expectation {
static const Expectation crash = Expectation("Crash", ExpectationGroup.crash);
static const Expectation timeout =
Expectation("Timeout", ExpectationGroup.timeout);
static const Expectation timeout = Expectation(
"Timeout",
ExpectationGroup.timeout,
);
static const Expectation fail = Expectation("Fail", ExpectationGroup.fail);
@@ -56,20 +58,23 @@ class Expectation {
}
class ExpectationSet {
static const ExpectationSet defaultExpectations = ExpectationSet(
<String, Expectation>{
"pass": Expectation.pass,
"crash": Expectation.crash,
"timeout": Expectation.timeout,
"fail": Expectation.fail,
"skip": Expectation.skip,
"missingcompiletimeerror":
Expectation("MissingCompileTimeError", ExpectationGroup.fail),
"missingruntimeerror":
Expectation("MissingRuntimeError", ExpectationGroup.fail),
"runtimeerror": Expectation("RuntimeError", ExpectationGroup.fail),
},
);
static const ExpectationSet defaultExpectations =
ExpectationSet(<String, Expectation>{
"pass": Expectation.pass,
"crash": Expectation.crash,
"timeout": Expectation.timeout,
"fail": Expectation.fail,
"skip": Expectation.skip,
"missingcompiletimeerror": Expectation(
"MissingCompileTimeError",
ExpectationGroup.fail,
),
"missingruntimeerror": Expectation(
"MissingRuntimeError",
ExpectationGroup.fail,
),
"runtimeerror": Expectation("RuntimeError", ExpectationGroup.fail),
});
final Map<String, Expectation> internalMap;
@@ -81,8 +86,9 @@ class ExpectationSet {
}
factory ExpectationSet.fromJsonList(List data) {
Map<String, Expectation> internalMap =
Map<String, Expectation>.from(defaultExpectations.internalMap);
Map<String, Expectation> internalMap = Map<String, Expectation>.from(
defaultExpectations.internalMap,
);
for (Map map in data) {
String? name;
String? group;
@@ -115,14 +121,7 @@ class ExpectationSet {
}
}
enum ExpectationGroup {
crash,
fail,
meta,
pass,
skip,
timeout,
}
enum ExpectationGroup { crash, fail, meta, pass, skip, timeout }
ExpectationGroup groupFromString(String name) {
switch (name) {
pkg/testing/lib/src/log.dart
+91 -28
View File
@@ -39,17 +39,39 @@ void enableVerboseOutput() {
}
abstract class Logger {
void logTestStart(int completed, int failed, int total, Suite suite,
TestDescription description);
void logTestStart(
int completed,
int failed,
int total,
Suite suite,
TestDescription description,
);
void logTestComplete(int completed, int failed, int total, Suite suite,
TestDescription description);
void logTestComplete(
int completed,
int failed,
int total,
Suite suite,
TestDescription description,
);
void logStepStart(int completed, int failed, int total, Suite suite,
TestDescription description, Step step);
void logStepStart(
int completed,
int failed,
int total,
Suite suite,
TestDescription description,
Step step,
);
void logStepComplete(int completed, int failed, int total, Suite suite,
TestDescription description, Step step);
void logStepComplete(
int completed,
int failed,
int total,
Suite suite,
TestDescription description,
Step step,
);
void logProgress(String message);
@@ -57,11 +79,19 @@ abstract class Logger {
void logNumberedLines(String text);
void logExpectedResult(Suite suite, TestDescription description,
Result result, Set<Expectation> expectedOutcomes);
void logExpectedResult(
Suite suite,
TestDescription description,
Result result,
Set<Expectation> expectedOutcomes,
);
void logUnexpectedResult(Suite suite, TestDescription description,
Result result, Set<Expectation> expectedOutcomes);
void logUnexpectedResult(
Suite suite,
TestDescription description,
Result result,
Set<Expectation> expectedOutcomes,
);
void logSuiteStarted(Suite suite);
@@ -79,12 +109,22 @@ class StdoutLogger implements Logger {
const StdoutLogger();
@override
void logTestStart(int completed, int failed, int total, Suite? suite,
TestDescription? description) {}
void logTestStart(
int completed,
int failed,
int total,
Suite? suite,
TestDescription? description,
) {}
@override
void logTestComplete(int completed, int failed, int total, Suite? suite,
TestDescription? description) {
void logTestComplete(
int completed,
int failed,
int total,
Suite? suite,
TestDescription? description,
) {
String message = formatProgress(completed, failed, total);
if (suite != null) {
message += ": ${formatTestDescription(suite, description!)}";
@@ -93,8 +133,14 @@ class StdoutLogger implements Logger {
}
@override
void logStepStart(int completed, int failed, int total, Suite? suite,
TestDescription description, Step step) {
void logStepStart(
int completed,
int failed,
int total,
Suite? suite,
TestDescription description,
Step step,
) {
String message = formatProgress(completed, failed, total);
if (suite != null) {
message += ": ${formatTestDescription(suite, description)} ${step.name}";
@@ -106,8 +152,14 @@ class StdoutLogger implements Logger {
}
@override
void logStepComplete(int completed, int failed, int total, Suite? suite,
TestDescription description, Step step) {
void logStepComplete(
int completed,
int failed,
int total,
Suite? suite,
TestDescription description,
Step step,
) {
if (!step.isAsync) return;
String message = formatProgress(completed, failed, total);
if (suite != null) {
@@ -154,12 +206,20 @@ class StdoutLogger implements Logger {
}
@override
void logExpectedResult(Suite suite, TestDescription description,
Result result, Set<Expectation> expectedOutcomes) {}
void logExpectedResult(
Suite suite,
TestDescription description,
Result result,
Set<Expectation> expectedOutcomes,
) {}
@override
void logUnexpectedResult(Suite suite, TestDescription description,
Result result, Set<Expectation> expectedOutcomes) {
void logUnexpectedResult(
Suite suite,
TestDescription description,
Result result,
Set<Expectation> expectedOutcomes,
) {
print("${eraseLine}UNEXPECTED: ${suite.name}/${description.shortName}");
Uri? statusFile = suite.statusFile;
if (statusFile != null) {
@@ -167,8 +227,10 @@ class StdoutLogger implements Logger {
if (result.outcome == Expectation.pass) {
print("The test unexpectedly passed, please update $path.");
} else {
print("The test had the outcome ${result.outcome}, but the status file "
"($path) allows these outcomes: ${expectedOutcomes.join(' ')}");
print(
"The test had the outcome ${result.outcome}, but the status file "
"($path) allows these outcomes: ${expectedOutcomes.join(' ')}",
);
}
}
String log = result.log;
@@ -218,8 +280,9 @@ String numberedLines(String text) {
String fill = " " * pad;
for (String line in lines) {
String paddedLineNumber = "$fill$lineNumber";
paddedLineNumber =
paddedLineNumber.substring(paddedLineNumber.length - pad);
paddedLineNumber = paddedLineNumber.substring(
paddedLineNumber.length - pad,
);
result.write("$paddedLineNumber: $line");
lineNumber++;
}
pkg/testing/lib/src/run.dart
+46 -21
View File
@@ -46,13 +46,16 @@ Future<TestRoot> computeTestRoot(String? configurationPath, Uri? base) {
/// The optional argument [configurationPath] should be used when
/// `testing.json` isn't located in the current working directory and is a path
/// relative to [me] which defaults to `Platform.script`.
Future<void> runMe(List<String> arguments, CreateContext f,
{String? configurationPath,
Uri? me,
int shards = 1,
int shard = 0,
int? limitTo,
Logger logger = const StdoutLogger()}) {
Future<void> runMe(
List<String> arguments,
CreateContext f, {
String? configurationPath,
Uri? me,
int shards = 1,
int shard = 0,
int? limitTo,
Logger logger = const StdoutLogger(),
}) {
me ??= Platform.script;
return withErrorHandling(() async {
TestRoot testRoot = await computeTestRoot(configurationPath, me);
@@ -61,8 +64,14 @@ Future<void> runMe(List<String> arguments, CreateContext f,
for (Chain suite in testRoot.toolChains) {
if (me == suite.source) {
ChainContext context = await f(suite, cl.environment);
await context.run(suite, Set<String>.from(cl.selectors),
shards: shards, shard: shard, limitTo: limitTo, logger: logger);
await context.run(
suite,
Set<String>.from(cl.selectors),
shards: shards,
shard: shard,
limitTo: limitTo,
logger: logger,
);
}
}
}, logger: logger);
@@ -94,15 +103,23 @@ Future<void> runMe(List<String> arguments, CreateContext f,
/// The optional argument [configurationPath] should be used when
/// `testing.json` isn't located in the current working directory and is a path
/// relative to `Uri.base`.
Future<void> run(List<String> arguments, List<String> suiteNames,
[String? configurationPath]) {
Future<void> run(
List<String> arguments,
List<String> suiteNames, [
String? configurationPath,
]) {
return withErrorHandling(() async {
TestRoot root = await computeTestRoot(configurationPath, Uri.base);
List<Suite> suites = root.suites
.where((Suite suite) => suiteNames.contains(suite.name))
.toList();
SuiteRunner runner = SuiteRunner(
suites, <String, String>{}, const <String>[], <String>{}, <String>{});
suites,
<String, String>{},
const <String>[],
<String>{},
<String>{},
);
String? program = await runner.generateDartProgram();
await runner.analyze(root.packages);
if (program != null) {
@@ -116,12 +133,16 @@ Future<void> runProgram(String program, Uri packages) async {
const StdoutLogger().logNumberedLines(program);
Uri dataUri = Uri.dataFromString(program);
ReceivePort exitPort = ReceivePort();
Isolate isolate = await Isolate.spawnUri(dataUri, <String>[], null,
paused: true,
onExit: exitPort.sendPort,
errorsAreFatal: false,
checked: true,
packageConfig: packages);
Isolate isolate = await Isolate.spawnUri(
dataUri,
<String>[],
null,
paused: true,
onExit: exitPort.sendPort,
errorsAreFatal: false,
checked: true,
packageConfig: packages,
);
List? error;
var subscription = isolate.errors.listen((data) {
error = data;
@@ -150,9 +171,13 @@ class SuiteRunner {
final List<Uri> testUris = <Uri>[];
SuiteRunner(this.suites, this.environment, Iterable<String> selectors,
this.selectedSuites, this.skippedSuites)
: selectors = selectors.toList(growable: false);
SuiteRunner(
this.suites,
this.environment,
Iterable<String> selectors,
this.selectedSuites,
this.skippedSuites,
) : selectors = selectors.toList(growable: false);
bool shouldRunSuite(Suite suite) {
return !skippedSuites.contains(suite.name) &&
pkg/testing/lib/src/run_tests.dart
+56 -37
View File
@@ -33,11 +33,13 @@ class CommandLine {
Set<String> get skip => commaSeparated("--skip=");
Set<String> commaSeparated(String prefix) {
return Set<String>.from(options.expand((String s) {
if (!s.startsWith(prefix)) return const [];
s = s.substring(prefix.length);
return s.split(",");
}));
return Set<String>.from(
options.expand((String s) {
if (!s.startsWith(prefix)) return const [];
s = s.substring(prefix.length);
return s.split(",");
}),
);
}
Map<String, String> get environment {
@@ -91,8 +93,10 @@ class CommandLine {
List<FileSystemEntity> candidates = await test
.list(recursive: true, followLinks: false)
.where((FileSystemEntity entity) {
return entity is File && entity.uri.path.endsWith("/testing.json");
}).toList();
return entity is File &&
entity.uri.path.endsWith("/testing.json");
})
.toList();
switch (candidates.length) {
case 0:
fail("Couldn't locate: '$configurationPath'.");
@@ -103,18 +107,22 @@ class CommandLine {
break;
default:
fail("Usage: run_tests.dart [$configPrefix=configuration_file]\n"
"Where configuration_file is one of:\n "
"${candidates.map((file) => file.path).join('\n ')}");
fail(
"Usage: run_tests.dart [$configPrefix=configuration_file]\n"
"Where configuration_file is one of:\n "
"${candidates.map((file) => file.path).join('\n ')}",
);
return null;
}
}
}
}
const StdoutLogger()
.logMessage("Reading configuration file '$configurationPath'.");
Uri? configuration =
await Isolate.resolvePackageUri(Uri.base.resolve(configurationPath));
const StdoutLogger().logMessage(
"Reading configuration file '$configurationPath'.",
);
Uri? configuration = await Isolate.resolvePackageUri(
Uri.base.resolve(configurationPath),
);
if (configuration == null || !await File.fromUri(configuration).exists()) {
fail("Couldn't locate: '$configurationPath'.");
return null;
@@ -130,8 +138,9 @@ class CommandLine {
arguments = arguments.sublist(index + 1);
} else {
options = arguments.where((argument) => argument.startsWith("-")).toSet();
arguments =
arguments.where((argument) => !argument.startsWith("-")).toList();
arguments = arguments
.where((argument) => !argument.startsWith("-"))
.toList();
}
return CommandLine(options, arguments);
}
@@ -156,7 +165,12 @@ Future<void> main(List<String> arguments) {
}
TestRoot root = await TestRoot.fromUri(configuration);
SuiteRunner runner = SuiteRunner(
root.suites, environment, cl.selectors, cl.selectedSuites, cl.skip);
root.suites,
environment,
cl.selectors,
cl.selectedSuites,
cl.skip,
);
String? program = await runner.generateDartProgram();
bool hasAnalyzerSuites = await runner.analyze(root.packages);
Stopwatch sw = Stopwatch()..start();
@@ -171,24 +185,29 @@ Future<void> main(List<String> arguments) {
});
}
Future<void> runTests(Map<String, Function> tests) =>
withErrorHandling<void>(() async {
int completed = 0;
for (String name in tests.keys) {
const StdoutLogger()
.logTestStart(completed, 0, tests.length, null, null);
StringBuffer sb = StringBuffer();
try {
await runGuarded(() {
print("Running test $name");
return tests[name]!();
}, printLineOnStdout: sb.writeln);
const StdoutLogger().logMessage(sb);
} catch (e) {
print(sb);
rethrow;
}
const StdoutLogger()
.logTestComplete(++completed, 0, tests.length, null, null);
Future<void> runTests(Map<String, Function> tests) => withErrorHandling<void>(
() async {
int completed = 0;
for (String name in tests.keys) {
const StdoutLogger().logTestStart(completed, 0, tests.length, null, null);
StringBuffer sb = StringBuffer();
try {
await runGuarded(() {
print("Running test $name");
return tests[name]!();
}, printLineOnStdout: sb.writeln);
const StdoutLogger().logMessage(sb);
} catch (e) {
print(sb);
rethrow;
}
});
const StdoutLogger().logTestComplete(
++completed,
0,
tests.length,
null,
null,
);
}
},
);
pkg/testing/lib/src/status_file_parser.dart
+6 -2
View File
@@ -9,7 +9,9 @@ import "dart:io";
import 'expectation.dart' show Expectation, ExpectationSet;
TestExpectations readTestExpectations(
List<String> statusFilePaths, ExpectationSet expectationSet) {
List<String> statusFilePaths,
ExpectationSet expectationSet,
) {
TestExpectations testExpectations = TestExpectations(expectationSet);
for (String path in statusFilePaths) {
readTestExpectationsInto(testExpectations, path);
@@ -18,7 +20,9 @@ TestExpectations readTestExpectations(
}
void readTestExpectationsInto(
TestExpectations expectations, String statusFilePath) {
TestExpectations expectations,
String statusFilePath,
) {
File file = File(statusFilePath);
for (String line in file.readAsLinesSync()) {
// Remove comments if any.
pkg/testing/lib/src/stdio_process.dart
+22 -12
View File
@@ -28,25 +28,35 @@ class StdioProcess {
return Result<int>.pass(exitCode);
} else {
return Result<int>(
exitCode, ExpectationSet.defaultExpectations["RuntimeError"], output);
exitCode,
ExpectationSet.defaultExpectations["RuntimeError"],
output,
);
}
}
static StreamTransformer<String, String> transformToStdio(Stdout stdio) {
return StreamTransformer<String, String>.fromHandlers(
handleData: (String data, EventSink<String> sink) {
sink.add(data);
stdio.write(data);
});
handleData: (String data, EventSink<String> sink) {
sink.add(data);
stdio.write(data);
},
);
}
static Future<StdioProcess> run(String executable, List<String> arguments,
{String? input,
Duration? timeout = const Duration(seconds: 60),
bool suppressOutput = true,
bool runInShell = false}) async {
Process process =
await Process.start(executable, arguments, runInShell: runInShell);
static Future<StdioProcess> run(
String executable,
List<String> arguments, {
String? input,
Duration? timeout = const Duration(seconds: 60),
bool suppressOutput = true,
bool runInShell = false,
}) async {
Process process = await Process.start(
executable,
arguments,
runInShell: runInShell,
);
Timer? timer;
StringBuffer sb = StringBuffer();
if (timeout != null) {
pkg/testing/lib/src/test_description.dart
+5 -2
View File
@@ -48,8 +48,11 @@ class FileBasedTestDescription extends TestDescription {
sink.writeln('.main,');
}
static FileBasedTestDescription? from(Uri root, FileSystemEntity entity,
{Pattern? pattern}) {
static FileBasedTestDescription? from(
Uri root,
FileSystemEntity entity, {
Pattern? pattern,
}) {
if (entity is! File) return null;
pattern ??= "_test.dart";
String path = entity.uri.path;
pkg/testing/lib/src/zone_helper.dart
+3 -2
View File
@@ -75,8 +75,9 @@ Future runGuarded(
errorPort.close();
Isolate.current.setErrorsFatal(true);
Isolate.current.removeErrorListener(errorPort.sendPort);
return acknowledgeControlMessages(Isolate.current)
.then((_) => errorFuture);
return acknowledgeControlMessages(
Isolate.current,
).then((_) => errorFuture);
});
});
}
pkg/testing/test/analyze_test.dart
+5 -2
View File
@@ -7,6 +7,9 @@ import "package:testing/src/run_tests.dart" as testing show main;
Future<void> main() {
// This method is async, but keeps a port open to prevent the VM from exiting
// prematurely.
return testing.main(
<String>["--config=pkg/testing/testing.json", "--verbose", "analyze"]);
return testing.main(<String>[
"--config=pkg/testing/testing.json",
"--verbose",
"analyze",
]);
}