Alexander Thomas
14b1d2feb8
This adds a "--deflake" flag to test.dart. Deflaking will only run if that flag is set. Deflaking is time consuming and delays the information that users want to see when most of the time they know that the result is not caused by flakiness. A typical session may look like this: * Run test.dart with a broad test selector without deflaking. * Re-run test.dart with a narrow test selector for suspected flakes with "--deflake". This second step is optional. Additionally, a new "--report-flakes" flag is added that can be used to report test failures for tests known to be flaky. Change-Id: I543d0b40c32065eb0a50338c55e7050b7887abce Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/102381 Reviewed-by: Jonas Termansen <sortie@google.com>
610 lines
23 KiB
Dart
Executable File
610 lines
23 KiB
Dart
Executable File
#!/usr/bin/env dart
|
|
// Copyright (c) 2018, the Dart project authors. Please see the AUTHORS file
|
|
// for details. All rights reserved. Use of this source code is governed by a
|
|
// BSD-style license that can be found in the LICENSE file.
|
|
|
|
// Run tests like on the given builder and/or named configuration.
|
|
|
|
import 'dart:async';
|
|
import 'dart:collection';
|
|
import 'dart:convert';
|
|
import 'dart:io';
|
|
|
|
import 'package:args/args.dart';
|
|
|
|
import 'bots/results.dart';
|
|
|
|
const int deflakingCount = 5;
|
|
|
|
/// Quotes a string in shell single quote mode. This function produces a single
|
|
/// shell argument that evaluates to the exact string provided, handling any
|
|
/// special characters in the input string. Shell single quote mode works uses
|
|
/// the single quote character as the delimiter and uses the characters
|
|
/// in-between verbatim without any special processing. To insert the single
|
|
/// quote character itself, escape single quote mode, insert an escaped single
|
|
/// quote, and then return to single quote mode.
|
|
///
|
|
/// Examples:
|
|
/// foo becomes 'foo'
|
|
/// foo bar becomes 'foo bar'
|
|
/// foo\ bar becomes 'foo\ bar'
|
|
/// foo's bar becomes 'foo '\''s bar'
|
|
/// foo "b"ar becomes 'foo "b"'
|
|
/// foo
|
|
/// bar becomes 'foo
|
|
/// bar'
|
|
String shellSingleQuote(String string) {
|
|
return "'${string.replaceAll("'", "'\\''")}'";
|
|
}
|
|
|
|
/// Like [shellSingleQuote], but if the string only contains safe ASCII
|
|
/// characters, don't quote it. Note that it's not always safe to omit the
|
|
/// quotes even if the string only has safe characters, as doing so might match
|
|
/// a shell keyword or a shell builtin in the first argument in a command. It
|
|
/// should be safe to use this for the second argument onwards in a command.
|
|
String simpleShellSingleQuote(String string) {
|
|
return new RegExp(r"^[a-zA-Z0-9%+,./:_-]*$").hasMatch(string)
|
|
? string
|
|
: shellSingleQuote(string);
|
|
}
|
|
|
|
/// Runs a process and exits likewise if the process exits non-zero.
|
|
Future<ProcessResult> runProcess(String executable, List<String> arguments,
|
|
{bool runInShell = false}) async {
|
|
final processResult =
|
|
await Process.run(executable, arguments, runInShell: runInShell);
|
|
if (processResult.exitCode != 0) {
|
|
final command =
|
|
[executable, ...arguments].map(simpleShellSingleQuote).join(" ");
|
|
throw new Exception("Command exited ${processResult.exitCode}: $command\n"
|
|
"${processResult.stdout}\n${processResult.stderr}");
|
|
}
|
|
return processResult;
|
|
}
|
|
|
|
/// Runs a process and exits likewise if the process exits non-zero, but let the
|
|
/// child process inherit out stdio handles.
|
|
Future<ProcessResult> runProcessInheritStdio(
|
|
String executable, List<String> arguments,
|
|
{bool runInShell = false}) async {
|
|
final process = await Process.start(executable, arguments,
|
|
mode: ProcessStartMode.inheritStdio, runInShell: runInShell);
|
|
final exitCode = await process.exitCode;
|
|
final processResult = new ProcessResult(process.pid, exitCode, "", "");
|
|
if (processResult.exitCode != 0) {
|
|
final command =
|
|
[executable, ...arguments].map(simpleShellSingleQuote).join(" ");
|
|
throw new Exception("Command exited ${processResult.exitCode}: $command");
|
|
}
|
|
return processResult;
|
|
}
|
|
|
|
/// Returns the operating system of a builder.
|
|
String systemOfBuilder(String builder) {
|
|
return builder.split("-").firstWhere(
|
|
(component) => ["linux", "mac", "win"].contains(component),
|
|
orElse: () => null);
|
|
}
|
|
|
|
/// Returns the product mode of a builder.
|
|
String modeOfBuilder(String builder) {
|
|
return builder.split("-").firstWhere(
|
|
(component) => ["debug", "product", "release"].contains(component),
|
|
orElse: () => null);
|
|
}
|
|
|
|
/// Returns the machine architecture of a builder.
|
|
String archOfBuilder(String builder) {
|
|
return builder.split("-").firstWhere(
|
|
(component) => [
|
|
"arm",
|
|
"arm64",
|
|
"armsimdbc",
|
|
"armsimdbc64",
|
|
"ia32",
|
|
"simarm",
|
|
"simarm64",
|
|
"simdbc",
|
|
"simdbc64",
|
|
"x64",
|
|
].contains(component),
|
|
orElse: () => null);
|
|
}
|
|
|
|
/// Returns the runtime environment of a builder.
|
|
String runtimeOfBuilder(String builder) {
|
|
return builder.split("-").firstWhere(
|
|
(component) => ["chrome", "d8", "edge", "firefox", "ie11", "safari"]
|
|
.contains(component),
|
|
orElse: () => null);
|
|
}
|
|
|
|
/// Expands a variable in a test matrix step command.
|
|
String expandVariable(String string, String variable, String value) {
|
|
return string.replaceAll("\${$variable}", value ?? "");
|
|
}
|
|
|
|
/// Expands all variables in a test matrix step command.
|
|
String expandVariables(String string, String builder) {
|
|
string = expandVariable(string, "system", systemOfBuilder(builder));
|
|
string = expandVariable(string, "mode", modeOfBuilder(builder));
|
|
string = expandVariable(string, "arch", archOfBuilder(builder));
|
|
string = expandVariable(string, "runtime", runtimeOfBuilder(builder));
|
|
return string;
|
|
}
|
|
|
|
/// Finds the branch of a builder given the list of branches.
|
|
String branchOfBuilder(String builder, List<String> branches) {
|
|
return branches.where((branch) => branch != "master").firstWhere(
|
|
(branch) => builder.endsWith("-$branch"),
|
|
orElse: () => "master");
|
|
}
|
|
|
|
/// Finds the named configuration to test according to the test matrix
|
|
/// information and the command line options.
|
|
bool resolveNamedConfiguration(
|
|
List<String> branches,
|
|
List<dynamic> buildersConfigurations,
|
|
String requestedBranch,
|
|
String requestedNamedConfiguration,
|
|
String requestedBuilder,
|
|
Set<String> outputNamedConfiguration,
|
|
Set<String> outputBuilders) {
|
|
bool foundBuilder = false;
|
|
for (final builderConfiguration in buildersConfigurations) {
|
|
for (final builder in builderConfiguration["builders"]) {
|
|
if (requestedBuilder != null && builder != requestedBuilder) {
|
|
continue;
|
|
}
|
|
final branch = branchOfBuilder(builder, branches);
|
|
if (branch != requestedBranch) {
|
|
if (requestedBuilder == null) {
|
|
continue;
|
|
}
|
|
stderr.writeln("error: Builder $requestedBuilder is on branch $branch "
|
|
"rather than $requestedBranch");
|
|
stderr.writeln("error: To compare with that branch, use: -B $branch");
|
|
return false;
|
|
}
|
|
foundBuilder = true;
|
|
final steps = (builderConfiguration["steps"] as List).cast<Map>();
|
|
final testSteps = steps
|
|
.where((step) =>
|
|
!step.containsKey("script") || step["script"] == "tools/test.py")
|
|
.toList();
|
|
for (final step in testSteps) {
|
|
final arguments = step["arguments"]
|
|
.map((argument) => expandVariables(argument, builder))
|
|
.toList();
|
|
final namedConfiguration = arguments
|
|
.firstWhere((argument) => (argument as String).startsWith("-n"))
|
|
.substring(2);
|
|
if (requestedNamedConfiguration == null ||
|
|
requestedNamedConfiguration == namedConfiguration) {
|
|
outputNamedConfiguration.add(namedConfiguration);
|
|
outputBuilders.add(builder);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if (requestedBuilder != null && !foundBuilder) {
|
|
stderr.writeln("error: Builder $requestedBuilder doesn't exist");
|
|
return false;
|
|
}
|
|
if (requestedBuilder != null &&
|
|
requestedNamedConfiguration == null &&
|
|
outputNamedConfiguration.isEmpty) {
|
|
stderr.writeln("error: Builder $requestedBuilder isn't testing any named "
|
|
"configurations");
|
|
return false;
|
|
}
|
|
if (requestedBuilder != null &&
|
|
requestedNamedConfiguration != null &&
|
|
outputNamedConfiguration.isEmpty) {
|
|
stderr.writeln("error: The builder $requestedBuilder isn't testing the "
|
|
"named configuration $requestedNamedConfiguration");
|
|
return false;
|
|
}
|
|
if (requestedNamedConfiguration != null && outputBuilders.isEmpty) {
|
|
stderr.writeln("error: The named configuration "
|
|
"$requestedNamedConfiguration isn't tested on any builders");
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
/// Locates the merge base between head and the [branch] on the given [remote].
|
|
/// If a particular [commit] was requested, use that.
|
|
Future<String> findMergeBase(
|
|
String commit, String remote, String branch) async {
|
|
if (commit != null) {
|
|
return commit;
|
|
}
|
|
final arguments = ["merge-base", "$remote/$branch", "HEAD"];
|
|
final result =
|
|
await Process.run("git", arguments, runInShell: Platform.isWindows);
|
|
if (result.exitCode != 0) {
|
|
throw new Exception("Failed to run: git ${arguments.join(' ')}\n"
|
|
"stdout:\n${result.stdout}\n"
|
|
"stderr:\n${result.stderr}\n");
|
|
}
|
|
return LineSplitter.split(result.stdout).first;
|
|
}
|
|
|
|
/// Exception thrown when looking up the build for a commit failed.
|
|
class CommitNotBuiltException implements Exception {
|
|
final String reason;
|
|
|
|
CommitNotBuiltException(this.reason);
|
|
|
|
String toString() => reason;
|
|
}
|
|
|
|
/// The result after searching for a build of a commit.
|
|
class BuildSearchResult {
|
|
final int build;
|
|
final String commit;
|
|
|
|
BuildSearchResult(this.build, this.commit);
|
|
}
|
|
|
|
/// Locates the build number of the [commit] on the [builder], or throws an
|
|
/// exception if the builder hasn't built the commit.
|
|
Future<BuildSearchResult> searchForBuild(String builder, String commit) async {
|
|
final requestUrl = Uri.parse(
|
|
"https://cr-buildbucket.appspot.com/_ah/api/buildbucket/v1/search"
|
|
"?bucket=luci.dart.ci.sandbox"
|
|
"&tag=builder%3A$builder"
|
|
"&tag=buildset%3Acommit%2Fgit%2F$commit"
|
|
"&fields=builds(status%2Ctags%2Curl)");
|
|
final client = new HttpClient();
|
|
final request = await client.getUrl(requestUrl);
|
|
final response = await request.close();
|
|
final Map<String, dynamic> object = await response
|
|
.transform(new Utf8Decoder())
|
|
.transform(new JsonDecoder())
|
|
.first;
|
|
client.close();
|
|
final builds = object["builds"];
|
|
if (builds == null || builds.isEmpty) {
|
|
throw new CommitNotBuiltException(
|
|
"Builder $builder hasn't built commit $commit");
|
|
}
|
|
final build = builds.last;
|
|
final tags = (build["tags"] as List).cast<String>();
|
|
final buildAddressTag =
|
|
tags.firstWhere((tag) => tag.startsWith("build_address:"));
|
|
final buildAddress = buildAddressTag.substring("build_address:".length);
|
|
if (build["status"] != "COMPLETED") {
|
|
throw new CommitNotBuiltException(
|
|
"Build $buildAddress isn't completed yet");
|
|
}
|
|
return new BuildSearchResult(int.parse(buildAddress.split("/").last), commit);
|
|
}
|
|
|
|
Future<BuildSearchResult> searchForApproximateBuild(
|
|
String builder, String commit) async {
|
|
try {
|
|
return await searchForBuild(builder, commit);
|
|
} on CommitNotBuiltException catch (e) {
|
|
print("Warning: $e, searching for an inexact previous build...");
|
|
final int limit = 25;
|
|
final arguments = [
|
|
"rev-list",
|
|
"$commit~$limit..$commit~1",
|
|
"--first-parent",
|
|
"--topo-order"
|
|
];
|
|
final processResult = await Process.run("git", arguments, runInShell: true);
|
|
if (processResult.exitCode != 0) {
|
|
throw new Exception("Failed to list potential commits: git $arguments\n"
|
|
"exitCode: ${processResult.exitCode}\n"
|
|
"stdout: ${processResult.stdout}\n"
|
|
"stdout: ${processResult.stderr}\n");
|
|
}
|
|
for (final fallbackCommit in LineSplitter.split(processResult.stdout)) {
|
|
try {
|
|
return await searchForBuild(builder, fallbackCommit);
|
|
} catch (e) {
|
|
print(
|
|
"Warning: Searching for inexact baseline build: $e, continuing...");
|
|
}
|
|
}
|
|
throw new CommitNotBuiltException(
|
|
"Failed to locate approximate baseline results for "
|
|
"$commit in past $limit commits");
|
|
}
|
|
}
|
|
|
|
void main(List<String> args) async {
|
|
final parser = new ArgParser();
|
|
parser.addOption("builder",
|
|
abbr: "b", help: "Run tests like on the given builder");
|
|
parser.addOption("branch",
|
|
abbr: "B",
|
|
help: "Select the builders building this branch",
|
|
defaultsTo: "master");
|
|
parser.addOption("commit", abbr: "C", help: "Compare with this commit");
|
|
parser.addFlag("deflake",
|
|
help: "Re-run failing newly tests $deflakingCount times.");
|
|
parser.addFlag("report-flakes",
|
|
help: "Report test failures for tests known to be flaky.\n"
|
|
"This ignores all flakiness data from CI but flakes\n"
|
|
"detected by --deflake will remain hidden");
|
|
parser.addFlag("list-configurations",
|
|
help: "Output list of configurations.", negatable: false);
|
|
parser.addOption("named-configuration",
|
|
abbr: "n",
|
|
help: "The named test configuration that supplies the\nvalues for all "
|
|
"test options, specifying how tests\nshould be run.");
|
|
parser.addOption("local-configuration",
|
|
abbr: "N",
|
|
help: "Use a different named configuration for local\ntesting than the "
|
|
"named configuration the baseline\nresults were downloaded for. The "
|
|
"results may be\ninexact if the baseline configuration is "
|
|
"different.");
|
|
parser.addOption("remote",
|
|
abbr: "R",
|
|
help: "Compare with this remote and git branch",
|
|
defaultsTo: "origin");
|
|
parser.addFlag("help", help: "Show the program usage.", negatable: false);
|
|
|
|
final options = parser.parse(args);
|
|
if (options["help"] ||
|
|
(options["builder"] == null &&
|
|
options["named-configuration"] == null &&
|
|
!options["list-configurations"])) {
|
|
print("""
|
|
Usage: test.dart -b [BUILDER] -n [CONFIGURATION] [OPTION]... [--]
|
|
[TEST.PY OPTION]... [SELECTOR]...
|
|
|
|
Run tests and compare with the results on the given builder. Either the -n or
|
|
the -b option, or both, must be used. Any options following -- and non-option
|
|
arguments will be forwarded to test.py invocations. The specified named
|
|
configuration's results will be downloaded from the specified builder. If only a
|
|
named configuration is specified, the results are downloaded from the
|
|
appropriate builders. If only a builder is specified, the default named
|
|
configuration is used if the builder only has a single named configuration.
|
|
Otherwise the available named configurations are listed.
|
|
|
|
See the documentation at https://goto.google.com/dart-status-file-free-workflow
|
|
|
|
${parser.usage}""");
|
|
return;
|
|
}
|
|
|
|
if (options["list-configurations"]) {
|
|
final process = await Process.start(
|
|
"python", ["tools/test.py", "--list-configurations"],
|
|
mode: ProcessStartMode.inheritStdio, runInShell: Platform.isWindows);
|
|
exitCode = await process.exitCode;
|
|
return;
|
|
}
|
|
|
|
// Locate gsutil.py.
|
|
gsutilPy =
|
|
Platform.script.resolve("../third_party/gsutil/gsutil.py").toFilePath();
|
|
|
|
// Load the test matrix.
|
|
final scriptPath = Platform.script.toFilePath();
|
|
final testMatrixPath =
|
|
scriptPath.substring(0, scriptPath.length - "test.dart".length) +
|
|
"bots/test_matrix.json";
|
|
final testMatrix = jsonDecode(await new File(testMatrixPath).readAsString());
|
|
final branches = (testMatrix["branches"] as List).cast<String>();
|
|
final buildersConfigurations =
|
|
testMatrix["builder_configurations"] as List<dynamic>;
|
|
|
|
// Determine what named configuration to run and which builders to download
|
|
// existing results from.
|
|
final namedConfigurations = new SplayTreeSet<String>();
|
|
final builders = new SplayTreeSet<String>();
|
|
if (!resolveNamedConfiguration(
|
|
branches,
|
|
buildersConfigurations,
|
|
options["branch"],
|
|
options["named-configuration"],
|
|
options["builder"],
|
|
namedConfigurations,
|
|
builders)) {
|
|
exitCode = 1;
|
|
return;
|
|
}
|
|
if (namedConfigurations.length > 1) {
|
|
final builder = builders.single;
|
|
stderr.writeln(
|
|
"error: The builder $builder is testing multiple named configurations");
|
|
stderr.writeln(
|
|
"error: Please select the desired named configuration using -n:");
|
|
for (final namedConfiguration in namedConfigurations) {
|
|
stderr.writeln(" -n $namedConfiguration");
|
|
}
|
|
exitCode = 1;
|
|
return;
|
|
}
|
|
final namedConfiguration = namedConfigurations.single;
|
|
final localConfiguration =
|
|
options["local-configuration"] as String ?? namedConfiguration;
|
|
for (final builder in builders) {
|
|
if (localConfiguration != namedConfiguration) {
|
|
print("Testing the named configuration $localConfiguration "
|
|
"compared with builder $builder's configuration $namedConfiguration");
|
|
} else {
|
|
print("Testing the named configuration $localConfiguration "
|
|
"compared with builder $builder");
|
|
}
|
|
}
|
|
|
|
// Find out where the current HEAD branched.
|
|
final commit = await findMergeBase(
|
|
options["commit"], options["remote"], options["branch"]);
|
|
print("Base commit is $commit");
|
|
|
|
// Store the downloaded results and our test results in a temporary directory.
|
|
final outDirectory = await Directory.systemTemp.createTemp("test.dart.");
|
|
try {
|
|
final mergedResults = <String, Map<String, dynamic>>{};
|
|
final mergedFlaky = <String, Map<String, dynamic>>{};
|
|
|
|
// Use the buildbucket API to search for builds of the right commit.
|
|
final inexactBuilds = new SplayTreeMap<String, String>();
|
|
for (final builder in builders) {
|
|
// Download the previous results and flakiness info from cloud storage.
|
|
print("Finding build on builder $builder to compare with...");
|
|
final buildSearchResult =
|
|
await searchForApproximateBuild(builder, commit);
|
|
if (buildSearchResult.commit != commit) {
|
|
print("Warning: Using commit ${buildSearchResult.commit} "
|
|
"as baseline instead of $commit for $builder");
|
|
inexactBuilds[builder] = buildSearchResult.commit;
|
|
}
|
|
final buildNumber = buildSearchResult.build.toString();
|
|
print("Downloading results from builder $builder build $buildNumber...");
|
|
await cpGsutil(buildFileCloudPath(builder, buildNumber, "results.json"),
|
|
"${outDirectory.path}/previous.json");
|
|
if (!options["report-flakes"]) {
|
|
await cpGsutil(buildFileCloudPath(builder, buildNumber, "flaky.json"),
|
|
"${outDirectory.path}/flaky.json");
|
|
}
|
|
print("Downloaded baseline results from builder $builder");
|
|
// Merge the results for the builders.
|
|
if (builders.length > 1) {
|
|
mergedResults
|
|
.addAll(await loadResultsMap("${outDirectory.path}/previous.json"));
|
|
if (!options["report-flakes"]) {
|
|
mergedFlaky
|
|
.addAll(await loadResultsMap("${outDirectory.path}/flaky.json"));
|
|
}
|
|
}
|
|
}
|
|
|
|
// Write out the merged results for the builders.
|
|
if (builders.length > 1) {
|
|
await new File("${outDirectory.path}/previous.json").writeAsString(
|
|
mergedResults.values.map((data) => jsonEncode(data) + "\n").join(""));
|
|
}
|
|
|
|
// Ensure that there is a flaky.json even if it wasn't downloaded.
|
|
if (builders.length > 1 || options["report-flakes"]) {
|
|
await new File("${outDirectory.path}/flaky.json").writeAsString(
|
|
mergedFlaky.values.map((data) => jsonEncode(data) + "\n").join(""));
|
|
}
|
|
|
|
// Override the named configuration in the baseline data if needed.
|
|
if (namedConfiguration != localConfiguration) {
|
|
for (final path in [
|
|
"${outDirectory.path}/previous.json",
|
|
"${outDirectory.path}/flaky.json"
|
|
]) {
|
|
final results = await loadResultsMap(path);
|
|
final records = results.values
|
|
.where((r) => r["configuration"] == namedConfiguration)
|
|
.toList()
|
|
..forEach((r) => r["configuration"] = localConfiguration);
|
|
await new File(path).writeAsString(
|
|
records.map((data) => jsonEncode(data) + "\n").join(""));
|
|
}
|
|
}
|
|
|
|
// Run the tests.
|
|
final arguments = [
|
|
"--named-configuration=$localConfiguration",
|
|
"--output-directory=${outDirectory.path}",
|
|
"--clean-exit",
|
|
"--silent-failures",
|
|
"--write-results",
|
|
"--write-logs",
|
|
...options.rest,
|
|
];
|
|
print("".padLeft(80, "="));
|
|
print("Running tests");
|
|
print("".padLeft(80, "="));
|
|
await runProcessInheritStdio("python", ["tools/test.py", ...arguments],
|
|
runInShell: Platform.isWindows);
|
|
|
|
if (options["deflake"]) {
|
|
await deflake(outDirectory, localConfiguration, options.rest);
|
|
}
|
|
|
|
// Write out the final comparison.
|
|
print("".padLeft(80, "="));
|
|
print("Test Results");
|
|
print("".padLeft(80, "="));
|
|
final compareOutput = await runProcess(Platform.resolvedExecutable, [
|
|
"tools/bots/compare_results.dart",
|
|
"--human",
|
|
"--verbose",
|
|
"--changed",
|
|
"--failing",
|
|
"--passing",
|
|
"--flakiness-data=${outDirectory.path}/flaky.json",
|
|
"--logs=${outDirectory.path}/logs.json",
|
|
"${outDirectory.path}/previous.json",
|
|
"${outDirectory.path}/results.json",
|
|
]);
|
|
if (compareOutput.stdout == "") {
|
|
print("There were no test failures.");
|
|
} else {
|
|
stdout.write(compareOutput.stdout);
|
|
}
|
|
if (inexactBuilds.isNotEmpty) {
|
|
print("");
|
|
inexactBuilds.forEach((String builder, String inexactCommit) => print(
|
|
"Warning: Results may be inexact because commit ${inexactCommit} "
|
|
"was used as the baseline for $builder instead of $commit"));
|
|
}
|
|
} finally {
|
|
await outDirectory.delete(recursive: true);
|
|
}
|
|
}
|
|
|
|
void deflake(Directory outDirectory, String localConfiguration,
|
|
List<String> testPyArgs) async {
|
|
// Find the list of tests to deflake.
|
|
final deflakeListOutput = await runProcess(Platform.resolvedExecutable, [
|
|
"tools/bots/compare_results.dart",
|
|
"--changed",
|
|
"--failing",
|
|
"--passing",
|
|
"--flakiness-data=${outDirectory.path}/flaky.json",
|
|
"${outDirectory.path}/previous.json",
|
|
"${outDirectory.path}/results.json",
|
|
]);
|
|
final deflakeListPath = "${outDirectory.path}/deflake.list";
|
|
final deflakeListFile = new File(deflakeListPath);
|
|
await deflakeListFile.writeAsString(deflakeListOutput.stdout);
|
|
|
|
// Deflake the changed tests.
|
|
final deflakingResultsPaths = <String>[];
|
|
for (int i = 1; deflakeListOutput.stdout != "" && i <= deflakingCount; i++) {
|
|
print("".padLeft(80, "="));
|
|
print("Running deflaking iteration $i");
|
|
print("".padLeft(80, "="));
|
|
final deflakeDirectory = new Directory("${outDirectory.path}/$i");
|
|
await deflakeDirectory.create();
|
|
final deflakeArguments = <String>[
|
|
"--named-configuration=$localConfiguration",
|
|
"--output-directory=${deflakeDirectory.path}",
|
|
"--clean-exit",
|
|
"--silent-failures",
|
|
"--write-results",
|
|
"--test-list=$deflakeListPath",
|
|
...testPyArgs,
|
|
];
|
|
await runProcessInheritStdio(
|
|
"python", ["tools/test.py", ...deflakeArguments],
|
|
runInShell: Platform.isWindows);
|
|
deflakingResultsPaths.add("${deflakeDirectory.path}/results.json");
|
|
}
|
|
|
|
// Update the flakiness information based on what we've learned.
|
|
print("Updating flakiness information...");
|
|
await runProcess(Platform.resolvedExecutable, [
|
|
"tools/bots/update_flakiness.dart",
|
|
"--input=${outDirectory.path}/flaky.json",
|
|
"--output=${outDirectory.path}/flaky.json",
|
|
"${outDirectory.path}/results.json",
|
|
...deflakingResultsPaths,
|
|
]);
|
|
}
|