dart-lang/sdk
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[infra] Do not mark tests as no longer flaky on try builds

Browse Source 
Adds a --noforgive flag to update_flakiness.dart, which will
be used on try builds.  Fixes lint warnings.

Change-Id: I88d6327b8292cb107f3ad512ee345e9ee27ccbbc
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/159902
Reviewed-by: Karl Klose <karlklose@google.com>
This commit is contained in:
William Hesse
2020-08-24 18:26:01 +00:00
parent 351a1cc489
commit 677feef159
1 changed files with 45 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/bots/update_flakiness.dart
+45 -60
View File
@@ -12,105 +12,90 @@ import 'package:args/args.dart';
import 'package:test_runner/bot_results.dart';
main(List<String> args) async {
final parser = new ArgParser();
void main(List<String> args) async {
final parser = ArgParser();
parser.addFlag('help', help: 'Show the program usage.', negatable: false);
parser.addOption('input', abbr: 'i', help: "Input flakiness file.");
parser.addOption('output', abbr: 'o', help: "Output flakiness file.");
parser.addOption('build-id', help: "Logdog ID of this buildbot run");
parser.addOption('commit', help: "Commit hash of this buildbot run");
parser.addOption('input', abbr: 'i', help: 'Input flakiness file.');
parser.addOption('output', abbr: 'o', help: 'Output flakiness file.');
parser.addOption('build-id', help: 'Logdog ID of this buildbot run');
parser.addOption('commit', help: 'Commit hash of this buildbot run');
parser.addFlag('no-forgive', help: 'Don\'t remove any flaky records');
final options = parser.parse(args);
if (options["help"]) {
print("""
if (options['help']) {
print('''
Usage: update_flakiness.dart [OPTION]... [RESULT-FILE]...
Update the flakiness data with a set of fresh results.
The options are as follows:
${parser.usage}""");
${parser.usage}''');
return;
}
final parameters = options.rest;
// Load the existing flakiness data, if any.
final data = options["input"] != null
? await loadResultsMap(options["input"])
final data = options['input'] != null
? await loadResultsMap(options['input'])
: <String, Map<String, dynamic>>{};
// Incrementally update the flakiness data with each observed result.
for (final path in parameters) {
final results = await loadResults(path);
for (final resultObject in results) {
final String configuration = resultObject["configuration"];
final String name = resultObject["name"];
final String result = resultObject["result"];
final key = "$configuration:$name";
newMap() => <String, dynamic>{};
final Map<String, dynamic> testData = data.putIfAbsent(key, newMap);
testData["configuration"] = configuration;
testData["name"] = name;
testData["expected"] = resultObject["expected"];
final outcomes = testData.putIfAbsent("outcomes", () => []);
final String configuration = resultObject['configuration'];
final String name = resultObject['name'];
final String result = resultObject['result'];
final key = '$configuration:$name';
Map<String, dynamic> newMap() => {};
final testData = data.putIfAbsent(key, newMap);
testData['configuration'] = configuration;
testData['name'] = name;
testData['expected'] = resultObject['expected'];
final outcomes = testData.putIfAbsent('outcomes', () => []);
final time = DateTime.now().toIso8601String();
if (!outcomes.contains(result)) {
outcomes
..add(result)
..sort();
testData["last_new_result_seen"] = time;
testData['last_new_result_seen'] = time;
}
if (testData["current"] == result) {
testData["current_counter"]++;
if (testData['current'] == result) {
testData['current_counter']++;
} else {
testData["current"] = result;
testData["current_counter"] = 1;
testData['current'] = result;
testData['current_counter'] = 1;
}
final occurrences = testData.putIfAbsent("occurrences", newMap);
final occurrences = testData.putIfAbsent('occurrences', newMap);
occurrences.putIfAbsent(result, () => 0);
occurrences[result]++;
final firstSeen = testData.putIfAbsent("first_seen", newMap);
final firstSeen = testData.putIfAbsent('first_seen', newMap);
firstSeen.putIfAbsent(result, () => time);
final lastSeen = testData.putIfAbsent("last_seen", newMap);
final lastSeen = testData.putIfAbsent('last_seen', newMap);
lastSeen[result] = time;
final matches = testData.putIfAbsent("matches", newMap);
// TODO: Temporarily fill in the matches field for all other outcomes.
// Remove this when all the builders have run at least once.
for (final outcome in occurrences.keys) {
matches[outcome] = resultObject["expected"] == "Fail"
? ["Fail", "CompileTimeError", "RuntimeError"].contains(outcome)
: resultObject["expected"] == outcome;
}
matches[result] = resultObject["matches"];
final matches = testData.putIfAbsent('matches', newMap);
matches[result] = resultObject['matches'];
if (options["build-id"] != null) {
final buildIds = testData.putIfAbsent("build_ids", newMap);
buildIds[result] = options["build-id"];
if (options['build-id'] != null) {
final buildIds = testData.putIfAbsent('build_ids', newMap);
buildIds[result] = options['build-id'];
}
if (options["commit"] != null) {
final commits = testData.putIfAbsent("commits", newMap);
commits[result] = options["commit"];
if (options['commit'] != null) {
final commits = testData.putIfAbsent('commits', newMap);
commits[result] = options['commit'];
}
}
}
// Write out the new flakiness data, containing all the tests known to have
// multiple outcomes.
final sink = options["output"] != null
? new File(options["output"]).openWrite()
: stdout;
final keys = new List<String>.from(data.keys)..sort();
// Write out the new flakiness data.
final sink =
options['output'] != null ? File(options['output']).openWrite() : stdout;
final keys = data.keys.toList()..sort();
for (final key in keys) {
final testData = data[key];
if (testData["outcomes"].length < 2) continue;
// TODO: Temporarily discard entries for old tests that don't run. Remove
// this when all the builders have run at least once.
if (!testData.containsKey("matches")) {
continue;
}
// Forgive tests that have become deterministic again. If they flake less
// than once in a 100 (p<1%), then if they flake again, the probability of
// them getting past 5 runs of deflaking is 1%^5 = 0.00000001%.
if (100 <= testData["current_counter"]) {
if (testData['outcomes'].length < 2) continue;
// Forgive tests that have been stable for 100 builds.
if (!options['no-forgive'] && testData['current_counter'] >= 100) {
continue;
}
sink.writeln(jsonEncode(testData));