diff --git a/tools/bots/update_flakiness.dart b/tools/bots/update_flakiness.dart index 71bf4023b6b..2f4f680a6d9 100755 --- a/tools/bots/update_flakiness.dart +++ b/tools/bots/update_flakiness.dart @@ -12,105 +12,90 @@ import 'package:args/args.dart'; import 'package:test_runner/bot_results.dart'; -main(List args) async { - final parser = new ArgParser(); +void main(List args) async { + final parser = ArgParser(); parser.addFlag('help', help: 'Show the program usage.', negatable: false); - parser.addOption('input', abbr: 'i', help: "Input flakiness file."); - parser.addOption('output', abbr: 'o', help: "Output flakiness file."); - parser.addOption('build-id', help: "Logdog ID of this buildbot run"); - parser.addOption('commit', help: "Commit hash of this buildbot run"); + parser.addOption('input', abbr: 'i', help: 'Input flakiness file.'); + parser.addOption('output', abbr: 'o', help: 'Output flakiness file.'); + parser.addOption('build-id', help: 'Logdog ID of this buildbot run'); + parser.addOption('commit', help: 'Commit hash of this buildbot run'); + parser.addFlag('no-forgive', help: 'Don\'t remove any flaky records'); final options = parser.parse(args); - if (options["help"]) { - print(""" + if (options['help']) { + print(''' Usage: update_flakiness.dart [OPTION]... [RESULT-FILE]... Update the flakiness data with a set of fresh results. The options are as follows: -${parser.usage}"""); +${parser.usage}'''); return; } final parameters = options.rest; // Load the existing flakiness data, if any. - final data = options["input"] != null - ? await loadResultsMap(options["input"]) + final data = options['input'] != null + ? await loadResultsMap(options['input']) : >{}; // Incrementally update the flakiness data with each observed result. for (final path in parameters) { final results = await loadResults(path); for (final resultObject in results) { - final String configuration = resultObject["configuration"]; - final String name = resultObject["name"]; - final String result = resultObject["result"]; - final key = "$configuration:$name"; - newMap() => {}; - final Map testData = data.putIfAbsent(key, newMap); - testData["configuration"] = configuration; - testData["name"] = name; - testData["expected"] = resultObject["expected"]; - final outcomes = testData.putIfAbsent("outcomes", () => []); + final String configuration = resultObject['configuration']; + final String name = resultObject['name']; + final String result = resultObject['result']; + final key = '$configuration:$name'; + Map newMap() => {}; + final testData = data.putIfAbsent(key, newMap); + testData['configuration'] = configuration; + testData['name'] = name; + testData['expected'] = resultObject['expected']; + final outcomes = testData.putIfAbsent('outcomes', () => []); final time = DateTime.now().toIso8601String(); if (!outcomes.contains(result)) { outcomes ..add(result) ..sort(); - testData["last_new_result_seen"] = time; + testData['last_new_result_seen'] = time; } - if (testData["current"] == result) { - testData["current_counter"]++; + if (testData['current'] == result) { + testData['current_counter']++; } else { - testData["current"] = result; - testData["current_counter"] = 1; + testData['current'] = result; + testData['current_counter'] = 1; } - final occurrences = testData.putIfAbsent("occurrences", newMap); + final occurrences = testData.putIfAbsent('occurrences', newMap); occurrences.putIfAbsent(result, () => 0); occurrences[result]++; - final firstSeen = testData.putIfAbsent("first_seen", newMap); + final firstSeen = testData.putIfAbsent('first_seen', newMap); firstSeen.putIfAbsent(result, () => time); - final lastSeen = testData.putIfAbsent("last_seen", newMap); + final lastSeen = testData.putIfAbsent('last_seen', newMap); lastSeen[result] = time; - final matches = testData.putIfAbsent("matches", newMap); - // TODO: Temporarily fill in the matches field for all other outcomes. - // Remove this when all the builders have run at least once. - for (final outcome in occurrences.keys) { - matches[outcome] = resultObject["expected"] == "Fail" - ? ["Fail", "CompileTimeError", "RuntimeError"].contains(outcome) - : resultObject["expected"] == outcome; - } - matches[result] = resultObject["matches"]; + final matches = testData.putIfAbsent('matches', newMap); + matches[result] = resultObject['matches']; - if (options["build-id"] != null) { - final buildIds = testData.putIfAbsent("build_ids", newMap); - buildIds[result] = options["build-id"]; + if (options['build-id'] != null) { + final buildIds = testData.putIfAbsent('build_ids', newMap); + buildIds[result] = options['build-id']; } - if (options["commit"] != null) { - final commits = testData.putIfAbsent("commits", newMap); - commits[result] = options["commit"]; + if (options['commit'] != null) { + final commits = testData.putIfAbsent('commits', newMap); + commits[result] = options['commit']; } } } - // Write out the new flakiness data, containing all the tests known to have - // multiple outcomes. - final sink = options["output"] != null - ? new File(options["output"]).openWrite() - : stdout; - final keys = new List.from(data.keys)..sort(); + // Write out the new flakiness data. + final sink = + options['output'] != null ? File(options['output']).openWrite() : stdout; + final keys = data.keys.toList()..sort(); for (final key in keys) { final testData = data[key]; - if (testData["outcomes"].length < 2) continue; - // TODO: Temporarily discard entries for old tests that don't run. Remove - // this when all the builders have run at least once. - if (!testData.containsKey("matches")) { - continue; - } - // Forgive tests that have become deterministic again. If they flake less - // than once in a 100 (p<1%), then if they flake again, the probability of - // them getting past 5 runs of deflaking is 1%^5 = 0.00000001%. - if (100 <= testData["current_counter"]) { + if (testData['outcomes'].length < 2) continue; + // Forgive tests that have been stable for 100 builds. + if (!options['no-forgive'] && testData['current_counter'] >= 100) { continue; } sink.writeln(jsonEncode(testData));