Refactor results failures tool to allow sharing of code.

Combines the two util.dart files and moves getting result logs from arguments to
a helper file.

Adding spec-parser as compiler to environment.

Bug:
Change-Id: Ifc3b9aacaf98a2976d25ef594bac5c5823bed208
Reviewed-on: https://dart-review.googlesource.com/25260
Reviewed-by: Jonas Termansen <sortie@google.com>
This commit is contained in:
Morten Krogh-Jespersen
2017-12-08 01:06:28 +00:00
committed by Morten Krogh-jespersen
parent dcdd12cc1a
commit 6cbb00a889
17 changed files with 243 additions and 282 deletions
+42 -148
View File
@@ -2,18 +2,17 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:async';
import 'dart:io';
import 'package:args/command_runner.dart';
import 'package:args/args.dart';
import 'package:gardening/src/luci.dart';
import 'package:gardening/src/luci_api.dart';
import 'package:gardening/src/results/status_expectations.dart';
import 'package:gardening/src/results/status_files.dart';
import 'package:gardening/src/results/test_result_helper.dart';
import 'package:gardening/src/results/test_result_service.dart';
import 'package:gardening/src/util.dart';
import 'package:gardening/src/console_table.dart';
import 'package:gardening/src/results/result_models.dart' as models;
import 'package:gardening/src/results/util.dart';
import 'package:gardening/src/results/result_json_models.dart' as models;
import 'package:gardening/src/logdog.dart';
import 'package:gardening/src/logdog_rpc.dart';
import 'package:gardening/src/buildbucket.dart';
import 'package:gardening/src/extended_printer.dart';
@@ -34,17 +33,6 @@ OutputTable getOutputTable(ArgResults argResults) {
return new ConsoleTable(template: rows);
}
/// Determine if arguments is a CQ url or commit-number + patchset.
bool isCqInput(ArgResults argResults) {
if (argResults.rest.length == 1) {
return isSwarmingTaskUrl(argResults.rest.first);
}
if (argResults.rest.length == 2) {
return areNumbers(argResults.rest);
}
return false;
}
String howToUse(String command) {
return "Use by calling one of the following:\n\n"
"\tget $command <file> : for a local result.log file.\n"
@@ -56,93 +44,6 @@ String howToUse(String command) {
"\tget $command <builder_group> : for a builder group.\n";
}
/// Utility method to get a single test-result no matter what has been passed in
/// as arguments. The test-result can either be from a builder-group, a single
/// build on a builder or from a log.
Future<models.TestResult> getTestResult(ArgResults argResults) async {
if (argResults.rest.length == 0) {
print("No result.log file given as argument.");
print(howToUse(argResults.name));
return null;
}
var logger = createLogger();
var cache = createCacheFunction(logger);
var testResultService = new TestResultService(logger, cache);
String firstArgument = argResults.rest.first;
var luciApi = new LuciApi();
bool isBuilderGroup = (await getBuilderGroups(luciApi, DART_CLIENT, cache()))
.any((builder) => builder == firstArgument);
bool isBuilder = (await getAllBuilders(luciApi, DART_CLIENT, cache()))
.any((builder) => builder == firstArgument);
if (argResults.rest.length == 1) {
if (argResults.rest.first.startsWith("http")) {
return testResultService.fromLogdog(firstArgument);
} else if (isBuilderGroup) {
return testResultService.forBuilderGroup(firstArgument);
} else if (isBuilder) {
return testResultService.latestForBuilder(BUILDER_PROJECT, firstArgument);
}
}
var file = new File(argResults.rest.first);
if (await file.exists()) {
return testResultService.getFromFile(file);
}
if (argResults.rest.length == 2 &&
isBuilder &&
isNumber(argResults.rest[1])) {
var buildNumber = int.parse(argResults.rest[1]);
return testResultService.forBuild(
BUILDER_PROJECT, argResults.rest[0], buildNumber);
}
print("Too many arguments passed to command or arguments were incorrect.");
print(howToUse(argResults.name));
return null;
}
/// Utility method to get test results from the CQ.
Future<Iterable<BuildBucketTestResult>> getTestResultsFromCq(
ArgResults argResults) async {
if (argResults.rest.length == 0) {
print("No result.log file given as argument.");
print(howToUse(argResults.name));
return null;
}
var logger = createLogger();
var createCache = createCacheFunction(logger);
var testResultService = new TestResultService(logger, createCache);
String firstArgument = argResults.rest.first;
if (argResults.rest.length == 1) {
if (!isSwarmingTaskUrl(firstArgument)) {
print("URI does not match "
"`https://ci.chromium.org/swarming/task/<taskid>?server...`.");
print(howToUse(argResults.name));
return null;
}
String swarmingTaskId = getSwarmingTaskId(firstArgument);
return await testResultService.getFromSwarmingTaskId(swarmingTaskId);
}
if (argResults.rest.length == 2 && areNumbers(argResults.rest)) {
int changeNumber = int.parse(firstArgument);
int patchset = int.parse(argResults.rest.last);
return await testResultService.fromGerrit(changeNumber, patchset);
}
print("Too many arguments passed to command or arguments were incorrect.");
print(howToUse(argResults.name));
return null;
}
/// [GetCommand] handles when given command 'get' and expect a sub-command.
class GetCommand extends Command {
@override
@@ -162,7 +63,8 @@ class GetCommand extends Command {
/// returns a list of tests with their respective results.
class GetTestsWithResultCommand extends Command {
@override
String get description => "Get results for tests.";
String get description => "Get a list of tests with their respective "
"results from result.logs found from input.";
@override
String get name => "tests";
@@ -172,8 +74,9 @@ class GetTestsWithResultCommand extends Command {
}
Future run() async {
models.TestResult testResults = await getTestResult(argResults);
models.TestResult testResults = await getTestResult(argResults.rest);
if (testResults == null) {
print(howToUse("tests"));
return;
}
var outputTable = getOutputTable(argResults)
@@ -191,10 +94,11 @@ class GetTestsWithResultCommand extends Command {
/// 'result' and returns a list of tests with their result and expectations.
class GetTestsWithResultAndExpectationCommand extends Command {
@override
String get description => "Get results and expectations for tests.";
String get description => "Get a list of tests with their respective "
"results and expectations from result.logs found from input.";
@override
String get name => "results";
String get name => "tests-with-expectations";
GetTestsWithResultAndExpectationCommand() {
buildArgs(argParser);
@@ -203,17 +107,20 @@ class GetTestsWithResultAndExpectationCommand extends Command {
Future run() async {
models.TestResult testResult = null;
if (isCqInput(argResults)) {
if (isCqInput(argResults.rest)) {
Iterable<BuildBucketTestResult> buildBucketTestResults =
await getTestResultsFromCq(argResults);
Iterable<models.TestResult> testResults =
buildBucketTestResults.map((build) => build.testResult);
testResult = new models.TestResult()..combineWith(testResults);
await getTestResultsFromCq(argResults.rest);
if (buildBucketTestResults != null) {
testResult = buildBucketTestResults.fold<models.TestResult>(
new models.TestResult(),
(combined, buildResult) => combined..combineWith([buildResult]));
}
} else {
testResult = await getTestResult(argResults);
testResult = await getTestResult(argResults.rest);
}
if (testResult == null) {
print(howToUse("results"));
return;
}
@@ -241,7 +148,8 @@ class GetTestsWithResultAndExpectationCommand extends Command {
/// returns only the failing tests.
class GetTestFailuresCommand extends Command {
@override
String get description => "Get failures of tests.";
String get description => "Get a list of tests with their respective "
"results and expectations from result.logs found from input.";
@override
String get name => "failures";
@@ -250,52 +158,38 @@ class GetTestFailuresCommand extends Command {
buildArgs(argParser);
}
Future run() {
if (isCqInput(argResults)) {
return handleCqInput(argResults);
Future run() async {
List<models.TestResult> testResults = [];
if (isCqInput(argResults.rest)) {
var buildBucketResults = await getTestResultsFromCq(argResults.rest);
if (buildBucketResults == null) {
print(howToUse("failures"));
return;
}
testResults.addAll(buildBucketResults);
} else {
return handleBuildbotInput(argResults);
}
}
Future handleCqInput(ArgResults argResults) async {
Iterable<BuildBucketTestResult> buildBucketTestResults =
await getTestResultsFromCq(argResults);
if (buildBucketTestResults == null) {
return;
var testResult = await getTestResult(argResults.rest);
if (testResult == null) {
print(howToUse("failures"));
return;
}
testResults.add(testResult);
}
print("All result logs fetched.");
print("Calling test.py to find statuses for each test.");
print("");
for (var buildResult in buildBucketTestResults) {
printBuild(buildResult.build);
for (var testResult in testResults) {
if (testResult is BuildBucketTestResult) {
printBuild(testResult.build);
}
List<TestExpectationResult> results =
await getTestResultsWithExpectation(buildResult.testResult);
await getTestResultsWithExpectation(testResult);
printFailingTestExpectationResults(results);
print("");
}
}
Future handleBuildbotInput(ArgResults argResults) async {
models.TestResult testResult = await getTestResult(argResults);
if (testResult == null) {
return;
}
print("All result logs fetched.");
var estimatedTime =
new Duration(milliseconds: testResult.results.length * 100 ~/ 1000);
print("Calling test.py to find status files for the configuration and "
"the expectation for ${testResult.results.length} tests. ");
List<TestExpectationResult> withExpectations =
await getTestResultsWithExpectation(testResult);
printFailingTestExpectationResults(withExpectations);
print("");
}
}
/// Prints a test result.
+1 -1
View File
@@ -6,7 +6,7 @@ import 'dart:async';
import 'package:args/args.dart';
import 'package:args/command_runner.dart';
import 'package:gardening/src/results/configurations.dart';
import 'package:gardening/src/results/result_models.dart' as models;
import 'package:gardening/src/results/result_json_models.dart' as models;
import 'package:gardening/src/results/testpy_wrapper.dart';
/// Helper function to add all standard arguments to the [argParser].
+1 -22
View File
@@ -12,15 +12,11 @@ import 'package:gardening/src/logger.dart';
import 'package:gardening/src/luci.dart';
import 'package:gardening/src/luci_api.dart';
import 'package:gardening/src/results/configuration_environment.dart';
import 'package:gardening/src/results/result_models.dart' as models;
import 'package:gardening/src/results/result_json_models.dart' as models;
import 'package:gardening/src/results/status_files.dart';
import 'package:gardening/src/results/test_result_service.dart';
import 'package:gardening/src/results/testpy_wrapper.dart';
import 'package:gardening/src/results/util.dart';
import 'package:gardening/src/util.dart';
import 'package:gardening/src/workflow/workflow.dart';
import 'results_status_workflow.dart';
/// Class [StatusCommand] handles the 'status' subcommand and provides
/// sub-commands for interacting with status files.
@@ -33,7 +29,6 @@ class StatusCommand extends Command {
StatusCommand() {
addSubcommand(new CheckStatusCommand());
addSubcommand(new UpdateStatusCommand());
}
}
@@ -201,19 +196,3 @@ class CheckStatusCommand extends Command {
}
}
}
/// Class [UpdateStatusCommand] handles the 'status update' subcommand and
/// updates status files.
class UpdateStatusCommand extends Command {
@override
String get description => "Update status files, from failure data and "
"existing status entries.";
@override
String get name => "update";
Future run() async {
var workflow = new Workflow();
return workflow.start(new AskForLogs());
}
}
@@ -6,8 +6,6 @@ import 'dart:async';
import 'dart:io';
import 'cache_new.dart';
import 'logdog_rpc.dart';
import 'results/util.dart';
import 'util.dart';
import 'buildbot_structures.dart';
@@ -7,7 +7,6 @@
/// Use this to detect flakiness of failures, especially timeouts.
import 'dart:async';
import 'dart:io';
import 'bot.dart';
import 'buildbot_structures.dart';
-1
View File
@@ -91,7 +91,6 @@ class LuciApi {
/// [_makeGetRequest] performs a get request to [uri].
Future<String> _makeGetRequest(Uri uri) async {
String uriString = uri.toString();
var request = await _client.getUrl(uri);
var response = await request.close();
if (response.statusCode != 200) {
@@ -7,7 +7,7 @@
// and also information about test-suites.
import 'package:status_file/environment.dart';
import 'result_models.dart';
import 'result_json_models.dart';
import 'configurations.dart';
typedef String _LookUpFunction(Configuration configuration);
@@ -45,6 +45,7 @@ final _variables = {
"minified": new _Variable.bool((c) => c.minified),
"mode": new _Variable((c) => c.mode, Mode.names),
"runtime": new _Variable(_runtimeName, Runtime.names),
"spec_parser": new _Variable.bool((c) => c.compiler == Compiler.specParser),
"strong": new _Variable.bool((c) => c.strong),
"system": new _Variable((c) => c.system, System.names),
"use_sdk": new _Variable.bool((c) => c.useSdk)
@@ -7,9 +7,9 @@ import 'dart:async';
import 'package:gardening/src/results/configuration_environment.dart';
import 'package:gardening/src/results/status_files.dart';
import 'result_models.dart';
import 'result_json_models.dart';
import 'testpy_wrapper.dart';
import 'util.dart';
import '../util.dart';
import 'package:status_file/expectation.dart';
/// Finds the expectation for each test found in the [testResult] results and
@@ -3,7 +3,6 @@
// BSD-style license that can be found in the LICENSE file.
import 'package:gardening/src/results/configuration_environment.dart';
import 'package:gardening/src/util.dart';
import 'package:status_file/status_file.dart';
class StatusFiles {
@@ -0,0 +1,92 @@
// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:async';
import 'dart:io';
import '../luci.dart';
import '../luci_api.dart';
import '../util.dart';
import 'result_json_models.dart';
import 'test_result_service.dart';
/// Utility method to get a single test-result no matter what has been passed in
/// as arguments. The test-result can either be from a builder-group, a single
/// build on a builder or from a log.
Future<TestResult> getTestResult(List<String> arguments) async {
if (arguments.isEmpty) {
print("No result.log file given as argument.");
return null;
}
var logger = createLogger();
var cache = createCacheFunction(logger);
var testResultService = new TestResultService(logger, cache);
String firstArgument = arguments.first;
var luciApi = new LuciApi();
bool isBuilderGroup = (await getBuilderGroups(luciApi, DART_CLIENT, cache()))
.any((builder) => builder == firstArgument);
bool isBuilder = (await getAllBuilders(luciApi, DART_CLIENT, cache()))
.any((builder) => builder == firstArgument);
if (arguments.length == 1) {
if (arguments.first.startsWith("http")) {
return testResultService.fromLogdog(firstArgument);
} else if (isBuilderGroup) {
return testResultService.forBuilderGroup(firstArgument);
} else if (isBuilder) {
return testResultService.latestForBuilder(BUILDER_PROJECT, firstArgument);
}
}
var file = new File(arguments.first);
if (await file.exists()) {
return testResultService.getFromFile(file);
}
if (arguments.length == 2 && isBuilder && isNumber(arguments.last)) {
var buildNumber = int.parse(arguments.last);
return testResultService.forBuild(
BUILDER_PROJECT, firstArgument, buildNumber);
}
print("Too many arguments passed to command or arguments were incorrect.");
return null;
}
/// Utility method to get test results from the CQ.
Future<Iterable<BuildBucketTestResult>> getTestResultsFromCq(
List<String> arguments) async {
if (arguments.isEmpty) {
print("No result.log file given as argument.");
return null;
}
var logger = createLogger();
var createCache = createCacheFunction(logger);
var testResultService = new TestResultService(logger, createCache);
String firstArgument = arguments.first;
if (arguments.length == 1) {
if (!isSwarmingTaskUrl(firstArgument)) {
print("URI does not match "
"`https://ci.chromium.org/swarming/task/<taskid>?server...`.");
return null;
}
String swarmingTaskId = getSwarmingTaskId(firstArgument);
return await testResultService.getFromSwarmingTaskId(swarmingTaskId);
}
if (arguments.length == 2 && areNumbers(arguments)) {
int changeNumber = int.parse(firstArgument);
int patchset = int.parse(arguments.last);
return await testResultService.fromGerrit(changeNumber, patchset);
}
print("Too many arguments passed to command or arguments were incorrect.");
return null;
}
@@ -6,7 +6,7 @@ import 'dart:io';
import 'dart:async';
import 'dart:convert';
import 'dart:core';
import 'result_models.dart';
import 'result_json_models.dart';
import '../logger.dart';
import '../cache_new.dart';
import '../logdog.dart';
@@ -14,7 +14,6 @@ import '../logdog_rpc.dart';
import '../luci_api.dart';
import '../luci.dart';
import '../buildbucket.dart';
import 'util.dart';
import '../util.dart';
/// [TestResultService] provides functions to obtain [TestResult]s from logs.
@@ -166,7 +165,7 @@ class TestResultService {
TestResult result = steps.fold(new TestResult(), (acc, buildStep) {
return acc..combineWith([buildStep.testResult]);
});
return new BuildBucketTestResult(build, result);
return new BuildBucketTestResult(build)..combineWith([result]);
});
}
@@ -228,10 +227,9 @@ class TestResultService {
}
/// Class that keeps track of a try build and the corresponding test result.
class BuildBucketTestResult {
class BuildBucketTestResult extends TestResult {
final BuildBucketBuild build;
final TestResult testResult;
BuildBucketTestResult(this.build, this.testResult);
BuildBucketTestResult(this.build);
}
/// Class that keeps track of a test step and test result.
@@ -2,11 +2,9 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:io';
import 'dart:async';
import 'package:path/path.dart' as path;
import 'result_models.dart';
import 'util.dart';
import 'result_json_models.dart';
import '../util.dart';
/// Calls test.py with arguments gathered from a specific [configuration] and
-92
View File
@@ -1,92 +0,0 @@
// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'result_models.dart';
import 'dart:io';
const String BUILDER_PROJECT = "chromium";
/// [PathHelper] is a utility class holding information about static paths.
class PathHelper {
static String testPyPath() {
var root = sdkRepositoryRoot();
return "${root}/tools/test.py";
}
static String _sdkRepositoryRoot;
static String sdkRepositoryRoot() {
return _sdkRepositoryRoot ??=
_findRoot(new Directory.fromUri(Platform.script));
}
static String _findRoot(Directory current) {
if (current.path.endsWith("sdk")) {
return current.path;
}
if (current.parent == null) {
print("Could not find the dart sdk folder. "
"Please run the tool in the root of the dart-sdk local repository.");
exit(1);
}
return _findRoot(current.parent);
}
}
/// Tests if all strings passed in [stringsToTest] are integers.
bool areNumbers(Iterable<String> stringsToTest) {
RegExp isNumberRegExp = new RegExp(r"^\d+$");
return stringsToTest
.every((string) => isNumberRegExp.firstMatch(string) != null);
}
bool isNumber(String stringToTest) {
bool succeeded = true;
int.parse(stringToTest, onError: (String) {
succeeded = false;
return 0;
});
return succeeded;
}
/// Gets if the [url] is a swarming task url.
bool isSwarmingTaskUrl(String url) {
return url.startsWith("https://ci.chromium.org/swarming");
}
/// Gets the swarming task id from the [url].
String getSwarmingTaskId(String url) {
RegExp swarmingTaskIdInPathRegExp =
new RegExp(r"https:\/\/ci\.chromium\.org\/swarming\/task\/(.*)\?server");
Match swarmingTaskIdMatch = swarmingTaskIdInPathRegExp.firstMatch(url);
if (swarmingTaskIdMatch == null) {
return null;
}
return swarmingTaskIdMatch.group(1);
}
/// Returns the test-suite for [name].
String getSuiteNameForTest(String name) {
var reg = new RegExp(r"^(.*?)\/.*$");
var match = reg.firstMatch(name);
if (match == null) {
return null;
}
return match.group(1);
}
/// Returns the qualified name (what to use in status-files) for a test with
/// [name].
String getQualifiedNameForTest(String name) {
if (name.startsWith("cc/")) {
return name;
}
return name.substring(name.indexOf("/") + 1);
}
/// Returns the reproduction command for test.py based on the [configuration]
/// and [name].
String getReproductionCommand(Configuration configuration, String name) {
var allArgs = configuration.toArgs(includeSelectors: false)..add(name);
return "${PathHelper.testPyPath()} ${allArgs.join(' ')}";
}
-2
View File
@@ -2,8 +2,6 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:async';
/// [Try] is similar to Haskell monad, where
/// a computation may throw an exception.
/// There is no checking of passing null into
+98
View File
@@ -7,6 +7,7 @@ import 'dart:convert';
import 'dart:io';
import 'package:args/args.dart';
import 'results/result_json_models.dart';
import 'cache.dart';
import 'cache_new.dart';
@@ -246,3 +247,100 @@ Future<ProcessResult> runPython(String script, List<String> args) {
/// Regular expression matches a Linux or Windows new line character.
final RegExp newLine = new RegExp(r'\r\n|\n');
/// Determine if arguments is a CQ url or commit-number + patchset.
bool isCqInput(List<String> arguments) {
if (arguments.length == 1) {
return isSwarmingTaskUrl(arguments.first);
}
if (arguments.length == 2) {
return areNumbers(arguments);
}
return false;
}
const String BUILDER_PROJECT = "chromium";
/// [PathHelper] is a utility class holding information about static paths.
class PathHelper {
static String testPyPath() {
var root = sdkRepositoryRoot();
return "${root}/tools/test.py";
}
static String _sdkRepositoryRoot;
static String sdkRepositoryRoot() {
return _sdkRepositoryRoot ??=
_findRoot(new Directory.fromUri(Platform.script));
}
static String _findRoot(Directory current) {
if (current.path.endsWith("sdk")) {
return current.path;
}
if (current.parent == null) {
print("Could not find the dart sdk folder. "
"Please run the tool in the root of the dart-sdk local repository.");
exit(1);
}
return _findRoot(current.parent);
}
}
/// Tests if all strings passed in [stringsToTest] are integers.
bool areNumbers(Iterable<String> stringsToTest) {
RegExp isNumberRegExp = new RegExp(r"^\d+$");
return stringsToTest
.every((string) => isNumberRegExp.firstMatch(string) != null);
}
bool isNumber(String stringToTest) {
bool succeeded = true;
int.parse(stringToTest, onError: (String) {
succeeded = false;
return 0;
});
return succeeded;
}
/// Gets if the [url] is a swarming task url.
bool isSwarmingTaskUrl(String url) {
return url.startsWith("https://ci.chromium.org/swarming");
}
/// Gets the swarming task id from the [url].
String getSwarmingTaskId(String url) {
RegExp swarmingTaskIdInPathRegExp =
new RegExp(r"https:\/\/ci\.chromium\.org\/swarming\/task\/(.*)\?server");
Match swarmingTaskIdMatch = swarmingTaskIdInPathRegExp.firstMatch(url);
if (swarmingTaskIdMatch == null) {
return null;
}
return swarmingTaskIdMatch.group(1);
}
/// Returns the test-suite for [name].
String getSuiteNameForTest(String name) {
var reg = new RegExp(r"^(.*?)\/.*$");
var match = reg.firstMatch(name);
if (match == null) {
return null;
}
return match.group(1);
}
/// Returns the qualified name (what to use in status-files) for a test with
/// [name].
String getQualifiedNameForTest(String name) {
if (name.startsWith("cc/")) {
return name;
}
return name.substring(name.indexOf("/") + 1);
}
/// Returns the reproduction command for test.py based on the [configuration]
/// and [name].
String getReproductionCommand(Configuration configuration, String name) {
var allArgs = configuration.toArgs(includeSelectors: false)..add(name);
return "${PathHelper.testPyPath()} ${allArgs.join(' ')}";
}