dart-lang/sdk
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Remove obsolete log types from test.py

Browse Source 
Removes the flaky log, result.log result format, test outcome log,
and --append-logs option from test.py. They are replaced by the
new --write-results and --write-logs options.

The debug log is now always appended to, if it is written to a file.

Change-Id: I20dbbc1db86ef23a19ac4ea02067f18ae4906b3d
Reviewed-on: https://dart-review.googlesource.com/c/85440
Reviewed-by: Jonas Termansen <sortie@google.com>
This commit is contained in:
William Hesse
2018-11-28 11:15:04 +00:00
parent 498994487b
commit f780b6ea84
5 changed files with 3 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/testing/dart/configuration.dart
-6
View File
@@ -26,7 +26,6 @@ class TestConfiguration {
this.progress,
this.selectors,
this.testList,
this.appendLogs,
this.repeat,
this.batch,
this.batchDart2JS,
@@ -42,8 +41,6 @@ class TestConfiguration {
this.skipCompilation,
this.useKernelBytecode,
this.writeDebugLog,
this.writeTestOutcomeLog,
this.writeResultLog,
this.writeResults,
this.writeLogs,
this.drtPath,
@@ -79,7 +76,6 @@ class TestConfiguration {
// Boolean flags.
final bool appendLogs;
final bool batch;
final bool batchDart2JS;
final bool copyCoreDumps;
@@ -95,8 +91,6 @@ class TestConfiguration {
final bool skipCompilation;
final bool useKernelBytecode;
final bool writeDebugLog;
final bool writeTestOutcomeLog;
final bool writeResultLog;
final bool writeResults;
final bool writeLogs;
final bool printPassingStdout;
tools/testing/dart/options.dart
-17
View File
@@ -249,20 +249,9 @@ compact, color, line, verbose, silent, status, buildbot, diff''',
new _Option.bool('noBatch', 'Do not run tests in batch mode.', hide: true),
new _Option.bool('dart2js_batch', 'Run dart2js tests in batch mode.',
hide: true),
new _Option.bool(
'append_logs', 'Do not delete old logs but rather append to them.',
hide: true),
new _Option.bool('write_debug_log',
'Don\'t write debug messages to stdout but rather to a logfile.',
hide: true),
new _Option.bool('write_test_outcome_log',
'Write test outcomes to a "${TestUtils.testOutcomeFileName}" file.',
hide: true),
new _Option.bool(
'write_result_log',
'Write test results to a "${TestUtils.resultLogFileName}" json file '
'located at the debug_output_directory.',
hide: true),
new _Option.bool(
'write_results',
'Write results to a "${TestUtils.resultsFileName}" json file '
@@ -341,7 +330,6 @@ compiler.''',
/// For printing out reproducing command lines, we don't want to add these
/// options.
static final _blacklistedOptions = [
'append_logs',
'build_directory',
'debug_output_directory',
'chrome',
@@ -364,8 +352,6 @@ compiler.''',
'verbose',
'write_logs',
'write_debug_log',
'write_test_outcome_log',
'write_result_log',
'write_results',
].toSet();
@@ -680,7 +666,6 @@ compiler.''',
progress: Progress.find(data["progress"] as String),
selectors: selectors,
testList: data["test_list_contents"] as List<String>,
appendLogs: data["append_logs"] as bool,
repeat: data["repeat"] as int,
batch: !(data["noBatch"] as bool),
batchDart2JS: data["dart2js_batch"] as bool,
@@ -696,8 +681,6 @@ compiler.''',
skipCompilation: data["skip_compilation"] as bool,
useKernelBytecode: compiler == Compiler.dartkb,
writeDebugLog: data["write_debug_log"] as bool,
writeTestOutcomeLog: data["write_test_outcome_log"] as bool,
writeResultLog: data["write_result_log"] as bool,
writeResults: data["write_results"] as bool,
writeLogs: data["write_logs"] as bool,
drtPath: data["drt"] as String,
tools/testing/dart/test_configurations.dart
+1 -25
View File
@@ -60,25 +60,10 @@ Future testConfigurations(List<TestConfiguration> configurations) async {
var printTiming = firstConf.printTiming;
var listTests = firstConf.listTests;
var listStatusFiles = firstConf.listStatusFiles;
var reportInJson = firstConf.reportInJson;
Browser.resetBrowserConfiguration = firstConf.resetBrowser;
if (!firstConf.appendLogs) {
var files = [
new File(TestUtils.flakyFileName),
new File(TestUtils.testOutcomeFileName)
];
for (var file in files) {
if (file.existsSync()) {
file.deleteSync();
}
}
}
DebugLogger.init(firstConf.writeDebugLog ? TestUtils.debugLogFilePath : null,
append: firstConf.appendLogs);
DebugLogger.init(firstConf.writeDebugLog ? TestUtils.debugLogFilePath : null);
// Print the configurations being run by this execution of
// test.dart. However, don't do it if the silent progress indicator
@@ -202,7 +187,6 @@ Future testConfigurations(List<TestConfiguration> configurations) async {
eventListener.add(new StatusFileUpdatePrinter());
}
eventListener.add(new SummaryPrinter());
eventListener.add(new FlakyLogWriter());
if (printFailures) {
// The buildbot has it's own failure summary since it needs to wrap it
// into '@@@'-annotated sections.
@@ -223,14 +207,6 @@ Future testConfigurations(List<TestConfiguration> configurations) async {
}
}
if (firstConf.writeTestOutcomeLog) {
eventListener.add(new TestOutcomeLogWriter());
}
if (firstConf.writeResultLog) {
eventListener.add(new ResultLogWriter(firstConf.outputDirectory));
}
if (firstConf.writeResults) {
eventListener.add(new ResultWriter(firstConf, startTime, startStopwatch));
}
tools/testing/dart/test_progress.dart
-181
View File
@@ -120,104 +120,6 @@ class IgnoredTestMonitor extends EventListener {
}
}
class FlakyLogWriter extends EventListener {
void done(TestCase test) {
if (test.isFlaky && test.result != Expectation.pass) {
var buf = new StringBuffer();
for (var l in _buildFailureOutput(test)) {
buf.write("$l\n");
}
_appendToFlakyFile(buf.toString());
}
}
void _appendToFlakyFile(String msg) {
var file = new File(TestUtils.flakyFileName);
var fd = file.openSync(mode: FileMode.append);
fd.writeStringSync(msg);
fd.closeSync();
}
}
class TestOutcomeLogWriter extends EventListener {
/*
* The ".test-outcome.log" file contains one line for every executed test.
* Such a line is an encoded JSON data structure of the following form:
* The durations are double values in milliseconds.
*
* {
* name: 'co19/LibTest/math/x',
* configuration: {
* mode : 'release',
* compiler : 'dart2js',
* ....
* },
* test_result: {
* outcome: 'RuntimeError',
* expected_outcomes: ['Pass', 'Fail'],
* duration: 2600.64,
* command_results: [
* {
* name: 'dart2js',
* duration: 2400.44,
* },
* {
* name: 'ff',
* duration: 200.2,
* },
* ],
* }
* },
*/
IOSink _sink;
void done(TestCase test) {
var name = test.displayName;
var outcome = '${test.lastCommandOutput.result(test)}';
var expectations =
test.expectedOutcomes.map((expectation) => "$expectation").toList();
var commandResults = [];
double totalDuration = 0.0;
for (var command in test.commands) {
var output = test.commandOutputs[command];
if (output != null) {
double duration = output.time.inMicroseconds / 1000.0;
totalDuration += duration;
commandResults.add({
'name': command.displayName,
'duration': duration,
});
}
}
_writeTestOutcomeRecord({
'name': name,
'configuration': test.configuration.toSummaryMap(),
'test_result': {
'outcome': outcome,
'expected_outcomes': expectations,
'duration': totalDuration,
'command_results': commandResults,
},
});
}
void allDone() {
if (_sink != null) _sink.close();
}
void _writeTestOutcomeRecord(Map record) {
// TODO(mkroghj) change the location of this file
// to be in the debug_output_directory
// if the current location is not used.
if (_sink == null) {
_sink = new File(TestUtils.testOutcomeFileName)
.openWrite(mode: FileMode.append);
}
_sink.write("${jsonEncode(record)}\n");
}
}
class UnexpectedCrashLogger extends EventListener {
final archivedBinaries = <String, String>{};
@@ -772,89 +674,6 @@ String _buildSummaryEnd(Formatter formatter, int failedTests) {
}
}
class ResultLogWriter extends EventListener {
Map<String, Map> _configurations = {};
List<Map> _results = [];
String _outputDirectory;
ResultLogWriter(this._outputDirectory);
void allTestsKnown() {
// Write an empty result log file, that will be overwritten if any tests
// are actually run, when the allDone event handler is invoked.
writeToFile({}, []);
}
void done(TestCase test) {
// We try to find an existing configuration, so as to not duplicate this
// for each test.
var thisConf = test.configuration.toSummaryMap();
String key = _configurations.keys.firstWhere(
(key) => identical(_configurations[key], thisConf), orElse: () {
var newKey = "conf${_configurations.length + 1}";
_configurations[newKey] = thisConf;
return newKey;
});
var commands = test.commands.map((command) {
var output = test.commandOutputs[command];
if (output == null) {
return {'name': command.displayName};
}
return {
'name': command.displayName,
'exitCode': output.exitCode,
'timeout': output.hasTimedOut,
'duration': output.time.inMilliseconds
};
}).toList();
// Compute inlined expectations.
var inlineExpectations = <String>[];
if (test.hasStaticWarning) {
inlineExpectations.add("static-type-warning");
}
if (test.hasRuntimeError) {
inlineExpectations.add("runtime-error");
}
if (test.hasSyntaxError) {
inlineExpectations.add("syntax-error");
}
if (test.hasCompileError) {
inlineExpectations.add("compile-time-error");
}
if (test.hasCompileErrorIfChecked) {
inlineExpectations.add("checked-compile-time-error");
}
if (test.isNegativeIfChecked) {
inlineExpectations.add("dynamic-type-error");
}
_results.add({
'configuration': key,
'name': test.displayName,
'result': test.lastCommandOutput.result(test).toString(),
'test_expectation': inlineExpectations,
'flaky': test.isFlaky,
'negative': test.isNegative,
'commands': commands
});
}
void allDone() {
writeToFile(_configurations, _results);
}
void writeToFile(Map<String, Map> configurations, List<Map> results) {
if (_outputDirectory != null) {
var path = new Path(_outputDirectory);
var file =
new File(path.append(TestUtils.resultLogFileName).toNativePath());
file.createSync(recursive: true);
file.writeAsStringSync(
jsonEncode({'configurations': configurations, 'results': results}));
}
}
}
/// Writes a results.json file with a line for each test.
/// Each line is a json map with the test name and result and expected result.
/// Also writes a run.json file with a json map containing the configuration
tools/testing/dart/utils.dart
+2 -18
View File
@@ -37,10 +37,9 @@ class DebugLogger {
/**
* If [path] was null, the DebugLogger will write messages to stdout.
*/
static void init(Path path, {bool append: false}) {
static void init(Path path) {
if (path != null) {
var mode = append ? FileMode.append : FileMode.write;
_sink = new File(path.toNativePath()).openWrite(mode: mode);
_sink = new File(path.toNativePath()).openWrite(mode: FileMode.append);
}
}
@@ -462,21 +461,6 @@ class TestUtils {
static final debugLogFilePath = new Path(".debug.log");
/// If a flaky test failed, information about it (test name, stdin, stdout)
/// is written to this file.
///
/// This is useful for debugging flaky tests. When running on a buildbot, the
/// file can be made visible in the waterfall UI.
static const flakyFileName = ".flaky.log";
/// If test.py was invoked with '--write-test-outcome-log' it will write
/// test outcomes to this file.
static const testOutcomeFileName = ".test-outcome.log";
/// If test.py was invoked with '--write-result-log' it will write
/// test outcomes to this file in the '--output-directory'.
static const resultLogFileName = "result.log";
/// If test.py was invoked with '--write-results' it will write
/// test outcomes to this file in the '--output-directory'.
static const resultsFileName = "results.json";