Alexander Thomas
182d55dfda
This adds support for BabelJS to test.py's dart2js compiler configuration. When --babel is specified directly or through a named configuration, test.py will run an additional step after dart2js that post-processes the javascript output by running it through BabelJS with the specified Babel configuration. BabelJS is added to the DEPS in its standalone form. d8 is used to run BabelJS standalone to avoid adding a dependency on NodeJS. d8 can only write to stdout but not to files or stderr, which makes it necessary to change the test_runner to handle commands that expect their output to be piped to a file. Changes: * Add --babel option to test.py. * Add babel option to pkg/smith. * Switch IE11 builder to use babel transformation. * Fix option list comparison bugs in pkg/smith. * Change dart2js compiler configuration to generate files using the test name rather than just "out.js" (update test that relied on this). * Remove runtime_configuration dependency on test_suite. * Remove obsolete blocks adding --preview-dart-2 dart2js arguments. * Make dart2js' compiler configuration more like DDC's. * Remove createCommand method that is no longer used. * Remove support for "OtherResources" which was only used for dart:isolate tests on dart2js and DDC. * Skip co19_2 tests that are slow to transform with babel. * Simplify the timeout handling in the test runner with Future.timeout. Change-Id: I32e4917b2a57ecbe684538e40d744f0101c552a0 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/90402 Commit-Queue: Alexander Thomas <athom@google.com> Reviewed-by: Sigmund Cherem <sigmund@google.com>
1780 lines
58 KiB
Dart
1780 lines
58 KiB
Dart
// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
|
|
// for details. All rights reserved. Use of this source code is governed by a
|
|
// BSD-style license that can be found in the LICENSE file.
|
|
|
|
/**
|
|
* Classes and methods for executing tests.
|
|
*
|
|
* This module includes:
|
|
* - Managing parallel execution of tests, including timeout checks.
|
|
* - Evaluating the output of each test as pass/fail/crash/timeout.
|
|
*/
|
|
import 'dart:async';
|
|
import 'dart:collection';
|
|
import 'dart:convert';
|
|
// We need to use the 'io' prefix here, otherwise io.exitCode will shadow
|
|
// CommandOutput.exitCode in subclasses of CommandOutput.
|
|
import 'dart:io' as io;
|
|
import 'dart:math' as math;
|
|
|
|
import "package:status_file/expectation.dart";
|
|
|
|
import 'android.dart';
|
|
import 'browser_controller.dart';
|
|
import 'command.dart';
|
|
import 'command_output.dart';
|
|
import 'configuration.dart';
|
|
import 'dependency_graph.dart';
|
|
import 'repository.dart';
|
|
import 'runtime_configuration.dart';
|
|
import 'test_progress.dart';
|
|
import 'test_suite.dart';
|
|
import 'utils.dart';
|
|
|
|
const int unhandledCompilerExceptionExitCode = 253;
|
|
const int parseFailExitCode = 245;
|
|
const int slowTimeoutMultiplier = 4;
|
|
const int extraSlowTimeoutMultiplier = 8;
|
|
const int nonUtfFakeExitCode = 0xFFFD;
|
|
|
|
const cannotOpenDisplayMessage = 'Gtk-WARNING **: cannot open display';
|
|
const failedToRunCommandMessage = 'Failed to run command. return code=1';
|
|
|
|
typedef void TestCaseEvent(TestCase testCase);
|
|
typedef void ExitCodeEvent(int exitCode);
|
|
typedef void EnqueueMoreWork(ProcessQueue queue);
|
|
typedef void Action();
|
|
typedef Future<AdbCommandResult> StepFunction();
|
|
|
|
/// Some IO tests use these variables and get confused if the host environment
|
|
/// variables are inherited so they are excluded.
|
|
const _excludedEnvironmentVariables = const [
|
|
'http_proxy',
|
|
'https_proxy',
|
|
'no_proxy',
|
|
'HTTP_PROXY',
|
|
'HTTPS_PROXY',
|
|
'NO_PROXY'
|
|
];
|
|
|
|
/**
|
|
* TestCase contains all the information needed to run a test and evaluate
|
|
* its output. Running a test involves starting a separate process, with
|
|
* the executable and arguments given by the TestCase, and recording its
|
|
* stdout and stderr output streams, and its exit code. TestCase only
|
|
* contains static information about the test; actually running the test is
|
|
* performed by [ProcessQueue] using a [RunningProcess] object.
|
|
*
|
|
* The output information is stored in a [CommandOutput] instance contained
|
|
* in TestCase.commandOutputs. The last CommandOutput instance is responsible
|
|
* for evaluating if the test has passed, failed, crashed, or timed out, and the
|
|
* TestCase has information about what the expected result of the test should
|
|
* be.
|
|
*
|
|
* The TestCase has a callback function, [completedHandler], that is run when
|
|
* the test is completed.
|
|
*/
|
|
class TestCase extends UniqueObject {
|
|
// Flags set in _expectations from the optional argument info.
|
|
static final int HAS_RUNTIME_ERROR = 1 << 0;
|
|
static final int HAS_SYNTAX_ERROR = 1 << 1;
|
|
static final int HAS_COMPILE_ERROR = 1 << 2;
|
|
static final int HAS_STATIC_WARNING = 1 << 3;
|
|
static final int HAS_CRASH = 1 << 4;
|
|
/**
|
|
* A list of commands to execute. Most test cases have a single command.
|
|
* Dart2js tests have two commands, one to compile the source and another
|
|
* to execute it. Some isolate tests might even have three, if they require
|
|
* compiling multiple sources that are run in isolation.
|
|
*/
|
|
List<Command> commands;
|
|
Map<Command, CommandOutput> commandOutputs =
|
|
new Map<Command, CommandOutput>();
|
|
|
|
TestConfiguration configuration;
|
|
String displayName;
|
|
int _expectations = 0;
|
|
int hash = 0;
|
|
Set<Expectation> expectedOutcomes;
|
|
|
|
TestCase(this.displayName, this.commands, this.configuration,
|
|
this.expectedOutcomes,
|
|
{TestInformation info}) {
|
|
// A test case should do something.
|
|
assert(commands.isNotEmpty);
|
|
|
|
if (info != null) {
|
|
_setExpectations(info);
|
|
hash = (info?.originTestPath?.relativeTo(Repository.dir)?.toString())
|
|
.hashCode;
|
|
}
|
|
}
|
|
|
|
void _setExpectations(TestInformation info) {
|
|
// We don't want to keep the entire (large) TestInformation structure,
|
|
// so we copy the needed bools into flags set in a single integer.
|
|
if (info.hasRuntimeError) _expectations |= HAS_RUNTIME_ERROR;
|
|
if (info.hasSyntaxError) _expectations |= HAS_SYNTAX_ERROR;
|
|
if (info.hasCrash) _expectations |= HAS_CRASH;
|
|
if (info.hasCompileError || info.hasSyntaxError) {
|
|
_expectations |= HAS_COMPILE_ERROR;
|
|
}
|
|
if (info.hasStaticWarning) _expectations |= HAS_STATIC_WARNING;
|
|
}
|
|
|
|
TestCase indexedCopy(int index) {
|
|
var newCommands = commands.map((c) => c.indexedCopy(index)).toList();
|
|
return new TestCase(
|
|
displayName, newCommands, configuration, expectedOutcomes)
|
|
.._expectations = _expectations
|
|
..hash = hash;
|
|
}
|
|
|
|
bool get hasRuntimeError => _expectations & HAS_RUNTIME_ERROR != 0;
|
|
bool get hasStaticWarning => _expectations & HAS_STATIC_WARNING != 0;
|
|
bool get hasSyntaxError => _expectations & HAS_SYNTAX_ERROR != 0;
|
|
bool get hasCompileError => _expectations & HAS_COMPILE_ERROR != 0;
|
|
bool get hasCrash => _expectations & HAS_CRASH != 0;
|
|
bool get isNegative =>
|
|
hasCompileError ||
|
|
hasRuntimeError && configuration.runtime != Runtime.none ||
|
|
displayName.contains("negative_test");
|
|
|
|
bool get unexpectedOutput {
|
|
var outcome = this.result;
|
|
return !expectedOutcomes.any((expectation) {
|
|
return outcome.canBeOutcomeOf(expectation);
|
|
});
|
|
}
|
|
|
|
Expectation get result => lastCommandOutput.result(this);
|
|
Expectation get realResult => lastCommandOutput.realResult(this);
|
|
Expectation get realExpected {
|
|
if (hasCrash) {
|
|
return Expectation.crash;
|
|
}
|
|
if (configuration.compiler == Compiler.specParser) {
|
|
if (hasSyntaxError) {
|
|
return Expectation.syntaxError;
|
|
}
|
|
} else if (hasCompileError) {
|
|
if (hasRuntimeError && configuration.runtime != Runtime.none) {
|
|
return Expectation.fail;
|
|
}
|
|
return Expectation.compileTimeError;
|
|
}
|
|
if (hasRuntimeError) {
|
|
if (configuration.runtime != Runtime.none) {
|
|
return Expectation.runtimeError;
|
|
}
|
|
return Expectation.pass;
|
|
}
|
|
if (displayName.contains("negative_test")) {
|
|
return Expectation.fail;
|
|
}
|
|
if (configuration.compiler == Compiler.dart2analyzer && hasStaticWarning) {
|
|
return Expectation.staticWarning;
|
|
}
|
|
return Expectation.pass;
|
|
}
|
|
|
|
CommandOutput get lastCommandOutput {
|
|
if (commandOutputs.length == 0) {
|
|
throw new Exception("CommandOutputs is empty, maybe no command was run? ("
|
|
"displayName: '$displayName', "
|
|
"configurationString: '$configurationString')");
|
|
}
|
|
return commandOutputs[commands[commandOutputs.length - 1]];
|
|
}
|
|
|
|
Command get lastCommandExecuted {
|
|
if (commandOutputs.length == 0) {
|
|
throw new Exception("CommandOutputs is empty, maybe no command was run? ("
|
|
"displayName: '$displayName', "
|
|
"configurationString: '$configurationString')");
|
|
}
|
|
return commands[commandOutputs.length - 1];
|
|
}
|
|
|
|
int get timeout {
|
|
var result = configuration.timeout;
|
|
if (expectedOutcomes.contains(Expectation.slow)) {
|
|
result *= slowTimeoutMultiplier;
|
|
} else if (expectedOutcomes.contains(Expectation.extraSlow)) {
|
|
result *= extraSlowTimeoutMultiplier;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
String get configurationString {
|
|
var compiler = configuration.compiler.name;
|
|
var runtime = configuration.runtime.name;
|
|
var mode = configuration.mode.name;
|
|
var arch = configuration.architecture.name;
|
|
var checked = configuration.isChecked ? '-checked' : '';
|
|
return "$compiler-$runtime$checked ${mode}_$arch";
|
|
}
|
|
|
|
List<String> get batchTestArguments {
|
|
assert(commands.last is ProcessCommand);
|
|
return (commands.last as ProcessCommand).arguments;
|
|
}
|
|
|
|
bool get isFlaky {
|
|
if (expectedOutcomes.contains(Expectation.skip) ||
|
|
expectedOutcomes.contains(Expectation.skipByDesign)) {
|
|
return false;
|
|
}
|
|
|
|
return expectedOutcomes
|
|
.where((expectation) => expectation.isOutcome)
|
|
.length >
|
|
1;
|
|
}
|
|
|
|
bool get isFinished {
|
|
return commandOutputs.length > 0 &&
|
|
(!lastCommandOutput.successful ||
|
|
commands.length == commandOutputs.length);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* An OutputLog records the output from a test, but truncates it if
|
|
* it is longer than MAX_HEAD characters, and just keeps the head and
|
|
* the last TAIL_LENGTH characters of the output.
|
|
*/
|
|
class OutputLog implements StreamConsumer<List<int>> {
|
|
static const int MAX_HEAD = 500 * 1024;
|
|
static const int TAIL_LENGTH = 10 * 1024;
|
|
List<int> head = <int>[];
|
|
List<int> tail;
|
|
List<int> complete;
|
|
bool dataDropped = false;
|
|
bool hasNonUtf8 = false;
|
|
StreamSubscription _subscription;
|
|
|
|
OutputLog();
|
|
|
|
void add(List<int> data) {
|
|
if (complete != null) {
|
|
throw new StateError("Cannot add to OutputLog after calling toList");
|
|
}
|
|
if (tail == null) {
|
|
head.addAll(data);
|
|
if (head.length > MAX_HEAD) {
|
|
tail = head.sublist(MAX_HEAD);
|
|
head.length = MAX_HEAD;
|
|
}
|
|
} else {
|
|
tail.addAll(data);
|
|
}
|
|
if (tail != null && tail.length > 2 * TAIL_LENGTH) {
|
|
tail = _truncatedTail();
|
|
dataDropped = true;
|
|
}
|
|
}
|
|
|
|
List<int> _truncatedTail() => tail.length > TAIL_LENGTH
|
|
? tail.sublist(tail.length - TAIL_LENGTH)
|
|
: tail;
|
|
|
|
void _checkUtf8(List<int> data) {
|
|
try {
|
|
utf8.decode(data, allowMalformed: false);
|
|
} on FormatException {
|
|
hasNonUtf8 = true;
|
|
String malformed = utf8.decode(data, allowMalformed: true);
|
|
data
|
|
..clear()
|
|
..addAll(utf8.encode(malformed))
|
|
..addAll("""
|
|
|
|
*****************************************************************************
|
|
|
|
test.dart: The output of this test contained non-UTF8 formatted data.
|
|
|
|
*****************************************************************************
|
|
|
|
"""
|
|
.codeUnits);
|
|
}
|
|
}
|
|
|
|
List<int> toList() {
|
|
if (complete == null) {
|
|
complete = head;
|
|
if (dataDropped) {
|
|
complete.addAll("""
|
|
|
|
*****************************************************************************
|
|
|
|
test.dart: Data was removed due to excessive length. If you need the limit to
|
|
be increased, please contact dart-engprod or file an issue.
|
|
|
|
*****************************************************************************
|
|
|
|
"""
|
|
.codeUnits);
|
|
complete.addAll(_truncatedTail());
|
|
} else if (tail != null) {
|
|
complete.addAll(tail);
|
|
}
|
|
head = null;
|
|
tail = null;
|
|
_checkUtf8(complete);
|
|
}
|
|
return complete;
|
|
}
|
|
|
|
@override
|
|
Future addStream(Stream<List<int>> stream) {
|
|
_subscription = stream.listen(this.add);
|
|
return _subscription.asFuture();
|
|
}
|
|
|
|
@override
|
|
Future close() {
|
|
toList();
|
|
return _subscription?.cancel();
|
|
}
|
|
|
|
Future cancel() {
|
|
return _subscription?.cancel();
|
|
}
|
|
}
|
|
|
|
// An [OutputLog] that tees the output to a file as well.
|
|
class FileOutputLog extends OutputLog {
|
|
io.File _outputFile;
|
|
io.IOSink _sink;
|
|
|
|
FileOutputLog(this._outputFile);
|
|
|
|
@override
|
|
void add(List<int> data) {
|
|
super.add(data);
|
|
_sink ??= _outputFile.openWrite();
|
|
_sink.add(data);
|
|
}
|
|
|
|
@override
|
|
Future close() {
|
|
return Future.wait([
|
|
super.close(),
|
|
if (_sink != null) _sink.flush().whenComplete(_sink.close)
|
|
]);
|
|
}
|
|
|
|
@override
|
|
Future cancel() {
|
|
return Future.wait([
|
|
super.cancel(),
|
|
if (_sink != null) _sink.flush().whenComplete(_sink.close)
|
|
]);
|
|
}
|
|
}
|
|
|
|
// Helper to get a list of all child pids for a parent process.
|
|
// The first element of the list is the parent pid.
|
|
Future<List<int>> _getPidList(int pid, List<String> diagnostics) async {
|
|
var pids = [pid];
|
|
List<String> lines;
|
|
var startLine = 0;
|
|
if (io.Platform.isLinux || io.Platform.isMacOS) {
|
|
var result =
|
|
await io.Process.run("pgrep", ["-P", "${pids[0]}"], runInShell: true);
|
|
lines = (result.stdout as String).split('\n');
|
|
} else if (io.Platform.isWindows) {
|
|
var result = await io.Process.run(
|
|
"wmic",
|
|
[
|
|
"process",
|
|
"where",
|
|
"(ParentProcessId=${pids[0]})",
|
|
"get",
|
|
"ProcessId"
|
|
],
|
|
runInShell: true);
|
|
lines = (result.stdout as String).split('\n');
|
|
// Skip first line containing header "ProcessId".
|
|
startLine = 1;
|
|
} else {
|
|
assert(false);
|
|
}
|
|
if (lines.length > startLine) {
|
|
for (var i = startLine; i < lines.length; ++i) {
|
|
var pid = int.tryParse(lines[i]);
|
|
if (pid != null) pids.add(pid);
|
|
}
|
|
} else {
|
|
diagnostics.add("Could not find child pids");
|
|
diagnostics.addAll(lines);
|
|
}
|
|
return pids;
|
|
}
|
|
|
|
/**
|
|
* A RunningProcess actually runs a test, getting the command lines from
|
|
* its [TestCase], starting the test process (and first, a compilation
|
|
* process if the TestCase is a [BrowserTestCase]), creating a timeout
|
|
* timer, and recording the results in a new [CommandOutput] object, which it
|
|
* attaches to the TestCase. The lifetime of the RunningProcess is limited
|
|
* to the time it takes to start the process, run the process, and record
|
|
* the result; there are no pointers to it, so it should be available to
|
|
* be garbage collected as soon as it is done.
|
|
*/
|
|
class RunningProcess {
|
|
ProcessCommand command;
|
|
int timeout;
|
|
bool timedOut = false;
|
|
DateTime startTime;
|
|
int pid;
|
|
OutputLog stdout;
|
|
OutputLog stderr = OutputLog();
|
|
StreamConsumer stdoutConsumer;
|
|
List<String> diagnostics = <String>[];
|
|
bool compilationSkipped = false;
|
|
Completer<CommandOutput> completer;
|
|
TestConfiguration configuration;
|
|
|
|
RunningProcess(this.command, this.timeout,
|
|
{this.configuration, io.File outputFile}) {
|
|
stdout = outputFile != null ? FileOutputLog(outputFile) : OutputLog();
|
|
}
|
|
|
|
Future<CommandOutput> run() {
|
|
completer = new Completer<CommandOutput>();
|
|
startTime = new DateTime.now();
|
|
_runCommand();
|
|
return completer.future;
|
|
}
|
|
|
|
void _runCommand() {
|
|
if (command.outputIsUpToDate) {
|
|
compilationSkipped = true;
|
|
_commandComplete(0);
|
|
} else {
|
|
var processEnvironment = _createProcessEnvironment();
|
|
var args = command.arguments;
|
|
var processFuture = io.Process.start(command.executable, args,
|
|
environment: processEnvironment,
|
|
workingDirectory: command.workingDirectory);
|
|
processFuture.then((io.Process process) {
|
|
var stdoutFuture = process.stdout.pipe(stdout);
|
|
var stderrFuture = process.stderr.pipe(stderr);
|
|
pid = process.pid;
|
|
|
|
// Close stdin so that tests that try to block on input will fail.
|
|
process.stdin.close();
|
|
timeoutHandler() async {
|
|
timedOut = true;
|
|
if (process != null) {
|
|
String executable;
|
|
if (io.Platform.isLinux) {
|
|
executable = 'eu-stack';
|
|
} else if (io.Platform.isMacOS) {
|
|
// Try to print stack traces of the timed out process.
|
|
// `sample` is a sampling profiler but we ask it sample for 1
|
|
// second with a 4 second delay between samples so that we only
|
|
// sample the threads once.
|
|
executable = '/usr/bin/sample';
|
|
} else if (io.Platform.isWindows) {
|
|
var isX64 = command.executable.contains("X64") ||
|
|
command.executable.contains("SIMARM64");
|
|
if (configuration.windowsSdkPath != null) {
|
|
executable = configuration.windowsSdkPath +
|
|
"\\Debuggers\\${isX64 ? 'x64' : 'x86'}\\cdb.exe";
|
|
diagnostics.add("Using $executable to print stack traces");
|
|
} else {
|
|
diagnostics.add("win_sdk_path not found");
|
|
}
|
|
} else {
|
|
diagnostics.add("Capturing stack traces on"
|
|
"${io.Platform.operatingSystem} not supported");
|
|
}
|
|
if (executable != null) {
|
|
var pids = await _getPidList(process.pid, diagnostics);
|
|
diagnostics.add("Process list including children: $pids");
|
|
for (pid in pids) {
|
|
List<String> arguments;
|
|
if (io.Platform.isLinux) {
|
|
arguments = ['-p $pid'];
|
|
} else if (io.Platform.isMacOS) {
|
|
arguments = ['$pid', '1', '4000', '-mayDie'];
|
|
} else if (io.Platform.isWindows) {
|
|
arguments = ['-p', '$pid', '-c', '!uniqstack;qd'];
|
|
} else {
|
|
assert(false);
|
|
}
|
|
diagnostics.add("Trying to capture stack trace for pid $pid");
|
|
try {
|
|
var result = await io.Process.run(executable, arguments);
|
|
diagnostics.addAll((result.stdout as String).split('\n'));
|
|
diagnostics.addAll((result.stderr as String).split('\n'));
|
|
} catch (error) {
|
|
diagnostics.add("Unable to capture stack traces: $error");
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!process.kill()) {
|
|
diagnostics.add("Unable to kill ${process.pid}");
|
|
}
|
|
}
|
|
}
|
|
|
|
// Wait for the process to finish or timeout
|
|
process.exitCode
|
|
.timeout(Duration(seconds: timeout), onTimeout: timeoutHandler)
|
|
.then((exitCode) {
|
|
// This timeout is used to close stdio to the subprocess once we got
|
|
// the exitCode. Sometimes descendants of the subprocess keep stdio
|
|
// handles alive even though the direct subprocess is dead.
|
|
Future.wait([stdoutFuture, stderrFuture]).timeout(MAX_STDIO_DELAY,
|
|
onTimeout: () async {
|
|
DebugLogger.warning(
|
|
"$MAX_STDIO_DELAY_PASSED_MESSAGE (command: $command)");
|
|
await stdout.cancel();
|
|
await stderr.cancel();
|
|
_commandComplete(exitCode);
|
|
return null;
|
|
}).then((_) {
|
|
if (stdout is FileOutputLog) {
|
|
// Prevent logging data that has already been written to a file
|
|
// and is unlikely too add value in the logs because the command
|
|
// succeeded.
|
|
stdout.complete = <int>[];
|
|
}
|
|
_commandComplete(exitCode);
|
|
});
|
|
});
|
|
}).catchError((e) {
|
|
// TODO(floitsch): should we try to report the stacktrace?
|
|
print("Process error:");
|
|
print(" Command: $command");
|
|
print(" Error: $e");
|
|
_commandComplete(-1);
|
|
return true;
|
|
});
|
|
}
|
|
}
|
|
|
|
void _commandComplete(int exitCode) {
|
|
var commandOutput = _createCommandOutput(command, exitCode);
|
|
completer.complete(commandOutput);
|
|
}
|
|
|
|
CommandOutput _createCommandOutput(ProcessCommand command, int exitCode) {
|
|
List<int> stdoutData = stdout.toList();
|
|
List<int> stderrData = stderr.toList();
|
|
if (stdout.hasNonUtf8 || stderr.hasNonUtf8) {
|
|
// If the output contained non-utf8 formatted data, then make the exit
|
|
// code non-zero if it isn't already.
|
|
if (exitCode == 0) {
|
|
exitCode = nonUtfFakeExitCode;
|
|
}
|
|
}
|
|
var commandOutput = createCommandOutput(
|
|
command,
|
|
exitCode,
|
|
timedOut,
|
|
stdoutData,
|
|
stderrData,
|
|
new DateTime.now().difference(startTime),
|
|
compilationSkipped,
|
|
pid);
|
|
commandOutput.diagnostics.addAll(diagnostics);
|
|
return commandOutput;
|
|
}
|
|
|
|
Map<String, String> _createProcessEnvironment() {
|
|
var environment = new Map<String, String>.from(io.Platform.environment);
|
|
|
|
if (command.environmentOverrides != null) {
|
|
for (var key in command.environmentOverrides.keys) {
|
|
environment[key] = command.environmentOverrides[key];
|
|
}
|
|
}
|
|
for (var excludedEnvironmentVariable in _excludedEnvironmentVariables) {
|
|
environment.remove(excludedEnvironmentVariable);
|
|
}
|
|
|
|
// TODO(terry): Needed for roll 50?
|
|
environment["GLIBCPP_FORCE_NEW"] = "1";
|
|
environment["GLIBCXX_FORCE_NEW"] = "1";
|
|
|
|
return environment;
|
|
}
|
|
}
|
|
|
|
class BatchRunnerProcess {
|
|
/// When true, the command line is passed to the test runner as a
|
|
/// JSON-encoded list of strings.
|
|
final bool _useJson;
|
|
|
|
Completer<CommandOutput> _completer;
|
|
ProcessCommand _command;
|
|
List<String> _arguments;
|
|
String _runnerType;
|
|
|
|
io.Process _process;
|
|
Map<String, String> _processEnvironmentOverrides;
|
|
Completer<Null> _stdoutCompleter;
|
|
Completer<Null> _stderrCompleter;
|
|
StreamSubscription<String> _stdoutSubscription;
|
|
StreamSubscription<String> _stderrSubscription;
|
|
Function _processExitHandler;
|
|
|
|
bool _currentlyRunning = false;
|
|
OutputLog _testStdout;
|
|
OutputLog _testStderr;
|
|
String _status;
|
|
DateTime _startTime;
|
|
Timer _timer;
|
|
int _testCount = 0;
|
|
|
|
BatchRunnerProcess({bool useJson: true}) : _useJson = useJson;
|
|
|
|
Future<CommandOutput> runCommand(String runnerType, ProcessCommand command,
|
|
int timeout, List<String> arguments) {
|
|
assert(_completer == null);
|
|
assert(!_currentlyRunning);
|
|
|
|
_completer = new Completer();
|
|
bool sameRunnerType = _runnerType == runnerType &&
|
|
_dictEquals(_processEnvironmentOverrides, command.environmentOverrides);
|
|
_runnerType = runnerType;
|
|
_currentlyRunning = true;
|
|
_command = command;
|
|
_arguments = arguments;
|
|
_processEnvironmentOverrides = command.environmentOverrides;
|
|
|
|
// TOOD(jmesserly): this restarts `dartdevc --batch` to work around a
|
|
// memory leak, see https://github.com/dart-lang/sdk/issues/30314.
|
|
var clearMemoryLeak = command is CompilationCommand &&
|
|
command.displayName == 'dartdevc' &&
|
|
++_testCount % 100 == 0;
|
|
if (_process == null) {
|
|
// Start process if not yet started.
|
|
_startProcess(() {
|
|
doStartTest(command, timeout);
|
|
});
|
|
} else if (!sameRunnerType || clearMemoryLeak) {
|
|
// Restart this runner with the right executable for this test if needed.
|
|
_processExitHandler = (_) {
|
|
_startProcess(() {
|
|
doStartTest(command, timeout);
|
|
});
|
|
};
|
|
_process.kill();
|
|
_stdoutSubscription.cancel();
|
|
_stderrSubscription.cancel();
|
|
} else {
|
|
doStartTest(command, timeout);
|
|
}
|
|
return _completer.future;
|
|
}
|
|
|
|
Future<bool> terminate() {
|
|
if (_process == null) return new Future.value(true);
|
|
var terminateCompleter = new Completer<bool>();
|
|
_processExitHandler = (_) {
|
|
terminateCompleter.complete(true);
|
|
};
|
|
_process.kill();
|
|
_stdoutSubscription.cancel();
|
|
_stderrSubscription.cancel();
|
|
|
|
return terminateCompleter.future;
|
|
}
|
|
|
|
void doStartTest(Command command, int timeout) {
|
|
_startTime = new DateTime.now();
|
|
_testStdout = new OutputLog();
|
|
_testStderr = new OutputLog();
|
|
_status = null;
|
|
_stdoutCompleter = new Completer();
|
|
_stderrCompleter = new Completer();
|
|
_timer = new Timer(new Duration(seconds: timeout), _timeoutHandler);
|
|
|
|
var line = _createArgumentsLine(_arguments, timeout);
|
|
_process.stdin.write(line);
|
|
_stdoutSubscription.resume();
|
|
_stderrSubscription.resume();
|
|
Future.wait([_stdoutCompleter.future, _stderrCompleter.future])
|
|
.then((_) => _reportResult());
|
|
}
|
|
|
|
String _createArgumentsLine(List<String> arguments, int timeout) {
|
|
if (_useJson) {
|
|
return "${jsonEncode(arguments)}\n";
|
|
} else {
|
|
return arguments.join(' ') + '\n';
|
|
}
|
|
}
|
|
|
|
void _reportResult() {
|
|
if (!_currentlyRunning) return;
|
|
// _status == '>>> TEST {PASS, FAIL, OK, CRASH, TIMEOUT, PARSE_FAIL}'
|
|
|
|
var outcome = _status.split(" ")[2];
|
|
var exitCode = 0;
|
|
if (outcome == "CRASH") exitCode = unhandledCompilerExceptionExitCode;
|
|
if (outcome == "PARSE_FAIL") exitCode = parseFailExitCode;
|
|
if (outcome == "FAIL" || outcome == "TIMEOUT") exitCode = 1;
|
|
var output = createCommandOutput(
|
|
_command,
|
|
exitCode,
|
|
(outcome == "TIMEOUT"),
|
|
_testStdout.toList(),
|
|
_testStderr.toList(),
|
|
new DateTime.now().difference(_startTime),
|
|
false);
|
|
assert(_completer != null);
|
|
_completer.complete(output);
|
|
_completer = null;
|
|
_currentlyRunning = false;
|
|
}
|
|
|
|
ExitCodeEvent makeExitHandler(String status) {
|
|
void handler(int exitCode) {
|
|
if (_currentlyRunning) {
|
|
if (_timer != null) _timer.cancel();
|
|
_status = status;
|
|
_stdoutSubscription.cancel();
|
|
_stderrSubscription.cancel();
|
|
_startProcess(_reportResult);
|
|
} else {
|
|
// No active test case running.
|
|
_process = null;
|
|
}
|
|
}
|
|
|
|
return handler;
|
|
}
|
|
|
|
void _timeoutHandler() {
|
|
_processExitHandler = makeExitHandler(">>> TEST TIMEOUT");
|
|
_process.kill();
|
|
}
|
|
|
|
void _startProcess(Action callback) {
|
|
assert(_command is ProcessCommand);
|
|
var executable = _command.executable;
|
|
var arguments = _command.batchArguments.toList();
|
|
arguments.add('--batch');
|
|
var environment = new Map<String, String>.from(io.Platform.environment);
|
|
if (_processEnvironmentOverrides != null) {
|
|
for (var key in _processEnvironmentOverrides.keys) {
|
|
environment[key] = _processEnvironmentOverrides[key];
|
|
}
|
|
}
|
|
var processFuture =
|
|
io.Process.start(executable, arguments, environment: environment);
|
|
processFuture.then((io.Process p) {
|
|
_process = p;
|
|
|
|
Stream<String> _stdoutStream =
|
|
_process.stdout.transform(utf8.decoder).transform(new LineSplitter());
|
|
_stdoutSubscription = _stdoutStream.listen((String line) {
|
|
if (line.startsWith('>>> TEST')) {
|
|
_status = line;
|
|
} else if (line.startsWith('>>> BATCH')) {
|
|
// ignore
|
|
} else if (line.startsWith('>>> ')) {
|
|
throw new Exception("Unexpected command from batch runner: '$line'.");
|
|
} else {
|
|
_testStdout.add(encodeUtf8(line));
|
|
_testStdout.add("\n".codeUnits);
|
|
}
|
|
if (_status != null) {
|
|
_stdoutSubscription.pause();
|
|
_timer.cancel();
|
|
_stdoutCompleter.complete(null);
|
|
}
|
|
});
|
|
_stdoutSubscription.pause();
|
|
|
|
Stream<String> _stderrStream =
|
|
_process.stderr.transform(utf8.decoder).transform(new LineSplitter());
|
|
_stderrSubscription = _stderrStream.listen((String line) {
|
|
if (line.startsWith('>>> EOF STDERR')) {
|
|
_stderrSubscription.pause();
|
|
_stderrCompleter.complete(null);
|
|
} else {
|
|
_testStderr.add(encodeUtf8(line));
|
|
_testStderr.add("\n".codeUnits);
|
|
}
|
|
});
|
|
_stderrSubscription.pause();
|
|
|
|
_processExitHandler = makeExitHandler(">>> TEST CRASH");
|
|
_process.exitCode.then((exitCode) {
|
|
_processExitHandler(exitCode);
|
|
});
|
|
|
|
_process.stdin.done.catchError((err) {
|
|
print('Error on batch runner input stream stdin');
|
|
print(' Previous test\'s status: $_status');
|
|
print(' Error: $err');
|
|
throw err;
|
|
});
|
|
callback();
|
|
}).catchError((e) {
|
|
// TODO(floitsch): should we try to report the stacktrace?
|
|
print("Process error:");
|
|
print(" Command: $executable ${arguments.join(' ')} ($_arguments)");
|
|
print(" Error: $e");
|
|
// If there is an error starting a batch process, chances are that
|
|
// it will always fail. So rather than re-trying a 1000+ times, we
|
|
// exit.
|
|
io.exit(1);
|
|
return true;
|
|
});
|
|
}
|
|
|
|
bool _dictEquals(Map a, Map b) {
|
|
if (a == null) return b == null;
|
|
if (b == null) return false;
|
|
if (a.length != b.length) return false;
|
|
for (var key in a.keys) {
|
|
if (a[key] != b[key]) return false;
|
|
}
|
|
return true;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* [TestCaseEnqueuer] takes a list of TestSuites, generates TestCases and
|
|
* builds a dependency graph of all commands in every TestSuite.
|
|
*
|
|
* It will maintain three helper data structures
|
|
* - command2node: A mapping from a [Command] to a node in the dependency graph
|
|
* - command2testCases: A mapping from [Command] to all TestCases that it is
|
|
* part of.
|
|
* - remainingTestCases: A set of TestCases that were enqueued but are not
|
|
* finished
|
|
*
|
|
* [Command] and it's subclasses all have hashCode/operator== methods defined
|
|
* on them, so we can safely use them as keys in Map/Set objects.
|
|
*/
|
|
class TestCaseEnqueuer {
|
|
final Graph<Command> graph;
|
|
final Function _onTestCaseAdded;
|
|
|
|
final command2node = <Command, Node<Command>>{};
|
|
final command2testCases = <Command, List<TestCase>>{};
|
|
final remainingTestCases = new Set<TestCase>();
|
|
|
|
TestCaseEnqueuer(this.graph, this._onTestCaseAdded);
|
|
|
|
void enqueueTestSuites(List<TestSuite> testSuites) {
|
|
// Cache information about test cases per test suite. For multiple
|
|
// configurations there is no need to repeatedly search the file
|
|
// system, generate tests, and search test files for options.
|
|
var testCache = <String, List<TestInformation>>{};
|
|
|
|
var iterator = testSuites.iterator;
|
|
void enqueueNextSuite() {
|
|
if (!iterator.moveNext()) {
|
|
// We're finished with building the dependency graph.
|
|
graph.seal();
|
|
} else {
|
|
iterator.current.forEachTest(_newTest, testCache, enqueueNextSuite);
|
|
}
|
|
}
|
|
|
|
enqueueNextSuite();
|
|
}
|
|
|
|
/// Adds a test case to the list of active test cases, and adds its commands
|
|
/// to the dependency graph of commands.
|
|
///
|
|
/// If the repeat flag is > 1, replicates the test case and its commands,
|
|
/// adding an index field with a distinct value to each of the copies.
|
|
///
|
|
/// Each copy of the test case depends on the previous copy of the test
|
|
/// case completing, with its first command having a dependency on the last
|
|
/// command of the previous copy of the test case. This dependency is
|
|
/// marked as a "timingDependency", so that it doesn't depend on the previous
|
|
/// test completing successfully, just on it completing.
|
|
void _newTest(TestCase testCase) {
|
|
Node<Command> lastNode;
|
|
for (int i = 0; i < testCase.configuration.repeat; ++i) {
|
|
if (i > 0) {
|
|
testCase = testCase.indexedCopy(i);
|
|
}
|
|
remainingTestCases.add(testCase);
|
|
bool isFirstCommand = true;
|
|
for (var command in testCase.commands) {
|
|
// Make exactly *one* node in the dependency graph for every command.
|
|
// This ensures that we never have two commands c1 and c2 in the graph
|
|
// with "c1 == c2".
|
|
var node = command2node[command];
|
|
if (node == null) {
|
|
var requiredNodes =
|
|
(lastNode != null) ? [lastNode] : <Node<Command>>[];
|
|
node = graph.add(command, requiredNodes,
|
|
timingDependency: isFirstCommand);
|
|
command2node[command] = node;
|
|
command2testCases[command] = <TestCase>[];
|
|
}
|
|
// Keep mapping from command to all testCases that refer to it
|
|
command2testCases[command].add(testCase);
|
|
|
|
lastNode = node;
|
|
isFirstCommand = false;
|
|
}
|
|
_onTestCaseAdded(testCase);
|
|
}
|
|
}
|
|
}
|
|
|
|
/*
|
|
* [CommandEnqueuer] will
|
|
* - change node.state to NodeState.Enqueuing as soon as all dependencies have
|
|
* a state of NodeState.Successful
|
|
* - change node.state to NodeState.UnableToRun if one or more dependencies
|
|
* have a state of NodeState.Failed/NodeState.UnableToRun.
|
|
*/
|
|
class CommandEnqueuer {
|
|
static const _initStates = const [NodeState.initialized, NodeState.waiting];
|
|
|
|
static const _finishedStates = const [
|
|
NodeState.successful,
|
|
NodeState.failed,
|
|
NodeState.unableToRun
|
|
];
|
|
|
|
final Graph<Command> _graph;
|
|
|
|
CommandEnqueuer(this._graph) {
|
|
_graph.added.listen(_changeNodeStateIfNecessary);
|
|
|
|
_graph.changed.listen((event) {
|
|
if (event.from == NodeState.waiting ||
|
|
event.from == NodeState.processing) {
|
|
if (_finishedStates.contains(event.to)) {
|
|
for (var dependentNode in event.node.neededFor) {
|
|
_changeNodeStateIfNecessary(dependentNode);
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
// Called when either a new node was added or if one of it's dependencies
|
|
// changed it's state.
|
|
void _changeNodeStateIfNecessary(Node<Command> node) {
|
|
if (_initStates.contains(node.state)) {
|
|
bool allDependenciesFinished =
|
|
node.dependencies.every((dep) => _finishedStates.contains(dep.state));
|
|
bool anyDependenciesUnsuccessful = node.dependencies.any((dep) =>
|
|
[NodeState.failed, NodeState.unableToRun].contains(dep.state));
|
|
bool allDependenciesSuccessful =
|
|
node.dependencies.every((dep) => dep.state == NodeState.successful);
|
|
|
|
var newState = NodeState.waiting;
|
|
if (allDependenciesSuccessful ||
|
|
(allDependenciesFinished && node.timingDependency)) {
|
|
newState = NodeState.enqueuing;
|
|
} else if (anyDependenciesUnsuccessful) {
|
|
newState = NodeState.unableToRun;
|
|
}
|
|
if (node.state != newState) {
|
|
_graph.changeState(node, newState);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/*
|
|
* [CommandQueue] will listen for nodes entering the NodeState.ENQUEUING state,
|
|
* queue them up and run them. While nodes are processed they will be in the
|
|
* NodeState.PROCESSING state. After running a command, the node will change
|
|
* to a state of NodeState.Successful or NodeState.Failed.
|
|
*
|
|
* It provides a synchronous stream [completedCommands] which provides the
|
|
* [CommandOutputs] for the finished commands.
|
|
*
|
|
* It provides a [done] future, which will complete once there are no more
|
|
* nodes left in the states Initialized/Waiting/Enqueing/Processing
|
|
* and the [executor] has cleaned up it's resources.
|
|
*/
|
|
class CommandQueue {
|
|
final Graph<Command> graph;
|
|
final CommandExecutor executor;
|
|
final TestCaseEnqueuer enqueuer;
|
|
|
|
final Queue<Command> _runQueue = new Queue<Command>();
|
|
final _commandOutputStream = new StreamController<CommandOutput>(sync: true);
|
|
final _completer = new Completer<Null>();
|
|
|
|
int _numProcesses = 0;
|
|
int _maxProcesses;
|
|
int _numBrowserProcesses = 0;
|
|
int _maxBrowserProcesses;
|
|
bool _finishing = false;
|
|
bool _verbose = false;
|
|
|
|
CommandQueue(this.graph, this.enqueuer, this.executor, this._maxProcesses,
|
|
this._maxBrowserProcesses, this._verbose) {
|
|
graph.changed.listen((event) {
|
|
if (event.to == NodeState.enqueuing) {
|
|
assert(event.from == NodeState.initialized ||
|
|
event.from == NodeState.waiting);
|
|
graph.changeState(event.node, NodeState.processing);
|
|
var command = event.node.data;
|
|
if (event.node.dependencies.isNotEmpty) {
|
|
_runQueue.addFirst(command);
|
|
} else {
|
|
_runQueue.add(command);
|
|
}
|
|
Timer.run(() => _tryRunNextCommand());
|
|
} else if (event.to == NodeState.unableToRun) {
|
|
_checkDone();
|
|
}
|
|
});
|
|
|
|
// We're finished if the graph is sealed and all nodes are in a finished
|
|
// state (Successful, Failed or UnableToRun).
|
|
// So we're calling '_checkDone()' to check whether that condition is met
|
|
// and we can cleanup.
|
|
graph.sealed.listen((event) {
|
|
_checkDone();
|
|
});
|
|
}
|
|
|
|
Stream<CommandOutput> get completedCommands => _commandOutputStream.stream;
|
|
|
|
Future get done => _completer.future;
|
|
|
|
void _tryRunNextCommand() {
|
|
_checkDone();
|
|
|
|
if (_numProcesses < _maxProcesses && !_runQueue.isEmpty) {
|
|
Command command = _runQueue.removeFirst();
|
|
var isBrowserCommand = command is BrowserTestCommand;
|
|
|
|
if (isBrowserCommand && _numBrowserProcesses == _maxBrowserProcesses) {
|
|
// If there is no free browser runner, put it back into the queue.
|
|
_runQueue.add(command);
|
|
// Don't lose a process.
|
|
new Timer(new Duration(milliseconds: 100), _tryRunNextCommand);
|
|
return;
|
|
}
|
|
|
|
_numProcesses++;
|
|
if (isBrowserCommand) _numBrowserProcesses++;
|
|
|
|
var node = enqueuer.command2node[command];
|
|
Iterable<TestCase> testCases = enqueuer.command2testCases[command];
|
|
// If a command is part of many TestCases we set the timeout to be
|
|
// the maximum over all [TestCase.timeout]s. At some point, we might
|
|
// eliminate [TestCase.timeout] completely and move it to [Command].
|
|
int timeout = testCases
|
|
.map((TestCase test) => test.timeout)
|
|
.fold(0, (int a, b) => math.max(a, b));
|
|
|
|
if (_verbose) {
|
|
print('Running "${command.displayName}" command: $command');
|
|
}
|
|
|
|
executor.runCommand(node, command, timeout).then((CommandOutput output) {
|
|
assert(command == output.command);
|
|
|
|
_commandOutputStream.add(output);
|
|
if (output.canRunDependendCommands) {
|
|
graph.changeState(node, NodeState.successful);
|
|
} else {
|
|
graph.changeState(node, NodeState.failed);
|
|
}
|
|
|
|
_numProcesses--;
|
|
if (isBrowserCommand) _numBrowserProcesses--;
|
|
|
|
// Don't lose a process
|
|
Timer.run(() => _tryRunNextCommand());
|
|
});
|
|
}
|
|
}
|
|
|
|
void _checkDone() {
|
|
if (!_finishing &&
|
|
_runQueue.isEmpty &&
|
|
_numProcesses == 0 &&
|
|
graph.isSealed &&
|
|
graph.stateCount(NodeState.initialized) == 0 &&
|
|
graph.stateCount(NodeState.waiting) == 0 &&
|
|
graph.stateCount(NodeState.enqueuing) == 0 &&
|
|
graph.stateCount(NodeState.processing) == 0) {
|
|
_finishing = true;
|
|
executor.cleanup().then((_) {
|
|
_completer.complete();
|
|
_commandOutputStream.close();
|
|
});
|
|
}
|
|
}
|
|
|
|
void dumpState() {
|
|
print("");
|
|
print("CommandQueue state:");
|
|
print(" Processes: used: $_numProcesses max: $_maxProcesses");
|
|
print(" BrowserProcesses: used: $_numBrowserProcesses "
|
|
"max: $_maxBrowserProcesses");
|
|
print(" Finishing: $_finishing");
|
|
print(" Queue (length = ${_runQueue.length}):");
|
|
for (var command in _runQueue) {
|
|
print(" $command");
|
|
}
|
|
}
|
|
}
|
|
|
|
/*
|
|
* [CommandExecutor] is responsible for executing commands. It will make sure
|
|
* that the following two constraints are satisfied
|
|
* - [:numberOfProcessesUsed <= maxProcesses:]
|
|
* - [:numberOfBrowserProcessesUsed <= maxBrowserProcesses:]
|
|
*
|
|
* It provides a [runCommand] method which will complete with a
|
|
* [CommandOutput] object.
|
|
*
|
|
* It provides a [cleanup] method to free all the allocated resources.
|
|
*/
|
|
abstract class CommandExecutor {
|
|
Future cleanup();
|
|
// TODO(kustermann): The [timeout] parameter should be a property of Command
|
|
Future<CommandOutput> runCommand(
|
|
Node<Command> node, covariant Command command, int timeout);
|
|
}
|
|
|
|
class CommandExecutorImpl implements CommandExecutor {
|
|
final TestConfiguration globalConfiguration;
|
|
final int maxProcesses;
|
|
final int maxBrowserProcesses;
|
|
AdbDevicePool adbDevicePool;
|
|
|
|
// For dart2js and analyzer batch processing,
|
|
// we keep a list of batch processes.
|
|
final _batchProcesses = new Map<String, List<BatchRunnerProcess>>();
|
|
// We keep a BrowserTestRunner for every configuration.
|
|
final _browserTestRunners = new Map<TestConfiguration, BrowserTestRunner>();
|
|
|
|
bool _finishing = false;
|
|
|
|
CommandExecutorImpl(
|
|
this.globalConfiguration, this.maxProcesses, this.maxBrowserProcesses,
|
|
{this.adbDevicePool});
|
|
|
|
Future cleanup() {
|
|
assert(!_finishing);
|
|
_finishing = true;
|
|
|
|
Future _terminateBatchRunners() {
|
|
var futures = <Future>[];
|
|
for (var runners in _batchProcesses.values) {
|
|
futures.addAll(runners.map((runner) => runner.terminate()));
|
|
}
|
|
return Future.wait(futures);
|
|
}
|
|
|
|
Future _terminateBrowserRunners() {
|
|
var futures =
|
|
_browserTestRunners.values.map((runner) => runner.terminate());
|
|
return Future.wait(futures);
|
|
}
|
|
|
|
return Future.wait([
|
|
_terminateBatchRunners(),
|
|
_terminateBrowserRunners(),
|
|
]);
|
|
}
|
|
|
|
Future<CommandOutput> runCommand(node, Command command, int timeout) {
|
|
assert(!_finishing);
|
|
|
|
Future<CommandOutput> runCommand(int retriesLeft) {
|
|
return _runCommand(command, timeout).then((CommandOutput output) {
|
|
if (retriesLeft > 0 && shouldRetryCommand(output)) {
|
|
DebugLogger.warning("Rerunning Command: ($retriesLeft "
|
|
"attempt(s) remains) [cmd: $command]");
|
|
return runCommand(retriesLeft - 1);
|
|
} else {
|
|
return new Future.value(output);
|
|
}
|
|
});
|
|
}
|
|
|
|
return runCommand(command.maxNumRetries);
|
|
}
|
|
|
|
Future<CommandOutput> _runCommand(Command command, int timeout) {
|
|
if (command is BrowserTestCommand) {
|
|
return _startBrowserControllerTest(command, timeout);
|
|
} else if (command is VMKernelCompilationCommand) {
|
|
// For now, we always run vm_compile_to_kernel in batch mode.
|
|
var name = command.displayName;
|
|
assert(name == 'vm_compile_to_kernel');
|
|
return _getBatchRunner(name)
|
|
.runCommand(name, command, timeout, command.arguments);
|
|
} else if (command is CompilationCommand &&
|
|
globalConfiguration.batchDart2JS &&
|
|
command.displayName == 'dart2js') {
|
|
return _getBatchRunner("dart2js")
|
|
.runCommand("dart2js", command, timeout, command.arguments);
|
|
} else if (command is AnalysisCommand && globalConfiguration.batch) {
|
|
return _getBatchRunner(command.displayName)
|
|
.runCommand(command.displayName, command, timeout, command.arguments);
|
|
} else if (command is CompilationCommand &&
|
|
(command.displayName == 'dartdevc' ||
|
|
command.displayName == 'dartdevk' ||
|
|
command.displayName == 'fasta') &&
|
|
globalConfiguration.batch) {
|
|
return _getBatchRunner(command.displayName)
|
|
.runCommand(command.displayName, command, timeout, command.arguments);
|
|
} else if (command is ScriptCommand) {
|
|
return command.run();
|
|
} else if (command is AdbPrecompilationCommand ||
|
|
command is AdbDartkCommand) {
|
|
assert(adbDevicePool != null);
|
|
return adbDevicePool.acquireDevice().then((AdbDevice device) async {
|
|
try {
|
|
if (command is AdbPrecompilationCommand) {
|
|
return await _runAdbPrecompilationCommand(device, command, timeout);
|
|
} else {
|
|
return await _runAdbDartkCommand(
|
|
device, command as AdbDartkCommand, timeout);
|
|
}
|
|
} finally {
|
|
await adbDevicePool.releaseDevice(device);
|
|
}
|
|
});
|
|
} else if (command is VmBatchCommand) {
|
|
var name = command.displayName;
|
|
return _getBatchRunner(command.displayName + command.dartFile)
|
|
.runCommand(name, command, timeout, command.arguments);
|
|
} else if (command is CompilationCommand &&
|
|
command.displayName == 'babel') {
|
|
return new RunningProcess(command, timeout,
|
|
configuration: globalConfiguration,
|
|
outputFile: io.File(command.outputFile))
|
|
.run();
|
|
} else if (command is ProcessCommand) {
|
|
return new RunningProcess(command, timeout,
|
|
configuration: globalConfiguration)
|
|
.run();
|
|
} else {
|
|
throw new ArgumentError("Unknown command type ${command.runtimeType}.");
|
|
}
|
|
}
|
|
|
|
Future<CommandOutput> _runAdbPrecompilationCommand(
|
|
AdbDevice device, AdbPrecompilationCommand command, int timeout) async {
|
|
var runner = command.precompiledRunnerFilename;
|
|
var processTest = command.processTestFilename;
|
|
var testdir = command.precompiledTestDirectory;
|
|
var arguments = command.arguments;
|
|
var devicedir = DartPrecompiledAdbRuntimeConfiguration.DeviceDir;
|
|
var deviceTestDir = DartPrecompiledAdbRuntimeConfiguration.DeviceTestDir;
|
|
|
|
// We copy all the files which the vm precompiler puts into the test
|
|
// directory.
|
|
List<String> files = new io.Directory(testdir)
|
|
.listSync()
|
|
.map((file) => file.path)
|
|
.map((path) => path.substring(path.lastIndexOf('/') + 1))
|
|
.toList();
|
|
|
|
var timeoutDuration = new Duration(seconds: timeout);
|
|
|
|
var steps = <StepFunction>[];
|
|
|
|
steps.add(() => device.runAdbShellCommand(['rm', '-Rf', deviceTestDir]));
|
|
steps.add(() => device.runAdbShellCommand(['mkdir', '-p', deviceTestDir]));
|
|
steps.add(() =>
|
|
device.pushCachedData(runner, '$devicedir/dart_precompiled_runtime'));
|
|
steps.add(
|
|
() => device.pushCachedData(processTest, '$devicedir/process_test'));
|
|
steps.add(() => device.runAdbShellCommand([
|
|
'chmod',
|
|
'777',
|
|
'$devicedir/dart_precompiled_runtime $devicedir/process_test'
|
|
]));
|
|
|
|
for (var file in files) {
|
|
steps.add(() => device
|
|
.runAdbCommand(['push', '$testdir/$file', '$deviceTestDir/$file']));
|
|
}
|
|
|
|
steps.add(() => device.runAdbShellCommand(
|
|
[
|
|
'$devicedir/dart_precompiled_runtime',
|
|
]..addAll(arguments),
|
|
timeout: timeoutDuration));
|
|
|
|
var stopwatch = new Stopwatch()..start();
|
|
var writer = new StringBuffer();
|
|
|
|
await device.waitForBootCompleted();
|
|
await device.waitForDevice();
|
|
|
|
AdbCommandResult result;
|
|
for (var i = 0; i < steps.length; i++) {
|
|
var fun = steps[i];
|
|
var commandStopwatch = new Stopwatch()..start();
|
|
result = await fun();
|
|
|
|
writer.writeln("Executing ${result.command}");
|
|
if (result.stdout.length > 0) {
|
|
writer.writeln("Stdout:\n${result.stdout.trim()}");
|
|
}
|
|
if (result.stderr.length > 0) {
|
|
writer.writeln("Stderr:\n${result.stderr.trim()}");
|
|
}
|
|
writer.writeln("ExitCode: ${result.exitCode}");
|
|
writer.writeln("Time: ${commandStopwatch.elapsed}");
|
|
writer.writeln("");
|
|
|
|
// If one command fails, we stop processing the others and return
|
|
// immediately.
|
|
if (result.exitCode != 0) break;
|
|
}
|
|
return createCommandOutput(command, result.exitCode, result.timedOut,
|
|
utf8.encode('$writer'), [], stopwatch.elapsed, false);
|
|
}
|
|
|
|
Future<CommandOutput> _runAdbDartkCommand(
|
|
AdbDevice device, AdbDartkCommand command, int timeout) async {
|
|
final String buildPath = command.buildPath;
|
|
final String processTest = command.processTestFilename;
|
|
final String hostKernelFile = command.kernelFile;
|
|
final List<String> arguments = command.arguments;
|
|
final String devicedir = DartkAdbRuntimeConfiguration.DeviceDir;
|
|
final String deviceTestDir = DartkAdbRuntimeConfiguration.DeviceTestDir;
|
|
|
|
final timeoutDuration = new Duration(seconds: timeout);
|
|
|
|
final steps = <StepFunction>[];
|
|
|
|
steps.add(() => device.runAdbShellCommand(['rm', '-Rf', deviceTestDir]));
|
|
steps.add(() => device.runAdbShellCommand(['mkdir', '-p', deviceTestDir]));
|
|
steps.add(
|
|
() => device.pushCachedData("${buildPath}/dart", '$devicedir/dart'));
|
|
steps.add(() => device
|
|
.runAdbCommand(['push', hostKernelFile, '$deviceTestDir/out.dill']));
|
|
|
|
for (final String lib in command.extraLibraries) {
|
|
final String libname = "lib${lib}.so";
|
|
steps.add(() => device.runAdbCommand(
|
|
['push', '${buildPath}/$libname', '$deviceTestDir/$libname']));
|
|
}
|
|
|
|
steps.add(() => device.runAdbShellCommand(
|
|
[
|
|
'export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:$deviceTestDir;'
|
|
'$devicedir/dart',
|
|
]..addAll(arguments),
|
|
timeout: timeoutDuration));
|
|
|
|
final stopwatch = new Stopwatch()..start();
|
|
final writer = new StringBuffer();
|
|
|
|
await device.waitForBootCompleted();
|
|
await device.waitForDevice();
|
|
|
|
AdbCommandResult result;
|
|
for (var i = 0; i < steps.length; i++) {
|
|
var step = steps[i];
|
|
var commandStopwatch = new Stopwatch()..start();
|
|
result = await step();
|
|
|
|
writer.writeln("Executing ${result.command}");
|
|
if (result.stdout.length > 0) {
|
|
writer.writeln("Stdout:\n${result.stdout.trim()}");
|
|
}
|
|
if (result.stderr.length > 0) {
|
|
writer.writeln("Stderr:\n${result.stderr.trim()}");
|
|
}
|
|
writer.writeln("ExitCode: ${result.exitCode}");
|
|
writer.writeln("Time: ${commandStopwatch.elapsed}");
|
|
writer.writeln("");
|
|
|
|
// If one command fails, we stop processing the others and return
|
|
// immediately.
|
|
if (result.exitCode != 0) break;
|
|
}
|
|
return createCommandOutput(command, result.exitCode, result.timedOut,
|
|
utf8.encode('$writer'), [], stopwatch.elapsed, false);
|
|
}
|
|
|
|
BatchRunnerProcess _getBatchRunner(String identifier) {
|
|
// Start batch processes if needed
|
|
var runners = _batchProcesses[identifier];
|
|
if (runners == null) {
|
|
runners = new List<BatchRunnerProcess>(maxProcesses);
|
|
for (int i = 0; i < maxProcesses; i++) {
|
|
runners[i] = new BatchRunnerProcess(useJson: identifier == "fasta");
|
|
}
|
|
_batchProcesses[identifier] = runners;
|
|
}
|
|
|
|
for (var runner in runners) {
|
|
if (!runner._currentlyRunning) return runner;
|
|
}
|
|
throw new Exception('Unable to find inactive batch runner.');
|
|
}
|
|
|
|
Future<CommandOutput> _startBrowserControllerTest(
|
|
BrowserTestCommand browserCommand, int timeout) {
|
|
var completer = new Completer<CommandOutput>();
|
|
|
|
var callback = (BrowserTestOutput output) {
|
|
completer.complete(new BrowserCommandOutput(browserCommand, output));
|
|
};
|
|
|
|
var browserTest = new BrowserTest(browserCommand.url, callback, timeout);
|
|
_getBrowserTestRunner(browserCommand.configuration).then((testRunner) {
|
|
testRunner.enqueueTest(browserTest);
|
|
});
|
|
|
|
return completer.future;
|
|
}
|
|
|
|
Future<BrowserTestRunner> _getBrowserTestRunner(
|
|
TestConfiguration configuration) async {
|
|
if (_browserTestRunners[configuration] == null) {
|
|
var testRunner = new BrowserTestRunner(
|
|
configuration, globalConfiguration.localIP, maxBrowserProcesses);
|
|
if (globalConfiguration.isVerbose) {
|
|
testRunner.logger = DebugLogger.info;
|
|
}
|
|
_browserTestRunners[configuration] = testRunner;
|
|
await testRunner.start();
|
|
}
|
|
return _browserTestRunners[configuration];
|
|
}
|
|
}
|
|
|
|
bool shouldRetryCommand(CommandOutput output) {
|
|
if (!output.successful) {
|
|
List<String> stdout, stderr;
|
|
|
|
decodeOutput() {
|
|
if (stdout == null && stderr == null) {
|
|
stdout = decodeUtf8(output.stderr).split("\n");
|
|
stderr = decodeUtf8(output.stderr).split("\n");
|
|
}
|
|
}
|
|
|
|
final command = output.command;
|
|
|
|
// The dartk batch compiler sometimes runs out of memory. In such a case we
|
|
// will retry running it.
|
|
if (command is VMKernelCompilationCommand) {
|
|
if (output.hasCrashed) {
|
|
bool containsOutOfMemoryMessage(String line) {
|
|
return line.contains('Exhausted heap space, trying to allocat');
|
|
}
|
|
|
|
decodeOutput();
|
|
if (stdout.any(containsOutOfMemoryMessage) ||
|
|
stderr.any(containsOutOfMemoryMessage)) {
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (io.Platform.operatingSystem == 'linux') {
|
|
decodeOutput();
|
|
// No matter which command we ran: If we get failures due to the
|
|
// "xvfb-run" issue 7564, try re-running the test.
|
|
bool containsFailureMsg(String line) {
|
|
return line.contains(cannotOpenDisplayMessage) ||
|
|
line.contains(failedToRunCommandMessage);
|
|
}
|
|
|
|
if (stdout.any(containsFailureMsg) || stderr.any(containsFailureMsg)) {
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/*
|
|
* [TestCaseCompleter] will listen for
|
|
* NodeState.Processing -> NodeState.{Successful,Failed} state changes and
|
|
* will complete a TestCase if it is finished.
|
|
*
|
|
* It provides a stream [finishedTestCases], which will stream all TestCases
|
|
* once they're finished. After all TestCases are done, the stream will be
|
|
* closed.
|
|
*/
|
|
class TestCaseCompleter {
|
|
static const _completedStates = const [
|
|
NodeState.failed,
|
|
NodeState.successful
|
|
];
|
|
|
|
final Graph<Command> _graph;
|
|
final TestCaseEnqueuer _enqueuer;
|
|
final CommandQueue _commandQueue;
|
|
|
|
final Map<Command, CommandOutput> _outputs = {};
|
|
final StreamController<TestCase> _controller = new StreamController();
|
|
bool _closed = false;
|
|
|
|
TestCaseCompleter(this._graph, this._enqueuer, this._commandQueue) {
|
|
var finishedRemainingTestCases = false;
|
|
|
|
// Store all the command outputs -- they will be delivered synchronously
|
|
// (i.e. before state changes in the graph)
|
|
_commandQueue.completedCommands.listen((CommandOutput output) {
|
|
_outputs[output.command] = output;
|
|
}, onDone: () {
|
|
_completeTestCasesIfPossible(new List.from(_enqueuer.remainingTestCases));
|
|
finishedRemainingTestCases = true;
|
|
assert(_enqueuer.remainingTestCases.isEmpty);
|
|
_checkDone();
|
|
});
|
|
|
|
// Listen for NodeState.Processing -> NodeState.{Successful,Failed}
|
|
// changes.
|
|
_graph.changed.listen((event) {
|
|
if (event.from == NodeState.processing && !finishedRemainingTestCases) {
|
|
var command = event.node.data;
|
|
|
|
assert(_completedStates.contains(event.to));
|
|
assert(_outputs[command] != null);
|
|
|
|
_completeTestCasesIfPossible(_enqueuer.command2testCases[command]);
|
|
_checkDone();
|
|
}
|
|
});
|
|
|
|
// Listen also for GraphSealedEvents. If there is not a single node in the
|
|
// graph, we still want to finish after the graph was sealed.
|
|
_graph.sealed.listen((_) {
|
|
if (!_closed && _enqueuer.remainingTestCases.isEmpty) {
|
|
_controller.close();
|
|
_closed = true;
|
|
}
|
|
});
|
|
}
|
|
|
|
Stream<TestCase> get finishedTestCases => _controller.stream;
|
|
|
|
void _checkDone() {
|
|
if (!_closed && _graph.isSealed && _enqueuer.remainingTestCases.isEmpty) {
|
|
_controller.close();
|
|
_closed = true;
|
|
}
|
|
}
|
|
|
|
void _completeTestCasesIfPossible(Iterable<TestCase> testCases) {
|
|
// Update TestCases with command outputs
|
|
for (TestCase test in testCases) {
|
|
for (var icommand in test.commands) {
|
|
var output = _outputs[icommand];
|
|
if (output != null) {
|
|
test.commandOutputs[icommand] = output;
|
|
}
|
|
}
|
|
}
|
|
|
|
void completeTestCase(TestCase testCase) {
|
|
if (_enqueuer.remainingTestCases.contains(testCase)) {
|
|
_controller.add(testCase);
|
|
_enqueuer.remainingTestCases.remove(testCase);
|
|
} else {
|
|
DebugLogger.error("${testCase.displayName} would be finished twice");
|
|
}
|
|
}
|
|
|
|
for (var testCase in testCases) {
|
|
// Ask the [testCase] if it's done. Note that we assume, that
|
|
// [TestCase.isFinished] will return true if all commands were executed
|
|
// or if a previous one failed.
|
|
if (testCase.isFinished) {
|
|
completeTestCase(testCase);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
class ProcessQueue {
|
|
TestConfiguration _globalConfiguration;
|
|
|
|
Function _allDone;
|
|
final Graph<Command> _graph = new Graph();
|
|
List<EventListener> _eventListener;
|
|
|
|
ProcessQueue(
|
|
this._globalConfiguration,
|
|
int maxProcesses,
|
|
int maxBrowserProcesses,
|
|
DateTime startTime,
|
|
List<TestSuite> testSuites,
|
|
this._eventListener,
|
|
this._allDone,
|
|
[bool verbose = false,
|
|
AdbDevicePool adbDevicePool]) {
|
|
void setupForListing(TestCaseEnqueuer testCaseEnqueuer) {
|
|
_graph.sealed.listen((_) {
|
|
var testCases = testCaseEnqueuer.remainingTestCases.toList();
|
|
testCases.sort((a, b) => a.displayName.compareTo(b.displayName));
|
|
|
|
print("\nGenerating all matching test cases ....\n");
|
|
|
|
for (TestCase testCase in testCases) {
|
|
eventFinishedTestCase(testCase);
|
|
var outcomes = testCase.expectedOutcomes.map((o) => '$o').toList()
|
|
..sort();
|
|
print("${testCase.displayName} "
|
|
"Expectations: ${outcomes.join(', ')} "
|
|
"Configuration: '${testCase.configurationString}'");
|
|
}
|
|
eventAllTestsKnown();
|
|
});
|
|
}
|
|
|
|
TestCaseEnqueuer testCaseEnqueuer;
|
|
CommandQueue commandQueue;
|
|
|
|
void setupForRunning(TestCaseEnqueuer testCaseEnqueuer) {
|
|
Timer _debugTimer;
|
|
// If we haven't seen a single test finishing during a 10 minute period
|
|
// something is definitely wrong, so we dump the debugging information.
|
|
final debugTimerDuration = const Duration(minutes: 10);
|
|
|
|
void cancelDebugTimer() {
|
|
if (_debugTimer != null) {
|
|
_debugTimer.cancel();
|
|
}
|
|
}
|
|
|
|
void resetDebugTimer() {
|
|
cancelDebugTimer();
|
|
_debugTimer = new Timer(debugTimerDuration, () {
|
|
print("The debug timer of test.dart expired. Please report this issue"
|
|
" to whesse@ and provide the following information:");
|
|
print("");
|
|
print("Graph is sealed: ${_graph.isSealed}");
|
|
print("");
|
|
_graph.dumpCounts();
|
|
print("");
|
|
var unfinishedNodeStates = [
|
|
NodeState.initialized,
|
|
NodeState.waiting,
|
|
NodeState.enqueuing,
|
|
NodeState.processing
|
|
];
|
|
|
|
for (var nodeState in unfinishedNodeStates) {
|
|
if (_graph.stateCount(nodeState) > 0) {
|
|
print("Commands in state '$nodeState':");
|
|
print("=================================");
|
|
print("");
|
|
for (var node in _graph.nodes) {
|
|
if (node.state == nodeState) {
|
|
var command = node.data;
|
|
var testCases = testCaseEnqueuer.command2testCases[command];
|
|
print(" Command: $command");
|
|
for (var testCase in testCases) {
|
|
print(" Enqueued by: ${testCase.configurationString} "
|
|
"-- ${testCase.displayName}");
|
|
}
|
|
print("");
|
|
}
|
|
}
|
|
print("");
|
|
print("");
|
|
}
|
|
}
|
|
|
|
if (commandQueue != null) {
|
|
commandQueue.dumpState();
|
|
}
|
|
});
|
|
}
|
|
|
|
// When the graph building is finished, notify event listeners.
|
|
_graph.sealed.listen((_) {
|
|
eventAllTestsKnown();
|
|
});
|
|
|
|
// Queue commands as they become "runnable"
|
|
new CommandEnqueuer(_graph);
|
|
|
|
// CommandExecutor will execute commands
|
|
var executor = new CommandExecutorImpl(
|
|
_globalConfiguration, maxProcesses, maxBrowserProcesses,
|
|
adbDevicePool: adbDevicePool);
|
|
|
|
// Run "runnable commands" using [executor] subject to
|
|
// maxProcesses/maxBrowserProcesses constraint
|
|
commandQueue = new CommandQueue(_graph, testCaseEnqueuer, executor,
|
|
maxProcesses, maxBrowserProcesses, verbose);
|
|
|
|
// Finish test cases when all commands were run (or some failed)
|
|
var testCaseCompleter =
|
|
new TestCaseCompleter(_graph, testCaseEnqueuer, commandQueue);
|
|
testCaseCompleter.finishedTestCases.listen((TestCase finishedTestCase) {
|
|
resetDebugTimer();
|
|
|
|
eventFinishedTestCase(finishedTestCase);
|
|
}, onDone: () {
|
|
// Wait until the commandQueue/exectutor is done (it may need to stop
|
|
// batch runners, browser controllers, ....)
|
|
commandQueue.done.then((_) {
|
|
cancelDebugTimer();
|
|
eventAllTestsDone();
|
|
});
|
|
});
|
|
|
|
resetDebugTimer();
|
|
}
|
|
|
|
// Build up the dependency graph
|
|
testCaseEnqueuer = new TestCaseEnqueuer(_graph, (TestCase newTestCase) {
|
|
eventTestAdded(newTestCase);
|
|
});
|
|
|
|
// Either list or run the tests
|
|
if (_globalConfiguration.listTests) {
|
|
setupForListing(testCaseEnqueuer);
|
|
} else {
|
|
setupForRunning(testCaseEnqueuer);
|
|
}
|
|
|
|
// Start enqueing all TestCases
|
|
testCaseEnqueuer.enqueueTestSuites(testSuites);
|
|
}
|
|
|
|
void eventFinishedTestCase(TestCase testCase) {
|
|
for (var listener in _eventListener) {
|
|
listener.done(testCase);
|
|
}
|
|
}
|
|
|
|
void eventTestAdded(TestCase testCase) {
|
|
for (var listener in _eventListener) {
|
|
listener.testAdded();
|
|
}
|
|
}
|
|
|
|
void eventAllTestsKnown() {
|
|
for (var listener in _eventListener) {
|
|
listener.allTestsKnown();
|
|
}
|
|
}
|
|
|
|
void eventAllTestsDone() {
|
|
for (var listener in _eventListener) {
|
|
listener.allDone();
|
|
}
|
|
_allDone();
|
|
}
|
|
}
|