efortuna@google.com
c6179a50c0
BUG= Review URL: https://codereview.chromium.org//12225109 git-svn-id: https://dart.googlecode.com/svn/branches/bleeding_edge/dart@18295 260f80e4-7a28-3924-810f-c04153c831b5
1920 lines
68 KiB
Dart
1920 lines
68 KiB
Dart
// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
|
|
// for details. All rights reserved. Use of this source code is governed by a
|
|
// BSD-style license that can be found in the LICENSE file.
|
|
|
|
/**
|
|
* Classes and methods for enumerating and preparing tests.
|
|
*
|
|
* This library includes:
|
|
*
|
|
* - Creating tests by listing all the Dart files in certain directories,
|
|
* and creating [TestCase]s for those files that meet the relevant criteria.
|
|
* - Preparing tests, including copying files and frameworks to temporary
|
|
* directories, and computing the command line and arguments to be run.
|
|
*/
|
|
library test_suite;
|
|
|
|
import "dart:async";
|
|
import "dart:io";
|
|
import "dart:isolate";
|
|
import "dart:uri";
|
|
import "drt_updater.dart";
|
|
import "multitest.dart";
|
|
import "status_file_parser.dart";
|
|
import "test_runner.dart";
|
|
import "utils.dart";
|
|
|
|
// TODO(efortuna,whess): Remove this import.
|
|
import 'vendored_pkg/path/path.dart' as pathLib;
|
|
|
|
part "browser_test.dart";
|
|
|
|
|
|
// TODO(rnystrom): Add to dart:core?
|
|
/**
|
|
* A simple function that tests [arg] and returns `true` or `false`.
|
|
*/
|
|
typedef bool Predicate<T>(T arg);
|
|
|
|
typedef void CreateTest(Path filePath,
|
|
bool hasCompileError,
|
|
bool hasRuntimeError,
|
|
{bool isNegativeIfChecked,
|
|
bool hasFatalTypeErrors,
|
|
Set<String> multitestOutcome});
|
|
|
|
typedef void VoidFunction();
|
|
|
|
/**
|
|
* Calls [function] asynchronously. Returns a future that completes with the
|
|
* result of the function. If the function is `null`, returns a future that
|
|
* completes immediately with `null`.
|
|
*/
|
|
Future asynchronously(function()) {
|
|
if (function == null) return new Future.immediate(null);
|
|
|
|
var completer = new Completer();
|
|
new Timer(0, (_) {
|
|
completer.complete(function());
|
|
});
|
|
|
|
return completer.future;
|
|
}
|
|
|
|
/** A completer that waits until all added [Future]s complete. */
|
|
// TODO(rnystrom): Copied from web_components. Remove from here when it gets
|
|
// added to dart:core. (See #6626.)
|
|
class FutureGroup {
|
|
const _FINISHED = -1;
|
|
int _pending = 0;
|
|
Completer<List> _completer = new Completer<List>();
|
|
final List<Future> futures = <Future>[];
|
|
bool wasCompleted = false;
|
|
|
|
/**
|
|
* Wait for [task] to complete (assuming this barrier has not already been
|
|
* marked as completed, otherwise you'll get an exception indicating that a
|
|
* future has already been completed).
|
|
*/
|
|
void add(Future task) {
|
|
if (_pending == _FINISHED) {
|
|
throw new Exception("FutureFutureAlreadyCompleteException");
|
|
}
|
|
_pending++;
|
|
var handledTaskFuture = task.catchError((e) {
|
|
if (!wasCompleted) {
|
|
_completer.completeError(e.error, task.stackTrace);
|
|
wasCompleted = true;
|
|
}
|
|
}).then((_) {
|
|
_pending--;
|
|
if (_pending == 0) {
|
|
_pending = _FINISHED;
|
|
if (!wasCompleted) {
|
|
_completer.complete(futures);
|
|
wasCompleted = true;
|
|
}
|
|
}
|
|
});
|
|
futures.add(handledTaskFuture);
|
|
}
|
|
|
|
Future<List> get future => _completer.future;
|
|
}
|
|
|
|
/**
|
|
* A TestSuite represents a collection of tests. It creates a [TestCase]
|
|
* object for each test to be run, and passes the test cases to a callback.
|
|
*
|
|
* Most TestSuites represent a directory or directory tree containing tests,
|
|
* and a status file containing the expected results when these tests are run.
|
|
*/
|
|
abstract class TestSuite {
|
|
final Map configuration;
|
|
final String suiteName;
|
|
|
|
TestSuite(this.configuration, this.suiteName);
|
|
|
|
/**
|
|
* Whether or not binaries should be found in the root build directory or
|
|
* in the built SDK.
|
|
*/
|
|
bool get useSdk {
|
|
// The pub suite always uses the SDK.
|
|
// TODO(rnystrom): Eventually, all test suites should run out of the SDK
|
|
// and this check should go away.
|
|
if (suiteName == 'pub') return true;
|
|
|
|
return configuration['use_sdk'];
|
|
}
|
|
|
|
/**
|
|
* The output directory for this suite's configuration.
|
|
*/
|
|
String get buildDir => TestUtils.buildDir(configuration);
|
|
|
|
/**
|
|
* The path to the compiler for this suite's configuration. Returns `null` if
|
|
* no compiler should be used.
|
|
*/
|
|
String get compilerPath {
|
|
if (configuration['compiler'] == 'none') {
|
|
return null; // No separate compiler for dartium tests.
|
|
}
|
|
var name;
|
|
switch (configuration['compiler']) {
|
|
case 'dartc':
|
|
name = executablePath;
|
|
case 'dart2js':
|
|
case 'dart2dart':
|
|
var prefix = 'sdk/bin/';
|
|
String suffix = getExecutableSuffix(configuration['compiler']);
|
|
if (configuration['host_checked']) {
|
|
// The script dart2js_developer is not included in the
|
|
// shipped SDK, that is the script is not installed in
|
|
// "$buildDir/dart-sdk/bin/"
|
|
name = '$prefix/dart2js_developer$suffix';
|
|
} else {
|
|
if (configuration['use_sdk']) {
|
|
prefix = '$buildDir/dart-sdk/bin/';
|
|
}
|
|
name = '${prefix}dart2js$suffix';
|
|
}
|
|
break;
|
|
default:
|
|
throw "Unknown compiler for: ${configuration['compiler']}";
|
|
}
|
|
if (!(new File(name)).existsSync() && !configuration['list']) {
|
|
throw "Executable '$name' does not exist";
|
|
}
|
|
return name;
|
|
}
|
|
|
|
/**
|
|
* The path to the executable used to run this suite's tests.
|
|
*/
|
|
String get executablePath {
|
|
var suffix = getExecutableSuffix(configuration['compiler']);
|
|
switch (configuration['compiler']) {
|
|
case 'none':
|
|
if (useSdk) {
|
|
return '$buildDir/dart-sdk/bin/dart$suffix';
|
|
}
|
|
return '$buildDir/dart$suffix';
|
|
case 'dartc':
|
|
return '$buildDir/analyzer/bin/dart_analyzer$suffix';
|
|
default:
|
|
throw "Unknown executable for: ${configuration['compiler']}";
|
|
}
|
|
}
|
|
|
|
/**
|
|
* The file name of the d8 executable.
|
|
*/
|
|
String get d8FileName {
|
|
var suffix = getExecutableSuffix('d8');
|
|
var d8 = '$buildDir/d8$suffix';
|
|
TestUtils.ensureExists(d8, configuration);
|
|
return d8;
|
|
}
|
|
|
|
String get dartShellFileName {
|
|
var name = configuration['dart'];
|
|
if (name == '') {
|
|
name = executablePath;
|
|
}
|
|
|
|
TestUtils.ensureExists(name, configuration);
|
|
return name;
|
|
}
|
|
|
|
String get jsShellFileName {
|
|
var executableSuffix = getExecutableSuffix('jsshell');
|
|
var executable = 'jsshell$executableSuffix';
|
|
var jsshellDir = '${TestUtils.dartDir()}/tools/testing/bin';
|
|
return '$jsshellDir/$executable';
|
|
}
|
|
|
|
/**
|
|
* The file name of the Dart VM executable.
|
|
*/
|
|
String get vmFileName {
|
|
var suffix = getExecutableSuffix('vm');
|
|
var vm = '$buildDir/dart$suffix';
|
|
TestUtils.ensureExists(vm, configuration);
|
|
return vm;
|
|
}
|
|
|
|
/**
|
|
* The file extension (if any) that should be added to the given executable
|
|
* name for the current platform.
|
|
*/
|
|
String getExecutableSuffix(String executable) {
|
|
if (Platform.operatingSystem == 'windows') {
|
|
if (executable == 'd8' || executable == 'vm' || executable == 'none') {
|
|
return '.exe';
|
|
} else {
|
|
return '.bat';
|
|
}
|
|
}
|
|
return '';
|
|
}
|
|
|
|
/**
|
|
* Call the callback function onTest with a [TestCase] argument for each
|
|
* test in the suite. When all tests have been processed, call [onDone].
|
|
*
|
|
* The [testCache] argument provides a persistent store that can be used to
|
|
* cache information about the test suite, so that directories do not need
|
|
* to be listed each time.
|
|
*/
|
|
void forEachTest(TestCaseEvent onTest, Map testCache, [VoidFunction onDone]);
|
|
}
|
|
|
|
|
|
void ccTestLister() {
|
|
port.receive((String runnerPath, SendPort replyTo) {
|
|
Future processFuture = Process.start(runnerPath, ["--list"]);
|
|
processFuture.then((p) {
|
|
// Drain stderr to not leak resources.
|
|
p.stderr.onData = p.stderr.read;
|
|
StringInputStream stdoutStream = new StringInputStream(p.stdout);
|
|
var streamDone = false;
|
|
var processExited = false;
|
|
checkDone() {
|
|
if (streamDone && processExited) {
|
|
replyTo.send("");
|
|
}
|
|
}
|
|
stdoutStream.onLine = () {
|
|
String line = stdoutStream.readLine();
|
|
replyTo.send(line);
|
|
};
|
|
stdoutStream.onClosed = () {
|
|
streamDone = true;
|
|
checkDone();
|
|
};
|
|
p.onExit = (code) {
|
|
if (code < 0) {
|
|
print("Failed to list tests: $runnerPath --list");
|
|
replyTo.send("");
|
|
} else {
|
|
processExited = true;
|
|
checkDone();
|
|
}
|
|
};
|
|
port.close();
|
|
}).catchError((e) {
|
|
print("Failed to list tests: $runnerPath --list");
|
|
replyTo.send("");
|
|
return true;
|
|
});
|
|
});
|
|
}
|
|
|
|
|
|
/**
|
|
* A specialized [TestSuite] that runs tests written in C to unit test
|
|
* the Dart virtual machine and its API.
|
|
*
|
|
* The tests are compiled into a monolithic executable by the build step.
|
|
* The executable lists its tests when run with the --list command line flag.
|
|
* Individual tests are run by specifying them on the command line.
|
|
*/
|
|
class CCTestSuite extends TestSuite {
|
|
final String testPrefix;
|
|
String runnerPath;
|
|
final String dartDir;
|
|
List<String> statusFilePaths;
|
|
TestCaseEvent doTest;
|
|
VoidFunction doDone;
|
|
ReceivePort receiveTestName;
|
|
TestExpectations testExpectations;
|
|
|
|
CCTestSuite(Map configuration,
|
|
String suiteName,
|
|
String runnerName,
|
|
List<String> this.statusFilePaths,
|
|
{this.testPrefix: ''})
|
|
: super(configuration, suiteName),
|
|
dartDir = TestUtils.dartDir().toNativePath() {
|
|
runnerPath = '$buildDir/$runnerName';
|
|
}
|
|
|
|
void testNameHandler(String testName, ignore) {
|
|
if (testName == "") {
|
|
receiveTestName.close();
|
|
|
|
if (doDone != null) doDone();
|
|
} else {
|
|
// Only run the tests that match the pattern. Use the name
|
|
// "suiteName/testName" for cc tests.
|
|
RegExp pattern = configuration['selectors'][suiteName];
|
|
String constructedName = '$suiteName/$testPrefix$testName';
|
|
if (!pattern.hasMatch(constructedName)) return;
|
|
|
|
var expectations = testExpectations.expectations(
|
|
'$testPrefix$testName');
|
|
|
|
if (configuration["report"]) {
|
|
SummaryReport.add(expectations);
|
|
}
|
|
|
|
if (expectations.contains(SKIP)) return;
|
|
|
|
var args = TestUtils.standardOptions(configuration);
|
|
args.add(testName);
|
|
|
|
doTest(new TestCase(constructedName,
|
|
[new Command(runnerPath, args)],
|
|
configuration,
|
|
completeHandler,
|
|
expectations));
|
|
}
|
|
}
|
|
|
|
void forEachTest(TestCaseEvent onTest, Map testCache, [VoidFunction onDone]) {
|
|
doTest = onTest;
|
|
doDone = onDone;
|
|
|
|
var filesRead = 0;
|
|
void statusFileRead() {
|
|
filesRead++;
|
|
if (filesRead == statusFilePaths.length) {
|
|
receiveTestName = new ReceivePort();
|
|
var port = spawnFunction(ccTestLister);
|
|
port.send(runnerPath, receiveTestName.toSendPort());
|
|
receiveTestName.receive(testNameHandler);
|
|
}
|
|
}
|
|
|
|
testExpectations = new TestExpectations();
|
|
for (var statusFilePath in statusFilePaths) {
|
|
ReadTestExpectationsInto(testExpectations,
|
|
'$dartDir/$statusFilePath',
|
|
configuration,
|
|
statusFileRead);
|
|
}
|
|
}
|
|
|
|
void completeHandler(TestCase testCase) {
|
|
}
|
|
}
|
|
|
|
|
|
class TestInformation {
|
|
Path filePath;
|
|
Map optionsFromFile;
|
|
bool hasCompileError;
|
|
bool hasRuntimeError;
|
|
bool isNegativeIfChecked;
|
|
bool hasFatalTypeErrors;
|
|
Set<String> multitestOutcome;
|
|
|
|
TestInformation(this.filePath, this.optionsFromFile,
|
|
this.hasCompileError, this.hasRuntimeError,
|
|
this.isNegativeIfChecked, this.hasFatalTypeErrors,
|
|
this.multitestOutcome) {
|
|
Expect.isTrue(filePath.isAbsolute);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* A standard [TestSuite] implementation that searches for tests in a
|
|
* directory, and creates [TestCase]s that compile and/or run them.
|
|
*/
|
|
class StandardTestSuite extends TestSuite {
|
|
final Path suiteDir;
|
|
final List<String> statusFilePaths;
|
|
TestCaseEvent doTest;
|
|
TestExpectations testExpectations;
|
|
List<TestInformation> cachedTests;
|
|
final Path dartDir;
|
|
Predicate<String> isTestFilePredicate;
|
|
final bool listRecursively;
|
|
/**
|
|
* The set of servers that have been started to run these tests (Could be
|
|
* none).
|
|
*/
|
|
List serverList;
|
|
|
|
static final RegExp multiTestRegExp = new RegExp(r"/// [0-9][0-9]:(.*)");
|
|
|
|
StandardTestSuite(Map configuration,
|
|
String suiteName,
|
|
Path suiteDirectory,
|
|
this.statusFilePaths,
|
|
{this.isTestFilePredicate,
|
|
bool recursive: false,
|
|
this.serverList: const []})
|
|
: super(configuration, suiteName),
|
|
dartDir = TestUtils.dartDir(),
|
|
listRecursively = recursive,
|
|
suiteDir = TestUtils.dartDir().join(suiteDirectory);
|
|
|
|
/**
|
|
* Creates a test suite whose file organization matches an expected structure.
|
|
* To use this, your suite should look like:
|
|
*
|
|
* dart/
|
|
* path/
|
|
* to/
|
|
* mytestsuite/
|
|
* mytestsuite.status
|
|
* example1_test.dart
|
|
* example2_test.dart
|
|
* example3_test.dart
|
|
*
|
|
* The important parts:
|
|
*
|
|
* * The leaf directory name is the name of your test suite.
|
|
* * The status file uses the same name.
|
|
* * Test files are directly in that directory and end in "_test.dart".
|
|
*
|
|
* If you follow that convention, then you can construct one of these like:
|
|
*
|
|
* new StandardTestSuite.forDirectory(configuration, 'path/to/mytestsuite');
|
|
*
|
|
* instead of having to create a custom [StandardTestSuite] subclass. In
|
|
* particular, if you add 'path/to/mytestsuite' to [TEST_SUITE_DIRECTORIES]
|
|
* in test.dart, this will all be set up for you.
|
|
*
|
|
* The [StandardTestSuite] also optionally takes a list of servers that have
|
|
* been started up by the test harness, to be used by browser tests.
|
|
*/
|
|
factory StandardTestSuite.forDirectory(
|
|
Map configuration, Path directory, {List serverList : const []}) {
|
|
final name = directory.filename;
|
|
|
|
return new StandardTestSuite(configuration,
|
|
name, directory,
|
|
['$directory/$name.status', '$directory/${name}_dart2js.status'],
|
|
isTestFilePredicate: (filename) => filename.endsWith('_test.dart'),
|
|
recursive: true, serverList: serverList);
|
|
}
|
|
|
|
Collection<Uri> get dart2JsBootstrapDependencies {
|
|
if (!useSdk) return [];
|
|
|
|
var snapshotPath = TestUtils.absolutePath(new Path(buildDir).join(
|
|
new Path('dart-sdk/lib/_internal/compiler/'
|
|
'implementation/dart2js.dart.snapshot'))).toString();
|
|
return [new Uri.fromComponents(scheme: 'file', path: snapshotPath)];
|
|
}
|
|
|
|
/**
|
|
* The default implementation assumes a file is a test if
|
|
* it ends in "Test.dart".
|
|
*/
|
|
bool isTestFile(String filename) {
|
|
// Use the specified predicate, if provided.
|
|
if (isTestFilePredicate != null) return isTestFilePredicate(filename);
|
|
|
|
return filename.endsWith("Test.dart");
|
|
}
|
|
|
|
List<String> additionalOptions(Path filePath) => [];
|
|
|
|
void forEachTest(TestCaseEvent onTest, Map testCache, [VoidFunction onDone]) {
|
|
updateDartium().then((_) {
|
|
doTest = onTest;
|
|
|
|
return readExpectations();
|
|
}).then((expectations) {
|
|
testExpectations = expectations;
|
|
|
|
// Checked if we have already found and generated the tests for
|
|
// this suite.
|
|
if (!testCache.containsKey(suiteName)) {
|
|
cachedTests = testCache[suiteName] = [];
|
|
return enqueueTests();
|
|
} else {
|
|
// We rely on enqueueing completing asynchronously.
|
|
return asynchronously(() {
|
|
for (var info in testCache[suiteName]) {
|
|
enqueueTestCaseFromTestInformation(info);
|
|
}
|
|
});
|
|
}
|
|
}).then((_) {
|
|
if (onDone != null) onDone();
|
|
});
|
|
}
|
|
|
|
/**
|
|
* If DumpRenderTree/Dartium is required, and not yet updated, waits for
|
|
* the update then completes. Otherwise completes immediately.
|
|
*/
|
|
Future updateDartium() {
|
|
var completer = new Completer();
|
|
var updater = runtimeUpdater(configuration);
|
|
if (updater == null || updater.updated) {
|
|
return new Future.immediate(null);
|
|
}
|
|
|
|
Expect.isTrue(updater.isActive);
|
|
updater.onUpdated.add(() => completer.complete(null));
|
|
|
|
return completer.future;
|
|
}
|
|
|
|
/**
|
|
* Reads the status files and completes with the parsed expectations.
|
|
*/
|
|
Future<TestExpectations> readExpectations() {
|
|
var completer = new Completer();
|
|
var expectations = new TestExpectations();
|
|
|
|
var filesRead = 0;
|
|
void statusFileRead() {
|
|
filesRead++;
|
|
if (filesRead == statusFilePaths.length) {
|
|
completer.complete(expectations);
|
|
}
|
|
}
|
|
|
|
for (var statusFilePath in statusFilePaths) {
|
|
// [forDirectory] adds name_dart2js.status for all tests suites. Use it
|
|
// if it exists, but otherwise skip it and don't fail.
|
|
if (statusFilePath.endsWith('_dart2js.status')) {
|
|
var file = new File.fromPath(dartDir.append(statusFilePath));
|
|
if (!file.existsSync()) {
|
|
filesRead++;
|
|
continue;
|
|
}
|
|
}
|
|
|
|
ReadTestExpectationsInto(expectations,
|
|
dartDir.append(statusFilePath).toNativePath(),
|
|
configuration, statusFileRead);
|
|
}
|
|
|
|
return completer.future;
|
|
}
|
|
|
|
Future enqueueTests() {
|
|
Directory dir = new Directory.fromPath(suiteDir);
|
|
return dir.exists().then((exists) {
|
|
if (!exists) {
|
|
print('Directory containing tests missing: ${suiteDir.toNativePath()}');
|
|
return new Future.immediate(null);
|
|
} else {
|
|
var group = new FutureGroup();
|
|
enqueueDirectory(dir, group);
|
|
return group.future;
|
|
}
|
|
});
|
|
}
|
|
|
|
void enqueueDirectory(Directory dir, FutureGroup group) {
|
|
var listCompleter = new Completer();
|
|
group.add(listCompleter.future);
|
|
|
|
var lister = dir.list(recursive: listRecursively);
|
|
lister.onFile = (file) => enqueueFile(file, group);
|
|
lister.onDone = listCompleter.complete;
|
|
}
|
|
|
|
void enqueueFile(String filename, FutureGroup group) {
|
|
if (!isTestFile(filename)) return;
|
|
Path filePath = new Path(filename);
|
|
|
|
// Only run the tests that match the pattern.
|
|
RegExp pattern = configuration['selectors'][suiteName];
|
|
if (!pattern.hasMatch('$filePath')) return;
|
|
if (filePath.filename.endsWith('test_config.dart')) return;
|
|
|
|
var optionsFromFile = readOptionsFromFile(filePath);
|
|
CreateTest createTestCase = makeTestCaseCreator(optionsFromFile);
|
|
|
|
if (optionsFromFile['isMultitest']) {
|
|
group.add(doMultitest(filePath, buildDir, suiteDir, createTestCase));
|
|
} else {
|
|
createTestCase(filePath,
|
|
optionsFromFile['hasCompileError'],
|
|
optionsFromFile['hasRuntimeError']);
|
|
}
|
|
}
|
|
|
|
void enqueueTestCaseFromTestInformation(TestInformation info) {
|
|
var filePath = info.filePath;
|
|
var optionsFromFile = info.optionsFromFile;
|
|
var isNegative = info.hasCompileError;
|
|
if (info.hasRuntimeError && hasRuntime) {
|
|
isNegative = true;
|
|
}
|
|
|
|
// Look up expectations in status files using a test name generated
|
|
// from the test file's path.
|
|
String testName;
|
|
|
|
if (optionsFromFile['isMultitest']) {
|
|
// Multitests do not run on browsers.
|
|
if (TestUtils.isBrowserRuntime(configuration['runtime'])) return;
|
|
// Multitests are in [build directory]/generated_tests/... .
|
|
// The test name will be '[test filename (no extension)]/[multitest key].
|
|
String name = filePath.filenameWithoutExtension;
|
|
int middle = name.lastIndexOf('_');
|
|
testName = '${name.substring(0, middle)}/${name.substring(middle + 1)}';
|
|
} else {
|
|
// The test name is the relative path from the test suite directory to
|
|
// the test, with the .dart extension removed.
|
|
Expect.isTrue(filePath.toNativePath().startsWith(
|
|
suiteDir.toNativePath()));
|
|
var testNamePath = filePath.relativeTo(suiteDir);
|
|
Expect.isTrue(testNamePath.extension == 'dart');
|
|
if (testNamePath.extension == 'dart') {
|
|
testName = testNamePath.directoryPath.append(
|
|
testNamePath.filenameWithoutExtension).toString();
|
|
}
|
|
}
|
|
int shards = configuration['shards'];
|
|
if (shards > 1) {
|
|
int shard = configuration['shard'];
|
|
if (testName.hashCode % shards != shard - 1) {
|
|
return;
|
|
}
|
|
}
|
|
|
|
Set<String> expectations = testExpectations.expectations(testName);
|
|
if (info.hasCompileError &&
|
|
TestUtils.isBrowserRuntime(configuration['runtime']) &&
|
|
configuration['report']) {
|
|
SummaryReport.addCompileErrorSkipTest();
|
|
return;
|
|
}
|
|
if (configuration['report']) {
|
|
// Tests with multiple VMOptions are counted more than once.
|
|
for (var dummy in getVmOptions(optionsFromFile)) {
|
|
SummaryReport.add(expectations);
|
|
}
|
|
}
|
|
if (expectations.contains(SKIP)) return;
|
|
|
|
if (configuration['compiler'] != 'none' && info.hasCompileError) {
|
|
// If a compile-time error is expected, and we're testing a
|
|
// compiler, we never need to attempt to run the program (in a
|
|
// browser or otherwise).
|
|
enqueueStandardTest(info, testName, expectations);
|
|
} else if (TestUtils.isBrowserRuntime(configuration['runtime'])) {
|
|
bool isWrappingRequired = configuration['compiler'] != 'dart2js';
|
|
if (info.optionsFromFile['isMultiHtmlTest']) {
|
|
// A browser multi-test has multiple expectations for one test file.
|
|
// Find all the different sub-test expecations for one entire test file.
|
|
List<String> subtestNames = info.optionsFromFile['subtestNames'];
|
|
Map<String, Set<String>> multiHtmlTestExpectations = {};
|
|
for (String name in subtestNames) {
|
|
String fullTestName = '$testName/$name';
|
|
multiHtmlTestExpectations[fullTestName] =
|
|
testExpectations.expectations(fullTestName);
|
|
}
|
|
enqueueBrowserTest(info, testName, multiHtmlTestExpectations,
|
|
isWrappingRequired);
|
|
} else {
|
|
enqueueBrowserTest(info, testName, expectations, isWrappingRequired);
|
|
}
|
|
} else {
|
|
enqueueStandardTest(info, testName, expectations);
|
|
}
|
|
}
|
|
|
|
void enqueueStandardTest(TestInformation info,
|
|
String testName,
|
|
Set<String> expectations) {
|
|
bool isNegative = info.hasCompileError ||
|
|
(configuration['checked'] && info.isNegativeIfChecked);
|
|
if (info.hasRuntimeError && hasRuntime) {
|
|
isNegative = true;
|
|
}
|
|
|
|
if (configuration['compiler'] == 'dartc') {
|
|
// dartc can detect static type warnings by the
|
|
// format of the error line
|
|
if (info.hasFatalTypeErrors) {
|
|
isNegative = true;
|
|
}
|
|
}
|
|
|
|
var commonArguments = commonArgumentsFromFile(info.filePath,
|
|
info.optionsFromFile);
|
|
|
|
List<List<String>> vmOptionsList = getVmOptions(info.optionsFromFile);
|
|
Expect.isFalse(vmOptionsList.isEmpty, "empty vmOptionsList");
|
|
|
|
for (var vmOptions in vmOptionsList) {
|
|
doTest(new TestCase('$suiteName/$testName',
|
|
makeCommands(info, vmOptions, commonArguments),
|
|
configuration,
|
|
completeHandler,
|
|
expectations,
|
|
isNegative: isNegative,
|
|
info: info));
|
|
}
|
|
}
|
|
|
|
List<Command> makeCommands(TestInformation info, var vmOptions, var args) {
|
|
switch (configuration['compiler']) {
|
|
case 'dart2js':
|
|
args = new List.from(args);
|
|
String tempDir = createOutputDirectory(info.filePath, '');
|
|
args.add('--out=$tempDir/out.js');
|
|
|
|
List<Command> commands =
|
|
<Command>[new CompilationCommand("$tempDir/out.js",
|
|
!useSdk,
|
|
dart2JsBootstrapDependencies,
|
|
compilerPath,
|
|
args)];
|
|
if (info.hasCompileError) {
|
|
// Do not attempt to run the compiled result. A compilation
|
|
// error should be reported by the compilation command.
|
|
} else if (configuration['runtime'] == 'd8') {
|
|
commands.add(new Command(d8FileName, ['$tempDir/out.js']));
|
|
} else if (configuration['runtime'] == 'jsshell') {
|
|
commands.add(new Command(jsShellFileName, ['$tempDir/out.js']));
|
|
}
|
|
return commands;
|
|
|
|
case 'dart2dart':
|
|
args = new List.from(args);
|
|
args.add('--output-type=dart');
|
|
String tempDir = createOutputDirectory(info.filePath, '');
|
|
args.add('--out=$tempDir/out.dart');
|
|
|
|
List<Command> commands =
|
|
<Command>[new CompilationCommand("$tempDir/out.dart",
|
|
!useSdk,
|
|
dart2JsBootstrapDependencies,
|
|
compilerPath,
|
|
args)];
|
|
if (info.hasCompileError) {
|
|
// Do not attempt to run the compiled result. A compilation
|
|
// error should be reported by the compilation command.
|
|
} else if (configuration['runtime'] == 'vm') {
|
|
// TODO(antonm): support checked.
|
|
var vmArguments = new List.from(vmOptions);
|
|
vmArguments.addAll([
|
|
'--ignore-unrecognized-flags', '$tempDir/out.dart']);
|
|
commands.add(new Command(vmFileName, vmArguments));
|
|
} else {
|
|
throw 'Unsupported runtime ${configuration["runtime"]} for dart2dart';
|
|
}
|
|
return commands;
|
|
|
|
case 'none':
|
|
case 'dartc':
|
|
var arguments = new List.from(vmOptions);
|
|
arguments.addAll(args);
|
|
return <Command>[new Command(dartShellFileName, arguments)];
|
|
|
|
default:
|
|
throw 'Unknown compiler ${configuration["compiler"]}';
|
|
}
|
|
}
|
|
|
|
CreateTest makeTestCaseCreator(Map optionsFromFile) {
|
|
return (Path filePath,
|
|
bool hasCompileError,
|
|
bool hasRuntimeError,
|
|
{bool isNegativeIfChecked: false,
|
|
bool hasFatalTypeErrors: false,
|
|
Set<String> multitestOutcome: null}) {
|
|
// Cache the test information for each test case.
|
|
var info = new TestInformation(filePath,
|
|
optionsFromFile,
|
|
hasCompileError,
|
|
hasRuntimeError,
|
|
isNegativeIfChecked,
|
|
hasFatalTypeErrors,
|
|
multitestOutcome);
|
|
cachedTests.add(info);
|
|
enqueueTestCaseFromTestInformation(info);
|
|
};
|
|
}
|
|
|
|
/**
|
|
* The [StandardTestSuite] has support for tests that
|
|
* compile a test from Dart to JavaScript, and then run the resulting
|
|
* JavaScript. This function creates a working directory to hold the
|
|
* JavaScript version of the test, and copies the appropriate framework
|
|
* files to that directory. It creates a [BrowserTestCase], which has
|
|
* two sequential steps to be run by the [ProcessQueue] when the test is
|
|
* executed: a compilation step and an execution step, both with the
|
|
* appropriate executable and arguments. The [expectations] object can be
|
|
* either a Set<String> if the test is a regular test, or a Map<String
|
|
* subTestName, Set<String>> if we are running a browser multi-test (one
|
|
* compilation and many browser runs).
|
|
*/
|
|
void enqueueBrowserTest(TestInformation info,
|
|
String testName,
|
|
Object expectations,
|
|
bool isWrappingRequired) {
|
|
// TODO(kustermann/ricow): This method should be refactored.
|
|
Map optionsFromFile = info.optionsFromFile;
|
|
Path filePath = info.filePath;
|
|
String filename = filePath.toString();
|
|
bool isWebTest = optionsFromFile['containsDomImport'];
|
|
bool isLibraryDefinition = optionsFromFile['isLibraryDefinition'];
|
|
if (isWrappingRequired
|
|
&& !isLibraryDefinition && optionsFromFile['containsSourceOrImport']) {
|
|
print('Warning for $filename: Browser tests require #library '
|
|
'in any file that uses #import, #source, or #resource');
|
|
}
|
|
|
|
final String compiler = configuration['compiler'];
|
|
final String runtime = configuration['runtime'];
|
|
|
|
for (var vmOptions in getVmOptions(optionsFromFile)) {
|
|
// Create a unique temporary directory for each set of vmOptions.
|
|
// TODO(dart:429): Replace separate replaceAlls with a RegExp when
|
|
// replaceAll(RegExp, String) is implemented.
|
|
String optionsName = '';
|
|
if (getVmOptions(optionsFromFile).length > 1) {
|
|
optionsName = Strings.join(vmOptions, '-').replaceAll('-','')
|
|
.replaceAll('=','')
|
|
.replaceAll('/','');
|
|
}
|
|
final String tempDir = createOutputDirectory(info.filePath, optionsName);
|
|
|
|
String dartWrapperFilename = '$tempDir/test.dart';
|
|
String compiledDartWrapperFilename = '$tempDir/test.js';
|
|
|
|
String htmlPath = '$tempDir/test.html';
|
|
if (isWrappingRequired && !isWebTest) {
|
|
// test.dart will import the dart test directly, if it is a library,
|
|
// or indirectly through test_as_library.dart, if it is not.
|
|
Path dartLibraryFilename = filePath;
|
|
if (!isLibraryDefinition) {
|
|
dartLibraryFilename = new Path(tempDir).append(
|
|
'test_as_library.dart');
|
|
File file = new File(dartLibraryFilename.toNativePath());
|
|
RandomAccessFile dartLibrary = file.openSync(FileMode.WRITE);
|
|
dartLibrary.writeStringSync(
|
|
wrapDartTestInLibrary(filePath, file.name));
|
|
dartLibrary.closeSync();
|
|
}
|
|
|
|
File file = new File(dartWrapperFilename);
|
|
RandomAccessFile dartWrapper = file.openSync(FileMode.WRITE);
|
|
dartWrapper.writeStringSync(
|
|
dartTestWrapper(dartDir, file.name, dartLibraryFilename));
|
|
dartWrapper.closeSync();
|
|
} else {
|
|
dartWrapperFilename = filename;
|
|
// TODO(whesse): Once test.py is retired, adjust the relative path in
|
|
// the client/samples/dartcombat test to its css file, remove the
|
|
// "../../" from this path, and move this out of the isWebTest guard.
|
|
// Also remove getHtmlName, and just use test.html.
|
|
// TODO(efortuna): this shortening of htmlFilename is a band-aid until
|
|
// the above TODO gets fixed. Windows cannot have paths that are longer
|
|
// than 260 characters, and without this hack, we were running past the
|
|
// the limit.
|
|
String htmlFilename = getHtmlName(filename);
|
|
while ('$tempDir/../$htmlFilename'.length >= 260) {
|
|
htmlFilename = htmlFilename.substring(htmlFilename.length~/2);
|
|
}
|
|
htmlPath = '$tempDir/../$htmlFilename';
|
|
}
|
|
final String scriptPath = (compiler == 'none') ?
|
|
dartWrapperFilename : compiledDartWrapperFilename;
|
|
// Create the HTML file for the test.
|
|
RandomAccessFile htmlTest = new File(htmlPath).openSync(FileMode.WRITE);
|
|
String content = null;
|
|
Path dir = filePath.directoryPath;
|
|
String nameNoExt = filePath.filenameWithoutExtension;
|
|
Path pngPath = dir.append('$nameNoExt.png');
|
|
Path txtPath = dir.append('$nameNoExt.txt');
|
|
Path expectedOutput = null;
|
|
if (new File.fromPath(pngPath).existsSync()) {
|
|
expectedOutput = pngPath;
|
|
// TODO(efortuna): Unify path libraries in test.dart.
|
|
content = getHtmlLayoutContents(scriptType, pathLib.relative(scriptPath,
|
|
from: pathLib.dirname(htmlPath)));
|
|
} else if (new File.fromPath(txtPath).existsSync()) {
|
|
expectedOutput = txtPath;
|
|
content = getHtmlLayoutContents(scriptType, pathLib.relative(scriptPath,
|
|
from: pathLib.dirname(htmlPath)));
|
|
} else {
|
|
final htmlLocation = new Path(htmlPath);
|
|
content = getHtmlContents(
|
|
filename,
|
|
dartDir.append('pkg/unittest/lib/test_controller.js')
|
|
.relativeTo(htmlLocation),
|
|
dartDir.append('pkg/browser/lib/dart.js').relativeTo(htmlLocation),
|
|
scriptType,
|
|
new Path(scriptPath).relativeTo(htmlLocation));
|
|
}
|
|
htmlTest.writeStringSync(content);
|
|
htmlTest.closeSync();
|
|
|
|
// Construct the command(s) that compile all the inputs needed by the
|
|
// browser test. For running Dart in DRT, this will be noop commands.
|
|
List<Command> commands = [];
|
|
if (compiler != 'none') {
|
|
commands.add(_compileCommand(
|
|
dartWrapperFilename, compiledDartWrapperFilename,
|
|
compiler, tempDir, vmOptions, optionsFromFile));
|
|
|
|
// some tests require compiling multiple input scripts.
|
|
List<String> otherScripts = optionsFromFile['otherScripts'];
|
|
for (String name in otherScripts) {
|
|
Path namePath = new Path(name);
|
|
Expect.equals(namePath.extension, 'dart');
|
|
String baseName = namePath.filenameWithoutExtension;
|
|
Path fromPath = filePath.directoryPath.join(namePath);
|
|
commands.add(_compileCommand(
|
|
fromPath.toNativePath(), '$tempDir/$baseName.js',
|
|
compiler, tempDir, vmOptions, optionsFromFile));
|
|
}
|
|
}
|
|
|
|
// Variables for browser multi-tests.
|
|
List<String> subtestNames = info.optionsFromFile['subtestNames'];
|
|
TestCase multitestParentTest;
|
|
int subtestIndex = 0;
|
|
// Construct the command that executes the browser test
|
|
do {
|
|
List<Command> commandSet = new List<Command>.from(commands);
|
|
if (subtestIndex != 0) {
|
|
// NOTE: The first time we enter this loop, all the compilation
|
|
// commands will be executed. On subsequent loop iterations, we
|
|
// don't need to do any compilations. Thus we set "commandSet = []".
|
|
commandSet = [];
|
|
}
|
|
|
|
List<String> args = <String>[];
|
|
var basePath = TestUtils.dartDir().toString();
|
|
if (!htmlPath.startsWith('/') && !htmlPath.startsWith('http')) {
|
|
htmlPath = '/$htmlPath';
|
|
}
|
|
htmlPath = htmlPath.startsWith(basePath) ?
|
|
htmlPath.substring(basePath.length) : htmlPath;
|
|
String fullHtmlPath = htmlPath;
|
|
var searchStr = '?';
|
|
if (!htmlPath.startsWith('http')) {
|
|
// Note: If we run test.py with the "--list" option, no http servers
|
|
// will be started. Therefore serverList is an empty list in this
|
|
// case. So we use PORT/CROSS_ORIGIN_PORT instead of real ports.
|
|
var serverPort = "PORT";
|
|
var crossOriginPort = "CROSS_ORIGIN_PORT";
|
|
if (!configuration['list']) {
|
|
serverPort = serverList[0].port.toString();
|
|
crossOriginPort = serverList[1].port.toString();
|
|
}
|
|
fullHtmlPath = 'http://127.0.0.1:$serverPort$htmlPath${searchStr}'
|
|
'crossOriginPort=$crossOriginPort';
|
|
searchStr = '&';
|
|
}
|
|
if (info.optionsFromFile['isMultiHtmlTest']
|
|
&& subtestNames.length > 0) {
|
|
fullHtmlPath = '${fullHtmlPath}${searchStr}group='
|
|
'${subtestNames[subtestIndex]}';
|
|
}
|
|
|
|
if (TestUtils.usesWebDriver(runtime)) {
|
|
args = [
|
|
dartDir.append('tools/testing/run_selenium.py').toNativePath(),
|
|
'--browser=$runtime',
|
|
'--timeout=${configuration["timeout"] - 2}',
|
|
'--out="$fullHtmlPath"'];
|
|
if (runtime == 'dartium') {
|
|
args.add('--executable=$dartiumFilename');
|
|
}
|
|
if (subtestIndex != 0) {
|
|
args.add('--force-refresh');
|
|
}
|
|
commandSet.add(new Command('python', args));
|
|
} else {
|
|
Expect.isTrue(runtime == "drt");
|
|
|
|
var dartFlags = [];
|
|
var dumpRenderTreeOptions = [];
|
|
|
|
dumpRenderTreeOptions.add('--no-timeout');
|
|
|
|
if (compiler == 'none' || compiler == 'dart2dart') {
|
|
dartFlags.add('--ignore-unrecognized-flags');
|
|
if (configuration["checked"]) {
|
|
dartFlags.add('--enable_asserts');
|
|
dartFlags.add("--enable_type_checks");
|
|
}
|
|
dartFlags.addAll(vmOptions);
|
|
}
|
|
|
|
if (expectedOutput != null) {
|
|
if (expectedOutput.toNativePath().endsWith('.png')) {
|
|
// pixel tests are specified by running DRT "foo.html'-p"
|
|
dumpRenderTreeOptions.add('--notree');
|
|
fullHtmlPath = "${fullHtmlPath}'-p";
|
|
}
|
|
}
|
|
commandSet.add(new DumpRenderTreeCommand(dumpRenderTreeFilename,
|
|
fullHtmlPath,
|
|
dumpRenderTreeOptions,
|
|
dartFlags,
|
|
expectedOutput));
|
|
}
|
|
|
|
// Create BrowserTestCase and queue it.
|
|
String testDisplayName = '$suiteName/$testName';
|
|
var testCase;
|
|
if (info.optionsFromFile['isMultiHtmlTest']) {
|
|
testDisplayName = '$testDisplayName/${subtestNames[subtestIndex]}';
|
|
testCase = new BrowserTestCase(testDisplayName,
|
|
commandSet, configuration, completeHandler,
|
|
expectations['$testName/${subtestNames[subtestIndex]}'],
|
|
info, info.hasCompileError || info.hasRuntimeError,
|
|
subtestIndex != 0);
|
|
} else {
|
|
testCase = new BrowserTestCase(testDisplayName,
|
|
commandSet, configuration, completeHandler, expectations,
|
|
info, info.hasCompileError || info.hasRuntimeError, false);
|
|
}
|
|
if (subtestIndex == 0) {
|
|
multitestParentTest = testCase;
|
|
} else {
|
|
multitestParentTest.addObserver(testCase);
|
|
}
|
|
doTest(testCase);
|
|
subtestIndex++;
|
|
} while(subtestIndex < subtestNames.length);
|
|
}
|
|
}
|
|
|
|
/** Helper to create a compilation command for a single input file. */
|
|
Command _compileCommand(String inputFile, String outputFile,
|
|
String compiler, String dir, vmOptions, optionsFromFile) {
|
|
String executable = compilerPath;
|
|
List<String> args = TestUtils.standardOptions(configuration);
|
|
switch (compiler) {
|
|
case 'dart2js':
|
|
case 'dart2dart':
|
|
String packageRoot =
|
|
packageRootArgument(optionsFromFile['packageRoot']);
|
|
if (packageRoot != null) {
|
|
args.add(packageRoot);
|
|
}
|
|
if (compiler == 'dart2dart') args.add('--out=$outputFile');
|
|
args.add('--out=$outputFile');
|
|
args.add(inputFile);
|
|
break;
|
|
default:
|
|
Expect.fail('unimplemented compiler $compiler');
|
|
}
|
|
if (executable.endsWith('.dart')) {
|
|
// Run the compiler script via the Dart VM.
|
|
args.insertRange(0, 1, executable);
|
|
executable = dartShellFileName;
|
|
}
|
|
if (['dart2js', 'dart2dart'].contains(configuration['compiler'])) {
|
|
return new CompilationCommand(outputFile,
|
|
!useSdk,
|
|
dart2JsBootstrapDependencies,
|
|
compilerPath,
|
|
args);
|
|
}
|
|
return new Command(executable, args);
|
|
}
|
|
|
|
/**
|
|
* Create a directory for the generated test. If a Dart language test
|
|
* needs to be run in a browser, the Dart test needs to be embedded in
|
|
* an HTML page, with a testing framework based on scripting and DOM events.
|
|
* These scripts and pages are written to a generated_test directory
|
|
* inside the build directory of the checkout.
|
|
*
|
|
* Those tests which are already HTML web applications (web tests), with
|
|
* resources including CSS files and HTML files, need to be compiled into
|
|
* a work directory where the relative URLS to the resources work.
|
|
* We use a subdirectory of the build directory that is the same number
|
|
* of levels down in the checkout as the original path of the web test.
|
|
*/
|
|
String createOutputDirectory(Path testPath, String optionsName) {
|
|
Path relative = testPath.relativeTo(TestUtils.dartDir());
|
|
relative = relative.directoryPath.append(relative.filenameWithoutExtension);
|
|
String testUniqueName = relative.toString().replaceAll('/', '_');
|
|
if (!optionsName.isEmpty) {
|
|
testUniqueName = '$testUniqueName-$optionsName';
|
|
}
|
|
|
|
// Create '[build dir]/generated_tests/$compiler-$runtime/$testUniqueName',
|
|
// including any intermediate directories that don't exist.
|
|
// If the tests are run in checked or minified mode we add that to the
|
|
// '$compile-$runtime' directory name.
|
|
var checked = configuration['checked'] ? '-checked' : '';
|
|
var minified = configuration['minified'] ? '-minified' : '';
|
|
var dirName = "${configuration['compiler']}-${configuration['runtime']}"
|
|
"$checked$minified";
|
|
Path generatedTestPath = new Path(dartDir.toNativePath())
|
|
.append(buildDir)
|
|
.append('generated_tests')
|
|
.append(dirName)
|
|
.append(testUniqueName);
|
|
|
|
TestUtils.mkdirRecursive(new Path('.'), generatedTestPath);
|
|
return new File.fromPath(generatedTestPath).fullPathSync()
|
|
.replaceAll('\\', '/');
|
|
}
|
|
|
|
String get scriptType {
|
|
switch (configuration['compiler']) {
|
|
case 'none':
|
|
case 'dart2dart':
|
|
return 'application/dart';
|
|
case 'dart2js':
|
|
case 'dartc':
|
|
return 'text/javascript';
|
|
default:
|
|
Expect.fail('Non-web runtime, so no scriptType for: '
|
|
'${configuration["compiler"]}');
|
|
return null;
|
|
}
|
|
}
|
|
|
|
bool get hasRuntime {
|
|
switch(configuration['runtime']) {
|
|
case 'none':
|
|
return false;
|
|
default:
|
|
return true;
|
|
}
|
|
}
|
|
|
|
String getHtmlName(String filename) {
|
|
var cleanFilename = filename.replaceAll('/', '_')
|
|
.replaceAll(':', '_')
|
|
.replaceAll('\\', '_');
|
|
|
|
return "$cleanFilename"
|
|
"${configuration['compiler']}-${configuration['runtime']}.html";
|
|
}
|
|
|
|
String get dumpRenderTreeFilename {
|
|
if (configuration['drt'] != '') {
|
|
return configuration['drt'];
|
|
}
|
|
if (Platform.operatingSystem == 'macos') {
|
|
return dartDir.append('/client/tests/drt/DumpRenderTree.app/Contents/'
|
|
'MacOS/DumpRenderTree').toNativePath();
|
|
}
|
|
return dartDir.append('client/tests/drt/DumpRenderTree').toNativePath();
|
|
}
|
|
|
|
String get dartiumFilename {
|
|
if (configuration['dartium'] != '') {
|
|
return configuration['dartium'];
|
|
}
|
|
if (Platform.operatingSystem == 'macos') {
|
|
return dartDir.append('client/tests/dartium/Chromium.app/Contents/'
|
|
'MacOS/Chromium').toNativePath();
|
|
}
|
|
return dartDir.append('client/tests/dartium/chrome').toNativePath();
|
|
}
|
|
|
|
void completeHandler(TestCase testCase) {
|
|
}
|
|
|
|
List<String> commonArgumentsFromFile(Path filePath, Map optionsFromFile) {
|
|
List args = TestUtils.standardOptions(configuration);
|
|
|
|
String packageRoot = packageRootArgument(optionsFromFile['packageRoot']);
|
|
if (packageRoot != null) {
|
|
args.add(packageRoot);
|
|
}
|
|
args.addAll(additionalOptions(filePath));
|
|
if (configuration['compiler'] == 'dartc') {
|
|
args.add('--error_format');
|
|
args.add('machine');
|
|
}
|
|
|
|
bool isMultitest = optionsFromFile["isMultitest"];
|
|
List<String> dartOptions = optionsFromFile["dartOptions"];
|
|
List<List<String>> vmOptionsList = getVmOptions(optionsFromFile);
|
|
Expect.isTrue(!isMultitest || dartOptions == null);
|
|
if (dartOptions == null) {
|
|
args.add(filePath.toNativePath());
|
|
} else {
|
|
var executable_name = dartOptions[0];
|
|
// TODO(ager): Get rid of this hack when the runtime checkout goes away.
|
|
var file = new File(executable_name);
|
|
if (!file.existsSync()) {
|
|
executable_name = '../$executable_name';
|
|
Expect.isTrue(new File(executable_name).existsSync());
|
|
dartOptions[0] = executable_name;
|
|
}
|
|
args.addAll(dartOptions);
|
|
}
|
|
|
|
return args;
|
|
}
|
|
|
|
String packageRoot(String packageRootFromFile) {
|
|
if (packageRootFromFile == "none") {
|
|
return null;
|
|
}
|
|
String packageRoot = packageRootFromFile;
|
|
if (packageRootFromFile == null) {
|
|
packageRoot = "$buildDir/packages/";
|
|
}
|
|
return packageRoot;
|
|
}
|
|
|
|
String packageRootArgument(String packageRootFromFile) {
|
|
var packageRootPath = packageRoot(packageRootFromFile);
|
|
if (packageRootPath == null) {
|
|
return null;
|
|
}
|
|
return "--package-root=$packageRootPath";
|
|
}
|
|
|
|
/**
|
|
* Special options for individual tests are currently specified in various
|
|
* ways: with comments directly in test files, by using certain imports, or by
|
|
* creating additional files in the test directories.
|
|
*
|
|
* Here is a list of options that are used by 'test.dart' today:
|
|
* - Flags can be passed to the vm or dartium process that runs the test by
|
|
* adding a comment to the test file:
|
|
*
|
|
* // VMOptions=--flag1 --flag2
|
|
*
|
|
* - Flags can be passed to the dart script that contains the test also
|
|
* using comments, as follows:
|
|
*
|
|
* // DartOptions=--flag1 --flag2
|
|
*
|
|
* - For tests that depend on compiling other files with dart2js (e.g.
|
|
* isolate tests that use multiple source scripts), you can specify
|
|
* additional files to compile using a comment too, as follows:
|
|
*
|
|
* // OtherScripts=file1.dart file2.dart
|
|
*
|
|
* - You can indicate whether a test is treated as a web-only test by
|
|
* using an explicit import to the dart:html library:
|
|
*
|
|
* #import('dart:html');
|
|
*
|
|
* Most tests are not web tests, but can (and will be) wrapped within
|
|
* another script file to test them also on browser environments (e.g.
|
|
* language and corelib tests are run this way). We deduce that if this
|
|
* import is specified, the test was intended to be a web test and no
|
|
* wrapping is necessary.
|
|
*
|
|
* - You can convert DRT web-tests into layout-web-tests by specifying a
|
|
* test expectation file. An expectation file is located in the same
|
|
* location as the test, it has the same file name, except for the extension
|
|
* (which can be either .txt or .png).
|
|
*
|
|
* When there are no expectation files, 'test.dart' assumes tests fail if
|
|
* the process return a non-zero exit code (in the case of web tests, we
|
|
* check for PASS/FAIL indications in the test output).
|
|
*
|
|
* When there is an expectation file, tests are run differently: the test
|
|
* code is run to the end of the event loop and 'test.dart' takes a snapshot
|
|
* of what is rendered in the page at that moment. This snapshot is
|
|
* represented either in text form, if the expectation ends in .txt, or as
|
|
* an image, if the expectation ends in .png. 'test.dart' will compare the
|
|
* snapshot to the expectation file. When tests fail, 'test.dart' saves the
|
|
* new snapshot into a file so it can be visualized or copied over.
|
|
* Expectations can be recorded for the first time by creating an empty file
|
|
* with the right name (touch test_name_test.png), running the test, and
|
|
* executing the copy command printed by the test script.
|
|
*
|
|
* This method is static as the map is cached and shared amongst
|
|
* configurations, so it may not use [configuration].
|
|
*/
|
|
static Map readOptionsFromFile(Path filePath) {
|
|
if (filePath.segments().contains('co19')) {
|
|
return readOptionsFromCo19File(filePath);
|
|
}
|
|
RegExp testOptionsRegExp = new RegExp(r"// VMOptions=(.*)");
|
|
RegExp dartOptionsRegExp = new RegExp(r"// DartOptions=(.*)");
|
|
RegExp otherScriptsRegExp = new RegExp(r"// OtherScripts=(.*)");
|
|
RegExp packageRootRegExp = new RegExp(r"// PackageRoot=(.*)");
|
|
RegExp multiHtmlTestRegExp =
|
|
new RegExp(r"useHtmlIndividualConfiguration()");
|
|
RegExp staticTypeRegExp =
|
|
new RegExp(r"/// ([0-9][0-9]:){0,1}\s*static type warning");
|
|
RegExp compileTimeRegExp =
|
|
new RegExp(r"/// ([0-9][0-9]:){0,1}\s*compile-time error");
|
|
RegExp staticCleanRegExp = new RegExp(r"// @static-clean");
|
|
RegExp leadingHashRegExp = new RegExp(r"^#", multiLine: true);
|
|
RegExp isolateStubsRegExp = new RegExp(r"// IsolateStubs=(.*)");
|
|
// TODO(gram) Clean these up once the old directives are not supported.
|
|
RegExp domImportRegExp =
|
|
new RegExp(r"^[#]?import.*dart:html", multiLine: true);
|
|
RegExp libraryDefinitionRegExp =
|
|
new RegExp(r"^[#]?library[\( ]", multiLine: true);
|
|
RegExp sourceOrImportRegExp =
|
|
new RegExp("^(#source|#import|part)[ \t]+[\('\"]", multiLine: true);
|
|
|
|
var bytes = new File.fromPath(filePath).readAsBytesSync();
|
|
String contents = decodeUtf8(bytes);
|
|
bytes = null;
|
|
|
|
// Find the options in the file.
|
|
List<List> result = new List<List>();
|
|
List<String> dartOptions;
|
|
String packageRoot;
|
|
bool isStaticClean = false;
|
|
|
|
Iterable<Match> matches = testOptionsRegExp.allMatches(contents);
|
|
for (var match in matches) {
|
|
result.add(match[1].split(' ').where((e) => e != '').toList());
|
|
}
|
|
if (result.isEmpty) result.add([]);
|
|
|
|
matches = dartOptionsRegExp.allMatches(contents);
|
|
for (var match in matches) {
|
|
if (dartOptions != null) {
|
|
throw new Exception(
|
|
'More than one "// DartOptions=" line in test $filePath');
|
|
}
|
|
dartOptions = match[1].split(' ').where((e) => e != '').toList();
|
|
}
|
|
|
|
matches = packageRootRegExp.allMatches(contents);
|
|
for (var match in matches) {
|
|
if (packageRoot != null) {
|
|
throw new Exception(
|
|
'More than one "// PackageRoot=" line in test $filePath');
|
|
}
|
|
packageRoot = match[1];
|
|
if (packageRoot != 'none') {
|
|
// PackageRoot=none means that no package-root option should be given.
|
|
packageRoot = '${filePath.directoryPath.join(new Path(packageRoot))}';
|
|
}
|
|
}
|
|
|
|
matches = staticCleanRegExp.allMatches(contents);
|
|
for (var match in matches) {
|
|
if (isStaticClean) {
|
|
throw new Exception(
|
|
'More than one "// @static-clean=" line in test $filePath');
|
|
}
|
|
isStaticClean = true;
|
|
}
|
|
|
|
List<String> otherScripts = new List<String>();
|
|
matches = otherScriptsRegExp.allMatches(contents);
|
|
for (var match in matches) {
|
|
otherScripts.addAll(match[1].split(' ').where((e) => e != '').toList());
|
|
}
|
|
|
|
bool isMultitest = multiTestRegExp.hasMatch(contents);
|
|
bool isMultiHtmlTest = multiHtmlTestRegExp.hasMatch(contents);
|
|
bool containsLeadingHash = leadingHashRegExp.hasMatch(contents);
|
|
Match isolateMatch = isolateStubsRegExp.firstMatch(contents);
|
|
String isolateStubs = isolateMatch != null ? isolateMatch[1] : '';
|
|
bool containsDomImport = domImportRegExp.hasMatch(contents);
|
|
bool isLibraryDefinition = libraryDefinitionRegExp.hasMatch(contents);
|
|
bool containsSourceOrImport = sourceOrImportRegExp.hasMatch(contents);
|
|
int numStaticTypeAnnotations = 0;
|
|
for (var i in staticTypeRegExp.allMatches(contents)) {
|
|
numStaticTypeAnnotations++;
|
|
}
|
|
int numCompileTimeAnnotations = 0;
|
|
for (var i in compileTimeRegExp.allMatches(contents)) {
|
|
numCompileTimeAnnotations++;
|
|
}
|
|
|
|
// Note: This is brittle. It's the age-old problem of having a context free
|
|
// language but the means to easily identify the construct is a regular
|
|
// expression, aka impossible. Therefore we just make an approximation of
|
|
// the number of top-level "group(...)" occurrences. This assumes you import
|
|
// unittest with no prefix and always directly call "group(". It only uses
|
|
// top-level "groups" so tests running nested groups will be no-ops.
|
|
RegExp numTests = new RegExp(r"\s*[^/]\s*group\('[^,']*");
|
|
List<String> subtestNames = [];
|
|
Iterator matchesIter = numTests.allMatches(contents).iterator;
|
|
while(matchesIter.moveNext() && isMultiHtmlTest) {
|
|
String fullMatch = matchesIter.current.group(0);
|
|
subtestNames.add(fullMatch.substring(fullMatch.indexOf("'") + 1));
|
|
}
|
|
|
|
return { "vmOptions": result,
|
|
"dartOptions": dartOptions,
|
|
"packageRoot": packageRoot,
|
|
"hasCompileError": false,
|
|
"hasRuntimeError": false,
|
|
"isStaticClean" : isStaticClean,
|
|
"otherScripts": otherScripts,
|
|
"isMultitest": isMultitest,
|
|
"isMultiHtmlTest": isMultiHtmlTest,
|
|
"subtestNames": subtestNames,
|
|
"containsLeadingHash": containsLeadingHash,
|
|
"isolateStubs": isolateStubs,
|
|
"containsDomImport": containsDomImport,
|
|
"isLibraryDefinition": isLibraryDefinition,
|
|
"containsSourceOrImport": containsSourceOrImport,
|
|
"numStaticTypeAnnotations": numStaticTypeAnnotations,
|
|
"numCompileTimeAnnotations": numCompileTimeAnnotations };
|
|
}
|
|
|
|
List<List<String>> getVmOptions(Map optionsFromFile) {
|
|
var COMPILERS = const ['none', 'dart2dart'];
|
|
var RUNTIMES = const ['none', 'vm', 'drt', 'dartium'];
|
|
var needsVmOptions = COMPILERS.contains(configuration['compiler']) &&
|
|
RUNTIMES.contains(configuration['runtime']);
|
|
if (!needsVmOptions) return [[]];
|
|
final vmOptions = optionsFromFile['vmOptions'];
|
|
if (configuration['compiler'] != 'dart2dart') return vmOptions;
|
|
// Temporary workaround for race in test suite: tests with different
|
|
// vm options are still compiled into the same output file which
|
|
// may lead to reads from empty files.
|
|
return [vmOptions[0]];
|
|
}
|
|
|
|
/**
|
|
* Read options from a co19 test file.
|
|
*
|
|
* The reason this is different from [readOptionsFromFile] is that
|
|
* co19 is developed based on a contract which defines certain test
|
|
* tags. These tags may appear unused, but should not be removed
|
|
* without consulting with the co19 team.
|
|
*
|
|
* Also, [readOptionsFromFile] recognizes a number of additional
|
|
* tags that are not appropriate for use in general tests of
|
|
* conformance to the Dart language. Any Dart implementation must
|
|
* pass the co19 test suite as is, and not require extra flags,
|
|
* environment variables, configuration files, etc.
|
|
*/
|
|
static Map readOptionsFromCo19File(Path filePath) {
|
|
String contents = decodeUtf8(new File.fromPath(filePath).readAsBytesSync());
|
|
|
|
bool hasCompileError = contents.contains("@compile-error");
|
|
bool hasRuntimeError = contents.contains("@runtime-error");
|
|
bool hasDynamicTypeError = contents.contains("@dynamic-type-error");
|
|
bool hasStaticWarning = contents.contains("@static-warning");
|
|
bool isMultitest = multiTestRegExp.hasMatch(contents);
|
|
|
|
if (hasDynamicTypeError) {
|
|
// TODO(ahe): Remove this warning when co19 no longer uses this tag.
|
|
|
|
// @dynamic-type-error has been replaced by tests that use
|
|
// tests/co19/src/Utils/dynamic_check.dart to dynamically detect
|
|
// if a test is running in checked mode or not and change its
|
|
// expectations accordingly.
|
|
|
|
// Using stderr.writeString to avoid breaking dartc/junit_tests
|
|
// which parses the output of the --list option.
|
|
stderr.writeString(
|
|
"Warning: deprecated @dynamic-type-error tag used in $filePath\n");
|
|
}
|
|
|
|
return {
|
|
"vmOptions": <List>[[]],
|
|
"dartOptions": null,
|
|
"packageRoot": null,
|
|
"hasCompileError": hasCompileError,
|
|
"hasRuntimeError": hasRuntimeError,
|
|
"isStaticClean" : !hasStaticWarning,
|
|
"otherScripts": <String>[],
|
|
"isMultitest": isMultitest,
|
|
"isMultiHtmlTest": false,
|
|
"subtestNames": <String>[],
|
|
"containsLeadingHash": false,
|
|
"isolateStubs": '',
|
|
"containsDomImport": false,
|
|
"isLibraryDefinition": false,
|
|
"containsSourceOrImport": false,
|
|
"numStaticTypeAnnotations": 0,
|
|
"numCompileTimeAnnotations": 0,
|
|
};
|
|
}
|
|
}
|
|
|
|
|
|
class DartcCompilationTestSuite extends StandardTestSuite {
|
|
List<String> _testDirs;
|
|
|
|
DartcCompilationTestSuite(Map configuration,
|
|
String suiteName,
|
|
String directoryPath,
|
|
List<String> this._testDirs,
|
|
List<String> expectations)
|
|
: super(configuration,
|
|
suiteName,
|
|
new Path(directoryPath),
|
|
expectations);
|
|
|
|
List<String> additionalOptions(Path filePath) {
|
|
return ['--fatal-warnings', '--fatal-type-errors'];
|
|
}
|
|
|
|
Future enqueueTests() {
|
|
var group = new FutureGroup();
|
|
|
|
for (String testDir in _testDirs) {
|
|
Directory dir = new Directory.fromPath(suiteDir.append(testDir));
|
|
if (dir.existsSync()) {
|
|
enqueueDirectory(dir, group);
|
|
}
|
|
}
|
|
|
|
return group.future;
|
|
}
|
|
}
|
|
|
|
|
|
class JUnitTestSuite extends TestSuite {
|
|
String directoryPath;
|
|
String statusFilePath;
|
|
final String dartDir;
|
|
String classPath;
|
|
List<String> testClasses;
|
|
TestCaseEvent doTest;
|
|
VoidFunction doDone;
|
|
TestExpectations testExpectations;
|
|
|
|
JUnitTestSuite(Map configuration,
|
|
String suiteName,
|
|
String this.directoryPath,
|
|
String this.statusFilePath)
|
|
: super(configuration, suiteName),
|
|
dartDir = TestUtils.dartDir().toNativePath();
|
|
|
|
bool isTestFile(String filename) => filename.endsWith("Tests.java") &&
|
|
!filename.contains('com/google/dart/compiler/vm') &&
|
|
!filename.contains('com/google/dart/corelib/SharedTests.java');
|
|
|
|
void forEachTest(TestCaseEvent onTest,
|
|
Map testCacheIgnored,
|
|
[VoidFunction onDone]) {
|
|
doTest = onTest;
|
|
doDone = onDone;
|
|
|
|
if (configuration['compiler'] != 'dartc') {
|
|
// Do nothing. Asynchronously report that the suite is enqueued.
|
|
asynchronously(doDone);
|
|
return;
|
|
}
|
|
RegExp pattern = configuration['selectors']['dartc'];
|
|
if (!pattern.hasMatch('junit_tests')) {
|
|
asynchronously(doDone);
|
|
return;
|
|
}
|
|
|
|
computeClassPath();
|
|
testClasses = <String>[];
|
|
// Do not read the status file.
|
|
// All exclusions are hardcoded in this script, as they are in testcfg.py.
|
|
processDirectory();
|
|
}
|
|
|
|
void processDirectory() {
|
|
directoryPath = '$dartDir/$directoryPath';
|
|
Directory dir = new Directory(directoryPath);
|
|
|
|
var lister = dir.list(recursive: true);
|
|
lister.onFile = processFile;
|
|
lister.onDone = createTest;
|
|
}
|
|
|
|
void processFile(String filename) {
|
|
if (!isTestFile(filename)) return;
|
|
|
|
int index = filename.indexOf('compiler/javatests/com/google/dart');
|
|
if (index != -1) {
|
|
String testRelativePath =
|
|
filename.substring(index + 'compiler/javatests/'.length,
|
|
filename.length - '.java'.length);
|
|
String testClass = testRelativePath.replaceAll('/', '.');
|
|
testClasses.add(testClass);
|
|
}
|
|
}
|
|
|
|
void createTest(successIgnored) {
|
|
var sdkDir = "$buildDir/dart-sdk".trim();
|
|
List<String> args = <String>[
|
|
'-ea',
|
|
'-classpath', classPath,
|
|
'-Dcom.google.dart.sdk=$sdkDir',
|
|
'-Dcom.google.dart.corelib.SharedTests.test_py=$dartDir/tools/test.py',
|
|
'org.junit.runner.JUnitCore'];
|
|
args.addAll(testClasses);
|
|
|
|
// Lengthen the timeout for JUnit tests. It is normal for them
|
|
// to run for a few minutes.
|
|
Map updatedConfiguration = new Map();
|
|
configuration.forEach((key, value) {
|
|
updatedConfiguration[key] = value;
|
|
});
|
|
updatedConfiguration['timeout'] *= 3;
|
|
doTest(new TestCase(suiteName,
|
|
[new Command('java', args)],
|
|
updatedConfiguration,
|
|
completeHandler,
|
|
new Set<String>.from([PASS])));
|
|
doDone();
|
|
}
|
|
|
|
void completeHandler(TestCase testCase) {
|
|
}
|
|
|
|
void computeClassPath() {
|
|
classPath = Strings.join(
|
|
['$buildDir/analyzer/util/analyzer/dart_analyzer.jar',
|
|
'$buildDir/analyzer/dart_analyzer_tests.jar',
|
|
// Third party libraries.
|
|
'$dartDir/third_party/args4j/2.0.12/args4j-2.0.12.jar',
|
|
'$dartDir/third_party/guava/r09/guava-r09.jar',
|
|
'$dartDir/third_party/rhino/1_7R3/js.jar',
|
|
'$dartDir/third_party/hamcrest/v1_3/hamcrest-core-1.3.0RC2.jar',
|
|
'$dartDir/third_party/hamcrest/v1_3/hamcrest-generator-1.3.0RC2.jar',
|
|
'$dartDir/third_party/hamcrest/v1_3/hamcrest-integration-1.3.0RC2.jar',
|
|
'$dartDir/third_party/hamcrest/v1_3/hamcrest-library-1.3.0RC2.jar',
|
|
'$dartDir/third_party/junit/v4_8_2/junit.jar'],
|
|
Platform.operatingSystem == 'windows'? ';': ':'); // Path separator.
|
|
}
|
|
}
|
|
|
|
class LastModifiedCache {
|
|
Map<String, Date> _cache = <String, Date>{};
|
|
|
|
/**
|
|
* Returns the last modified date of the given [uri].
|
|
*
|
|
* The return value will be cached for future queries. If [uri] is a local
|
|
* file, it's last modified [Date] will be returned. If the file does not
|
|
* exist, null will be returned instead.
|
|
* In case [uri] is not a local file, this method will always return
|
|
* the current date.
|
|
*/
|
|
Date getLastModified(Uri uri) {
|
|
if (uri.scheme == "file") {
|
|
if (_cache.containsKey(uri.path)) {
|
|
return _cache[uri.path];
|
|
}
|
|
var file = new File(new Path(uri.path).toNativePath());
|
|
_cache[uri.path] = file.existsSync() ? file.lastModifiedSync() : null;
|
|
return _cache[uri.path];
|
|
}
|
|
return new Date.now();
|
|
}
|
|
}
|
|
|
|
class TestUtils {
|
|
/**
|
|
* The libraries in this directory relies on finding various files
|
|
* relative to the 'test.dart' script in '.../dart/tools/test.dart'. If
|
|
* the main script using 'test_suite.dart' is not there, the main
|
|
* script must set this to '.../dart/tools/test.dart'.
|
|
*/
|
|
static String testScriptPath = new Options().script;
|
|
static LastModifiedCache lastModifiedCache = new LastModifiedCache();
|
|
static Path currentWorkingDirectory =
|
|
new Path(new Directory.current().path);
|
|
/**
|
|
* Creates a directory using a [relativePath] to an existing
|
|
* [base] directory if that [relativePath] does not already exist.
|
|
*/
|
|
static Directory mkdirRecursive(Path base, Path relativePath) {
|
|
if (relativePath.isAbsolute) {
|
|
base = new Path('/');
|
|
}
|
|
Directory dir = new Directory.fromPath(base);
|
|
Expect.isTrue(dir.existsSync(),
|
|
"Expected ${dir} to already exist");
|
|
var segments = relativePath.segments();
|
|
for (String segment in segments) {
|
|
base = base.append(segment);
|
|
if (base.toString() == "/$segment" &&
|
|
segment.length == 2 &&
|
|
segment.endsWith(':')) {
|
|
// Skip the directory creation for a path like "/E:".
|
|
continue;
|
|
}
|
|
dir = new Directory.fromPath(base);
|
|
if (!dir.existsSync()) {
|
|
dir.createSync();
|
|
}
|
|
Expect.isTrue(dir.existsSync(), "Failed to create ${dir.path}");
|
|
}
|
|
return dir;
|
|
}
|
|
|
|
/**
|
|
* Copy a [source] file to a new place.
|
|
* Assumes that the directory for [dest] already exists.
|
|
*/
|
|
static Future copyFile(Path source, Path dest) {
|
|
var output = new File.fromPath(dest).openOutputStream();
|
|
new File.fromPath(source).openInputStream().pipe(output);
|
|
var completer = new Completer();
|
|
output.onClosed = (){ completer.complete(null); };
|
|
return completer.future;
|
|
}
|
|
|
|
static Path debugLogfile() {
|
|
return new Path(".debug.log");
|
|
}
|
|
|
|
static String flakyFileName() {
|
|
// If a flaky test did fail, infos about it (i.e. test name, stdin, stdout)
|
|
// will be written to this file. This is useful for the debugging of
|
|
// flaky tests.
|
|
// When running on a built bot, the file can be made visible in the
|
|
// waterfall UI.
|
|
return ".flaky.log";
|
|
}
|
|
|
|
static void ensureExists(String filename, Map configuration) {
|
|
if (!configuration['list'] && !(new File(filename).existsSync())) {
|
|
throw "Executable '$filename' does not exist";
|
|
}
|
|
}
|
|
|
|
static Path absolutePath(Path path) {
|
|
if (!path.isAbsolute) {
|
|
return currentWorkingDirectory.join(path);
|
|
}
|
|
return path;
|
|
}
|
|
|
|
static String outputDir(Map configuration) {
|
|
var result = '';
|
|
var system = configuration['system'];
|
|
if (system == 'linux') {
|
|
result = 'out/';
|
|
} else if (system == 'macos') {
|
|
result = 'xcodebuild/';
|
|
} else if (system == 'windows') {
|
|
result = 'build/';
|
|
}
|
|
return result;
|
|
}
|
|
|
|
static Path dartDir() {
|
|
File scriptFile = new File(testScriptPath);
|
|
Path scriptPath = new Path(scriptFile.fullPathSync());
|
|
return scriptPath.directoryPath.directoryPath;
|
|
}
|
|
|
|
static List<String> standardOptions(Map configuration) {
|
|
List args = ["--ignore-unrecognized-flags"];
|
|
if (configuration["checked"]) {
|
|
args.add('--enable_asserts');
|
|
args.add("--enable_type_checks");
|
|
}
|
|
String compiler = configuration["compiler"];
|
|
if (compiler == "dart2js" || compiler == "dart2dart") {
|
|
args = [];
|
|
if (configuration["checked"]) {
|
|
args.add('--enable-checked-mode');
|
|
}
|
|
// args.add("--verbose");
|
|
if (!isBrowserRuntime(configuration['runtime'])) {
|
|
args.add("--allow-mock-compilation");
|
|
args.add("--categories=all");
|
|
}
|
|
}
|
|
if ((compiler == "dart2js" || compiler == "dart2dart") &&
|
|
configuration["minified"]) {
|
|
args.add("--minify");
|
|
}
|
|
return args;
|
|
}
|
|
|
|
static bool usesWebDriver(String runtime) {
|
|
const BROWSERS = const [
|
|
'dartium',
|
|
'ie9',
|
|
'ie10',
|
|
'safari',
|
|
'opera',
|
|
'chrome',
|
|
'ff'
|
|
];
|
|
return BROWSERS.contains(runtime);
|
|
}
|
|
|
|
static bool isBrowserRuntime(String runtime) =>
|
|
runtime == 'drt' || TestUtils.usesWebDriver(runtime);
|
|
|
|
static bool isJsCommandLineRuntime(String runtime) =>
|
|
const ['d8', 'jsshell'].contains(runtime);
|
|
|
|
static String buildDir(Map configuration) {
|
|
if (configuration['build_directory'] != '') {
|
|
return configuration['build_directory'];
|
|
}
|
|
var outputDir = '';
|
|
var system = configuration['system'];
|
|
if (system == 'linux') {
|
|
outputDir = 'out/';
|
|
} else if (system == 'macos') {
|
|
outputDir = 'xcodebuild/';
|
|
} else if (system == 'windows') {
|
|
outputDir = 'build/';
|
|
}
|
|
return "$outputDir${configurationDir(configuration)}";
|
|
}
|
|
|
|
static String configurationDir(Map configuration) {
|
|
// For regular dart checkouts, the configDir by default is mode+arch.
|
|
// For Dartium, the configDir by default is mode (as defined by the Chrome
|
|
// build setup). We can detect this because in the dartium checkout, the
|
|
// "output" directory is a sibling of the dart directory instead of a child.
|
|
var mode = (configuration['mode'] == 'debug') ? 'Debug' : 'Release';
|
|
var arch = configuration['arch'].toUpperCase();
|
|
if (currentWorkingDirectory != dartDir()) {
|
|
return '$mode$arch';
|
|
} else {
|
|
return mode;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Returns the path to the dart binary checked into the repo, used for
|
|
* bootstrapping test.dart.
|
|
*/
|
|
static Path get dartTestExecutable {
|
|
var path = '${TestUtils.dartDir()}/tools/testing/bin/'
|
|
'${Platform.operatingSystem}/dart';
|
|
if (Platform.operatingSystem == 'windows') {
|
|
path = '$path.exe';
|
|
}
|
|
return new Path(path);
|
|
}
|
|
}
|
|
|
|
class SummaryReport {
|
|
static int total = 0;
|
|
static int skipped = 0;
|
|
static int noCrash = 0;
|
|
static int pass = 0;
|
|
static int failOk = 0;
|
|
static int fail = 0;
|
|
static int crash = 0;
|
|
static int timeout = 0;
|
|
static int compileErrorSkip = 0;
|
|
|
|
static void add(Set<String> expectations) {
|
|
++total;
|
|
if (expectations.contains(SKIP)) {
|
|
++skipped;
|
|
} else {
|
|
if (expectations.contains(PASS) && expectations.contains(FAIL) &&
|
|
!expectations.contains(CRASH) && !expectations.contains(OK)) {
|
|
++noCrash;
|
|
}
|
|
if (expectations.contains(PASS) && expectations.length == 1) {
|
|
++pass;
|
|
}
|
|
if (expectations.containsAll([FAIL, OK]) && expectations.length == 2) {
|
|
++failOk;
|
|
}
|
|
if (expectations.contains(FAIL) && expectations.length == 1) {
|
|
++fail;
|
|
}
|
|
if (expectations.contains(CRASH) && expectations.length == 1) {
|
|
++crash;
|
|
}
|
|
if (expectations.contains(TIMEOUT)) {
|
|
++timeout;
|
|
}
|
|
}
|
|
}
|
|
|
|
static void addCompileErrorSkipTest() {
|
|
total++;
|
|
compileErrorSkip++;
|
|
}
|
|
|
|
static void printReport() {
|
|
if (total == 0) return;
|
|
String report = """Total: $total tests
|
|
* $skipped tests will be skipped
|
|
* $noCrash tests are expected to be flaky but not crash
|
|
* $pass tests are expected to pass
|
|
* $failOk tests are expected to fail that we won't fix
|
|
* $fail tests are expected to fail that we should fix
|
|
* $crash tests are expected to crash that we should fix
|
|
* $timeout tests are allowed to timeout
|
|
* $compileErrorSkip tests are skipped on browsers due to compile-time error
|
|
""";
|
|
print(report);
|
|
}
|
|
}
|