Martin Kustermann
a50dfd6698
Change-Id: Ibb492283c2b53762cb4f74da52c3f005cc282d85 Reviewed-on: https://dart-review.googlesource.com/c/78745 Reviewed-by: Alexander Thomas <athom@google.com> Commit-Queue: Martin Kustermann <kustermann@google.com>
884 lines
32 KiB
Dart
884 lines
32 KiB
Dart
// Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file
|
|
// for details. All rights reserved. Use of this source code is governed by a
|
|
// BSD-style license that can be found in the LICENSE file.
|
|
|
|
import 'dart:io';
|
|
|
|
import 'package:smith/smith.dart';
|
|
|
|
import 'configuration.dart';
|
|
import 'path.dart';
|
|
import 'repository.dart';
|
|
import 'utils.dart';
|
|
|
|
const _defaultTestSelectors = const [
|
|
'samples',
|
|
'standalone',
|
|
'standalone_2',
|
|
'corelib',
|
|
'corelib_2',
|
|
'co19',
|
|
'language',
|
|
'language_2',
|
|
'isolate',
|
|
'vm',
|
|
'html',
|
|
'benchmark_smoke',
|
|
'utils',
|
|
'lib',
|
|
'lib_2',
|
|
'analyze_library',
|
|
'service',
|
|
'kernel',
|
|
'observatory_ui'
|
|
];
|
|
|
|
/// Specifies a single command line option.
|
|
///
|
|
/// The name of the specification is used as the key for the option in the Map
|
|
/// returned from the [TestOptionParser] parse method.
|
|
class _Option {
|
|
// TODO(rnystrom): Some string options use "" to mean "no value" and others
|
|
// use null. Clean that up.
|
|
_Option(this.name, this.description,
|
|
{String abbr, List<String> values, String defaultsTo, bool hide})
|
|
: abbreviation = abbr,
|
|
values = values ?? [],
|
|
defaultValue = defaultsTo,
|
|
type = _OptionValueType.string,
|
|
verboseOnly = hide ?? false;
|
|
|
|
_Option.bool(this.name, this.description, {String abbr, bool hide})
|
|
: abbreviation = abbr,
|
|
values = [],
|
|
defaultValue = false,
|
|
type = _OptionValueType.bool,
|
|
verboseOnly = hide ?? false;
|
|
|
|
_Option.int(this.name, this.description,
|
|
{String abbr, int defaultsTo, bool hide})
|
|
: abbreviation = abbr,
|
|
values = [],
|
|
defaultValue = defaultsTo,
|
|
type = _OptionValueType.int,
|
|
verboseOnly = hide ?? false;
|
|
|
|
final String name;
|
|
final String description;
|
|
final String abbreviation;
|
|
final List<String> values;
|
|
final Object defaultValue;
|
|
final _OptionValueType type;
|
|
|
|
/// Only show this option in the verbose help.
|
|
final bool verboseOnly;
|
|
|
|
/// Gets the shortest command line argument used to refer to this option.
|
|
String get shortCommand => abbreviation != null ? "-$abbreviation" : command;
|
|
|
|
/// Gets the canonical long command line argument used to refer to this
|
|
/// option.
|
|
String get command => "--${name.replaceAll('_', '-')}";
|
|
}
|
|
|
|
enum _OptionValueType { bool, int, string }
|
|
|
|
/// Parses command line arguments and produces a test runner configuration.
|
|
class OptionsParser {
|
|
static final List<_Option> _options = [
|
|
new _Option('mode', 'Mode in which to run the tests.',
|
|
abbr: 'm', values: ['all']..addAll(Mode.names)),
|
|
new _Option(
|
|
'compiler',
|
|
'''How the Dart code should be compiled or statically processed.
|
|
|
|
none: Do not compile the Dart code.
|
|
precompiler: Compile into AOT snapshot before running the test.
|
|
dart2js: Compile to JavaScript using dart2js.
|
|
dart2analyzer: Perform static analysis on Dart code using the analyzer.
|
|
compareAnalyzerCfe: Compare analyzer and common front end representations.
|
|
app_jit: Compile the Dart code into an app snapshot.
|
|
app_jitk: Compile the Dart code into Kernel and then into an app snapshot.
|
|
dartk: Compile the Dart code into Kernel before running test.
|
|
dartkb: Compile the Dart code into Kernel with bytecode before running test.
|
|
dartkp: Compile the Dart code into Kernel and then Kernel into AOT
|
|
snapshot before running the test.
|
|
spec_parser: Parse Dart code using the specification parser.''',
|
|
abbr: 'c',
|
|
values: Compiler.names),
|
|
new _Option(
|
|
'runtime',
|
|
'''Where the tests should be run.
|
|
vm: Run Dart code on the standalone Dart VM.
|
|
flutter: Run Dart code on the Flutter engine.
|
|
dart_precompiled: Run a precompiled snapshot on the VM without a JIT.
|
|
d8: Run JavaScript from the command line using v8.
|
|
jsshell: Run JavaScript from the command line using Firefox js-shell.
|
|
|
|
firefox:
|
|
chrome:
|
|
safari:
|
|
ie9:
|
|
ie10:
|
|
ie11:
|
|
chromeOnAndroid: Run JavaScript in the specified browser.
|
|
|
|
self_check: Pass each test or its compiled output to every file under
|
|
`pkg` whose name ends with `_self_check.dart`. Each test is
|
|
given to the self_check tester as a filename on stdin using
|
|
the batch-mode protocol.
|
|
|
|
none: No runtime, compile only.''',
|
|
abbr: 'r',
|
|
values: Runtime.names),
|
|
new _Option(
|
|
'arch',
|
|
'''The architecture to run tests for.
|
|
|
|
Allowed values are:
|
|
all
|
|
ia32, x64
|
|
arm, armv6, armv5te, arm64,
|
|
simarm, simarmv6, simarmv5te, simarm64,
|
|
simdbc, simdbc64''',
|
|
abbr: 'a',
|
|
values: ['all']..addAll(Architecture.names),
|
|
defaultsTo: Architecture.x64.name,
|
|
hide: true),
|
|
new _Option('system', 'The operating system to run tests on.',
|
|
abbr: 's',
|
|
values: System.names,
|
|
defaultsTo: Platform.operatingSystem,
|
|
hide: true),
|
|
new _Option(
|
|
'named_configuration',
|
|
'''The named test configuration that supplies the values for all
|
|
test options, specifying how tests should be run. Unimplemented
|
|
currently.''',
|
|
abbr: 'n',
|
|
hide: true),
|
|
new _Option.bool('checked', 'Run tests in checked mode.'),
|
|
new _Option.bool('strong', 'Deprecated, no-op.', hide: true),
|
|
// TODO(sigmund): rename flag once we migrate all dart2js bots to the test
|
|
// matrix.
|
|
new _Option.bool('host_checked', 'Run compiler with assertions enabled.',
|
|
hide: true),
|
|
new _Option.bool('minified', 'Enable minification in the compiler.',
|
|
hide: true),
|
|
new _Option.bool(
|
|
'csp', 'Run tests under Content Security Policy restrictions.',
|
|
hide: true),
|
|
new _Option.bool('fast_startup', 'Pass the --fast-startup flag to dart2js.',
|
|
hide: true),
|
|
new _Option.bool('fast_tests',
|
|
'Only run tests that are not marked `Slow` or `Timeout`.'),
|
|
new _Option.bool('enable_asserts',
|
|
'Pass the --enable-asserts flag to dart2js or to the vm.'),
|
|
new _Option.bool('no_preview_dart_2',
|
|
'Enable legacy Dart 1 behavior for some runtimes and compilers.',
|
|
hide: true),
|
|
new _Option.bool('use_cfe', 'Pass the --use-cfe flag to analyzer',
|
|
hide: true),
|
|
new _Option.bool('analyzer_use_fasta_parser',
|
|
'Pass the --use-fasta-parser flag to analyzer',
|
|
hide: true),
|
|
|
|
// TODO(sigmund): replace dart2js_with_kernel with preview-dart-2.
|
|
new _Option.bool(
|
|
'dart2js_with_kernel', 'Pass the --use-kernel flag to dart2js.',
|
|
hide: true),
|
|
new _Option.bool(
|
|
'dart2js_old_frontend', 'Pass the --use-old-frontend flag to dart2js.',
|
|
hide: true),
|
|
new _Option.bool('hot_reload', 'Run hot reload stress tests.', hide: true),
|
|
new _Option.bool(
|
|
'hot_reload_rollback', 'Run hot reload rollback stress tests.',
|
|
hide: true),
|
|
new _Option.bool(
|
|
'use_blobs', 'Use mmap instead of shared libraries for precompilation.',
|
|
hide: true),
|
|
new _Option.int('timeout', 'Timeout in seconds.', abbr: 't'),
|
|
new _Option(
|
|
'progress',
|
|
'''Progress indication mode.
|
|
|
|
Allowed values are:
|
|
compact, color, line, verbose, silent, status, buildbot, diff''',
|
|
abbr: 'p',
|
|
values: Progress.names,
|
|
defaultsTo: Progress.compact.name),
|
|
new _Option('step_name', 'Step name for use by -pbuildbot.', hide: true),
|
|
new _Option.bool('report',
|
|
'Print a summary report of the number of tests, by expectation.',
|
|
hide: true),
|
|
new _Option.int('tasks', 'The number of parallel tasks to run.',
|
|
abbr: 'j', defaultsTo: Platform.numberOfProcessors),
|
|
new _Option.int('shards',
|
|
'The number of instances that the tests will be sharded over.',
|
|
defaultsTo: 1, hide: true),
|
|
new _Option.int(
|
|
'shard', 'The index of this instance when running in sharded mode.',
|
|
defaultsTo: 1, hide: true),
|
|
new _Option.bool('help', 'Print list of options.', abbr: 'h'),
|
|
new _Option.int('repeat', 'How many times each test is run', defaultsTo: 1),
|
|
new _Option.bool('verbose', 'Verbose output.', abbr: 'v'),
|
|
new _Option.bool('verify-ir', 'Verify kernel IR.', hide: true),
|
|
new _Option.bool('no-tree-shake', 'Disable kernel IR tree shaking.',
|
|
hide: true),
|
|
new _Option.bool('list', 'List tests only, do not run them.'),
|
|
new _Option.bool('list_status_files',
|
|
'List status files for test-suites. Do not run any test suites.',
|
|
hide: true),
|
|
new _Option.bool('report_in_json',
|
|
'When listing with --list, output result summary in JSON.',
|
|
hide: true),
|
|
new _Option.bool('time', 'Print timing information after running tests.'),
|
|
new _Option('dart', 'Path to dart executable.', hide: true),
|
|
new _Option('flutter', 'Path to flutter executable.', hide: true),
|
|
new _Option('firefox', 'Path to firefox browser executable.', hide: true),
|
|
new _Option('chrome', 'Path to chrome browser executable.', hide: true),
|
|
new _Option('safari', 'Path to safari browser executable.', hide: true),
|
|
new _Option.bool('use_sdk', '''Use compiler or runtime from the SDK.'''),
|
|
// TODO(rnystrom): This does not appear to be used. Remove?
|
|
new _Option('build_directory',
|
|
'The name of the build directory, where products are placed.',
|
|
hide: true),
|
|
new _Option('output_directory',
|
|
'The name of the output directory for storing log files.',
|
|
defaultsTo: "logs", hide: true),
|
|
new _Option.bool('noBatch', 'Do not run tests in batch mode.', hide: true),
|
|
new _Option.bool('dart2js_batch', 'Run dart2js tests in batch mode.',
|
|
hide: true),
|
|
new _Option.bool(
|
|
'append_logs', 'Do not delete old logs but rather append to them.',
|
|
hide: true),
|
|
new _Option.bool('write_debug_log',
|
|
'Don\'t write debug messages to stdout but rather to a logfile.',
|
|
hide: true),
|
|
new _Option.bool('write_test_outcome_log',
|
|
'Write test outcomes to a "${TestUtils.testOutcomeFileName}" file.',
|
|
hide: true),
|
|
new _Option.bool(
|
|
'write_result_log',
|
|
'Write test results to a "${TestUtils.resultLogFileName}" json file '
|
|
'located at the debug_output_directory.',
|
|
hide: true),
|
|
new _Option.bool(
|
|
'write_results',
|
|
'Write results to a "${TestUtils.resultsFileName}" json file '
|
|
'located at the debug_output_directory.',
|
|
hide: true),
|
|
new _Option.bool(
|
|
'reset_browser_configuration',
|
|
'''Browser specific reset of configuration.
|
|
|
|
Warning: Using this option may remove your bookmarks and other
|
|
settings.''',
|
|
hide: true),
|
|
new _Option.bool(
|
|
'copy_coredumps',
|
|
'''If we see a crash that we did not expect, copy the core dumps to
|
|
"/tmp".''',
|
|
hide: true),
|
|
new _Option(
|
|
'local_ip',
|
|
'''IP address the HTTP servers should listen on. This address is also
|
|
used for browsers to connect to.''',
|
|
defaultsTo: '127.0.0.1',
|
|
hide: true),
|
|
new _Option.int('test_server_port', 'Port for test http server.',
|
|
defaultsTo: 0, hide: true),
|
|
new _Option.int('test_server_cross_origin_port',
|
|
'Port for test http server cross origin.',
|
|
defaultsTo: 0, hide: true),
|
|
new _Option.int('test_driver_port', 'Port for http test driver server.',
|
|
defaultsTo: 0, hide: true),
|
|
new _Option.int(
|
|
'test_driver_error_port', 'Port for http test driver server errors.',
|
|
defaultsTo: 0, hide: true),
|
|
new _Option('test_list', 'File containing a list of tests to be executed'),
|
|
new _Option(
|
|
'builder_tag',
|
|
'''Machine specific options that is not captured by the regular test
|
|
options. Used to be able to make sane updates to the status files.''',
|
|
hide: true),
|
|
new _Option('vm_options', 'Extra options to send to the VM when running.',
|
|
hide: true),
|
|
new _Option(
|
|
'dart2js_options', 'Extra options for dart2js compilation step.',
|
|
hide: true),
|
|
new _Option(
|
|
'suite_dir', 'Additional directory to add to the testing matrix.',
|
|
hide: true),
|
|
new _Option('package_root', 'The package root to use for testing.',
|
|
hide: true),
|
|
new _Option('packages', 'The package spec file to use for testing.',
|
|
hide: true),
|
|
new _Option(
|
|
'exclude_suite',
|
|
'''Exclude suites from default selector, only works when no selector
|
|
has been specified on the command line.''',
|
|
hide: true),
|
|
new _Option.bool(
|
|
'skip_compilation',
|
|
'''
|
|
Skip the compilation step, using the compilation artifacts left in
|
|
the output folder from a previous run. This flag will often cause
|
|
false positves and negatives, but can be useful for quick and
|
|
dirty offline testing when not making changes that affect the
|
|
compiler.''',
|
|
hide: true),
|
|
new _Option.bool('print_passing_stdout',
|
|
'Print the stdout of passing, as well as failing, tests.',
|
|
hide: true)
|
|
];
|
|
|
|
/// For printing out reproducing command lines, we don't want to add these
|
|
/// options.
|
|
static final _blacklistedOptions = [
|
|
'append_logs',
|
|
'build_directory',
|
|
'debug_output_directory',
|
|
'chrome',
|
|
'copy_coredumps',
|
|
'dart',
|
|
'flutter',
|
|
'drt',
|
|
'exclude_suite',
|
|
'firefox',
|
|
'local_ip',
|
|
'progress',
|
|
'report',
|
|
'safari',
|
|
'shard',
|
|
'shards',
|
|
'step_name',
|
|
'tasks',
|
|
'time',
|
|
'verbose',
|
|
'write_debug_log',
|
|
'write_test_outcome_log',
|
|
'write_result_log',
|
|
'write_results',
|
|
].toSet();
|
|
|
|
/// Parses a list of strings as test options.
|
|
///
|
|
/// Returns a list of configurations in which to run the tests.
|
|
/// Configurations are maps mapping from option keys to values. When
|
|
/// encountering the first non-option string, the rest of the arguments are
|
|
/// stored in the returned Map under the 'rest' key.
|
|
List<TestConfiguration> parse(List<String> arguments) {
|
|
// Help supersedes all other arguments.
|
|
if (arguments.contains("--help") || arguments.contains("-h")) {
|
|
_printHelp(
|
|
verbose: arguments.contains("--verbose") || arguments.contains("-v"));
|
|
return null;
|
|
}
|
|
// Dart1 mode has been deprecated.
|
|
if (arguments.contains("--no-preview-dart-2")) {
|
|
return null;
|
|
}
|
|
|
|
var configuration = <String, dynamic>{};
|
|
|
|
// Fill in configuration with arguments passed to the test script.
|
|
for (var i = 0; i < arguments.length; i++) {
|
|
var arg = arguments[i];
|
|
|
|
// Extract name and value for options.
|
|
String command;
|
|
String value;
|
|
_Option option;
|
|
|
|
if (arg.startsWith("--")) {
|
|
// A long option name.
|
|
var equals = arg.indexOf("=");
|
|
if (equals != -1) {
|
|
// A long option with a value, like "--arch=ia32".
|
|
command = arg.substring(0, equals);
|
|
value = arg.substring(equals + 1);
|
|
} else {
|
|
command = arg;
|
|
}
|
|
|
|
option = _findByName(command.substring(2));
|
|
} else if (arg.startsWith("-")) {
|
|
// An abbreviated option.
|
|
if (arg.length == 1) {
|
|
_fail('Missing option name after "-".');
|
|
}
|
|
|
|
command = arg.substring(0, 2);
|
|
|
|
if (arg.length > 2) {
|
|
// An abbreviated option followed by a value, like "-aia32".
|
|
value = arg.substring(2);
|
|
}
|
|
|
|
option = _findByAbbreviation(command.substring(1));
|
|
} else {
|
|
// The argument does not start with "-" or "--" and is therefore not an
|
|
// option. Use it as a test selector pattern.
|
|
var patterns = configuration.putIfAbsent("selectors", () => <String>[]);
|
|
patterns.add(arg);
|
|
continue;
|
|
}
|
|
|
|
if (option == null) {
|
|
_fail('Unknown command line option "$command".');
|
|
}
|
|
|
|
// If we need a value, look at the next argument.
|
|
if (value == null && option.type != _OptionValueType.bool) {
|
|
if (i + 1 >= arguments.length) {
|
|
_fail('Missing value for command line option "$command".');
|
|
}
|
|
value = arguments[++i];
|
|
}
|
|
|
|
// Multiple uses of a flag are an error, because there is no naturally
|
|
// correct way to handle conflicting options.
|
|
if (configuration.containsKey(option.name)) {
|
|
_fail('Already have value for command line option "$command".');
|
|
}
|
|
|
|
// Parse the value for the option.
|
|
switch (option.type) {
|
|
case _OptionValueType.bool:
|
|
if (value != null) {
|
|
_fail('Boolean flag "$command" does not take a value.');
|
|
}
|
|
|
|
configuration[option.name] = true;
|
|
break;
|
|
|
|
case _OptionValueType.int:
|
|
try {
|
|
configuration[option.name] = int.parse(value);
|
|
} on FormatException {
|
|
_fail('Integer value expected for option "$command".');
|
|
}
|
|
break;
|
|
|
|
case _OptionValueType.string:
|
|
// Validate against the allowed values.
|
|
if (!option.values.isEmpty) {
|
|
for (var v in value.split(",")) {
|
|
if (!option.values.contains(v)) {
|
|
_fail('Unknown value "$v" for command line option "$command".');
|
|
}
|
|
}
|
|
}
|
|
|
|
// TODO(rnystrom): Store as a list instead of a comma-delimited
|
|
// string.
|
|
configuration[option.name] = value;
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Apply default values for unspecified options.
|
|
for (var option in _options) {
|
|
if (!configuration.containsKey(option.name)) {
|
|
configuration[option.name] = option.defaultValue;
|
|
}
|
|
}
|
|
|
|
// Fetch list of tests to run, if option is present.
|
|
if (configuration['test_list'] is String) {
|
|
configuration['test_list_contents'] =
|
|
File(configuration['test_list'] as String).readAsLinesSync();
|
|
}
|
|
|
|
return _createConfigurations(configuration);
|
|
}
|
|
|
|
/// Prints [message] and exits with a non-zero exit code.
|
|
void _fail(String message) {
|
|
print(message);
|
|
exit(1);
|
|
}
|
|
|
|
/// Given a set of parsed option values, returns the list of command line
|
|
/// arguments that would reproduce that configuration.
|
|
List<String> _reproducingCommand(Map<String, dynamic> data) {
|
|
var arguments = <String>[];
|
|
|
|
for (var option in _options) {
|
|
var name = option.name;
|
|
if (!data.containsKey(name) || _blacklistedOptions.contains(name)) {
|
|
continue;
|
|
}
|
|
|
|
var value = data[name];
|
|
if (data[name] == option.defaultValue ||
|
|
(name == 'packages' &&
|
|
value == Repository.uri.resolve('.packages').toFilePath())) {
|
|
continue;
|
|
}
|
|
|
|
arguments.add(option.shortCommand);
|
|
if (option.type != _OptionValueType.bool) {
|
|
arguments.add(value.toString());
|
|
}
|
|
}
|
|
|
|
return arguments;
|
|
}
|
|
|
|
List<TestConfiguration> _createConfigurations(
|
|
Map<String, dynamic> configuration) {
|
|
var selectors = _expandSelectors(configuration);
|
|
|
|
// Put observatory_ui in a configuration with its own packages override.
|
|
// Only one value in the configuration map is mutable:
|
|
if (selectors.containsKey('observatory_ui')) {
|
|
if (selectors.length == 1) {
|
|
configuration['packages'] = Repository.uri
|
|
.resolve('runtime/observatory/.packages')
|
|
.toFilePath();
|
|
} else {
|
|
// Make a new configuration whose selectors map only contains
|
|
// observatory_ui, and remove observatory_ui from the original
|
|
// selectors. The only mutable value in the map is the selectors, so a
|
|
// shallow copy is safe.
|
|
var observatoryConfiguration =
|
|
new Map<String, dynamic>.from(configuration);
|
|
var observatorySelectors = {
|
|
'observatory_ui': selectors['observatory_ui']
|
|
};
|
|
selectors.remove('observatory_ui');
|
|
|
|
// Set the packages flag.
|
|
observatoryConfiguration['packages'] = Repository.uri
|
|
.resolve('runtime/observatory/.packages')
|
|
.toFilePath();
|
|
|
|
return _expandConfigurations(configuration, selectors)
|
|
..addAll(_expandConfigurations(
|
|
observatoryConfiguration, observatorySelectors));
|
|
}
|
|
}
|
|
|
|
return _expandConfigurations(configuration, selectors);
|
|
}
|
|
|
|
/// Recursively expands a configuration with multiple values per key into a
|
|
/// list of configurations with exactly one value per key.
|
|
List<TestConfiguration> _expandConfigurations(
|
|
Map<String, dynamic> data, Map<String, RegExp> selectors) {
|
|
var result = <TestConfiguration>[];
|
|
|
|
// Handles a string option containing a space-separated list of words.
|
|
listOption(String name) {
|
|
var value = data[name] as String;
|
|
if (value == null) return const <String>[];
|
|
return value
|
|
.split(" ")
|
|
.map((s) => s.trim())
|
|
.where((s) => s.isNotEmpty)
|
|
.toList();
|
|
}
|
|
|
|
var dart2jsOptions = listOption("dart2js_options");
|
|
var vmOptions = listOption("vm_options");
|
|
|
|
// JSON reporting implies listing and reporting.
|
|
if (data['report_in_json'] as bool) {
|
|
data['list'] = true;
|
|
data['report'] = true;
|
|
}
|
|
|
|
// Use verbose progress indication for verbose output unless buildbot
|
|
// progress indication is requested.
|
|
if ((data['verbose'] as bool) &&
|
|
(data['progress'] as String) != 'buildbot') {
|
|
data['progress'] = 'verbose';
|
|
}
|
|
|
|
var runtimeNames = data["runtime"] as String;
|
|
var runtimes = <Runtime>[];
|
|
if (runtimeNames != null) {
|
|
runtimes.addAll(runtimeNames.split(",").map(Runtime.find));
|
|
}
|
|
|
|
var compilerNames = data["compiler"] as String;
|
|
var compilers = <Compiler>[];
|
|
if (compilerNames != null) {
|
|
compilers.addAll(compilerNames.split(",").map(Compiler.find));
|
|
}
|
|
|
|
// Pick default compilers or runtimes if only one or the other is provided.
|
|
if (runtimes.isEmpty) {
|
|
if (compilers.isEmpty) {
|
|
runtimes = [Runtime.vm];
|
|
compilers = [Compiler.dartk];
|
|
} else {
|
|
// Pick a runtime for each compiler.
|
|
runtimes.addAll(compilers.map((compiler) => compiler.defaultRuntime));
|
|
}
|
|
} else if (compilers.isEmpty) {
|
|
// Pick a compiler for each runtime.
|
|
compilers.addAll(runtimes.map((runtime) => runtime.defaultCompiler));
|
|
}
|
|
|
|
// Expand runtimes.
|
|
for (var runtime in runtimes) {
|
|
// Start installing the runtime if needed.
|
|
|
|
// Expand architectures.
|
|
var architectures = data["arch"] as String;
|
|
if (architectures == "all") {
|
|
architectures = "ia32,x64,simarm,simarm64,simdbc64";
|
|
}
|
|
|
|
for (var architectureName in architectures.split(",")) {
|
|
var architecture = Architecture.find(architectureName);
|
|
|
|
// Expand compilers.
|
|
for (var compiler in compilers) {
|
|
// Expand modes.
|
|
String modes = data["mode"] ?? compiler.defaultMode.name;
|
|
if (modes == "all") modes = "debug,release,product";
|
|
for (var modeName in modes.split(",")) {
|
|
var mode = Mode.find(modeName);
|
|
var system = System.find(data["system"] as String);
|
|
var namedConfiguration =
|
|
getNamedConfiguration(data["named_configuration"] as String);
|
|
var innerConfiguration = namedConfiguration ??
|
|
new Configuration("custom configuration", architecture,
|
|
compiler, mode, runtime, system,
|
|
timeout: data["timeout"] as int,
|
|
enableAsserts: data["enable_asserts"] as bool,
|
|
useAnalyzerCfe: data["use_cfe"] as bool,
|
|
useAnalyzerFastaParser:
|
|
data["analyzer_use_fasta_parser"] as bool,
|
|
useBlobs: data["use_blobs"] as bool,
|
|
useSdk: data["use_sdk"] as bool,
|
|
useFastStartup: data["fast_startup"] as bool,
|
|
useDart2JSWithKernel: data["dart2js_with_kernel"] as bool,
|
|
useDart2JSOldFrontEnd: data["dart2js_old_frontend"] as bool,
|
|
useHotReload: data["hot_reload"] as bool,
|
|
useHotReloadRollback: data["hot_reload_rollback"] as bool,
|
|
isChecked: data["checked"] as bool,
|
|
isHostChecked: data["host_checked"] as bool,
|
|
isCsp: data["csp"] as bool,
|
|
isMinified: data["minified"] as bool,
|
|
vmOptions: vmOptions,
|
|
builderTag: data["builder_tag"] as String,
|
|
previewDart2: true);
|
|
var configuration = new TestConfiguration(
|
|
configuration: innerConfiguration,
|
|
progress: Progress.find(data["progress"] as String),
|
|
selectors: selectors,
|
|
testList: data["test_list_contents"] as List<String>,
|
|
appendLogs: data["append_logs"] as bool,
|
|
repeat: data["repeat"] as int,
|
|
batch: !(data["noBatch"] as bool),
|
|
batchDart2JS: data["dart2js_batch"] as bool,
|
|
copyCoreDumps: data["copy_coredumps"] as bool,
|
|
isVerbose: data["verbose"] as bool,
|
|
listTests: data["list"] as bool,
|
|
listStatusFiles: data["list_status_files"] as bool,
|
|
printTiming: data["time"] as bool,
|
|
printReport: data["report"] as bool,
|
|
reportInJson: data["report_in_json"] as bool,
|
|
resetBrowser: data["reset_browser_configuration"] as bool,
|
|
skipCompilation: data["skip_compilation"] as bool,
|
|
useKernelBytecode: compiler == Compiler.dartkb,
|
|
writeDebugLog: data["write_debug_log"] as bool,
|
|
writeTestOutcomeLog: data["write_test_outcome_log"] as bool,
|
|
writeResultLog: data["write_result_log"] as bool,
|
|
writeResults: data["write_results"] as bool,
|
|
drtPath: data["drt"] as String,
|
|
chromePath: data["chrome"] as String,
|
|
safariPath: data["safari"] as String,
|
|
firefoxPath: data["firefox"] as String,
|
|
dartPath: data["dart"] as String,
|
|
dartPrecompiledPath: data["dart_precompiled"] as String,
|
|
flutterPath: data["flutter"] as String,
|
|
taskCount: data["tasks"] as int,
|
|
shardCount: data["shards"] as int,
|
|
shard: data["shard"] as int,
|
|
stepName: data["step_name"] as String,
|
|
testServerPort: data["test_server_port"] as int,
|
|
testServerCrossOriginPort:
|
|
data['test_server_cross_origin_port'] as int,
|
|
testDriverErrorPort: data["test_driver_error_port"] as int,
|
|
localIP: data["local_ip"] as String,
|
|
dart2jsOptions: dart2jsOptions,
|
|
packages: data["packages"] as String,
|
|
packageRoot: data["package_root"] as String,
|
|
suiteDirectory: data["suite_dir"] as String,
|
|
outputDirectory: data["output_directory"] as String,
|
|
reproducingArguments: _reproducingCommand(data),
|
|
fastTestsOnly: data["fast_tests"] as bool,
|
|
printPassingStdout: data["print_passing_stdout"] as bool);
|
|
|
|
if (configuration.validate()) {
|
|
result.add(configuration);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (result.length > 1 && data["named_configuration"] != null) {
|
|
_fail("Named configuration cannot be used with multiple values for "
|
|
"arch, compiler, mode, or runtime");
|
|
}
|
|
return result;
|
|
}
|
|
|
|
/// Expands the test selectors into a suite name and a simple regular
|
|
/// expression to be used on the full path of a test file in that test suite.
|
|
///
|
|
/// If no selectors are explicitly given, uses the default suite patterns.
|
|
Map<String, RegExp> _expandSelectors(Map<String, dynamic> configuration) {
|
|
var selectors = configuration['selectors'];
|
|
|
|
if (selectors == null) {
|
|
if (configuration['suite_dir'] != null) {
|
|
var suitePath = Path(configuration['suite_dir'] as String);
|
|
selectors = [suitePath.filename];
|
|
} else if (configuration['test_list_contents'] != null) {
|
|
selectors = (configuration['test_list_contents'] as List<String>)
|
|
.map((t) => t.split('/').first)
|
|
.toSet()
|
|
.toList();
|
|
} else {
|
|
selectors = _defaultTestSelectors.toList();
|
|
}
|
|
|
|
var excludeSuites = configuration['exclude_suite'] != null
|
|
? configuration['exclude_suite'].split(',')
|
|
: [];
|
|
for (var exclude in excludeSuites) {
|
|
if ((selectors as List).contains(exclude)) {
|
|
selectors.remove(exclude);
|
|
} else {
|
|
print("Warning: default selectors does not contain $exclude");
|
|
}
|
|
}
|
|
}
|
|
|
|
var selectorMap = <String, RegExp>{};
|
|
for (var i = 0; i < (selectors as List).length; i++) {
|
|
var pattern = selectors[i] as String;
|
|
var suite = pattern;
|
|
var slashLocation = pattern.indexOf('/');
|
|
if (slashLocation != -1) {
|
|
suite = pattern.substring(0, slashLocation);
|
|
pattern = pattern.substring(slashLocation + 1);
|
|
pattern = pattern.replaceAll('*', '.*');
|
|
} else {
|
|
pattern = ".?";
|
|
}
|
|
if (selectorMap.containsKey(suite)) {
|
|
_fail("Error: '$suite/$pattern'. Only one test selection"
|
|
" pattern is allowed to start with '$suite/'");
|
|
}
|
|
selectorMap[suite] = new RegExp(pattern);
|
|
}
|
|
|
|
return selectorMap;
|
|
}
|
|
|
|
/// Print out usage information.
|
|
void _printHelp({bool verbose}) {
|
|
var buffer = new StringBuffer();
|
|
|
|
buffer.writeln('''The Dart SDK's internal test runner.
|
|
|
|
Usage: dart tools/test.dart [options] [selector]
|
|
|
|
The optional selector limits the tests that will be run. For example, the
|
|
selector "language/issue", or equivalently "language/*issue*", limits to test
|
|
files matching the regexp ".*issue.*\\.dart" in the "tests/language" directory.
|
|
|
|
If you specify only a runtime ("-r"), then an appropriate default compiler will
|
|
be chosen for that runtime. Likewise, if you specify only a compiler ("-c"),
|
|
then a matching runtime is chosen. If neither compiler nor runtime is selected,
|
|
the test is run directly from source on the VM.
|
|
|
|
Options:''');
|
|
|
|
for (var option in _options) {
|
|
if (!verbose && option.verboseOnly) continue;
|
|
|
|
if (option.abbreviation != null) {
|
|
buffer.write("-${option.abbreviation}, ");
|
|
} else {
|
|
buffer.write(" ");
|
|
}
|
|
|
|
buffer.write(option.command);
|
|
|
|
switch (option.type) {
|
|
case _OptionValueType.bool:
|
|
// No value.
|
|
break;
|
|
case _OptionValueType.int:
|
|
buffer.write("=<integer>");
|
|
break;
|
|
case _OptionValueType.string:
|
|
if (option.values.length > 6) {
|
|
// If there are many options, they won't fit nicely in one line and
|
|
// should be instead listed in the description.
|
|
buffer.write("=<...>");
|
|
} else if (option.values.isNotEmpty) {
|
|
buffer.write("=<${option.values.join('|')}>");
|
|
} else {
|
|
buffer.write("=<string>");
|
|
}
|
|
break;
|
|
}
|
|
|
|
if (option.type != _OptionValueType.bool &&
|
|
option.defaultValue != null &&
|
|
option.defaultValue != "") {
|
|
buffer.write(" (defaults to ${option.defaultValue})");
|
|
}
|
|
|
|
buffer.writeln();
|
|
buffer
|
|
.writeln(" ${option.description.replaceAll('\n', '\n ')}");
|
|
buffer.writeln();
|
|
}
|
|
|
|
if (!verbose) {
|
|
buffer.write('Pass "--verbose" to see more options.');
|
|
}
|
|
|
|
print(buffer);
|
|
}
|
|
|
|
_Option _findByAbbreviation(String abbreviation) {
|
|
for (var option in _options) {
|
|
if (abbreviation == option.abbreviation) return option;
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
_Option _findByName(String name) {
|
|
for (var option in _options) {
|
|
if (name == option.name) return option;
|
|
|
|
// Allow hyphens instead of underscores as the separator since they are
|
|
// more common for command line flags.
|
|
if (name == option.name.replaceAll("_", "-")) return option;
|
|
}
|
|
|
|
return null;
|
|
}
|
|
}
|
|
|
|
Configuration getNamedConfiguration(String template) {
|
|
if (template == null) return null;
|
|
TestMatrix testMatrix = TestMatrix.fromPath("tools/bots/test_matrix.json");
|
|
return testMatrix.configurations
|
|
.singleWhere((c) => c.name == template, orElse: () => null);
|
|
}
|