diff --git a/runtime/tests/vm/vm.status b/runtime/tests/vm/vm.status index d324552eb9e..f18405b8255 100644 --- a/runtime/tests/vm/vm.status +++ b/runtime/tests/vm/vm.status @@ -156,9 +156,6 @@ dart/optimized_stacktrace_line_and_column_test: RuntimeError # minifying they can be renamed, which is issue 7953. dart/inline_stack_frame_test: RuntimeError, Pass # Issue 7953 -[ $compiler == dart2js && $cps_ir && $checked ] -dart/*: Skip # checked mode + cpsir not supported yet. Issue 25761 - [ $compiler == dart2js || $compiler == dart2analyzer ] # Data uri's not supported by dart2js or the analyzer. dart/data_uri*test: Skip diff --git a/samples/samples.status b/samples/samples.status index 4a9105ebddd..2c735a8064c 100644 --- a/samples/samples.status +++ b/samples/samples.status @@ -14,9 +14,6 @@ sample_extension: Skip [ $compiler == dart2js && $runtime == none ] *: Fail, Pass # TODO(ahe): Triage these tests. -[ $compiler == dart2js && $cps_ir && $checked ] -sample_extension: Crash # Unable to compile UnsupportedError.message. - [ $compiler == dart2analyzer ] build_dart: Skip diff --git a/tests/standalone/io/skipping_dart2js_compilations_test.dart b/tests/standalone/io/skipping_dart2js_compilations_test.dart index 8fc5fafcdad..992f4280f7a 100644 --- a/tests/standalone/io/skipping_dart2js_compilations_test.dart +++ b/tests/standalone/io/skipping_dart2js_compilations_test.dart @@ -20,10 +20,10 @@ import 'package:expect/expect.dart'; import 'package:path/path.dart'; import 'dart:async'; import 'dart:io'; +import '../../../tools/testing/dart/options.dart' as options; import '../../../tools/testing/dart/path.dart'; import '../../../tools/testing/dart/test_suite.dart' as suite; import '../../../tools/testing/dart/test_runner.dart' as runner; -import '../../../tools/testing/dart/test_options.dart' as options; import '../../../tools/testing/dart/utils.dart'; /** @@ -151,7 +151,7 @@ class CommandCompletedHandler { } runner.Command makeCompilationCommand(String testName, FileUtils fileUtils) { - var config = new options.TestOptionsParser().parse(['--timeout', '2'])[0]; + var config = new options.OptionsParser().parse(['--timeout', '2'])[0]; var createFileScript = Platform.script .resolve('skipping_dart2js_compilations_helper.dart') .toFilePath(); diff --git a/tests/standalone/io/test_runner_test.dart b/tests/standalone/io/test_runner_test.dart index af37fb4a9ae..90920e93221 100644 --- a/tests/standalone/io/test_runner_test.dart +++ b/tests/standalone/io/test_runner_test.dart @@ -6,11 +6,11 @@ import "dart:io"; import "dart:isolate"; import "dart:async"; import "../../../tools/testing/dart/expectation.dart"; +import "../../../tools/testing/dart/options.dart"; import "../../../tools/testing/dart/status_file.dart"; import "../../../tools/testing/dart/test_runner.dart"; import "../../../tools/testing/dart/test_suite.dart"; import "../../../tools/testing/dart/test_progress.dart" as progress; -import "../../../tools/testing/dart/test_options.dart"; import "process_test_util.dart"; final DEFAULT_TIMEOUT = 20; @@ -113,8 +113,7 @@ class CustomTestSuite extends TestSuite { } _makeTestCase(name, timeout, command, expectations) { - var configuration = - new TestOptionsParser().parse(['--timeout', '$timeout'])[0]; + var configuration = new OptionsParser().parse(['--timeout', '$timeout'])[0]; return new TestCase(name, [command], configuration, new Set.from(expectations)); } @@ -123,7 +122,7 @@ class CustomTestSuite extends TestSuite { void testProcessQueue() { var maxProcesses = 2; var maxBrowserProcesses = maxProcesses; - var config = new TestOptionsParser().parse(['--nobatch'])[0]; + var config = new OptionsParser().parse(['--noBatch'])[0]; new ProcessQueue( config, maxProcesses, diff --git a/tools/testing/dart/co19_test.dart b/tools/testing/dart/co19_test.dart index 5cbcddaf36b..29113a07ccf 100644 --- a/tools/testing/dart/co19_test.dart +++ b/tools/testing/dart/co19_test.dart @@ -18,7 +18,7 @@ library co19_test; import "dart:io"; -import "test_options.dart"; +import "options.dart"; import "test_suite.dart"; import "test_configurations.dart"; @@ -59,7 +59,7 @@ const List> COMMAND_LINES = const >[ void main(List args) { TestUtils.setDartDirUri(Platform.script.resolve('../../..')); - var optionsParser = new TestOptionsParser(); + var optionsParser = new OptionsParser(); var configurations = []; for (var commandLine in COMMAND_LINES) { var arguments = []; diff --git a/tools/testing/dart/compiler_configuration.dart b/tools/testing/dart/compiler_configuration.dart index fffd68135bc..aa6486bbb73 100644 --- a/tools/testing/dart/compiler_configuration.dart +++ b/tools/testing/dart/compiler_configuration.dart @@ -47,6 +47,12 @@ abstract class CompilerConfiguration { final bool isHostChecked; final bool useSdk; + /// Only some subclasses support this check, but we statically allow calling + /// it on [CompilerConfiguration]. + bool get useDfe { + throw new UnsupportedError("This compiler does not support DFE."); + } + // TODO(ahe): Remove this constructor and move the switch to // test_options.dart. We probably want to store an instance of // [CompilerConfiguration] in [configuration] there. @@ -62,7 +68,6 @@ abstract class CompilerConfiguration { bool isHostChecked = configuration['host_checked']; bool useSdk = configuration['use_sdk']; bool isCsp = configuration['csp']; - bool useCps = configuration['cps_ir']; bool useBlobs = configuration['use_blobs']; bool hotReload = configuration['hot_reload']; bool hotReloadRollback = configuration['hot_reload_rollback']; @@ -82,7 +87,6 @@ abstract class CompilerConfiguration { isDebug: isDebug, isChecked: isChecked, isHostChecked: isHostChecked, - useCps: useCps, useSdk: useSdk, isCsp: isCsp, useFastStartup: useFastStartup, @@ -107,7 +111,7 @@ abstract class CompilerConfiguration { useSdk: useSdk, hotReload: hotReload, hotReloadRollback: hotReloadRollback, - useDFE: true); + useDfe: true); case 'dartkp': return new PrecompilerCompilerConfiguration( isDebug: isDebug, @@ -115,7 +119,7 @@ abstract class CompilerConfiguration { arch: configuration['arch'], useBlobs: useBlobs, isAndroid: configuration['system'] == 'android', - useDFE: true); + useDfe: true); case 'none': return new NoneCompilerConfiguration( isDebug: isDebug, @@ -187,7 +191,7 @@ abstract class CompilerConfiguration { class NoneCompilerConfiguration extends CompilerConfiguration { final bool hotReload; final bool hotReloadRollback; - final bool useDFE; + final bool useDfe; NoneCompilerConfiguration( {bool isDebug, @@ -196,7 +200,7 @@ class NoneCompilerConfiguration extends CompilerConfiguration { bool useSdk, bool this.hotReload, bool this.hotReloadRollback, - this.useDFE: false}) + this.useDfe: false}) : super._subclass( isDebug: isDebug, isChecked: isChecked, @@ -214,7 +218,7 @@ class NoneCompilerConfiguration extends CompilerConfiguration { List originalArguments, CommandArtifact artifact) { List args = []; - if (useDFE) { + if (useDfe) { args.add('--dfe=${buildDir}/gen/kernel-service.dart.snapshot'); args.add('--platform=${buildDir}/patched_sdk/platform.dill'); } @@ -537,20 +541,15 @@ class Dart2xCompilerConfiguration extends CompilerConfiguration { /// Configuration for dart2js compiler. class Dart2jsCompilerConfiguration extends Dart2xCompilerConfiguration { final bool isCsp; - final bool useCps; final bool useFastStartup; final bool useKernel; final List extraDart2jsOptions; - // We cache the extended environment to save memory. - static Map cpsFlagCache; - static Map environmentOverridesCacheObject; Dart2jsCompilerConfiguration( {bool isDebug, bool isChecked, bool isHostChecked, bool useSdk, - bool this.useCps, bool this.isCsp, bool this.useFastStartup, this.useKernel, @@ -604,7 +603,7 @@ class PrecompilerCompilerConfiguration extends CompilerConfiguration { final String arch; final bool useBlobs; final bool isAndroid; - final bool useDFE; + final bool useDfe; PrecompilerCompilerConfiguration( {bool isDebug, @@ -612,7 +611,7 @@ class PrecompilerCompilerConfiguration extends CompilerConfiguration { this.arch, this.useBlobs, this.isAndroid, - this.useDFE: false}) + this.useDfe: false}) : super._subclass(isDebug: isDebug, isChecked: isChecked); int computeTimeoutMultiplier() { @@ -658,7 +657,7 @@ class PrecompilerCompilerConfiguration extends CompilerConfiguration { exec = "$buildDir/dart_bootstrap"; } var args = []; - if (useDFE) { + if (useDfe) { args.add('--dfe=utils/kernel-service/kernel-service.dart'); args.add('--platform=${buildDir}/patched_sdk/platform.dill'); } diff --git a/tools/testing/dart/main.dart b/tools/testing/dart/main.dart index 727cccf18b5..ff2a0d12641 100644 --- a/tools/testing/dart/main.dart +++ b/tools/testing/dart/main.dart @@ -22,8 +22,8 @@ /// `factory StandardTestSuite.forDirectory`. import "dart:io"; +import "options.dart"; import "test_configurations.dart"; -import "test_options.dart"; import "test_suite.dart"; /// Runs all of the tests specified by the given command line [arguments]. @@ -32,8 +32,8 @@ void main(List arguments) { TestUtils.setDartDirUri(Platform.script.resolve('../../..')); // Parse the command line arguments to a configuration. - var optionsParser = new TestOptionsParser(); - var configurations = optionsParser.parse(arguments); + var parser = new OptionsParser(); + var configurations = parser.parse(arguments); if (configurations == null || configurations.isEmpty) return; // Run all of the configured tests. diff --git a/tools/testing/dart/options.dart b/tools/testing/dart/options.dart new file mode 100644 index 00000000000..685e7c93df3 --- /dev/null +++ b/tools/testing/dart/options.dart @@ -0,0 +1,916 @@ +// Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:io'; +import 'drt_updater.dart'; +import 'test_suite.dart'; +import 'path.dart'; +import 'compiler_configuration.dart' show CompilerConfiguration; +import 'runtime_configuration.dart' show RuntimeConfiguration; + +const _defaultTestSelectors = const [ + 'samples', + 'standalone', + 'corelib', + 'co19', + 'language', + 'isolate', + 'vm', + 'html', + 'benchmark_smoke', + 'utils', + 'lib', + 'analyze_library', + 'service', + 'kernel', + 'observatory_ui' +]; + +/// Specifies a single command line option. +/// +/// The name of the specification is used as the key for the option in the Map +/// returned from the [TestOptionParser] parse method. +class _Option { + // TODO(rnystrom): Some string options use "" to mean "no value" and others + // use null. Clean that up. + _Option(this.name, this.description, + {String abbr, List values, String defaultsTo = ""}) + : abbreviation = abbr, + values = values ?? [], + defaultValue = defaultsTo, + type = _OptionValueType.string; + + _Option.bool(this.name, this.description, [this.abbreviation]) + : values = [], + defaultValue = false, + type = _OptionValueType.bool; + + _Option.int(this.name, this.description, {String abbr, int defaultsTo}) + : abbreviation = abbr, + values = [], + defaultValue = defaultsTo, + type = _OptionValueType.int; + + final String name; + final String description; + final String abbreviation; + final List values; + final Object defaultValue; + final _OptionValueType type; + + /// Gets the shortest command line argument used to refer to this option. + String get shortCommand => abbreviation != null ? "-$abbreviation" : command; + + /// Gets the canonical long command line argument used to refer to this + /// option. + String get command => "--${name.replaceAll('_', '-')}"; +} + +enum _OptionValueType { bool, int, string } + +/// Parses command line arguments and produces a test runner configuration. +class OptionsParser { + static final List<_Option> _options = [ + new _Option('mode', 'Mode in which to run the tests.', + abbr: 'm', + values: ['all', 'debug', 'release', 'product'], + defaultsTo: 'debug'), + new _Option( + 'compiler', + '''Specify any compilation step (if needed). + +none: Do not compile the Dart code (run native Dart code + on the VM). + (Only valid with runtimes vm, flutter, or drt.) + +precompiler: Compile into AOT snapshot before running the test. + (Only valid with runtime dart_precompiled.) + +dart2js: Compile dart code to JavaScript by running dart2js. + (Only valid with runtimes: d8, drt, chrome, safari, + ie9, ie10, ie11, firefox, opera, chromeOnAndroid, + and none [compile only].) + +dart2analyzer: Perform static analysis on Dart code by running the + analyzer. + (Only valid with runtime none.) + +app_jit: Compile the Dart code into an app snapshot before + running test. + (Only valid with dart_app runtime.) + +dartk: Compile the Dart source into Kernel before running + test. + +dartkp: Compile the Dart source into Kernel and then Kernel + into AOT snapshot before running the test. + (Only valid with runtime dart_precompiled.)''', + abbr: 'c', + values: [ + 'none', + 'precompiler', + 'dart2js', + 'dart2analyzer', + 'app_jit', + 'dartk', + 'dartkp' + ], + defaultsTo: 'none'), + new _Option( + 'runtime', + '''Where the tests should be run. +vm: Run Dart code on the standalone dart vm. + +flutter: Run Dart code on the flutter engine. + +dart_precompiled: Run a precompiled snapshot on a variant of the + standalone dart VM lacking a JIT. + +d8: Run JavaScript from the command line using v8. + +jsshell: Run JavaScript from the command line using + Firefox js-shell. + +drt: Run Dart or JavaScript in the headless version + of Chrome, Content shell. + +dartium: Run Dart or JavaScript in Dartium. + +ContentShellOnAndroid: Run Dart or JavaScript in Dartium content + shell on Android. + +DartiumOnAndroid: Run Dart or Javascript in Dartium on Android. + +ff: +chrome: +safari: +ie9: +ie10: +ie11: +opera: +chromeOnAndroid: Run JavaScript in the specified browser. + +self_check: Pass each test or its compiled output to every + file under `pkg` whose name ends with + `_self_check.dart`. Each test is given to the + self_check tester as a filename on stdin using + the batch-mode protocol. + +none: No runtime, compile only. (For example, used + for dart2analyzer static analysis tests).''', + abbr: 'r', + values: [ + 'vm', + 'flutter', + 'dart_precompiled', + 'd8', + 'jsshell', + 'drt', + 'dartium', + 'ff', + 'firefox', + 'chrome', + 'safari', + 'ie9', + 'ie10', + 'ie11', + 'opera', + 'chromeOnAndroid', + 'safarimobilesim', + 'ContentShellOnAndroid', + 'DartiumOnAndroid', + 'self_check', + 'none' + ], + defaultsTo: 'vm'), + new _Option( + 'arch', + '''The architecture to run tests for. + +Allowed values are: +all +ia32, x64 +arm, armv6, armv5te, arm64, +simarm, simarmv6, simarmv5te, simarm64, +mips, simmips +simdbc, simdbc64''', + abbr: 'a', + values: [ + 'all', + 'ia32', + 'x64', + 'arm', + 'armv6', + 'armv5te', + 'arm64', + 'mips', + 'simarm', + 'simarmv6', + 'simarmv5te', + 'simarm64', + 'simmips', + 'simdbc', + 'simdbc64', + ], + defaultsTo: 'x64'), + new _Option('system', 'The operating system to run tests on.', + abbr: 's', + values: ['linux', 'macos', 'windows', 'android'], + defaultsTo: Platform.operatingSystem), + new _Option.bool('checked', 'Run tests in checked mode.'), + new _Option.bool('strong', 'Run tests in strong mode.'), + new _Option.bool('host_checked', 'Run compiler in checked mode.'), + new _Option.bool('minified', 'Enable minification in the compiler.'), + new _Option.bool( + 'csp', 'Run tests under Content Security Policy restrictions.'), + new _Option.bool( + 'fast_startup', 'Pass the --fast-startup flag to dart2js.'), + new _Option.bool('dart2js_with_kernel', + 'Enable the internal pipeline in dart2js to use kernel.'), + new _Option.bool('hot_reload', 'Run hot reload stress tests.'), + new _Option.bool( + 'hot_reload_rollback', 'Run hot reload rollback stress tests.'), + new _Option.bool('use_blobs', + 'Use mmap instead of shared libraries for precompilation.'), + new _Option.int('timeout', 'Timeout in seconds.', + abbr: 't', defaultsTo: -1), + new _Option( + 'progress', + '''Progress indication mode. + +Allowed values are: +compact, color, line, verbose, silent, status, buildbot, diff +''', + abbr: 'p', + values: [ + 'compact', + 'color', + 'line', + 'verbose', + 'silent', + 'status', + 'buildbot', + 'diff' + ], + defaultsTo: 'compact'), + new _Option('step_name', 'Step name for use by -pbuildbot.', + defaultsTo: null), + new _Option.bool('report', + 'Print a summary report of the number of tests, by expectation.'), + new _Option.int('tasks', 'The number of parallel tasks to run.', + abbr: 'j', defaultsTo: Platform.numberOfProcessors), + new _Option.int('shards', + 'The number of instances that the tests will be sharded over.', + defaultsTo: 1), + new _Option.int( + 'shard', 'The index of this instance when running in sharded mode.', + defaultsTo: 1), + new _Option.bool('help', 'Print list of options.', 'h'), + new _Option.bool('verbose', 'Verbose output.', 'v'), + new _Option.bool('verify-ir', 'Verify kernel IR.'), + new _Option.bool('no-tree-shake', 'Disable kernel IR tree shaking.'), + new _Option.bool('list', 'List tests only, do not run them.'), + new _Option.bool('report_in_json', + 'When listing with --list, output result summary in JSON.'), + new _Option.bool('time', 'Print timing information after running tests.'), + new _Option('dart', 'Path to dart executable.'), + new _Option('flutter', 'Path to flutter executable.'), + new _Option( + 'drt', // TODO(antonm): fix the option name. + 'Path to content shell executable.'), + new _Option('dartium', 'Path to Dartium Chrome executable.'), + new _Option('firefox', 'Path to firefox browser executable.'), + new _Option('chrome', 'Path to chrome browser executable.'), + new _Option('safari', 'Path to safari browser executable.'), + new _Option.bool( + 'use_sdk', + '''Use compiler or runtime from the SDK. + +Normally, the compiler or runtimes in PRODUCT_DIR is tested, with +this option, the compiler or runtime in PRODUCT_DIR/dart-sdk/bin +is tested. + +(Note: currently only implemented for dart2js.)'''), + new _Option('build_directory', + 'The name of the build directory, where products are placed.'), + new _Option.bool('noBatch', 'Do not run tests in batch mode.', 'n'), + new _Option.bool('dart2js_batch', 'Run dart2js tests in batch mode.'), + new _Option.bool( + 'append_logs', 'Do not delete old logs but rather append to them.'), + new _Option.bool('write_debug_log', + 'Don\'t write debug messages to stdout but rather to a logfile.'), + new _Option.bool('write_test_outcome_log', + 'Write test outcomes to a "${TestUtils.testOutcomeFileName}" file.'), + new _Option.bool( + 'reset_browser_configuration', + '''Browser specific reset of configuration. + +Warning: Using this option may remove your bookmarks and other +settings.'''), + new _Option.bool( + 'copy_coredumps', + '''If we see a crash that we did not expect, copy the core dumps to +"/tmp".'''), + new _Option( + 'local_ip', + '''IP address the HTTP servers should listen on. This address is also +used for browsers to connect to.''', + defaultsTo: '127.0.0.1'), + new _Option.int('test_server_port', 'Port for test http server.', + defaultsTo: 0), + new _Option.int('test_server_cross_origin_port', + 'Port for test http server cross origin.', + defaultsTo: 0), + new _Option.int('test_driver_port', 'Port for http test driver server.', + defaultsTo: 0), + new _Option.int( + 'test_driver_error_port', 'Port for http test driver server errors.', + defaultsTo: 0), + new _Option('record_to_file', + 'Records all commands to be executed and writes to a file.', + defaultsTo: null), + new _Option( + 'replay_from_file', 'Replays a previously recorded list of commands.', + defaultsTo: null), + new _Option( + 'builder_tag', + '''Machine specific options that is not captured by the regular test +options. Used to be able to make sane updates to the status files.'''), + new _Option('vm_options', 'Extra options to send to the vm when running.', + defaultsTo: null), + new _Option( + 'dart2js_options', 'Extra options for dart2js compilation step.', + defaultsTo: null), + new _Option( + 'suite_dir', 'Additional directory to add to the testing matrix.', + defaultsTo: null), + new _Option('package_root', 'The package root to use for testing.', + defaultsTo: null), + new _Option('packages', 'The package spec file to use for testing.', + defaultsTo: null), + new _Option( + 'exclude_suite', + '''Exclude suites from default selector, only works when no selector +has been specified on the command line.''', + defaultsTo: null), + new _Option.bool( + 'skip-compilation', + ''' +Skip the compilation step, using the compilation artifacts left in +the output folder from a previous run. This flag will often cause +false positves and negatives, but can be useful for quick and +dirty offline testing when not making changes that affect the +compiler.''') + ]; + + /// For printing out reproducing command lines, we don't want to add these + /// options. + static final _blacklistedOptions = [ + 'append_logs', + 'build_directory', + 'chrome', + 'copy_coredumps', + 'dart', + 'flutter', + 'dartium', + 'drt', + 'exclude_suite', + 'firefox', + 'local_ip', + 'progress', + 'report', + 'safari', + 'shard', + 'shards', + 'step_name', + 'tasks', + 'time', + 'verbose', + 'write_debug_log', + 'write_test_outcome_log', + ].toSet(); + + /// Parses a list of strings as test options. + /// + /// Returns a list of configurations in which to run the tests. + /// Configurations are maps mapping from option keys to values. When + /// encountering the first non-option string, the rest of the arguments are + /// stored in the returned Map under the 'rest' key. + List parse(List arguments) { + // TODO(rnystrom): The builders on the buildbots still pass this even + // though it does nothing. Until those can be fixed, silently ignore the + // option. Remove this once the buildbot scripts are fixed. + if (arguments.contains("--failure-summary")) { + arguments = arguments.where((arg) => arg != "--failure-summary").toList(); + print('Note: Ignoring unsupported "--failure-summary" option.'); + } + + var configuration = {}; + + // Fill in configuration with arguments passed to the test script. + for (var i = 0; i < arguments.length; i++) { + var arg = arguments[i]; + + // Help supersedes all other arguments. + if (arg == "--help" || arg == "-h") { + _printHelp(); + return null; + } + + // Extract name and value for options. + String command; + String value; + _Option option; + + if (arg.startsWith("--")) { + // A long option name. + var equals = arg.indexOf("="); + if (equals != -1) { + // A long option with a value, like "--arch=ia32". + command = arg.substring(0, equals); + value = arg.substring(equals + 1); + } else { + command = arg; + } + + option = _findByName(command.substring(2)); + } else if (arg.startsWith("-")) { + // An abbreviated option. + if (arg.length == 1) { + _fail('Missing option name after "-".'); + } + + command = arg.substring(0, 2); + + if (arg.length > 2) { + // An abbreviated option followed by a value, like "-aia32". + value = arg.substring(2); + } + + option = _findByAbbreviation(command.substring(1)); + } else { + // The argument does not start with "-" or "--" and is therefore not an + // option. Use it as a test selector pattern. + var patterns = configuration.putIfAbsent("selectors", () => []); + patterns.add(arg); + continue; + } + + if (option == null) { + _fail('Unknown command line option "$command".'); + } + + // If we need a value, look at the next argument. + if (value == null && option.type != _OptionValueType.bool) { + if (i + 1 >= arguments.length) { + _fail('Missing value for command line option "$command".'); + } + value = arguments[++i]; + } + + // Multiple uses of a flag are an error, because there is no naturally + // correct way to handle conflicting options. + if (configuration.containsKey(option.name)) { + _fail('Already have value for command line option "$command".'); + } + + // Parse the value for the option. + switch (option.type) { + case _OptionValueType.bool: + if (value != null) { + _fail('Boolean flag "$command" does not take a value.'); + } + + configuration[option.name] = true; + break; + + case _OptionValueType.int: + try { + configuration[option.name] = int.parse(value); + } on FormatException { + _fail('Integer value expected for option "$command".'); + } + break; + + case _OptionValueType.string: + // Validate against the allowed values. + if (!option.values.isEmpty) { + for (var v in value.split(",")) { + if (!option.values.contains(v)) { + _fail('Unknown value "$v" for command line option "$command".'); + } + } + } + + // TODO(rnystrom): Store as a list instead of a comma-delimited + // string. + configuration[option.name] = value; + break; + } + } + + // Apply default values for unspecified options. + for (var option in _options) { + if (!configuration.containsKey(option.name)) { + configuration[option.name] = option.defaultValue; + } + } + + var expandedConfigs = _expandConfigurations(configuration); + var result = expandedConfigs.where(_isValidConfig).toList(); + for (var config in result) { + config['_reproducing_arguments_'] = _reproducingCommand(config); + } + + return result.isEmpty ? null : result; + } + + /// Prints [message] and exits with a non-zero exit code. + void _fail(String message) { + print(message); + exit(1); + } + + /// Given a configuration, returns the list of command line arguments that + /// would reproduce that configuration. + List _reproducingCommand(Map config) { + var arguments = []; + + for (var option in _options) { + var name = option.name; + if (!config.containsKey(name) || _blacklistedOptions.contains(name)) { + continue; + } + + var value = config[name]; + if (config[name] == option.defaultValue || + (name == 'packages' && + value == + TestUtils.dartDirUri.resolve('.packages').toFilePath())) { + continue; + } + + arguments.add(option.shortCommand); + if (option.type != _OptionValueType.bool) { + arguments.add(value.toString()); + } + } + + return arguments; + } + + /// Determines if a particular configuration has a valid combination of + /// compiler and runtime elements. + bool _isValidConfig(Map config) { + var isValid = true; + List validRuntimes; + switch (config['compiler']) { + case 'dart2js': + // Note: by adding 'none' as a configuration, if the user + // runs test.py -c dart2js -r drt,none the dart2js_none and + // dart2js_drt will be duplicating work. If later we don't need 'none' + // with dart2js, we should remove it from here. + validRuntimes = const [ + 'd8', + 'jsshell', + 'drt', + 'none', + 'dartium', + 'ff', + 'chrome', + 'safari', + 'ie9', + 'ie10', + 'ie11', + 'opera', + 'chromeOnAndroid', + 'safarimobilesim' + ]; + break; + case 'dart2analyzer': + validRuntimes = const ['none']; + break; + case 'app_jit': + case 'dartk': + validRuntimes = const ['vm', 'self_check', 'none']; + break; + case 'precompiler': + case 'dartkp': + validRuntimes = const ['dart_precompiled']; + break; + case 'none': + validRuntimes = const [ + 'vm', + 'flutter', + 'drt', + 'dartium', + 'ContentShellOnAndroid', + 'DartiumOnAndroid' + ]; + break; + } + + if (!validRuntimes.contains(config['runtime'])) { + isValid = false; + print("Warning: combination of compiler '${config['compiler']}' and " + "runtime '${config['runtime']}' is invalid. " + "Skipping this combination."); + } + + if (config['ie'] && Platform.operatingSystem != 'windows') { + isValid = false; + print("Warning: cannot run Internet Explorer on non-Windows operating" + " system."); + } + + if (config['shard'] < 1 || config['shard'] > config['shards']) { + isValid = false; + print("Error: shard index is ${config['shard']} out of " + "${config['shards']} shards"); + } + + if (config['runtime'] == 'flutter' && config['flutter'] == '') { + isValid = false; + print("-rflutter requires the flutter engine executable to " + "be specified using --flutter="); + } + + if (config['runtime'] == 'flutter' && config['arch'] != 'x64') { + isValid = false; + print("-rflutter is applicable only for --arch=x64"); + } + + return isValid; + } + + /// Recursively expands a configuration with multiple values per key into a + /// list of configurations with exactly one value per key. + List _expandConfigurations(Map configuration) { + // Expand the pseudo-values such as 'all'. + if (configuration['arch'] == 'all') { + configuration['arch'] = 'ia32,x64,simarm,simarm64,simmips,simdbc64'; + } + + if (configuration['mode'] == 'all') { + configuration['mode'] = 'debug,release,product'; + } + + if (configuration['report_in_json']) { + configuration['list'] = true; + configuration['report'] = true; + } + + // Use verbose progress indication for verbose output unless buildbot + // progress indication is requested. + if (configuration['verbose'] && configuration['progress'] != 'buildbot') { + configuration['progress'] = 'verbose'; + } + + // Create the artificial negative options that test status files + // expect. + configuration['unchecked'] = !configuration['checked']; + configuration['host_unchecked'] = !configuration['host_checked']; + configuration['unminified'] = !configuration['minified']; + configuration['nocsp'] = !configuration['csp']; + + String runtime = configuration['runtime']; + if (runtime == 'firefox') { + configuration['runtime'] == 'ff'; + } + + String compiler = configuration['compiler']; + configuration['browser'] = TestUtils.isBrowserRuntime(runtime); + configuration['analyzer'] = TestUtils.isCommandLineAnalyzer(compiler); + + // Set the javascript command line flag for less verbose status files. + configuration['jscl'] = TestUtils.isJsCommandLineRuntime(runtime); + + // Allow suppression that is valid for all ie versions + configuration['ie'] = runtime.startsWith('ie'); + + // Expand the test selectors into a suite name and a simple + // regular expressions to be used on the full path of a test file + // in that test suite. If no selectors are explicitly given use + // the default suite patterns. + var selectors = configuration['selectors']; + if (selectors is! Map) { + if (selectors == null) { + if (configuration['suite_dir'] != null) { + var suite_path = new Path(configuration['suite_dir']); + selectors = [suite_path.filename]; + } else { + selectors = _defaultTestSelectors.toList(); + } + + var excludeSuites = configuration['exclude_suite'] != null + ? configuration['exclude_suite'].split(',') + : []; + for (var exclude in excludeSuites) { + if (selectors.contains(exclude)) { + selectors.remove(exclude); + } else { + print("Warning: default selectors does not contain $exclude"); + } + } + } + var selectorMap = {}; + for (var i = 0; i < selectors.length; i++) { + var pattern = selectors[i]; + var suite = pattern; + var slashLocation = pattern.indexOf('/'); + if (slashLocation != -1) { + suite = pattern.substring(0, slashLocation); + pattern = pattern.substring(slashLocation + 1); + pattern = pattern.replaceAll('*', '.*'); + } else { + pattern = ".?"; + } + if (selectorMap.containsKey(suite)) { + print("Error: '$suite/$pattern'. Only one test selection" + " pattern is allowed to start with '$suite/'"); + exit(1); + } + selectorMap[suite] = new RegExp(pattern); + } + configuration['selectors'] = selectorMap; + } + + // Put observatory_ui in a configuration with its own packages override. + // Only one value in the configuration map is mutable: + selectors = configuration['selectors']; + if (selectors.containsKey('observatory_ui')) { + if (selectors.length == 1) { + configuration['packages'] = TestUtils.dartDirUri + .resolve('runtime/observatory/.packages') + .toFilePath(); + } else { + // Make a new configuration whose selectors map only contains + // observatory_ui, and remove the key from the original selectors. + // The only mutable value in the map is the selectors, so a + // shallow copy is safe. + var observatoryConfiguration = new Map.from(configuration); + observatoryConfiguration['selectors'] = { + 'observatory_ui': selectors['observatory_ui'] + }; + selectors.remove('observatory_ui'); + + // Set the packages flag. + observatoryConfiguration['packages'] = TestUtils.dartDirUri + .resolve('runtime/observatory/.packages') + .toFilePath(); + + // Return the expansions of both configurations. Neither will reach + // this line in the recursive call to _expandConfigurations. + return _expandConfigurations(configuration) + ..addAll(_expandConfigurations(observatoryConfiguration)); + } + } + + // Set the default package spec explicitly. + if (configuration['package_root'] == null && + configuration['packages'] == null) { + configuration['packages'] = + TestUtils.dartDirUri.resolve('.packages').toFilePath(); + } + + // Expand the architectures. + if (configuration['arch'].contains(',')) { + return _expandHelper('arch', configuration); + } + + // Expand modes. + if (configuration['mode'].contains(',')) { + return _expandHelper('mode', configuration); + } + + // Expand compilers. + if (configuration['compiler'].contains(',')) { + return _expandHelper('compiler', configuration); + } + + // Expand runtimes. + var runtimes = configuration['runtime']; + if (runtimes.contains(',')) { + return _expandHelper('runtime', configuration); + } else { + // All runtimes eventually go through this path, after expansion. + var updater = runtimeUpdater(configuration); + if (updater != null) { + updater.update(); + } + } + + // Adjust default timeout based on mode, compiler, and sometimes runtime. + if (configuration['timeout'] == -1) { + var isReload = + configuration['hot_reload'] || configuration['hot_reload_rollback']; + int compilerMulitiplier = + new CompilerConfiguration(configuration).computeTimeoutMultiplier(); + int runtimeMultiplier = new RuntimeConfiguration(configuration) + .computeTimeoutMultiplier( + mode: configuration['mode'], + isChecked: configuration['checked'], + isReload: isReload, + arch: configuration['arch']); + configuration['timeout'] = 60 * compilerMulitiplier * runtimeMultiplier; + } + + return [configuration]; + } + + /// Helper for _expandConfigurations. Creates a new configuration and adds it + /// to a list, for use in a case when a particular configuration has multiple + /// results (separated by a ','). + /// Arguments: + /// option: The particular test option we are expanding. + /// configuration: The map containing all test configuration information + /// specified. + List _expandHelper(String option, Map configuration) { + var result = []; + var configs = configuration[option]; + for (var config in configs.split(',')) { + var newConfiguration = new Map.from(configuration); + newConfiguration[option] = config; + result.addAll(_expandConfigurations(newConfiguration)); + } + return result; + } + + /// Print out usage information. + void _printHelp() { + var buffer = new StringBuffer(); + + buffer.writeln('''usage: dart test.dart [options] [selector] + +The optional selector limits the tests that will be run. +For example, the selector "language/issue", or equivalently +"language/*issue*", limits to test files matching the regexp +".*issue.*\\.dart" in the "tests/language" directory. + +Options:'''); + + for (var option in _options) { + if (option.abbreviation != null) { + buffer.write("-${option.abbreviation}, "); + } else { + buffer.write(" "); + } + + buffer.write(option.command); + + switch (option.type) { + case _OptionValueType.bool: + // No value. + break; + case _OptionValueType.int: + buffer.write("="); + break; + case _OptionValueType.string: + if (option.values.length > 6) { + // If there are many options, they won't fit nicely in one line and + // should be instead listed in the description. + buffer.write("=<...>"); + } else if (option.values.isNotEmpty) { + buffer.write("=<${option.values.join('|')}>"); + } else { + buffer.write("="); + } + break; + } + + if (option.type != _OptionValueType.bool && + option.defaultValue != null && + option.defaultValue != "") { + buffer.write(" (defaults to ${option.defaultValue})"); + } + + buffer.writeln(); + buffer + .writeln(" ${option.description.replaceAll('\n', '\n ')}"); + buffer.writeln(); + } + + print(buffer); + } + + _Option _findByAbbreviation(String abbreviation) { + for (var option in _options) { + if (abbreviation == option.abbreviation) return option; + } + + return null; + } + + _Option _findByName(String name) { + for (var option in _options) { + if (name == option.name) return option; + + // Allow hyphens instead of underscores as the separator since they are + // more common for command line flags. + if (name == option.name.replaceAll("_", "-")) return option; + } + + return null; + } +} diff --git a/tools/testing/dart/package_testing_support.dart b/tools/testing/dart/package_testing_support.dart index d4706635384..913d6282356 100644 --- a/tools/testing/dart/package_testing_support.dart +++ b/tools/testing/dart/package_testing_support.dart @@ -4,17 +4,15 @@ library package_testing_support; -import 'test_configurations.dart' show testConfigurations; - -import 'test_options.dart' show TestOptionsParser; - -import 'test_suite.dart' show TestUtils; +import 'options.dart'; +import 'test_configurations.dart'; +import 'test_suite.dart'; void main(List arguments) { TestUtils.setDartDirUri(Uri.base); var configurations = []; for (var argument in arguments) { - configurations.addAll(new TestOptionsParser().parse(argument.split(" "))); + configurations.addAll(new OptionsParser().parse(argument.split(" "))); } testConfigurations(configurations); } diff --git a/tools/testing/dart/test_options.dart b/tools/testing/dart/test_options.dart deleted file mode 100644 index 06ddec629ea..00000000000 --- a/tools/testing/dart/test_options.dart +++ /dev/null @@ -1,1012 +0,0 @@ -// Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -library test_options_parser; - -import "dart:io"; -import "drt_updater.dart"; -import "test_suite.dart"; -import "path.dart"; -import "compiler_configuration.dart" show CompilerConfiguration; -import "runtime_configuration.dart" show RuntimeConfiguration; - -const List defaultTestSelectors = const [ - 'samples', - 'standalone', - 'corelib', - 'co19', - 'language', - 'isolate', - 'vm', - 'html', - 'benchmark_smoke', - 'utils', - 'lib', - 'analyze_library', - 'service', - 'kernel', - 'observatory_ui' -]; - -/** - * Specification of a single test option. - * - * The name of the specification is used as the key for the option in - * the Map returned from the [TestOptionParser] parse method. - */ -class _TestOptionSpecification { - _TestOptionSpecification( - this.name, this.description, this.keys, this.values, this.defaultValue, - {this.type: 'string'}); - String name; - String description; - List keys; - List values; - Object defaultValue; - String type; -} - -/** - * Parser of test options. - */ -class TestOptionsParser { - /** - * Creates a test options parser initialized with the known options. - */ - TestOptionsParser() { - _options = [ - new _TestOptionSpecification('mode', 'Mode in which to run the tests', - ['-m', '--mode'], ['all', 'debug', 'release', 'product'], 'debug'), - new _TestOptionSpecification( - 'compiler', - '''Specify any compilation step (if needed). - - none: Do not compile the Dart code (run native Dart code on the VM). - (only valid with the following runtimes: vm, flutter, drt) - - precompiler: Compile into AOT snapshot before running the test. - (only valid with the dart_precompiled runtime) - - dart2js: Compile dart code to JavaScript by running dart2js. - (only valid with the following runtimes: d8, drt, chrome, - safari, ie9, ie10, ie11, firefox, opera, chromeOnAndroid, - none (compile only)), - - dart2analyzer: Perform static analysis on Dart code by running the analyzer - (only valid with the following runtimes: none) - - app_jit: Compile the Dart code into an app snapshot before running test - (only valid with dart_app runtime) - - dartk: Compile the Dart source into Kernel before running test. - - dartkp: Compiler the Dart source into Kernel and then Kernel into AOT - snapshot before running the test. - (only valid with the dart_precompiled runtime) - ''', - ['-c', '--compiler'], - [ - 'none', - 'precompiler', - 'dart2js', - 'dart2analyzer', - 'app_jit', - 'dartk', - 'dartkp' - ], - 'none'), - // TODO(antonm): fix the option drt. - new _TestOptionSpecification( - 'runtime', - '''Where the tests should be run. - vm: Run Dart code on the standalone dart vm. - - flutter: Run Dart code on the flutter engine. - - dart_precompiled: Run a precompiled snapshot on a variant of the standalone - dart vm lacking a JIT. - - d8: Run JavaScript from the command line using v8. - - jsshell: Run JavaScript from the command line using firefox js-shell. - - drt: Run Dart or JavaScript in the headless version of Chrome, - Content shell. - - dartium: Run Dart or JavaScript in Dartium. - - ContentShellOnAndroid: Run Dart or JavaScript in Dartium content shell - on Android. - - DartiumOnAndroid: Run Dart or Javascript in Dartium on Android. - - [ff | chrome | safari | ie9 | ie10 | ie11 | opera | chromeOnAndroid]: - Run JavaScript in the specified browser. - - self_check: Pass each test or its compiled output to every file under - `pkg` whose name ends with `_self_check.dart`. - Each test is given to the self_check tester as a filename on stdin using - the batch-mode protocol. - - none: No runtime, compile only (for example, used for dart2analyzer static - analysis tests).''', - ['-r', '--runtime'], - [ - 'vm', - 'flutter', - 'dart_precompiled', - 'd8', - 'jsshell', - 'drt', - 'dartium', - 'ff', - 'firefox', - 'chrome', - 'safari', - 'ie9', - 'ie10', - 'ie11', - 'opera', - 'chromeOnAndroid', - 'safarimobilesim', - 'ContentShellOnAndroid', - 'DartiumOnAndroid', - 'self_check', - 'none' - ], - 'vm'), - new _TestOptionSpecification( - 'arch', - 'The architecture to run tests for', - ['-a', '--arch'], - [ - 'all', - 'ia32', - 'x64', - 'arm', - 'armv6', - 'armv5te', - 'arm64', - 'mips', - 'simarm', - 'simarmv6', - 'simarmv5te', - 'simarm64', - 'simmips', - 'simdbc', - 'simdbc64', - ], - 'x64'), - new _TestOptionSpecification( - 'system', - 'The operating system to run tests on', - ['-s', '--system'], - ['linux', 'macos', 'windows', 'android'], - Platform.operatingSystem), - new _TestOptionSpecification( - 'checked', 'Run tests in checked mode', ['--checked'], [], false, - type: 'bool'), - new _TestOptionSpecification( - 'strong', 'Run tests in strong mode', ['--strong'], [], false, - type: 'bool'), - new _TestOptionSpecification('host_checked', - 'Run compiler in checked mode', ['--host-checked'], [], false, - type: 'bool'), - new _TestOptionSpecification('minified', - 'Enable minification in the compiler', ['--minified'], [], false, - type: 'bool'), - new _TestOptionSpecification( - 'csp', - 'Run tests under Content Security Policy restrictions', - ['--csp'], - [], - false, - type: 'bool'), - new _TestOptionSpecification( - 'cps_ir', - 'Run the compiler with the cps based backend', - ['--cps-ir'], - [], - false, - type: 'bool'), - new _TestOptionSpecification( - 'fast_startup', - 'Pass the --fast-startup flag to dart2js', - ['--fast-startup'], - [], - false, - type: 'bool'), - new _TestOptionSpecification( - 'dart2js_with_kernel', - 'Enable the internal pipeline in dart2js to use kernel', - ['--dart2js-with-kernel'], - [], - false, - type: 'bool'), - new _TestOptionSpecification('hot_reload', 'Run hot reload stress tests', - ['--hot-reload'], [], false, - type: 'bool'), - new _TestOptionSpecification( - 'hot_reload_rollback', - 'Run hot reload rollback stress tests', - ['--hot-reload-rollback'], - [], - false, - type: 'bool'), - new _TestOptionSpecification( - 'use_blobs', - 'Use mmap instead of shared libraries for precompilation', - ['--use-blobs'], - [], - false, - type: 'bool'), - new _TestOptionSpecification( - 'timeout', 'Timeout in seconds', ['-t', '--timeout'], [], -1, - type: 'int'), - new _TestOptionSpecification( - 'progress', - 'Progress indication mode', - ['-p', '--progress'], - [ - 'compact', - 'color', - 'line', - 'verbose', - 'silent', - 'status', - 'buildbot', - 'diff' - ], - 'compact'), - new _TestOptionSpecification('failure-summary', - 'Print failure summary at the end', ['--failure-summary'], [], false, - type: 'bool'), - new _TestOptionSpecification('step_name', - 'Step name for use by -pbuildbot', ['--step_name'], [], null), - new _TestOptionSpecification( - 'report', - 'Print a summary report of the number of tests, by expectation', - ['--report'], - [], - false, - type: 'bool'), - new _TestOptionSpecification( - 'tasks', - 'The number of parallel tasks to run', - ['-j', '--tasks'], - [], - Platform.numberOfProcessors, - type: 'int'), - new _TestOptionSpecification( - 'shards', - 'The number of instances that the tests will be sharded over', - ['--shards'], - [], - 1, - type: 'int'), - new _TestOptionSpecification( - 'shard', - 'The index of this instance when running in sharded mode', - ['--shard'], - [], - 1, - type: 'int'), - new _TestOptionSpecification( - 'help', 'Print list of options', ['-h', '--help'], [], false, - type: 'bool'), - new _TestOptionSpecification( - 'verbose', 'Verbose output', ['-v', '--verbose'], [], false, - type: 'bool'), - new _TestOptionSpecification( - 'verify-ir', 'Verify kernel IR', ['--verify-ir'], [], false, - type: 'bool'), - new _TestOptionSpecification('no-tree-shake', - 'Disable kernel IR tree shaking', ['--no-tree-shake'], [], false, - type: 'bool'), - new _TestOptionSpecification( - 'list', 'List tests only, do not run them', ['--list'], [], false, - type: 'bool'), - new _TestOptionSpecification( - 'report_in_json', - 'When doing list, output result summary in json only.', - ['--report-in-json'], - [], - false, - type: 'bool'), - new _TestOptionSpecification('time', - 'Print timing information after running tests', ['--time'], [], false, - type: 'bool'), - new _TestOptionSpecification( - 'dart', 'Path to dart executable', ['--dart'], [], ''), - new _TestOptionSpecification( - 'flutter', 'Path to flutter executable', ['--flutter'], [], ''), - new _TestOptionSpecification( - 'drt', // TODO(antonm): fix the option name. - 'Path to content shell executable', - ['--drt'], - [], - ''), - new _TestOptionSpecification('dartium', - 'Path to Dartium Chrome executable', ['--dartium'], [], ''), - new _TestOptionSpecification('firefox', - 'Path to firefox browser executable', ['--firefox'], [], ''), - new _TestOptionSpecification( - 'chrome', 'Path to chrome browser executable', ['--chrome'], [], ''), - new _TestOptionSpecification( - 'safari', 'Path to safari browser executable', ['--safari'], [], ''), - new _TestOptionSpecification( - 'use_sdk', - '''Use compiler or runtime from the SDK. - -Normally, the compiler or runtimes in PRODUCT_DIR is tested, with this -option, the compiler or runtime in PRODUCT_DIR/dart-sdk/bin is tested. - -Note: currently only implemented for dart2js.''', - ['--use-sdk'], - [], - false, - type: 'bool'), - new _TestOptionSpecification( - 'build_directory', - 'The name of the build directory, where products are placed.', - ['--build-directory'], - [], - ''), - new _TestOptionSpecification('noBatch', 'Do not run tests in batch mode', - ['-n', '--nobatch'], [], false, - type: 'bool'), - new _TestOptionSpecification('dart2js_batch', - 'Run dart2js tests in batch mode', ['--dart2js-batch'], [], false, - type: 'bool'), - new _TestOptionSpecification( - 'append_logs', - 'Do not delete old logs but rather append to them.', - ['--append_logs'], - [], - false, - type: 'bool'), - new _TestOptionSpecification( - 'write_debug_log', - 'Don\'t write debug messages to stdout but rather to a logfile.', - ['--write-debug-log'], - [], - false, - type: 'bool'), - new _TestOptionSpecification( - 'write_test_outcome_log', - 'Write the outcome of all tests executed to a ' - '"${TestUtils.testOutcomeFileName}" file.', - ['--write-test-outcome-log'], - [], - false, - type: 'bool'), - new _TestOptionSpecification( - 'reset_browser_configuration', - 'Browser specific reset of configuration. ' - 'WARNING: Using this option may remove your bookmarks and ' - 'other settings.', - ['--reset-browser-configuration'], - [], - false, - type: 'bool'), - new _TestOptionSpecification( - 'copy_coredumps', - 'If we see a crash that we did not expect, copy the core dumps. ' - 'to /tmp', - ['--copy-coredumps'], - [], - false, - type: 'bool'), - new _TestOptionSpecification( - 'local_ip', - 'IP address the http servers should listen on.' - 'This address is also used for browsers to connect.', - ['--local_ip'], - [], - '127.0.0.1'), - new _TestOptionSpecification('test_server_port', - 'Port for test http server.', ['--test_server_port'], [], 0, - type: 'int'), - new _TestOptionSpecification( - 'test_server_cross_origin_port', - 'Port for test http server cross origin.', - ['--test_server_cross_origin_port'], - [], - 0, - type: 'int'), - new _TestOptionSpecification('test_driver_port', - 'Port for http test driver server.', ['--test_driver_port'], [], 0, - type: 'int'), - new _TestOptionSpecification( - 'test_driver_error_port', - 'Port for http test driver server errors.', - ['--test_driver_error_port'], - [], - 0, - type: 'int'), - new _TestOptionSpecification( - 'record_to_file', - 'Records all the commands that need to be executed and writes it ' - 'out to a file.', - ['--record_to_file'], - [], - null), - new _TestOptionSpecification( - 'replay_from_file', - 'Records all the commands that need to be executed and writes it ' - 'out to a file.', - ['--replay_from_file'], - [], - null), - new _TestOptionSpecification( - 'builder_tag', - 'Machine specific options that is not captured by the regular ' - 'test options. Used to be able to make sane updates to the ' - 'status files.', - ['--builder-tag'], - [], - ''), - new _TestOptionSpecification( - 'vm_options', - 'Extra options to send to the vm when running', - ['--vm-options'], - [], - null), - new _TestOptionSpecification( - 'dart2js_options', - 'Extra options for dart2js compilation step', - ['--dart2js-options'], - [], - null), - new _TestOptionSpecification( - 'suite_dir', - 'Additional directory to add to the testing matrix', - ['--suite-dir'], - [], - null), - new _TestOptionSpecification('package_root', - 'The package root to use for testing.', ['--package-root'], [], null), - new _TestOptionSpecification( - 'packages', - 'The package spec file to use for testing.', - ['--packages'], - [], - null), - new _TestOptionSpecification( - 'exclude_suite', - 'Exclude suites from default selector, only works when no' - ' selector has been specified on the command line', - ['--exclude-suite'], - [], - null), - new _TestOptionSpecification( - 'skip-compilation', - 'Skip the compilation step, using the compilation artifacts left in ' - ' the output folder from a previous run.' - 'This flag will often cause false positves and negatives, but can be' - ' useful for quick-and-dirty offline testing when not making changes' - ' that affect the compiler.', - ['--skip-compilation'], - [], - false, - type: 'bool') - ]; - } - - /** - * Parse a list of strings as test options. - * - * Returns a list of configurations in which to run the - * tests. Configurations are maps mapping from option keys to - * values. When encountering the first non-option string, the rest - * of the arguments are stored in the returned Map under the 'rest' - * key. - */ - List parse(List arguments) { - var configuration = new Map(); - // Fill in configuration with arguments passed to the test script. - var numArguments = arguments.length; - for (var i = 0; i < numArguments; i++) { - // Extract name and value for options. - String arg = arguments[i]; - String name = ''; - String value = ''; - _TestOptionSpecification spec; - if (arg.startsWith('--')) { - if (arg == '--help') { - _printHelp(); - return null; - } - var split = arg.indexOf('='); - if (split == -1) { - name = arg; - spec = _getSpecification(name); - // Boolean options do not have a value. - if (spec.type != 'bool') { - if ((i + 1) >= arguments.length) { - print('No value supplied for option $name'); - return null; - } - value = arguments[++i]; - } - } else { - name = arg.substring(0, split); - spec = _getSpecification(name); - value = arg.substring(split + 1, arg.length); - } - } else if (arg.startsWith('-')) { - if (arg == '-h') { - _printHelp(); - return null; - } - if (arg.length > 2) { - name = arg.substring(0, 2); - spec = _getSpecification(name); - value = arg.substring(2, arg.length); - } else { - name = arg; - spec = _getSpecification(name); - // Boolean options do not have a value. - if (spec.type != 'bool') { - if ((i + 1) >= arguments.length) { - print('No value supplied for option $name'); - return null; - } - value = arguments[++i]; - } - } - } else { - // The argument does not start with '-' or '--' and is - // therefore not an option. We use it as a test selection - // pattern. - var patterns = configuration.putIfAbsent('selectors', () => []); - patterns.add(arg); - continue; - } - - // Multiple uses of a flag are an error, because there is no - // naturally correct way to handle conflicting options. - if (configuration.containsKey(spec.name)) { - print('Error: test.dart disallows multiple "--${spec.name}" flags'); - exit(1); - } - // Parse the value for the option. - if (spec.type == 'bool') { - if (!value.isEmpty) { - print('No value expected for bool option $name'); - exit(1); - } - configuration[spec.name] = true; - } else if (spec.type == 'int') { - try { - configuration[spec.name] = int.parse(value); - } catch (e) { - print('Integer value expected for int option $name'); - exit(1); - } - } else { - assert(spec.type == 'string'); - if (!spec.values.isEmpty) { - for (var v in value.split(',')) { - if (spec.values.lastIndexOf(v) == -1) { - print('Unknown value ($v) for option $name'); - exit(1); - } - } - } - configuration[spec.name] = value; - } - } - - // Apply default values for unspecified options. - for (var option in _options) { - if (!configuration.containsKey(option.name)) { - configuration[option.name] = option.defaultValue; - } - } - - List expandedConfigs = _expandConfigurations(configuration); - List result = expandedConfigs.where(_isValidConfig).toList(); - for (var config in result) { - config['_reproducing_arguments_'] = - _constructReproducingCommandArguments(config); - } - return result.isEmpty ? null : result; - } - - // For printing out reproducing command lines, we don't want to add these - // options. - final _blacklistedOptions = [ - 'append_logs', - 'build_directory', - 'chrome', - 'copy_coredumps', - 'dart', - 'flutter', - 'dartium', - 'drt', - 'exclude_suite', - 'failure-summary', - 'firefox', - 'local_ip', - 'progress', - 'report', - 'safari', - 'shard', - 'shards', - 'step_name', - 'tasks', - 'time', - 'verbose', - 'write_debug_log', - 'write_test_outcome_log', - ].toSet(); - - List _constructReproducingCommandArguments(Map config) { - var arguments = new List(); - for (var option in _options) { - var name = option.name; - if (!config.containsKey(name) || _blacklistedOptions.contains(name)) { - continue; - } - var value = config[name]; - if (config[name] == option.defaultValue || - (name == 'packages' && - value == - TestUtils.dartDirUri.resolve('.packages').toFilePath())) { - continue; - } - shortest(String a, String b) => a.length <= b.length ? a : b; - var key = option.keys.reduce(shortest); - if (option.type == 'bool') { - arguments.add(key); - } else if (key.startsWith('--')) { - // long version - arguments.add(key); - arguments.add("$value"); - } else { - // short version - assert(key.startsWith('-')); - arguments.add("$key$value"); - } - } - return arguments; - } - - /** - * Determine if a particular configuration has a valid combination of compiler - * and runtime elements. - */ - bool _isValidConfig(Map config) { - bool isValid = true; - List validRuntimes; - switch (config['compiler']) { - case 'dart2js': - // Note: by adding 'none' as a configuration, if the user - // runs test.py -c dart2js -r drt,none the dart2js_none and - // dart2js_drt will be duplicating work. If later we don't need 'none' - // with dart2js, we should remove it from here. - validRuntimes = const [ - 'd8', - 'jsshell', - 'drt', - 'none', - 'dartium', - 'ff', - 'chrome', - 'safari', - 'ie9', - 'ie10', - 'ie11', - 'opera', - 'chromeOnAndroid', - 'safarimobilesim' - ]; - break; - case 'dart2analyzer': - validRuntimes = const ['none']; - break; - case 'app_jit': - case 'dartk': - validRuntimes = const ['vm', 'self_check', 'none']; - break; - case 'precompiler': - case 'dartkp': - validRuntimes = const ['dart_precompiled']; - break; - case 'none': - validRuntimes = const [ - 'vm', - 'flutter', - 'drt', - 'dartium', - 'ContentShellOnAndroid', - 'DartiumOnAndroid' - ]; - break; - } - if (!validRuntimes.contains(config['runtime'])) { - isValid = false; - print("Warning: combination of compiler '${config['compiler']}' and " - "runtime '${config['runtime']}' is invalid. " - "Skipping this combination."); - } - if (config['ie'] && Platform.operatingSystem != 'windows') { - isValid = false; - print("Warning: cannot run Internet Explorer on non-Windows operating" - " system."); - } - if (config['shard'] < 1 || config['shard'] > config['shards']) { - isValid = false; - print("Error: shard index is ${config['shard']} out of " - "${config['shards']} shards"); - } - if ((config['runtime'] == 'flutter') && (config['flutter'] == '')) { - isValid = false; - print("-rflutter requires the flutter engine executable to " - "be specified using --flutter="); - } - if ((config['runtime'] == 'flutter') && (config['arch'] != 'x64')) { - isValid = false; - print("-rflutter is applicable only for --arch=x64"); - } - - return isValid; - } - - /** - * Recursively expand a configuration with multiple values per key - * into a list of configurations with exactly one value per key. - */ - List _expandConfigurations(Map configuration) { - // Expand the pseudo-values such as 'all'. - if (configuration['arch'] == 'all') { - configuration['arch'] = 'ia32,x64,simarm,simarm64,simmips,simdbc64'; - } - if (configuration['mode'] == 'all') { - configuration['mode'] = 'debug,release,product'; - } - - if (configuration['report_in_json']) { - configuration['list'] = true; - configuration['report'] = true; - } - - // Use verbose progress indication for verbose output unless buildbot - // progress indication is requested. - if (configuration['verbose'] && configuration['progress'] != 'buildbot') { - configuration['progress'] = 'verbose'; - } - - // Create the artificial negative options that test status files - // expect. - configuration['unchecked'] = !configuration['checked']; - configuration['host_unchecked'] = !configuration['host_checked']; - configuration['unminified'] = !configuration['minified']; - configuration['nocsp'] = !configuration['csp']; - - String runtime = configuration['runtime']; - if (runtime == 'firefox') { - configuration['runtime'] == 'ff'; - } - - String compiler = configuration['compiler']; - configuration['browser'] = TestUtils.isBrowserRuntime(runtime); - configuration['analyzer'] = TestUtils.isCommandLineAnalyzer(compiler); - - // Set the javascript command line flag for less verbose status files. - configuration['jscl'] = TestUtils.isJsCommandLineRuntime(runtime); - - // Allow suppression that is valid for all ie versions - configuration['ie'] = runtime.startsWith('ie'); - - // Expand the test selectors into a suite name and a simple - // regular expressions to be used on the full path of a test file - // in that test suite. If no selectors are explicitly given use - // the default suite patterns. - var selectors = configuration['selectors']; - if (selectors is! Map) { - if (selectors == null) { - if (configuration['suite_dir'] != null) { - var suite_path = new Path(configuration['suite_dir']); - selectors = [suite_path.filename]; - } else { - selectors = defaultTestSelectors.toList(); - } - - var excludeSuites = configuration['exclude_suite'] != null - ? configuration['exclude_suite'].split(',') - : []; - for (var exclude in excludeSuites) { - if (selectors.contains(exclude)) { - selectors.remove(exclude); - } else { - print("Warning: default selectors does not contain $exclude"); - } - } - } - var selectorMap = {}; - for (var i = 0; i < selectors.length; i++) { - var pattern = selectors[i]; - var suite = pattern; - var slashLocation = pattern.indexOf('/'); - if (slashLocation != -1) { - suite = pattern.substring(0, slashLocation); - pattern = pattern.substring(slashLocation + 1); - pattern = pattern.replaceAll('*', '.*'); - } else { - pattern = ".?"; - } - if (selectorMap.containsKey(suite)) { - print("Error: '$suite/$pattern'. Only one test selection" - " pattern is allowed to start with '$suite/'"); - exit(1); - } - selectorMap[suite] = new RegExp(pattern); - } - configuration['selectors'] = selectorMap; - } - - // Put observatory_ui in a configuration with its own packages override. - // Only one value in the configuration map is mutable: - selectors = configuration['selectors']; - if (selectors.containsKey('observatory_ui')) { - if (selectors.length == 1) { - configuration['packages'] = TestUtils.dartDirUri - .resolve('runtime/observatory/.packages') - .toFilePath(); - } else { - // Make a new configuration whose selectors map only contains - // observatory_ui, and remove the key from the original selectors. - // The only mutable value in the map is the selectors, so a - // shallow copy is safe. - var observatoryConfiguration = new Map.from(configuration); - observatoryConfiguration['selectors'] = { - 'observatory_ui': selectors['observatory_ui'] - }; - selectors.remove('observatory_ui'); - - // Set the packages flag. - observatoryConfiguration['packages'] = TestUtils.dartDirUri - .resolve('runtime/observatory/.packages') - .toFilePath(); - - // Return the expansions of both configurations. Neither will reach - // this line in the recursive call to _expandConfigurations. - return _expandConfigurations(configuration) - ..addAll(_expandConfigurations(observatoryConfiguration)); - } - } - // Set the default package spec explicitly. - if (configuration['package_root'] == null && - configuration['packages'] == null) { - configuration['packages'] = - TestUtils.dartDirUri.resolve('.packages').toFilePath(); - } - - // Expand the architectures. - if (configuration['arch'].contains(',')) { - return _expandHelper('arch', configuration); - } - - // Expand modes. - if (configuration['mode'].contains(',')) { - return _expandHelper('mode', configuration); - } - - // Expand compilers. - if (configuration['compiler'].contains(',')) { - return _expandHelper('compiler', configuration); - } - - // Expand runtimes. - var runtimes = configuration['runtime']; - if (runtimes.contains(',')) { - return _expandHelper('runtime', configuration); - } else { - // All runtimes eventually go through this path, after expansion. - var updater = runtimeUpdater(configuration); - if (updater != null) { - updater.update(); - } - } - - // Adjust default timeout based on mode, compiler, and sometimes runtime. - if (configuration['timeout'] == -1) { - var isReload = - configuration['hot_reload'] || configuration['hot_reload_rollback']; - int compilerMulitiplier = - new CompilerConfiguration(configuration).computeTimeoutMultiplier(); - int runtimeMultiplier = new RuntimeConfiguration(configuration) - .computeTimeoutMultiplier( - mode: configuration['mode'], - isChecked: configuration['checked'], - isReload: isReload, - arch: configuration['arch']); - configuration['timeout'] = 60 * compilerMulitiplier * runtimeMultiplier; - } - - return [configuration]; - } - - /** - * Helper for _expandConfigurations. Creates a new configuration and adds it - * to a list, for use in a case when a particular configuration has multiple - * results (separated by a ','). - * Arguments: - * option: The particular test option we are expanding. - * configuration: The map containing all test configuration information - * specified. - */ - List _expandHelper(String option, Map configuration) { - var result = new List(); - var configs = configuration[option]; - for (var config in configs.split(',')) { - var newConfiguration = new Map.from(configuration); - newConfiguration[option] = config; - result.addAll(_expandConfigurations(newConfiguration)); - } - return result; - } - - /** - * Print out usage information. - */ - void _printHelp() { - print('usage: dart test.dart [options] [selector]'); - print(''); - print('The optional selector limits the tests that will be run.'); - print('For example, the selector "language/issue", or equivalently'); - print('"language/*issue*", limits to test files matching the regexp'); - print('".*issue.*\\.dart" in the "tests/language" directory.'); - print(''); - print('Options:\n'); - for (var option in _options) { - print('${option.name}: ${option.description}.'); - for (var name in option.keys) { - assert(name.startsWith('-')); - var buffer = new StringBuffer(); - ; - buffer.write(name); - if (option.type == 'bool') { - assert(option.values.isEmpty); - } else { - buffer.write(name.startsWith('--') ? '=' : ' '); - if (option.type == 'int') { - assert(option.values.isEmpty); - buffer.write('n (default: ${option.defaultValue})'); - } else { - buffer.write('['); - bool first = true; - for (var value in option.values) { - if (!first) buffer.write(", "); - if (value == option.defaultValue) buffer.write('*'); - buffer.write(value); - first = false; - } - buffer.write(']'); - } - } - print(buffer.toString()); - } - print(''); - } - } - - /** - * Find the test option specification for a given option key. - */ - _TestOptionSpecification _getSpecification(String name) { - for (var option in _options) { - if (option.keys.contains(name)) { - return option; - } - } - print('Unknown test option $name'); - exit(1); - return null; // Unreachable. - } - - List<_TestOptionSpecification> _options; -} diff --git a/tools/testing/dart/test_suite.dart b/tools/testing/dart/test_suite.dart index 08362f36542..cf0af89723d 100644 --- a/tools/testing/dart/test_suite.dart +++ b/tools/testing/dart/test_suite.dart @@ -474,7 +474,7 @@ class CCTestSuite extends TestSuite { var args = TestUtils.standardOptions(configuration); var compilerConfiguration = new CompilerConfiguration(configuration); - if (compilerConfiguration.useDFE) { + if (compilerConfiguration.useDfe) { args.add('--use-dart-frontend'); // '--dfe' has to be the first argument for run_vm_test to pick it up. args.insert(0, '--dfe=$buildDir/gen/kernel-service.dart.snapshot'); @@ -2028,9 +2028,6 @@ class TestUtils { if (compiler == "dart2js" && configuration["csp"]) { args.add("--csp"); } - if (compiler == "dart2js" && configuration["cps_ir"]) { - args.add("--use-cps-ir"); - } if (compiler == "dart2js" && configuration["fast_startup"]) { args.add("--fast-startup"); }