From 8f03ae05ffeff853773997ecd3452bd51622e615 Mon Sep 17 00:00:00 2001 From: Jonas Termansen Date: Mon, 11 Mar 2019 18:04:59 +0000 Subject: [PATCH] [infra] Add --list-configurations option to test.dart. This option makes the available named configurations much more discoverable. Additionally this change expands the available named configurations by generalizing the operating system and processor architecture patterns in the named configurations. This change should ensure that nobody is doing any local testing that isn't covered by a named configuration. Change-Id: I776105955a86e9f0403ce07a3cdf971e4213646f Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/96320 Commit-Queue: Jonas Termansen Reviewed-by: Alexander Thomas --- tools/bots/test_matrix.json | 62 +++++++++++++++++---------------- tools/test.dart | 19 ++++++++-- tools/testing/dart/options.dart | 12 +++++++ 3 files changed, 60 insertions(+), 33 deletions(-) diff --git a/tools/bots/test_matrix.json b/tools/bots/test_matrix.json index 57ddeec70f5..766eb758ff9 100644 --- a/tools/bots/test_matrix.json +++ b/tools/bots/test_matrix.json @@ -195,7 +195,7 @@ "timeout": 240, "use-sdk": true }}, - "unittest-asserts-(release|debug)-(linux|mac|win)": { + "unittest-asserts-(debug|product|release)-(linux|mac|win)": { "options": { "compiler": "dartk", "enable-asserts": true, @@ -211,7 +211,7 @@ "runtime": "vm", "timeout": 240 }}, - "unittest-analyzer_use_fasta-linux": { + "unittest-analyzer_use_fasta-(linux|mac|win)": { "options": { "compiler": "none", "runtime": "vm", @@ -220,7 +220,7 @@ "vm-options": ["-DuseFastaParser=true"], "builder-tag": "analyzer_use_fasta" }}, - "dartk-asan-linux-release-(ia32|x64)": { + "dartk-asan-(linux|mac|win)-(debug|product|release)-(ia32|x64)": { "options": { "builder-tag": "asan", "timeout": 240 @@ -241,96 +241,98 @@ "options": { "use-sdk": true }}, - "dart2js-minified-csp-linux-chrome": { + "dart2js-minified-csp-(linux|mac|win)-chrome": { "options": { "minified": true, "csp": true, "use-sdk": true }}, - "dart2js-minified-linux-d8": { + "dart2js-minified-(linux|mac|win)-d8": { "options": { "minified": true, "use-sdk": true }}, - "dart2js-hostasserts-linux-ia32-d8": { + "dart2js-hostasserts-(linux|mac|win)-(ia32|x64)-d8": { "options": { "host-checked": true }}, - "dartkp-android-release-arm": { + "dartkp-android-(debug|product|release)-arm": { "options": { "use-blobs": true }}, - "dartkp-linux-release-(simarm|simarm64)": { + "dartkp-linux-(debug|product|release)-(simarm|simarm64)": { "options": { "use-blobs": true }}, - "dartkp-(win|mac)-release-simarm64": { + "dartkp-(win|mac)-(debug|product|release)-(simarm|simarm64)": { "options": { "use-blobs": true }}, - "dartkp-win-release-x64": { + "dartkp-win-(product|release)-x64": { "options": { "use-blobs": true }}, - "dartkp-linux-(product|release)-x64": { }, - "dartkp-obfuscate-linux-release-x64": { + "dartkp-win-debug-x64": { + "options": { + "use-blobs": true, + "vm-options": ["--no-enable-malloc-hooks"] + }}, + "dartkp-(linux|mac)-(product|release)-x64": { }, + "dartkp-obfuscate-(linux|mac|win)-(debug|product|release)-x64": { "options": { "builder-tag": "obfuscated", "vm-options": ["--obfuscate"] }}, - "dartkp-linux-debug-x64": { + "dartkp-(linux|mac)-debug-x64": { "options": { "vm-options": ["--no-enable-malloc-hooks"] }}, - "dartkp-no-bare-linux-(debug|release)-x64": { + "dartkp-no-bare-(linux|mac|win)-(debug|product|release)-x64": { "options": { "vm-options": ["--no-enable-malloc-hooks", "--no-use-bare-instructions"] }}, - "dartkp-no-bare-linux-(debug|release)-(simarm|simarm64)": { + "dartkp-no-bare-(linux|mac|win)-(debug|product|release)-(simarm|simarm64)": { "options": { "vm-options": ["--no-enable-malloc-hooks", "--no-use-bare-instructions"], "use-blobs": true }}, - "dartk-(linux|mac)-(debug|release)-(ia32|x64)": { }, - "dartk-checked-linux-release-x64": { + "dartk-(linux|mac|win)-(debug|product|release)-(ia32|x64)": { }, + "dartk-checked-(linux|mac|win)-(debug|product|release)-(ia32|x64)": { "options": { "checked": true }}, - "dartk-win-(debug|release)-(ia32|x64)": { }, - "dartk-(linux|mac|win)-product-x64": { }, - "dartk-(linux|mac)-(debug|release)-simdbc64": { }, - "dartk-linux-release-(arm64|simarm|simarm64)": { }, - "dartk-optcounter-linux-release-(ia32|x64)": { + "dartk-(linux|mac|win)-(debug|product|release)-(arm64|simarm|simarm64|simdbc64)": { }, + "dartk-optcounter-(linux|mac|win)-(debug|product|release)-(ia32|x64)": { "options": { "builder-tag": "optimization_counter_threshold", "vm-options": ["--optimization-counter-threshold=5"] }}, - "dartk-reload-linux-(debug|release)-x64": { + "dartk-reload-(linux|mac|win)-(debug|product|release)-(ia32|x64)": { "options": { "hot-reload": true }}, - "dartk-reload-mac-(debug|release)-simdbc64": { + "dartk-reload-mac-(debug|product|release)-simdbc64": { "options": { "hot-reload": true }}, - "dartk-reload-rollback-linux-(debug|release)-x64": { + "dartk-reload-rollback-(linux|mac|win)-(debug|product|release)-(ia32|x64)": { "options": { "hot-reload-rollback": true }}, - "app_jitk-linux-(debug|product|release)-x64": { }, - "dartkb-interpret-linux-(debug|release)-x64": { + "app_jitk-(linux|mac|win)-(debug|product|release)-(ia32|x64)": { }, + "dartkb-interpret-(linux|mac|win)-(debug|product|release)-(ia32|x64)": { "options": { "vm-options": ["--enable_interpreter", "--compilation-counter-threshold=-1"] }}, - "dartkb-mixed-linux-(debug|release)-x64": { + "dartkb-mixed-(linux|mac|win)-(debug|product|release)-(ia32|x64)": { "options": { "vm-options": ["--enable_interpreter"] }}, - "dartkb-compile-linux-(debug|release)-x64": { + "dartkb-compile-(linux|mac|win)-(debug|product|release)-(ia32|x64)": { "options": { "vm-options": ["--use_bytecode_compiler"] }}, - "(dartdevc|dartdevk)-checked-(linux|mac|win)-release-chrome": { + "(dartdevc|dartdevk)-checked-(linux|mac|win)-(debug|product|release)-chrome": { "options": { "checked": true, "use-sdk": true diff --git a/tools/test.dart b/tools/test.dart index 618c9fbb0fc..20950b44009 100755 --- a/tools/test.dart +++ b/tools/test.dart @@ -272,6 +272,8 @@ void main(List args) async { help: "Select the builders building this branch", defaultsTo: "master"); parser.addOption("commit", abbr: "C", help: "Compare with this commit"); + parser.addFlag("list-configurations", + help: "Output list of configurations.", negatable: false); parser.addOption("named-configuration", abbr: "n", help: "The named test configuration that supplies the\nvalues for all " @@ -279,8 +281,9 @@ void main(List args) async { parser.addOption("local-configuration", abbr: "N", help: "Use a different named configuration for local\ntesting than the " - "named configuration the baseline\nresults were downloaded for.\nThe" - "results may be inexact if the baseline configuration is different."); + "named configuration the baseline\nresults were downloaded for. The " + "results may be\ninexact if the baseline configuration is " + "different."); parser.addOption("remote", abbr: "R", help: "Compare with this remote and git branch", @@ -289,7 +292,9 @@ void main(List args) async { final options = parser.parse(args); if (options["help"] || - (options["builder"] == null && options["named-configuration"] == null)) { + (options["builder"] == null && + options["named-configuration"] == null && + !options["list-configurations"])) { print(""" Usage: test.dart -b [BUILDER] -n [CONFIGURATION] [OPTION]... [--] [TEST.PY OPTION]... [SELECTOR]... @@ -309,6 +314,14 @@ ${parser.usage}"""); return; } + if (options["list-configurations"]) { + final process = await Process.start( + "python", ["tools/test.py", "--list-configurations"], + mode: ProcessStartMode.inheritStdio, runInShell: Platform.isWindows); + exitCode = await process.exitCode; + return; + } + // Locate gsutil.py. gsutilPy = Platform.script.resolve("../third_party/gsutil/gsutil.py").toFilePath(); diff --git a/tools/testing/dart/options.dart b/tools/testing/dart/options.dart index e462b424b9b..564dbc8ec80 100644 --- a/tools/testing/dart/options.dart +++ b/tools/testing/dart/options.dart @@ -219,6 +219,7 @@ compact, color, line, verbose, silent, status, buildbot, diff''', new _Option.bool('no-tree-shake', 'Disable kernel IR tree shaking.', hide: true), new _Option.bool('list', 'List tests only, do not run them.'), + new _Option.bool('list-configurations', 'Output list of configurations.'), new _Option.bool('list_status_files', 'List status files for test-suites. Do not run any test suites.', hide: true), @@ -383,6 +384,17 @@ compiler.''', verbose: arguments.contains("--verbose") || arguments.contains("-v")); return null; } + if (arguments.contains("--list-configurations")) { + final testMatrixFile = "tools/bots/test_matrix.json"; + TestMatrix testMatrix = TestMatrix.fromPath(testMatrixFile); + for (final configuration in testMatrix.configurations + .map((configuration) => configuration.name) + .toList() + ..sort()) { + print(configuration); + } + return null; + } // Dart1 mode has been deprecated. if (arguments.contains("--no-preview-dart-2")) { return null;