diff --git a/pkg/test_runner/lib/src/environment.dart b/pkg/test_runner/lib/src/environment.dart index f012f12323c..4d90271539b 100644 --- a/pkg/test_runner/lib/src/environment.dart +++ b/pkg/test_runner/lib/src/environment.dart @@ -30,6 +30,7 @@ final _variables = { "jscl": _Variable.bool((c) => c.runtime.isJSCommandLine), "minified": _Variable.bool((c) => c.isMinified), "mode": _Variable((c) => c.mode.name, Mode.names), + "nnbd": _Variable((TestConfiguration c) => c.nnbdMode.name, NnbdMode.names), "runtime": _Variable(_runtimeName, _runtimeNames), "spec_parser": _Variable.bool((c) => c.compiler == Compiler.specParser), "system": _Variable(_systemName, _systemNames), diff --git a/runtime/tests/vm/vm.status b/runtime/tests/vm/vm.status index c228d58b9e0..7cab004cfe5 100644 --- a/runtime/tests/vm/vm.status +++ b/runtime/tests/vm/vm.status @@ -129,6 +129,9 @@ cc/StandaloneSnapshotSize: SkipByDesign # Imports dart:mirrors dart/redirection_type_shuffling_test: SkipByDesign # Imports dart:mirrors dart_2/redirection_type_shuffling_test: SkipByDesign # Imports dart:mirrors +[ $nnbd == legacy ] +dart/*: SkipByDesign # Migrated tests are not supposed to run on non-NNBD bots. + [ $system == android ] dart/sdk_hash_test: SkipByDesign # The test doesn't know location of cross-platform gen_snapshot dart_2/sdk_hash_test: SkipByDesign # The test doesn't know location of cross-platform gen_snapshot @@ -174,9 +177,6 @@ dart_2/data_uri_import_test/percentencoded: Crash dart_2/data_uri_import_test/utf16: Crash dart_2/data_uri_import_test/wrongmime: Crash -[ $builder_tag != bytecode_interpreter_nnbd && $builder_tag != bytecode_mixed_nnbd && $builder_tag != vm_nnbd ] -dart/*: SkipByDesign # Migrated tests are not supposed to run on non-NNBD bots. - [ $builder_tag == obfuscated && $compiler == dartkp ] dart/extension_names_test: SkipByDesign # No demangling (obfuscated). dart/extension_unnamed_names_test: SkipByDesign # No demangling (obfuscated). @@ -382,9 +382,6 @@ cc/StreamingFlowGraphBuilder_FlattenNestedStringInterp: SkipByDesign # Optimisat dart/entrypoints/*: SkipByDesign # These tests are for compiler optimizations and very sensitive to when functions are optimized, so they are disabled on hotreload and optcounter bots. dart_2/entrypoints/*: SkipByDesign # These tests are for compiler optimizations and very sensitive to when functions are optimized, so they are disabled on hotreload and optcounter bots. -[ $builder_tag == bytecode_interpreter_nnbd || $builder_tag == bytecode_mixed_nnbd || $builder_tag == vm_nnbd ] -dart_2/*: SkipByDesign # Legacy tests are not supposed to run on NNBD bots. - [ $builder_tag == crossword || $builder_tag == crossword_ast ] dart/bytecode_with_ast_in_aot_test: SkipByDesign # The test doesn't know location of cross-platform gen_snapshot dart/emit_aot_size_info_flag_test: SkipByDesign # The test itself cannot determine the location of gen_snapshot (only tools/test.py knows where it is). @@ -428,6 +425,9 @@ dart_2/use_bare_instructions_flag_test: SkipByDesign # This test is for VM AOT o [ $mode != debug || $runtime != dart_precompiled ] in_memory_elf_test: Skip # Tests a debug-mode flag to dart_precompiled_runtime. +[ $nnbd == strong || $nnbd == weak ] +dart_2/*: SkipByDesign # Legacy tests are not supposed to run on NNBD bots. + # It makes no sense to run any test that uses spawnURI under the simulator # as that would involve running CFE (the front end) in simulator mode # to compile the URI file specified in spawnURI code. diff --git a/tests/lib/lib_vm.status b/tests/lib/lib_vm.status index ae42a729641..dfe5cd6799f 100644 --- a/tests/lib/lib_vm.status +++ b/tests/lib/lib_vm.status @@ -75,6 +75,15 @@ mirrors/invocation_fuzz_test: Crash mirrors/library_uri_io_test: RuntimeError mirrors/library_uri_package_test: RuntimeError +[ $runtime == vm && $system == windows ] +isolate/detect_nullsafety_1_test: Skip # Issue 43583 +isolate/nnbd_spawn_autodetect_1_test: Skip # Issue 43583 +isolate/nnbd_spawn_autodetect_2_test: Skip # Issue 43583 +isolate/nnbd_spawnuri_autodetect_1_test: Skip # Issue 43583 +isolate/nnbd_spawnuri_autodetect_2_test: Skip # Issue 43583 +isolate/nnbd_spawnuri_autodetect_3_test: Skip # Issue 43583 +isolate/nnbd_spawnuri_autodetect_4_test: Skip # Issue 43583 + [ $runtime == vm && ($arch == simarm || $arch == simarmv6) ] convert/utf85_test: Skip # Pass, Slow Issue 12644. diff --git a/tools/bots/test_matrix.json b/tools/bots/test_matrix.json index bb2918c669c..605c4d523e5 100644 --- a/tools/bots/test_matrix.json +++ b/tools/bots/test_matrix.json @@ -630,6 +630,16 @@ "builder-tag": "crossword" } }, + "dartkp-weak-asserts-linux-(debug|product|release)-simarm-crossword": { + "options": { + "builder-tag": "crossword" + } + }, + "dartkp-strong-linux-(debug|product|release)-simarm-crossword": { + "options": { + "builder-tag": "crossword" + } + }, "dartkp-(win|mac)-(debug|product|release)-simarm-crossword": { "options": { "builder-tag": "crossword", @@ -703,13 +713,13 @@ "builder-tag": "vm_nnbd" } }, - "dartk-weak-asserts-(linux|mac|win)-(debug|product|release)-x64": { + "dartk-weak-asserts-(linux|mac|win)-(debug|product|release)-(ia32|simarm|simarm64|x64)": { "options": { "enable-asserts": true, "builder-tag": "vm_nnbd" } }, - "dartk-strong-(linux|mac|win)-(debug|product|release)-x64": { + "dartk-strong-(linux|mac|win)-(debug|product|release)-(ia32|simarm|simarm64|x64)": { "options": { "builder-tag": "vm_nnbd" } @@ -1077,8 +1087,11 @@ }, { "builders": [ + "vm-kernel-precomp-nnbd-linux-debug-x64", "vm-kernel-precomp-nnbd-linux-release-simarm64", - "vm-kernel-precomp-nnbd-linux-release-x64" + "vm-kernel-precomp-nnbd-linux-release-x64", + "vm-kernel-precomp-nnbd-mac-release-simarm64", + "vm-kernel-precomp-nnbd-win-release-x64" ], "meta": { "description": "This configuration is for the VM AOT nnbd builder group." @@ -1096,29 +1109,35 @@ "name": "vm nnbd tests in weak mode with asserts", "arguments": [ "-ndartkp-weak-asserts-${system}-${mode}-${arch}", + "benchmark_smoke", "corelib", "ffi", "language", "lib", - "standalone/io", + "samples", + "standalone", + "utils", "vm" ], "fileset": "vm-kernel", - "shards": 4 + "shards": 6 }, { "name": "vm nnbd tests in strong mode", "arguments": [ "-ndartkp-strong-${system}-${mode}-${arch}", + "benchmark_smoke", "corelib", "ffi", "language", "lib", - "standalone/io", + "samples", + "standalone", + "utils", "vm" ], "fileset": "vm-kernel", - "shards": 4 + "shards": 6 }, { "name": "co19 nnbd tests in strong mode", @@ -1127,7 +1146,7 @@ "co19" ], "fileset": "vm-kernel", - "shards": 6 + "shards": 10 }, { "name": "co19 nnbd tests in weak mode with asserts", @@ -1136,7 +1155,7 @@ "co19" ], "fileset": "vm-kernel", - "shards": 6 + "shards": 10 } ] }, @@ -1447,8 +1466,17 @@ }, { "builders": [ + "vm-kernel-nnbd-linux-debug-ia32", + "vm-kernel-nnbd-linux-debug-x64", + "vm-kernel-nnbd-linux-release-ia32", + "vm-kernel-nnbd-linux-release-simarm", + "vm-kernel-nnbd-linux-release-simarm64", "vm-kernel-nnbd-linux-release-x64", - "vm-kernel-nnbd-linux-debug-x64" + "vm-kernel-nnbd-mac-debug-x64", + "vm-kernel-nnbd-mac-release-x64", + "vm-kernel-nnbd-win-release-ia32", + "vm-kernel-nnbd-win-debug-x64", + "vm-kernel-nnbd-win-release-x64" ], "meta": { "description": "This configuration is for the VM nnbd builder group." @@ -1463,34 +1491,56 @@ ] }, { - "name": "vm nnbd test in weak mode with asserts", + "name": "vm nnbd tests in weak mode with asserts", "arguments": [ "-ndartk-weak-asserts-${system}-${mode}-${arch}", + "benchmark_smoke", "corelib", "ffi", "language", "lib", - "co19", - "standalone/io", + "samples", + "standalone", + "utils", "vm" ], "fileset": "vm-kernel", - "shards": 2 + "shards": 6 + }, + { + "name": "vm co19 nnbd tests in weak mode with asserts", + "arguments": [ + "-ndartk-weak-asserts-${system}-${mode}-${arch}", + "co19" + ], + "fileset": "vm-kernel", + "shards": 8 }, { "name": "vm nnbd tests in strong mode", "arguments": [ "-ndartk-strong-${system}-${mode}-${arch}", + "benchmark_smoke", "corelib", "ffi", "language", "lib", - "co19", - "standalone/io", + "samples", + "standalone", + "utils", "vm" ], "fileset": "vm-kernel", - "shards": 2 + "shards": 6 + }, + { + "name": "vm co19 nnbd tests in strong mode", + "arguments": [ + "-ndartk-strong-${system}-${mode}-${arch}", + "co19" + ], + "fileset": "vm-kernel", + "shards": 8 } ] }, @@ -3314,6 +3364,67 @@ "shards": 10 } ] + }, + { + "builders": [ + "vm-kernel-precomp-nnbd-linux-debug-simarm_x64", + "vm-kernel-precomp-nnbd-linux-release-simarm_x64" + ], + "meta": { + "description": "This configuration runs tests for the simarm_x64 architecture." + }, + "steps": [ + { + "name": "build dart for simarm_x64", + "script": "tools/build.py", + "arguments": [ + "gen_snapshot" + ] + }, + { + "name": "build dart for simarm", + "script": "tools/build.py", + "arguments": [ + "--arch=simarm", + "dart_precompiled_runtime", + "vm_platform" + ] + }, + { + "name": "vm weak tests", + "arguments": [ + "-ndartkp-weak-asserts-${system}-${mode}-simarm-crossword", + "benchmark_smoke", + "corelib", + "ffi", + "language", + "lib", + "samples", + "standalone", + "utils", + "vm" + ], + "fileset": "vm-kernel", + "shards": 10 + }, + { + "name": "vm tests", + "arguments": [ + "-ndartkp-strong-${system}-${mode}-simarm-crossword", + "benchmark_smoke", + "corelib", + "ffi", + "language", + "lib", + "samples", + "standalone", + "utils", + "vm" + ], + "fileset": "vm-kernel", + "shards": 10 + } + ] } ], "sanitizer_options": {