diff --git a/runtime/tests/concurrency/run_stress_test_shards.dart b/runtime/tests/concurrency/run_stress_test_shards.dart index 2d31d15d0b0..87eb9ab7568 100644 --- a/runtime/tests/concurrency/run_stress_test_shards.dart +++ b/runtime/tests/concurrency/run_stress_test_shards.dart @@ -12,6 +12,8 @@ import 'package:path/path.dart' as path; import '../vm/dart/snapshot_test_helper.dart'; +int crashCounter = 0; + void forwardStream(Stream> input, IOSink output) { // Print the information line-by-line. input @@ -22,14 +24,25 @@ void forwardStream(Stream> input, IOSink output) { }); } -Future run(String executable, List args) async { +class PotentialCrash { + final String test; + final int pid; + final List binaries; + PotentialCrash(this.test, this.pid, this.binaries); +} + +Future run( + String executable, List args, List crashes) async { print('Running "$executable ${args.join(' ')}"'); final Process process = await Process.start(executable, args); forwardStream(process.stdout, stdout); forwardStream(process.stderr, stderr); final int exitCode = await process.exitCode; if (exitCode != 0) { + final crashNr = crashCounter++; print('=> Running "$executable ${args.join(' ')}" failed with $exitCode'); + print('=> Possible crash $crashNr (pid: ${process.pid})'); + crashes.add(PotentialCrash('crash-$crashNr', process.pid, [executable])); io.exitCode = 255; // Make this shard fail. return false; } @@ -37,7 +50,7 @@ Future run(String executable, List args) async { } abstract class TestRunner { - Future runTest(); + Future runTest(List crashes); } class JitTestRunner extends TestRunner { @@ -46,8 +59,8 @@ class JitTestRunner extends TestRunner { JitTestRunner(this.buildDir, this.arguments); - Future runTest() async { - await run('$buildDir/dart', arguments); + Future runTest(List crashes) async { + await run('$buildDir/dart', arguments, crashes); } } @@ -58,19 +71,62 @@ class AotTestRunner extends TestRunner { AotTestRunner(this.buildDir, this.arguments, this.aotArguments); - Future runTest() async { + Future runTest(List crashes) async { await withTempDir((String dir) async { final elfFile = path.join(dir, 'app.elf'); - if (await run('$buildDir/gen_snapshot', - ['--snapshot-kind=app-aot-elf', '--elf=$elfFile', ...arguments])) { - await run( - '$buildDir/dart_precompiled_runtime', [...aotArguments, elfFile]); + if (await run( + '$buildDir/gen_snapshot', + ['--snapshot-kind=app-aot-elf', '--elf=$elfFile', ...arguments], + crashes)) { + await run('$buildDir/dart_precompiled_runtime', + [...aotArguments, elfFile], crashes); } }); } } +// Produces a name that tools/utils.py:BaseCoredumpArchiver supports. +String getArchiveName(String binary) { + final parts = binary.split(Platform.pathSeparator); + late String mode; + late String arch; + final buildDir = parts[1]; + for (final prefix in ['Release', 'Debug', 'Product']) { + if (buildDir.startsWith(prefix)) { + mode = prefix.toLowerCase(); + arch = buildDir.substring(prefix.length); + } + } + final name = parts.skip(2).join('__'); + return 'binary.${mode}_${arch}_${name}'; +} + +void writeUnexpectedCrashesFile(List crashes) { + // The format of this file is: + // + // test-name,pid,binary-file1,binary-file2,... + // + const unexpectedCrashesFile = 'unexpected-crashes'; + + final buffer = StringBuffer(); + final Set archivedBinaries = {}; + for (final crash in crashes) { + buffer.write('${crash.test},${crash.pid}'); + for (final binary in crash.binaries) { + final archivedName = getArchiveName(binary); + buffer.write(',$archivedName'); + if (!archivedBinaries.contains(archivedName)) { + File(binary).copySync(archivedName); + archivedBinaries.add(archivedName); + } + } + buffer.writeln(); + } + + File(unexpectedCrashesFile).writeAsStringSync(buffer.toString()); +} + const int tsanShards = 200; final configurations = [ @@ -120,11 +176,14 @@ main(List arguments) async { ..addOption('shards', help: 'number of shards used', defaultsTo: '1') ..addOption('shard', help: 'shard id', defaultsTo: '1') ..addOption('output-directory', - help: 'unused parameter to make sharding infra work', defaultsTo: ''); + help: 'unused parameter to make sharding infra work', defaultsTo: '') + ..addFlag('copy-coredumps', + help: 'whether to copy binaries for coredumps', defaultsTo: false); final options = parser.parse(arguments); final shards = int.parse(options['shards']); final shard = int.parse(options['shard']) - 1; + final copyCoredumps = options['copy-coredumps'] as bool; // Tasks will eventually be killed if they do not have any output for some // time. So we'll explicitly print something every 4 minutes. @@ -140,8 +199,12 @@ main(List arguments) async { thisShardsConfigurations.add(configurations[i]); } } + final crashes = []; for (final config in thisShardsConfigurations) { - await config.runTest(); + await config.runTest(crashes); + } + if (!crashes.isEmpty && copyCoredumps) { + writeUnexpectedCrashesFile(crashes); } } finally { timer.cancel(); diff --git a/tools/bots/test_matrix.json b/tools/bots/test_matrix.json index 892e142416c..8bcd617612e 100644 --- a/tools/bots/test_matrix.json +++ b/tools/bots/test_matrix.json @@ -3912,9 +3912,11 @@ }, { "name": "Run Isolate Stress Tests", - "script": "out/ReleaseX64/dart", + "script": "tools/run_with_coredumps_enabled.py", "arguments": [ - "runtime/tests/concurrency/run_stress_test_shards.dart" + "out/ReleaseX64/dart", + "runtime/tests/concurrency/run_stress_test_shards.dart", + "--copy-coredumps" ], "shards": 10, "fileset": "vm-kernel" diff --git a/tools/run_with_coredumps_enabled.py b/tools/run_with_coredumps_enabled.py new file mode 100755 index 00000000000..e69cb94a5c7 --- /dev/null +++ b/tools/run_with_coredumps_enabled.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. + +from contextlib import ExitStack +import subprocess +import sys + +import utils + + +def Main(): + args = sys.argv[1:] + + with ExitStack() as stack: + for ctx in utils.CoreDumpArchiver(args): + stack.enter_context(ctx) + exit_code = subprocess.call(args) + + utils.DiagnoseExitCode(exit_code, args) + return exit_code + + +if __name__ == '__main__': + sys.exit(Main())