[CFE/testing] Isolate v processes test script; make pkg:testing run more consistent with invalid shards info

E.g. compile to aot:
out/ReleaseX64/dart-sdk/bin/dart compile aot-snapshot pkg/front_end/test/isolates_v_processes.dart

And run with a different number of isolates:
$ out/ReleaseX64/dart-sdk/bin/dartaotruntime --deterministic pkg/front_end/test/isolates_v_processes.aot --isolates -j1
 => 0:01:13.683401
$ out/ReleaseX64/dart-sdk/bin/dartaotruntime --deterministic pkg/front_end/test/isolates_v_processes.aot --isolates -j2
 => 0:00:53.379932
$ out/ReleaseX64/dart-sdk/bin/dartaotruntime --deterministic pkg/front_end/test/isolates_v_processes.aot --isolates -j4
 => 0:01:12.864165

(you might change `const bool doPrint = false;` to `true`,
compile again, run with -j4 and see interesting stops when it runs)

...or you could try the same but using processes:
$ out/ReleaseX64/dart-sdk/bin/dartaotruntime --deterministic pkg/front_end/test/isolates_v_processes.aot --processes -j1
 => 0:01:17.590922
$ out/ReleaseX64/dart-sdk/bin/dartaotruntime --deterministic pkg/front_end/test/isolates_v_processes.aot --processes -j2
 => 0:00:44.055109
$ out/ReleaseX64/dart-sdk/bin/dartaotruntime --deterministic pkg/front_end/test/isolates_v_processes.aot --processes -j4
 => 0:00:26.360444

Change-Id: Ie161e7dfa0f29931af34058b7b043ce93b839218
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/375481
Commit-Queue: Jens Johansen <jensj@google.com>
Reviewed-by: Johnni Winther <johnniwinther@google.com>

pkg/front_end/presubmit_helper_spawn.dart

@@ -225,7 +225,7 @@ class ErrorNotingLogger implements Logger {
 
   @override
   void noticeFrameworkCatchError(error, StackTrace stackTrace) {
-    print("Framework Catch Error: $error\n$StackTrace");
+    print("Framework Catch Error: $error\n$stackTrace");
     gotFailure = true;
   }
 }

pkg/front_end/test/isolates_v_processes.dart

@@ -0,0 +1,162 @@
+// Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import "dart:io";
+import "dart:isolate";
+
+import "package:testing/src/suite.dart";
+import "package:testing/testing.dart";
+import 'package:testing/src/log.dart' show Logger, StdoutLogger;
+
+import "fasta/strong_suite.dart" as strong;
+
+const bool doPrint = false;
+
+enum What {
+  Process,
+  Isolate,
+  Direct,
+  Unknown;
+}
+
+Future<void> main(List<String> args) async {
+  int j = 2;
+  int x = 0;
+  What what = What.Unknown;
+
+  for (String arg in args) {
+    if (arg.startsWith("-j")) {
+      j = int.tryParse(arg.substring(2)) ?? (throw "invalid -j ($arg)");
+    } else if (arg.startsWith("-x")) {
+      x = int.tryParse(arg.substring(2)) ?? (throw "invalid -x ($arg)");
+    } else if (arg == "--isolates") {
+      what = What.Isolate;
+    } else if (arg == "--processes") {
+      what = What.Process;
+    } else if (arg == "--direct") {
+      what = What.Direct;
+    }
+  }
+  Stopwatch stopwatch = new Stopwatch()..start();
+  switch (what) {
+    case What.Process:
+      await useProcesses(j);
+    case What.Isolate:
+      await useIsolates(j);
+    case What.Direct:
+      await useDirect(j, x);
+    case What.Unknown:
+      throw "Specify with --isolates or --processes and optionally -j<num>";
+  }
+  print("All done after ${stopwatch.elapsed}");
+}
+
+Future<void> entry(List<int> args) async {
+  await runMe(
+    ["-DskipVm=true", "-DsemiFuzz=false"],
+    strong.createContext,
+    me: Platform.script.resolve("fasta/strong_suite.dart"),
+    configurationPath: "../../testing.json",
+    shards: args[0],
+    shard: args[1],
+    logger: doPrint ? const StdoutLogger() : const DevNullLogger(),
+  );
+}
+
+Future<void> useDirect(int shards, int shard) async {
+  List<Future> futures = [];
+  futures.add(entry([shards, shard]));
+  await Future.wait(futures);
+}
+
+Future<void> useIsolates(final int j) async {
+  Stopwatch stopwatch = new Stopwatch()..start();
+  print("Using $j isolates...");
+  List<Future> futures = [];
+  for (int i = 0; i <= j; i++) {
+    ReceivePort exitPort = new ReceivePort();
+    futures.add(exitPort.first.then((_) {
+      if (i == j) {
+        print("Isolate #$i (checking startup cost) finished after "
+            "${stopwatch.elapsed}");
+      } else {
+        print("Isolate #$i finished after ${stopwatch.elapsed}");
+      }
+    }));
+    await Isolate.spawn(entry, [j, i], onExit: exitPort.sendPort);
+  }
+  await Future.wait(futures);
+}
+
+Future<void> useProcesses(final int j) async {
+  Stopwatch stopwatch = new Stopwatch()..start();
+  print("Using $j processes...");
+  String script = Platform.script.toFilePath();
+  List<Future> futures = [];
+  for (int i = 0; i <= j; i++) {
+    futures.add(Process.run(Platform.resolvedExecutable, [
+      script,
+      "--direct",
+      "-j$j",
+      "-x$i",
+    ]).then((_) {
+      if (i == j) {
+        print("Process #$i (checking startup cost) finished after "
+            "${stopwatch.elapsed}");
+      } else {
+        print("Process #$i finished after ${stopwatch.elapsed}");
+      }
+    }));
+  }
+  await Future.wait(futures);
+}
+
+class DevNullLogger implements Logger {
+  const DevNullLogger();
+
+  @override
+  void logExpectedResult(Suite suite, TestDescription description,
+      Result result, Set<Expectation> expectedOutcomes) {}
+
+  @override
+  void logMessage(Object message) {}
+
+  @override
+  void logNumberedLines(String text) {}
+
+  @override
+  void logProgress(String message) {}
+
+  @override
+  void logStepComplete(int completed, int failed, int total, Suite suite,
+      TestDescription description, Step<dynamic, dynamic, ChainContext> step) {}
+
+  @override
+  void logStepStart(int completed, int failed, int total, Suite suite,
+      TestDescription description, Step<dynamic, dynamic, ChainContext> step) {}
+
+  @override
+  void logSuiteComplete(Suite suite) {}
+
+  @override
+  void logSuiteStarted(Suite suite) {}
+
+  @override
+  void logTestComplete(int completed, int failed, int total, Suite suite,
+      TestDescription description) {}
+
+  @override
+  void logTestStart(int completed, int failed, int total, Suite suite,
+      TestDescription description) {}
+
+  @override
+  void logUncaughtError(error, StackTrace stackTrace) {}
+
+  @override
+  void logUnexpectedResult(Suite suite, TestDescription description,
+      Result result, Set<Expectation> expectedOutcomes) {}
+
+  @override
+  void noticeFrameworkCatchError(error, StackTrace stackTrace) {}
+}

pkg/testing/lib/src/chain.dart

@@ -134,7 +134,11 @@ abstract class ChainContext {
         <String>[suite.statusFile!.toFilePath()], expectationSet);
     List<TestDescription> descriptions = await list(suite);
     descriptions.sort();
-    if (shards > 1) {
+
+    /// Hack: If not running with asserts running the (invalid) configuration
+    /// shards=1 shard>0 should behave as when running with the (invalid)
+    /// configuration shards>1 shard>=shards, i.e. it should run nothing.
+    if (shards > 1 || shard >= shards) {
       List<TestDescription> shardDescriptions = [];
       for (int index = 0; index < descriptions.length; index++) {
         if (index % shards == shard) {