[infra] Front-end unit tests: Check for timeout of each suite separately

This CL also improves the printed messages to include full suite name
and timings.

Change-Id: I7e2129fb27a91e6ab56269f0ef4ed2c80bf4e786
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/122788
Commit-Queue: Karl Klose <karlklose@google.com>
Reviewed-by: Jonas Termansen <sortie@google.com>

pkg/front_end/test/unit_test_suites.dart

@@ -4,7 +4,7 @@
 
 import 'dart:async' show Timer;
 import 'dart:convert' show jsonEncode;
-import 'dart:io' show File, Platform, exit, exitCode;
+import 'dart:io' show File, Platform, exitCode;
 import 'dart:isolate' show Isolate, ReceivePort, SendPort;
 
 import 'package:args/args.dart' show ArgParser;
@@ -96,7 +96,10 @@ class ResultLogger implements Logger {
       TestDescription description, Step step) {}
 
   @override
-  void logSuiteComplete() {}
+  void logSuiteStarted(testing.Suite suite) {}
+
+  @override
+  void logSuiteComplete(testing.Suite suite) {}
 
   handleTestResult(TestDescription testDescription, Result result,
       String fullSuiteName, bool matchedExpectations) {
@@ -252,36 +255,49 @@ main([List<String> arguments = const <String>[]]) async {
   List<String> results = [];
   List<String> logs = [];
   Options options = Options.parse(arguments);
-  Timer timer = Timer(timeoutDuration, () {
-    // TODO(karlklose): use timer for each suite.
-    // TODO(karlklose): report timeout on specific tests
-    print("Error: Test suite timed out!");
-    exit(1);
-  });
   ReceivePort resultsPort = new ReceivePort()
     ..listen((resultEntry) => results.add(resultEntry));
   ReceivePort logsPort = new ReceivePort()
     ..listen((logEntry) => logs.add(logEntry));
-  List<Future> futures = [];
+  List<Future<bool>> futures = [];
   // Run test suites and record the results and possible failure logs.
   for (Suite suite in suites) {
     // Start the test suite in a new isolate.
     ReceivePort exitPort = new ReceivePort();
+    String name = suite.name;
     SuiteConfiguration configuration = SuiteConfiguration(
-        suite.name,
+        name,
         resultsPort.sendPort,
         logsPort.sendPort,
         options.verbose,
         options.configurationName);
-    // TODO(karlklose): Implement --filter to select tests to run
-    // to implement deflaking (dartbug.com/38607).
-    await Isolate.spawn<SuiteConfiguration>(runSuite, configuration,
-        onExit: exitPort.sendPort);
-    futures.add(exitPort.first);
+    Future future = Future<bool>(() async {
+      Stopwatch stopwatch = Stopwatch()..start();
+      print("Running suite $name");
+      // TODO(karlklose): Implement --filter to select tests to run
+      // to implement deflaking (dartbug.com/38607).
+      Isolate isolate = await Isolate.spawn<SuiteConfiguration>(
+          runSuite, configuration,
+          onExit: exitPort.sendPort);
+      bool timedOut = false;
+      Timer timer = Timer(timeoutDuration, () {
+        timedOut = true;
+        print("Suite $name timed out after "
+            "${timeoutDuration.inMilliseconds}ms");
+        isolate.kill(priority: Isolate.immediate);
+      });
+      await exitPort.first;
+      timer.cancel();
+      if (!timedOut) {
+        print(
+            "Suite $name finished (took ${stopwatch.elapsedMilliseconds}ms).");
+      }
+      return timedOut;
+    });
+    futures.add(future);
   }
   // Wait for isolates to terminate and clean up.
-  await Future.wait(futures);
-  timer.cancel();
+  Iterable<bool> timeouts = await Future.wait(futures);
   resultsPort.close();
   logsPort.close();
   // Write results.json and logs.json.
@@ -291,8 +307,14 @@ main([List<String> arguments = const <String>[]]) async {
   await writeLinesToFile(logsJsonUri, logs);
   print("Log files written to ${resultJsonUri.toFilePath()} and"
       " ${logsJsonUri.toFilePath()}");
-  // The testing framework (pkg/testing) sets the exitCode to 1 if any test
-  // failed, so we reset it here to indicate that the test runner was
-  // successful.
-  exitCode = 0;
+  // Return with exit code 1 if at least one suite timed out.
+  bool timeout = timeouts.any((timeout) => timeout);
+  if (timeout) {
+    exitCode = 1;
+  } else {
+    // The testing framework (pkg/testing) sets the exitCode to 1 if any test
+    // failed, so we reset it here to indicate that the test runner was
+    // successful.
+    exitCode = 0;
+  }
 }

pkg/testing/lib/src/chain.dart

@@ -136,6 +136,7 @@ abstract class ChainContext {
     Map<TestDescription, Set<Expectation>> unexpectedOutcomes =
         <TestDescription, Set<Expectation>>{};
     int completed = 0;
+    logger.logSuiteStarted(suite);
     List<Future> futures = <Future>[];
     for (TestDescription description in descriptions) {
       String selector = "${suite.name}/${description.shortName}";
@@ -236,7 +237,7 @@ abstract class ChainContext {
       await doStep(description);
     }
     await Future.wait(futures);
-    logger.logSuiteComplete();
+    logger.logSuiteComplete(suite);
     if (unexpectedResults.isNotEmpty) {
       unexpectedResults.forEach((TestDescription description, Result result) {
         logger.logUnexpectedResult(

pkg/testing/lib/src/log.dart

@@ -63,7 +63,9 @@ abstract class Logger {
   void logUnexpectedResult(Suite suite, TestDescription description,
       Result result, Set<Expectation> expectedOutcomes);
 
-  void logSuiteComplete();
+  void logSuiteStarted(Suite suite);
+
+  void logSuiteComplete(Suite suite);
 
   void logUncaughtError(error, StackTrace stackTrace);
 }
@@ -167,7 +169,11 @@ class StdoutLogger implements Logger {
     }
   }
 
-  void logSuiteComplete() {
+  void logSuiteStarted(Suite suite) {
+    print("Running suite ${suite.name}...");
+  }
+
+  void logSuiteComplete(Suite suite) {
     if (!isVerbose) {
       print("");
     }

pkg/testing/lib/src/run.dart

@@ -62,7 +62,6 @@ Future<Null> runMe(List<String> arguments, CreateContext f,
     if (cl.verbose) enableVerboseOutput();
     for (Chain suite in testRoot.toolChains) {
       if (me == suite.source) {
-        print("Running suite ${suite.name}...");
         ChainContext context = await f(suite, cl.environment);
         await context.run(suite, new Set<String>.from(cl.selectors),
             shards: shards, shard: shard, logger: logger);

pkg/testing/lib/src/run_tests.dart

@@ -188,5 +188,4 @@ Future<void> runTests(Map<String, Function> tests) =>
         const StdoutLogger()
             .logTestComplete(++completed, 0, tests.length, null, null);
       }
-      const StdoutLogger().logSuiteComplete();
     });