diff --git a/pkg/front_end/test/unit_test_suites.dart b/pkg/front_end/test/unit_test_suites.dart index f4c316b89c0..ebc86c0272b 100644 --- a/pkg/front_end/test/unit_test_suites.dart +++ b/pkg/front_end/test/unit_test_suites.dart @@ -4,7 +4,7 @@ import 'dart:async' show Timer; import 'dart:convert' show jsonEncode; -import 'dart:io' show File, Platform, exit, exitCode; +import 'dart:io' show File, Platform, exitCode; import 'dart:isolate' show Isolate, ReceivePort, SendPort; import 'package:args/args.dart' show ArgParser; @@ -96,7 +96,10 @@ class ResultLogger implements Logger { TestDescription description, Step step) {} @override - void logSuiteComplete() {} + void logSuiteStarted(testing.Suite suite) {} + + @override + void logSuiteComplete(testing.Suite suite) {} handleTestResult(TestDescription testDescription, Result result, String fullSuiteName, bool matchedExpectations) { @@ -252,36 +255,49 @@ main([List arguments = const []]) async { List results = []; List logs = []; Options options = Options.parse(arguments); - Timer timer = Timer(timeoutDuration, () { - // TODO(karlklose): use timer for each suite. - // TODO(karlklose): report timeout on specific tests - print("Error: Test suite timed out!"); - exit(1); - }); ReceivePort resultsPort = new ReceivePort() ..listen((resultEntry) => results.add(resultEntry)); ReceivePort logsPort = new ReceivePort() ..listen((logEntry) => logs.add(logEntry)); - List futures = []; + List> futures = []; // Run test suites and record the results and possible failure logs. for (Suite suite in suites) { // Start the test suite in a new isolate. ReceivePort exitPort = new ReceivePort(); + String name = suite.name; SuiteConfiguration configuration = SuiteConfiguration( - suite.name, + name, resultsPort.sendPort, logsPort.sendPort, options.verbose, options.configurationName); - // TODO(karlklose): Implement --filter to select tests to run - // to implement deflaking (dartbug.com/38607). - await Isolate.spawn(runSuite, configuration, - onExit: exitPort.sendPort); - futures.add(exitPort.first); + Future future = Future(() async { + Stopwatch stopwatch = Stopwatch()..start(); + print("Running suite $name"); + // TODO(karlklose): Implement --filter to select tests to run + // to implement deflaking (dartbug.com/38607). + Isolate isolate = await Isolate.spawn( + runSuite, configuration, + onExit: exitPort.sendPort); + bool timedOut = false; + Timer timer = Timer(timeoutDuration, () { + timedOut = true; + print("Suite $name timed out after " + "${timeoutDuration.inMilliseconds}ms"); + isolate.kill(priority: Isolate.immediate); + }); + await exitPort.first; + timer.cancel(); + if (!timedOut) { + print( + "Suite $name finished (took ${stopwatch.elapsedMilliseconds}ms)."); + } + return timedOut; + }); + futures.add(future); } // Wait for isolates to terminate and clean up. - await Future.wait(futures); - timer.cancel(); + Iterable timeouts = await Future.wait(futures); resultsPort.close(); logsPort.close(); // Write results.json and logs.json. @@ -291,8 +307,14 @@ main([List arguments = const []]) async { await writeLinesToFile(logsJsonUri, logs); print("Log files written to ${resultJsonUri.toFilePath()} and" " ${logsJsonUri.toFilePath()}"); - // The testing framework (pkg/testing) sets the exitCode to 1 if any test - // failed, so we reset it here to indicate that the test runner was - // successful. - exitCode = 0; + // Return with exit code 1 if at least one suite timed out. + bool timeout = timeouts.any((timeout) => timeout); + if (timeout) { + exitCode = 1; + } else { + // The testing framework (pkg/testing) sets the exitCode to 1 if any test + // failed, so we reset it here to indicate that the test runner was + // successful. + exitCode = 0; + } } diff --git a/pkg/testing/lib/src/chain.dart b/pkg/testing/lib/src/chain.dart index 041b8c1c398..56d0b231f74 100644 --- a/pkg/testing/lib/src/chain.dart +++ b/pkg/testing/lib/src/chain.dart @@ -136,6 +136,7 @@ abstract class ChainContext { Map> unexpectedOutcomes = >{}; int completed = 0; + logger.logSuiteStarted(suite); List futures = []; for (TestDescription description in descriptions) { String selector = "${suite.name}/${description.shortName}"; @@ -236,7 +237,7 @@ abstract class ChainContext { await doStep(description); } await Future.wait(futures); - logger.logSuiteComplete(); + logger.logSuiteComplete(suite); if (unexpectedResults.isNotEmpty) { unexpectedResults.forEach((TestDescription description, Result result) { logger.logUnexpectedResult( diff --git a/pkg/testing/lib/src/log.dart b/pkg/testing/lib/src/log.dart index b15de1466e6..e916ba97970 100644 --- a/pkg/testing/lib/src/log.dart +++ b/pkg/testing/lib/src/log.dart @@ -63,7 +63,9 @@ abstract class Logger { void logUnexpectedResult(Suite suite, TestDescription description, Result result, Set expectedOutcomes); - void logSuiteComplete(); + void logSuiteStarted(Suite suite); + + void logSuiteComplete(Suite suite); void logUncaughtError(error, StackTrace stackTrace); } @@ -167,7 +169,11 @@ class StdoutLogger implements Logger { } } - void logSuiteComplete() { + void logSuiteStarted(Suite suite) { + print("Running suite ${suite.name}..."); + } + + void logSuiteComplete(Suite suite) { if (!isVerbose) { print(""); } diff --git a/pkg/testing/lib/src/run.dart b/pkg/testing/lib/src/run.dart index 12305bad32a..49d0b1261ef 100644 --- a/pkg/testing/lib/src/run.dart +++ b/pkg/testing/lib/src/run.dart @@ -62,7 +62,6 @@ Future runMe(List arguments, CreateContext f, if (cl.verbose) enableVerboseOutput(); for (Chain suite in testRoot.toolChains) { if (me == suite.source) { - print("Running suite ${suite.name}..."); ChainContext context = await f(suite, cl.environment); await context.run(suite, new Set.from(cl.selectors), shards: shards, shard: shard, logger: logger); diff --git a/pkg/testing/lib/src/run_tests.dart b/pkg/testing/lib/src/run_tests.dart index ecb28d436f3..b4ccd3be64a 100644 --- a/pkg/testing/lib/src/run_tests.dart +++ b/pkg/testing/lib/src/run_tests.dart @@ -188,5 +188,4 @@ Future runTests(Map tests) => const StdoutLogger() .logTestComplete(++completed, 0, tests.length, null, null); } - const StdoutLogger().logSuiteComplete(); });