diff --git a/pkg/front_end/test/fasta/testing/suite.dart b/pkg/front_end/test/fasta/testing/suite.dart index 2540ddded6c..ed80870c89f 100644 --- a/pkg/front_end/test/fasta/testing/suite.dart +++ b/pkg/front_end/test/fasta/testing/suite.dart @@ -153,6 +153,9 @@ export 'package:testing/testing.dart' show Chain, runMe; const String ENABLE_FULL_COMPILE = " full compile "; +const String UPDATE_EXPECTATIONS = "updateExpectations"; +const String UPDATE_COMMENTS = "updateComments"; + const String EXPECTATIONS = ''' [ { @@ -289,6 +292,9 @@ class FastaContext extends ChainContext with MatchContext { @override final bool updateExpectations; + @override + String get updateExpectationsOption => '${UPDATE_EXPECTATIONS}=true'; + @override final ExpectationSet expectationSet = new ExpectationSet.fromJsonList(jsonDecode(EXPECTATIONS)); @@ -687,8 +693,8 @@ class FastaContext extends ChainContext with MatchContext { bool weak = environment["weak"] == "true"; bool onlyCrashes = environment["onlyCrashes"] == "true"; bool ignoreExpectations = environment["ignoreExpectations"] == "true"; - bool updateExpectations = environment["updateExpectations"] == "true"; - bool updateComments = environment["updateComments"] == "true"; + bool updateExpectations = environment[UPDATE_EXPECTATIONS] == "true"; + bool updateComments = environment[UPDATE_COMMENTS] == "true"; bool skipVm = environment["skipVm"] == "true"; bool verify = environment["verify"] != "false"; bool kernelTextSerialization = @@ -1098,7 +1104,8 @@ class Outline extends Step { description, p, userLibraries, options, sourceTarget), context.expectationSet["InstrumentationMismatch"], instrumentation.problemsAsString, - null); + null, + autoFixCommand: '${UPDATE_COMMENTS}=true'); } } } diff --git a/pkg/front_end/test/fasta/textual_outline_suite.dart b/pkg/front_end/test/fasta/textual_outline_suite.dart index db5dda5c709..aeb3343c551 100644 --- a/pkg/front_end/test/fasta/textual_outline_suite.dart +++ b/pkg/front_end/test/fasta/textual_outline_suite.dart @@ -22,6 +22,7 @@ import 'package:testing/testing.dart' runMe; import '../utils/kernel_chain.dart' show MatchContext; +import 'testing/suite.dart' show UPDATE_EXPECTATIONS; const List> EXPECTATIONS = [ { @@ -52,6 +53,10 @@ main([List arguments = const []]) => class Context extends ChainContext with MatchContext { final bool updateExpectations; + + @override + String get updateExpectationsOption => '${UPDATE_EXPECTATIONS}=true'; + Context(this.updateExpectations); final List steps = const [ diff --git a/pkg/front_end/test/parser_suite.dart b/pkg/front_end/test/parser_suite.dart index 632e9eea382..79145fbff75 100644 --- a/pkg/front_end/test/parser_suite.dart +++ b/pkg/front_end/test/parser_suite.dart @@ -46,6 +46,7 @@ import 'package:testing/testing.dart' TestDescription, runMe; +import 'fasta/testing/suite.dart' show UPDATE_EXPECTATIONS; import 'utils/kernel_chain.dart' show MatchContext; import 'parser_test_listener.dart' show ParserTestListener; @@ -86,6 +87,10 @@ ScannerConfiguration scannerConfigurationNonNNBD = new ScannerConfiguration( class Context extends ChainContext with MatchContext { final bool updateExpectations; + + @override + String get updateExpectationsOption => '${UPDATE_EXPECTATIONS}=true'; + final bool addTrace; final bool annotateLines; final String suiteName; diff --git a/pkg/front_end/test/unit_test_suites.dart b/pkg/front_end/test/unit_test_suites.dart index 1f420476422..c106ebf11a5 100644 --- a/pkg/front_end/test/unit_test_suites.dart +++ b/pkg/front_end/test/unit_test_suites.dart @@ -45,9 +45,10 @@ class Options { final bool printFailureLog; final Uri outputDirectory; final String testFilter; + final List environmentOptions; Options(this.configurationName, this.verbose, this.printFailureLog, - this.outputDirectory, this.testFilter); + this.outputDirectory, this.testFilter, this.environmentOptions); static Options parse(List args) { var parser = new ArgParser() @@ -59,7 +60,9 @@ class Options { ..addFlag("verbose", abbr: "v", help: "print additional information", defaultsTo: false) ..addFlag("print", - abbr: "p", help: "print failure logs", defaultsTo: false); + abbr: "p", help: "print failure logs", defaultsTo: false) + ..addMultiOption('environment', + abbr: 'D', help: "environment options for the test suite"); var parsedArguments = parser.parse(args); String outputPath = parsedArguments["output-directory"] ?? "."; Uri outputDirectory = Uri.base.resolveUri(Uri.directory(outputPath)); @@ -75,7 +78,8 @@ class Options { parsedArguments["verbose"], parsedArguments["print"], outputDirectory, - filter); + filter, + parsedArguments['environment']); } } @@ -142,8 +146,17 @@ class ResultLogger implements Logger { if (result.trace != null) { failureLog = "$failureLog\n\n${result.trace}"; } - failureLog = "$failureLog\n\nRe-run this test: dart " - "pkg/front_end/test/unit_test_suites.dart -p $testName"; + if (result.autoFixCommand != null) { + failureLog = "$failureLog\n\n" + "To re-run this test, run:\n\n" + " dart pkg/front_end/test/unit_test_suites.dart -p $testName\n\n" + "To automatically update the test expectations, run:\n\n" + " dart pkg/front_end/test/unit_test_suites.dart -p $testName " + "-D${result.autoFixCommand}\n"; + } else { + failureLog = "$failureLog\n\nRe-run this test: dart " + "pkg/front_end/test/unit_test_suites.dart -p $testName"; + } String outcome = "${result.outcome}"; logsPort.send(jsonEncode({ "name": testName, @@ -287,6 +300,8 @@ class SuiteConfiguration { final bool printFailureLog; final String configurationName; final String testFilter; + final List environmentOptions; + const SuiteConfiguration( this.name, this.resultsPort, @@ -294,7 +309,8 @@ class SuiteConfiguration { this.verbose, this.printFailureLog, this.configurationName, - this.testFilter); + this.testFilter, + this.environmentOptions); } void runSuite(SuiteConfiguration configuration) { @@ -312,9 +328,11 @@ void runSuite(SuiteConfiguration configuration) { configuration.verbose, configuration.printFailureLog, configuration.configurationName); - runMe( - [if (configuration.testFilter != null) configuration.testFilter], - suite.createContext, + runMe([ + if (configuration.testFilter != null) configuration.testFilter, + if (configuration.environmentOptions != null) + for (String option in configuration.environmentOptions) '-D${option}', + ], suite.createContext, me: suiteUri, configurationPath: suite.testingRootPath, logger: logger, @@ -359,7 +377,8 @@ main([List arguments = const []]) async { options.verbose, options.printFailureLog, options.configurationName, - filter); + filter, + options.environmentOptions); Future future = Future(() async { Stopwatch stopwatch = Stopwatch()..start(); print("Running suite $name"); diff --git a/pkg/front_end/test/utils/kernel_chain.dart b/pkg/front_end/test/utils/kernel_chain.dart index d530ed8de31..804af3cf00d 100644 --- a/pkg/front_end/test/utils/kernel_chain.dart +++ b/pkg/front_end/test/utils/kernel_chain.dart @@ -66,6 +66,8 @@ final Uri platformBinariesLocation = computePlatformBinariesLocation(); abstract class MatchContext implements ChainContext { bool get updateExpectations; + String get updateExpectationsOption; + ExpectationSet get expectationSet; Expectation get expectationFileMismatch => @@ -93,7 +95,10 @@ abstract class MatchContext implements ChainContext { String diff = await runDiff(expectedFile.uri, actual); onMismatch ??= expectationFileMismatch; return new Result(output, onMismatch, - "$uri doesn't match ${expectedFile.uri}\n$diff", null); + "$uri doesn't match ${expectedFile.uri}\n$diff", null, + autoFixCommand: onMismatch == expectationFileMismatch + ? updateExpectationsOption + : null); } else { return new Result.pass(output); } @@ -108,7 +113,8 @@ abstract class MatchContext implements ChainContext { """ Please create file ${expectedFile.path} with this content: $actual""", - null); + null, + autoFixCommand: updateExpectationsOption); } } diff --git a/pkg/front_end/testcases/inference_new/const_invocation.dart b/pkg/front_end/testcases/inference_new/const_invocation.dart index 85251743edf..4e58d44822f 100644 --- a/pkg/front_end/testcases/inference_new/const_invocation.dart +++ b/pkg/front_end/testcases/inference_new/const_invocation.dart @@ -8,7 +8,7 @@ library test; typedef V F(U u); class Foo { - Bar get v1 => const /*@ typeArgs=Null? */ Bar(); + Bar get v1 => const /*@typeArgs=Null?*/ Bar(); Bar> get v2 => const /*@ typeArgs=List* */ Bar(); Bar> get v3 => const /*@ typeArgs=(Object*) ->* Null? */ Bar(); Bar, T>> get v4 => diff --git a/pkg/testing/lib/src/chain.dart b/pkg/testing/lib/src/chain.dart index e51bcb45e2b..a6c982d4e79 100644 --- a/pkg/testing/lib/src/chain.dart +++ b/pkg/testing/lib/src/chain.dart @@ -354,7 +354,12 @@ class Result { final List logs = []; - Result(this.output, this.outcome, this.error, this.trace); + /// If set, running the test with '-D$autoFixCommand' will automatically + /// update the test to match new expectations. + final String autoFixCommand; + + Result(this.output, this.outcome, this.error, this.trace, + {this.autoFixCommand}); Result.pass(O output) : this(output, Expectation.Pass, null, null);