diff --git a/pkg/front_end/test/fasta/testing/suite.dart b/pkg/front_end/test/fasta/testing/suite.dart index 7e4bc88fc3f..0dd5247c22d 100644 --- a/pkg/front_end/test/fasta/testing/suite.dart +++ b/pkg/front_end/test/fasta/testing/suite.dart @@ -294,6 +294,9 @@ class FastaContext extends ChainContext with MatchContext { @override String get updateExpectationsOption => '${UPDATE_EXPECTATIONS}=true'; + @override + bool get canBeFixWithUpdateExpectations => true; + @override final ExpectationSet expectationSet = new ExpectationSet.fromJsonList(jsonDecode(EXPECTATIONS)); @@ -1117,7 +1120,8 @@ class Outline extends Step { description, p, userLibraries, options, sourceTarget), context.expectationSet["InstrumentationMismatch"], instrumentation.problemsAsString, - autoFixCommand: '${UPDATE_COMMENTS}=true'); + autoFixCommand: '${UPDATE_COMMENTS}=true', + canBeFixWithUpdateExpectations: true); } } } diff --git a/pkg/front_end/test/fasta/textual_outline_suite.dart b/pkg/front_end/test/fasta/textual_outline_suite.dart index f0dbb30935b..8e3827762e5 100644 --- a/pkg/front_end/test/fasta/textual_outline_suite.dart +++ b/pkg/front_end/test/fasta/textual_outline_suite.dart @@ -59,6 +59,9 @@ class Context extends ChainContext with MatchContext { @override String get updateExpectationsOption => '${UPDATE_EXPECTATIONS}=true'; + @override + bool get canBeFixWithUpdateExpectations => true; + Context(this.updateExpectations); final List steps = const [ diff --git a/pkg/front_end/test/incremental_load_from_dill_suite.dart b/pkg/front_end/test/incremental_load_from_dill_suite.dart index dba99230171..70a536004c4 100644 --- a/pkg/front_end/test/incremental_load_from_dill_suite.dart +++ b/pkg/front_end/test/incremental_load_from_dill_suite.dart @@ -1120,7 +1120,8 @@ Result checkExpectFile(TestData data, int worldNum, "${extra}Unexpected serialized representation. " "Fix or update $uri to contain the below:\n\n" "$actualSerialized", - autoFixCommand: "updateExpectations=true"); + autoFixCommand: "updateExpectations=true", + canBeFixWithUpdateExpectations: true); } } return null; diff --git a/pkg/front_end/test/parser_suite.dart b/pkg/front_end/test/parser_suite.dart index 70bbcad857e..3bc79a2fd38 100644 --- a/pkg/front_end/test/parser_suite.dart +++ b/pkg/front_end/test/parser_suite.dart @@ -101,6 +101,9 @@ class Context extends ChainContext with MatchContext { @override String get updateExpectationsOption => '${UPDATE_EXPECTATIONS}=true'; + @override + bool get canBeFixWithUpdateExpectations => true; + final bool addTrace; final bool annotateLines; final String suiteName; diff --git a/pkg/front_end/test/unit_test_suites.dart b/pkg/front_end/test/unit_test_suites.dart index 0bd8b6646d1..8e256f67eaf 100644 --- a/pkg/front_end/test/unit_test_suites.dart +++ b/pkg/front_end/test/unit_test_suites.dart @@ -138,24 +138,30 @@ class ResultLogger implements Logger { "matches": matchedExpectations, })); if (!matchedExpectations) { - String failureLog = result.log; + StringBuffer sb = new StringBuffer(); + sb.write(result.log); if (result.error != null) { - failureLog = "$failureLog\n\n${result.error}"; + sb.write("\n\n${result.error}"); } if (result.trace != null) { - failureLog = "$failureLog\n\n${result.trace}"; + sb.write("\n\n${result.trace}"); } + sb.write("\n\nTo re-run this test, run:"); + sb.write("\n\n dart pkg/front_end/test/unit_test_suites.dart -p " + "$testName"); if (result.autoFixCommand != null) { - failureLog = "$failureLog\n\n" - "To re-run this test, run:\n\n" - " dart pkg/front_end/test/unit_test_suites.dart -p $testName\n\n" - "To automatically update the test expectations, run:\n\n" - " dart pkg/front_end/test/unit_test_suites.dart -p $testName " - "-D${result.autoFixCommand}\n"; - } else { - failureLog = "$failureLog\n\nRe-run this test: dart " - "pkg/front_end/test/unit_test_suites.dart -p $testName"; + sb.write("\n\nTo automatically update the test expectations, run:"); + sb.write("\n\n dart pkg/front_end/test/unit_test_suites.dart -p " + "$testName -D${result.autoFixCommand}"); + if (result.canBeFixWithUpdateExpectations) { + sb.write('\n\nTo update test expectations for all tests at once, ' + 'run:'); + sb.write('\n\n dart pkg/front_end/tool/update_expectations.dart'); + sb.write('\n\nNote that this takes a long time and should only be ' + 'used when many tests need updating.\n'); + } } + String failureLog = sb.toString(); String outcome = "${result.outcome}"; logsPort.send(jsonEncode({ "name": testName, diff --git a/pkg/front_end/test/utils/kernel_chain.dart b/pkg/front_end/test/utils/kernel_chain.dart index 397879a3c62..b76a823faa9 100644 --- a/pkg/front_end/test/utils/kernel_chain.dart +++ b/pkg/front_end/test/utils/kernel_chain.dart @@ -66,6 +66,8 @@ abstract class MatchContext implements ChainContext { String get updateExpectationsOption; + bool get canBeFixWithUpdateExpectations; + ExpectationSet get expectationSet; Expectation get expectationFileMismatch => @@ -96,7 +98,10 @@ abstract class MatchContext implements ChainContext { output, onMismatch, "$uri doesn't match ${expectedFile.uri}\n$diff", autoFixCommand: onMismatch == expectationFileMismatch ? updateExpectationsOption - : null); + : null, + canBeFixWithUpdateExpectations: + onMismatch == expectationFileMismatch && + canBeFixWithUpdateExpectations); } else { return new Result.pass(output); } diff --git a/pkg/testing/lib/src/chain.dart b/pkg/testing/lib/src/chain.dart index c6f6c16ef4b..1f7fc847ea1 100644 --- a/pkg/testing/lib/src/chain.dart +++ b/pkg/testing/lib/src/chain.dart @@ -358,8 +358,16 @@ class Result { /// update the test to match new expectations. final String autoFixCommand; + /// If set, the test can be fixed by running + /// + /// dart pkg/front_end/tool/update_expectations.dart + /// + final bool canBeFixWithUpdateExpectations; + Result(this.output, this.outcome, this.error, - {this.trace, this.autoFixCommand}); + {this.trace, + this.autoFixCommand, + this.canBeFixWithUpdateExpectations: false}); Result.pass(O output) : this(output, Expectation.Pass, null); @@ -384,7 +392,9 @@ class Result { Result copyWithOutput(O2 output) { return new Result(output, outcome, error, - trace: trace, autoFixCommand: autoFixCommand) + trace: trace, + autoFixCommand: autoFixCommand, + canBeFixWithUpdateExpectations: canBeFixWithUpdateExpectations) ..logs.addAll(logs); } }