[cfe] Report distinct error codes in incremental_load_from_dill testing

This changes the reported failure from Crash to something more specific
to the encounter error, like ExpectationFileMismatch. Validation errors
for the .yaml file still produce Crash, and so does a couple of the
checks on Component integrity that were also used outside
incremental_load_from_dill_suite.dart

Change-Id: I48aa8cb25198d24e986b0f354db2b370dc9b8ac8
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/167565
Commit-Queue: Johnni Winther <johnniwinther@google.com>
Reviewed-by: Jens Johansen <jensj@google.com>

pkg/front_end/test/fasta/testing/suite.dart

@@ -1104,7 +1104,6 @@ class Outline extends Step<TestDescription, ComponentResult, FastaContext> {
                     description, p, userLibraries, options, sourceTarget),
                 context.expectationSet["InstrumentationMismatch"],
                 instrumentation.problemsAsString,
-                null,
                 autoFixCommand: '${UPDATE_COMMENTS}=true');
           }
         }
@@ -1194,8 +1193,7 @@ class Transform extends Step<ComponentResult, ComponentResult, FastaContext> {
       return new Result<ComponentResult>(
           result,
           context.expectationSet["TransformVerificationError"],
-          errors.join('\n'),
-          null);
+          errors.join('\n'));
     }
     return pass(result);
   }
@@ -1236,8 +1234,8 @@ class Verify extends Step<ComponentResult, ComponentResult, FastaContext> {
       if (messages.isEmpty) {
         return pass(result);
       } else {
-        return new Result<ComponentResult>(null,
-            context.expectationSet["VerificationError"], "$messages", null);
+        return new Result<ComponentResult>(
+            null, context.expectationSet["VerificationError"], "$messages");
       }
     }, errorOnMissingInput: false);
   }

pkg/front_end/test/fasta/textual_outline_suite.dart

@@ -81,8 +81,8 @@ class TextualOutline extends Step<TestDescription, TestDescription, Context> {
           performModelling: modelled,
           addMarkerForUnknownForTest: modelled);
       if (result == null) {
-        return new Result(null, context.expectationSet["EmptyOutput"],
-            description.uri, StackTrace.current);
+        return new Result(
+            null, context.expectationSet["EmptyOutput"], description.uri);
       }
 
       // In an attempt to make it less sensitive to formatting first remove
@@ -122,8 +122,9 @@ class TextualOutline extends Step<TestDescription, TestDescription, Context> {
       if (expectMatch.outcome != Expectation.Pass) return expectMatch;
 
       if (formatterException != null) {
-        return new Result(null, context.expectationSet["FormatterCrash"],
-            formatterException, formatterExceptionSt);
+        return new Result(
+            null, context.expectationSet["FormatterCrash"], formatterException,
+            trace: formatterExceptionSt);
       }
     }
 

pkg/front_end/test/incremental_load_from_dill_suite.dart

@@ -74,7 +74,7 @@ import 'package:kernel/target/targets.dart'
 import 'package:kernel/text/ast_to_text.dart' show Printer, componentToString;
 
 import "package:testing/testing.dart"
-    show Chain, ChainContext, Result, Step, TestDescription, runMe;
+    show Chain, ChainContext, Expectation, Result, Step, TestDescription, runMe;
 
 import "package:vm/target/vm.dart" show VmTarget;
 
@@ -89,6 +89,43 @@ import 'utils/io_utils.dart' show computeRepoDir;
 main([List<String> arguments = const []]) =>
     runMe(arguments, createContext, configurationPath: "../testing.json");
 
+const Expectation ExpectationFileMismatch =
+    const Expectation.fail("ExpectationFileMismatch");
+const Expectation ExpectationFileMissing =
+    const Expectation.fail("ExpectationFileMissing");
+const Expectation MissingErrors = const Expectation.fail("MissingErrors");
+const Expectation UnexpectedErrors = const Expectation.fail("UnexpectedErrors");
+const Expectation MissingWarnings = const Expectation.fail("MissingWarnings");
+const Expectation UnexpectedWarnings =
+    const Expectation.fail("UnexpectedWarnings");
+const Expectation ClassHierarchyError =
+    const Expectation.fail("ClassHierarchyError");
+const Expectation NeededDillMismatch =
+    const Expectation.fail("NeededDillMismatch");
+const Expectation IncrementalSerializationError =
+    const Expectation.fail("IncrementalSerializationError");
+const Expectation ContentDataMismatch =
+    const Expectation.fail("ContentDataMismatch");
+const Expectation MissingInitializationError =
+    const Expectation.fail("MissingInitializationError");
+const Expectation UnexpectedInitializationError =
+    const Expectation.fail("UnexpectedInitializationError");
+const Expectation ReachableLibrariesError =
+    const Expectation.fail("ReachableLibrariesError");
+const Expectation UriToSourceError = const Expectation.fail("UriToSourceError");
+const Expectation MissingPlatformLibraries =
+    const Expectation.fail("MissingPlatformLibraries");
+const Expectation UnexpectedPlatformLibraries =
+    const Expectation.fail("UnexpectedPlatformLibraries");
+const Expectation UnexpectedRebuildBodiesOnly =
+    const Expectation.fail("UnexpectedRebuildBodiesOnly");
+const Expectation UnexpectedEntryToLibraryCount =
+    const Expectation.fail("UnexpectedEntryToLibraryCount");
+const Expectation LibraryCountMismatch =
+    const Expectation.fail("LibraryCountMismatch");
+const Expectation InitializedFromDillMismatch =
+    const Expectation.fail("InitializedFromDillMismatch");
+
 Future<Context> createContext(
     Chain suite, Map<String, String> environment) async {
   // Disable colors to ensure that expectation files are the same across
@@ -152,6 +189,7 @@ class RunCompilations extends Step<TestData, TestData, Context> {
   String get name => "run compilations";
 
   Future<Result<TestData>> run(TestData data, Context context) async {
+    Result<TestData> result;
     YamlMap map = data.map;
     Set<String> keys = new Set<String>.from(map.keys.cast<String>());
     keys.remove("type");
@@ -175,7 +213,7 @@ class RunCompilations extends Step<TestData, TestData, Context> {
           "trackWidgetCreation",
           "incrementalSerialization"
         ]);
-        await new NewWorldTest().newWorldTest(
+        result = await new NewWorldTest().newWorldTest(
           data,
           context,
           map["worlds"],
@@ -192,7 +230,7 @@ class RunCompilations extends Step<TestData, TestData, Context> {
     }
 
     if (keys.isNotEmpty) throw "Unknown toplevel keys: $keys";
-    return pass(data);
+    return result ?? pass(data);
   }
 }
 
@@ -340,7 +378,7 @@ class NewWorldTest {
     return output;
   }
 
-  Future<Null> newWorldTest(
+  Future<Result<TestData>> newWorldTest(
       TestData data,
       Context context,
       List worlds,
@@ -611,8 +649,9 @@ class NewWorldTest {
           }
         }
       }
-      performErrorAndWarningCheck(
-          world, gotError, formattedErrors, gotWarning, formattedWarnings);
+      Result<TestData> result = performErrorAndWarningCheck(world, data,
+          gotError, formattedErrors, gotWarning, formattedWarnings);
+      if (result != null) return result;
       if (world["expectInitializationError"] != null) {
         Set<String> seenInitializationError = seenDiagnosticCodes.intersection({
           "InitializeFromDillNotSelfContainedNoDump",
@@ -622,12 +661,16 @@ class NewWorldTest {
         });
         if (world["expectInitializationError"] == true) {
           if (seenInitializationError.isEmpty) {
-            throw "Expected to see an initialization error but didn't.";
+            return new Result<TestData>(data, MissingInitializationError,
+                "Expected to see an initialization error but didn't.");
           }
         } else if (world["expectInitializationError"] == false) {
           if (seenInitializationError.isNotEmpty) {
-            throw "Expected not to see an initialization error but did: "
-                "$seenInitializationError.";
+            return new Result<TestData>(
+                data,
+                UnexpectedInitializationError,
+                "Expected not to see an initialization error but did: "
+                "$seenInitializationError.");
           }
         } else {
           throw "Unsupported value for 'expectInitializationError': "
@@ -639,8 +682,11 @@ class NewWorldTest {
           fileSystem: gotError ? null : fs);
       print("Compile took ${stopwatch.elapsedMilliseconds} ms");
 
-      checkExpectedContent(world, component);
-      checkNeededDillLibraries(world, compiler.neededDillLibraries, base);
+      Result contentResult = checkExpectedContent(world, component);
+      if (contentResult != null) return contentResult.copyWithOutput(data);
+      result = checkNeededDillLibraries(
+          world, data, compiler.neededDillLibraries, base);
+      if (result != null) return result;
 
       if (!noFullComponent) {
         Set<Library> allLibraries = new Set<Library>();
@@ -648,13 +694,19 @@ class NewWorldTest {
           computeAllReachableLibrariesFor(lib, allLibraries);
         }
         if (allLibraries.length != component.libraries.length) {
-          Expect.fail("Expected for the reachable stuff to be equal to "
+          return new Result<TestData>(
+              data,
+              ReachableLibrariesError,
+              "Expected for the reachable stuff to be equal to "
               "${component.libraries} but it was $allLibraries");
         }
         Set<Library> tooMany = allLibraries.toSet()
           ..removeAll(component.libraries);
         if (tooMany.isNotEmpty) {
-          Expect.fail("Expected for the reachable stuff to be equal to "
+          return new Result<TestData>(
+              data,
+              ReachableLibrariesError,
+              "Expected for the reachable stuff to be equal to "
               "${component.libraries} but these were there too: $tooMany "
               "(and others were missing)");
         }
@@ -670,8 +722,11 @@ class NewWorldTest {
         for (String filename in world["uriToSourcesDoesntInclude"]) {
           Uri uri = base.resolve(filename);
           if (component.uriToSource[uri] != null) {
-            throw "Expected no uriToSource for $uri but found "
-                "${component.uriToSource[uri]}";
+            return new Result<TestData>(
+                data,
+                UriToSourceError,
+                "Expected no uriToSource for $uri but found "
+                "${component.uriToSource[uri]}");
           }
         }
       }
@@ -686,14 +741,20 @@ class NewWorldTest {
           // Dart scheme uris too.
           if (uri == null || uri.scheme == "org-dartlang-sdk") continue;
           if (!allowed.contains(uri)) {
-            throw "Expected no uriToSource for $uri but found "
-                "${component.uriToSource[uri]}";
+            return new Result<TestData>(
+                data,
+                UriToSourceError,
+                "Expected no uriToSource for $uri but found "
+                "${component.uriToSource[uri]}");
           }
         }
       }
 
-      checkExpectFile(data, worldNum, "", context, actualSerialized);
-      checkClassHierarchy(compiler, component, data, worldNum, context);
+      result = checkExpectFile(data, worldNum, "", context, actualSerialized);
+      if (result != null) return result;
+      result =
+          checkClassHierarchy(compiler, component, data, worldNum, context);
+      if (result != null) return result;
 
       int nonSyntheticLibraries = countNonSyntheticLibraries(component);
       int nonSyntheticPlatformLibraries =
@@ -701,36 +762,54 @@ class NewWorldTest {
       int syntheticLibraries = countSyntheticLibraries(component);
       if (world["expectsPlatform"] == true) {
         if (nonSyntheticPlatformLibraries < 5) {
-          throw "Expected to have at least 5 platform libraries "
+          return new Result<TestData>(
+              data,
+              MissingPlatformLibraries,
+              "Expected to have at least 5 platform libraries "
               "(actually, the entire sdk), "
-              "but got $nonSyntheticPlatformLibraries.";
+              "but got $nonSyntheticPlatformLibraries.");
         }
       } else {
         if (nonSyntheticPlatformLibraries != 0) {
-          throw "Expected to have 0 platform libraries "
-              "but got $nonSyntheticPlatformLibraries.";
+          return new Result<TestData>(
+              data,
+              UnexpectedPlatformLibraries,
+              "Expected to have 0 platform libraries "
+              "but got $nonSyntheticPlatformLibraries.");
         }
       }
       if (world["expectedLibraryCount"] != null) {
         if (nonSyntheticLibraries - nonSyntheticPlatformLibraries !=
             world["expectedLibraryCount"]) {
-          throw "Expected ${world["expectedLibraryCount"]} non-synthetic "
+          return new Result<TestData>(
+              data,
+              LibraryCountMismatch,
+              "Expected ${world["expectedLibraryCount"]} non-synthetic "
               "libraries, got "
               "${nonSyntheticLibraries - nonSyntheticPlatformLibraries} "
-              "(not counting platform libraries)";
+              "(not counting platform libraries)");
         }
       }
       if (world["expectedSyntheticLibraryCount"] != null) {
         if (syntheticLibraries != world["expectedSyntheticLibraryCount"]) {
-          throw "Expected ${world["expectedSyntheticLibraryCount"]} synthetic "
-              "libraries, got ${syntheticLibraries}";
+          return new Result<TestData>(
+              data,
+              LibraryCountMismatch,
+              "Expected ${world["expectedSyntheticLibraryCount"]} synthetic "
+              "libraries, got ${syntheticLibraries}");
         }
       }
 
       if (world["expectsRebuildBodiesOnly"] != null) {
         bool didRebuildBodiesOnly = compiler.rebuildBodiesCount > 0;
-        Expect.equals(world["expectsRebuildBodiesOnly"], didRebuildBodiesOnly,
-            "Whether we expected to rebuild bodies only.");
+        if (world["expectsRebuildBodiesOnly"] != didRebuildBodiesOnly) {
+          return new Result<TestData>(
+              data,
+              UnexpectedRebuildBodiesOnly,
+              "Expected didRebuildBodiesOnly="
+              "${world["expectsRebuildBodiesOnly"]}, "
+              "didRebuildBodiesOnly=${didRebuildBodiesOnly}.");
+        }
       }
 
       if (!noFullComponent) {
@@ -740,14 +819,20 @@ class NewWorldTest {
                 entries.contains(lib.fileUri))
             .toList();
         if (entryLib.length != entries.length) {
-          throw "Expected the entries to become libraries. "
+          return new Result<TestData>(
+              data,
+              UnexpectedEntryToLibraryCount,
+              "Expected the entries to become libraries. "
               "Got ${entryLib.length} libraries for the expected "
-              "${entries.length} entries.";
+              "${entries.length} entries.");
         }
       }
       if (compiler.initializedFromDill != expectInitializeFromDill) {
-        throw "Expected that initializedFromDill would be "
-            "$expectInitializeFromDill but was ${compiler.initializedFromDill}";
+        return new Result<TestData>(
+            data,
+            InitializedFromDillMismatch,
+            "Expected that initializedFromDill would be "
+            "$expectInitializeFromDill but was ${compiler.initializedFromDill}");
       }
 
       if (incrementalSerialization == true && compiler.initializedFromDill) {
@@ -772,8 +857,12 @@ class NewWorldTest {
           Expect.isNull(world["expectedInvalidatedUri"]);
         }
       }
-      List<int> incrementalSerializationBytes = checkIncrementalSerialization(
+      Result<List<int>> serializationResult = checkIncrementalSerialization(
           incrementalSerialization, component, incrementalSerializer, world);
+      if (!serializationResult.isPass) {
+        return serializationResult.copyWithOutput(data);
+      }
+      List<int> incrementalSerializationBytes = serializationResult.output;
 
       Set<String> prevFormattedErrors = formattedErrors.toSet();
       Set<String> prevFormattedWarnings = formattedWarnings.toSet();
@@ -791,8 +880,9 @@ class NewWorldTest {
             entryPoints: entries,
             fullComponent: true,
             simulateTransformer: world["simulateTransformer"]);
-        performErrorAndWarningCheck(
-            world, gotError, formattedErrors, gotWarning, formattedWarnings);
+        Result<TestData> result = performErrorAndWarningCheck(world, data,
+            gotError, formattedErrors, gotWarning, formattedWarnings);
+        if (result != null) return result;
         List<int> thisWholeComponent = util.postProcess(component2);
         print("*****\n\ncomponent2:\n"
             "${componentToStringSdkFiltered(component2)}\n\n\n");
@@ -801,16 +891,22 @@ class NewWorldTest {
             prevFormattedWarnings, formattedWarnings);
         newestWholeComponent = component2;
 
-        List<int> incrementalSerializationBytes2 =
-            checkIncrementalSerialization(incrementalSerialization, component2,
-                incrementalSerializer, world);
+        Result<List<int>> serializationResult = checkIncrementalSerialization(
+            incrementalSerialization, component2, incrementalSerializer, world);
+        if (!serializationResult.isPass) {
+          return serializationResult.copyWithOutput(data);
+        }
+        List<int> incrementalSerializationBytes2 = serializationResult.output;
 
         if ((incrementalSerializationBytes == null &&
                 incrementalSerializationBytes2 != null) ||
             (incrementalSerializationBytes != null &&
                 incrementalSerializationBytes2 == null)) {
-          throw "Incremental serialization gave results in one instance, "
-              "but not another.";
+          return new Result<TestData>(
+              data,
+              IncrementalSerializationError,
+              "Incremental serialization gave results in one instance, "
+              "but not another.");
         }
 
         if (incrementalSerializationBytes != null) {
@@ -837,22 +933,29 @@ class NewWorldTest {
           Procedure procedure = await compiler.compileExpression(
               expression, {}, [], "debugExpr", uri);
           if (gotError && !expectErrors) {
-            throw "Got error(s) on expression compilation: ${formattedErrors}.";
+            return new Result<TestData>(data, UnexpectedErrors,
+                "Got error(s) on expression compilation: ${formattedErrors}.");
           } else if (!gotError && expectErrors) {
-            throw "Didn't get any errors.";
+            return new Result<TestData>(
+                data, MissingErrors, "Didn't get any errors.");
           }
           if (gotWarning && !expectWarnings) {
-            throw "Got warning(s) on expression compilation: "
-                "${formattedWarnings}.";
+            return new Result<TestData>(
+                data,
+                UnexpectedWarnings,
+                "Got warning(s) on expression compilation: "
+                "${formattedWarnings}.");
           } else if (!gotWarning && expectWarnings) {
-            throw "Didn't get any warnings.";
+            return new Result<TestData>(
+                data, MissingWarnings, "Didn't get any warnings.");
           }
-          checkExpectFile(
+          Result<TestData> result = checkExpectFile(
               data,
               worldNum,
               ".expression.$expressionCompilationNum",
               context,
               nodeToString(procedure));
+          if (result != null) return result;
         }
       }
 
@@ -885,8 +988,9 @@ class NewWorldTest {
             entryPoints: entries,
             simulateTransformer: world["simulateTransformer"]);
         compilerFromScratch = null;
-        performErrorAndWarningCheck(
-            world, gotError, formattedErrors, gotWarning, formattedWarnings);
+        Result<TestData> result = performErrorAndWarningCheck(world, data,
+            gotError, formattedErrors, gotWarning, formattedWarnings);
+        if (result != null) return result;
         util.throwOnEmptyMixinBodies(component3);
         await util.throwOnInsufficientUriToSource(component3);
         print("Compile took ${stopwatch.elapsedMilliseconds} ms");
@@ -897,16 +1001,25 @@ class NewWorldTest {
         checkErrorsAndWarnings(prevFormattedErrors, formattedErrors,
             prevFormattedWarnings, formattedWarnings);
 
-        List<int> incrementalSerializationBytes3 =
-            checkIncrementalSerialization(incrementalSerialization, component3,
-                incrementalSerializer2, world);
+        Result<List<int>> serializationResult = checkIncrementalSerialization(
+            incrementalSerialization,
+            component3,
+            incrementalSerializer2,
+            world);
+        if (!serializationResult.isPass) {
+          return serializationResult.copyWithOutput(data);
+        }
+        List<int> incrementalSerializationBytes3 = serializationResult.output;
 
         if ((incrementalSerializationBytes == null &&
                 incrementalSerializationBytes3 != null) ||
             (incrementalSerializationBytes != null &&
                 incrementalSerializationBytes3 == null)) {
-          throw "Incremental serialization gave results in one instance, "
-              "but not another.";
+          return new Result<TestData>(
+              data,
+              IncrementalSerializationError,
+              "Incremental serialization gave results in one instance, "
+              "but not another.");
         }
 
         if (incrementalSerializationBytes != null) {
@@ -930,11 +1043,12 @@ class NewWorldTest {
         print("Continuing after debug break");
       }
     }
+    return new Result<TestData>.pass(data);
   }
 }
 
-void checkExpectFile(TestData data, int worldNum, String extraUriString,
-    Context context, String actualSerialized) {
+Result<TestData> checkExpectFile(TestData data, int worldNum,
+    String extraUriString, Context context, String actualSerialized) {
   Uri uri = data.loadedFrom.resolve(data.loadedFrom.pathSegments.last +
       ".world.$worldNum${extraUriString}.expect");
   String expected;
@@ -948,24 +1062,32 @@ void checkExpectFile(TestData data, int worldNum, String extraUriString,
     } else {
       String extra = "";
       if (expected == null) extra = "Expect file did not exist.\n";
-      throw "${extra}Unexpected serialized representation. "
+      return new Result<TestData>(
+          data,
+          expected == null ? ExpectationFileMissing : ExpectationFileMismatch,
+          "${extra}Unexpected serialized representation. "
           "Fix or update $uri to contain the below:\n\n"
-          "$actualSerialized";
+          "$actualSerialized",
+          autoFixCommand: "updateExpectations=true");
     }
   }
+  return null;
 }
 
 /// Check that the class hierarchy is up-to-date with reality.
 ///
 /// This has the option to do expect files, but it's disabled by default
 /// while we're trying to figure out if it's useful or not.
-void checkClassHierarchy(TestIncrementalCompiler compiler, Component component,
-    TestData data, int worldNum, Context context,
+Result<TestData> checkClassHierarchy(TestIncrementalCompiler compiler,
+    Component component, TestData data, int worldNum, Context context,
     {bool checkExpectFile: false}) {
   ClassHierarchy classHierarchy = compiler.getClassHierarchy();
   if (classHierarchy is! ClosedWorldClassHierarchy) {
-    throw "Expected the class hierarchy to be ClosedWorldClassHierarchy "
-        "but it wasn't. It was ${classHierarchy.runtimeType}";
+    return new Result<TestData>(
+        data,
+        ClassHierarchyError,
+        "Expected the class hierarchy to be ClosedWorldClassHierarchy "
+        "but it wasn't. It was ${classHierarchy.runtimeType}");
   }
   List<ForTestingClassInfo> classHierarchyData =
       (classHierarchy as ClosedWorldClassHierarchy).getTestingClassInfo();
@@ -973,7 +1095,8 @@ void checkClassHierarchy(TestIncrementalCompiler compiler, Component component,
       new Map<Class, ForTestingClassInfo>();
   for (ForTestingClassInfo info in classHierarchyData) {
     if (classHierarchyMap[info.classNode] != null) {
-      throw "Two entries for ${info.classNode}";
+      return new Result<TestData>(
+          data, ClassHierarchyError, "Two entries for ${info.classNode}");
     }
     classHierarchyMap[info.classNode] = info;
   }
@@ -986,28 +1109,35 @@ void checkClassHierarchy(TestIncrementalCompiler compiler, Component component,
       sb.writeln("  - Class ${c.name}");
 
       Set<Class> checkedSupertypes = <Class>{};
-      void checkSupertype(Supertype supertype) {
-        if (supertype == null) return;
+      Result<TestData> checkSupertype(Supertype supertype) {
+        if (supertype == null) return null;
         Class superclass = supertype.classNode;
         if (checkedSupertypes.add(superclass)) {
           Supertype asSuperClass =
               classHierarchy.getClassAsInstanceOf(c, superclass);
           if (asSuperClass == null) {
-            throw "${superclass} not found as a superclass of $c";
+            return new Result<TestData>(data, ClassHierarchyError,
+                "${superclass} not found as a superclass of $c");
           }
-          checkSupertype(superclass.supertype);
-          checkSupertype(superclass.mixedInType);
+          Result<TestData> result = checkSupertype(superclass.supertype);
+          if (result != null) return result;
+          result = checkSupertype(superclass.mixedInType);
+          if (result != null) return result;
           for (Supertype interface in superclass.implementedTypes) {
-            checkSupertype(interface);
+            result = checkSupertype(interface);
+            if (result != null) return result;
           }
         }
+        return null;
       }
 
-      checkSupertype(c.asThisSupertype);
+      Result<TestData> result = checkSupertype(c.asThisSupertype);
+      if (result != null) return result;
 
       ForTestingClassInfo info = classHierarchyMap[c];
       if (info == null) {
-        throw "Didn't find any class hierarchy info for $c";
+        return new Result<TestData>(data, ClassHierarchyError,
+            "Didn't find any class hierarchy info for $c");
       }
 
       if (info.lazyDeclaredGettersAndCalls != null) {
@@ -1022,21 +1152,30 @@ void checkClassHierarchy(TestIncrementalCompiler compiler, Component component,
           if (f.isStatic) continue;
           if (!f.hasImplicitGetter) continue;
           if (!members.remove(f)) {
-            throw "Didn't find ${f.name.text} in lazyDeclaredGettersAndCalls "
-                "for ${c.name} in ${library.importUri}";
+            return new Result<TestData>(
+                data,
+                ClassHierarchyError,
+                "Didn't find ${f.name.text} in lazyDeclaredGettersAndCalls "
+                "for ${c.name} in ${library.importUri}");
           }
         }
         for (Procedure p in c.procedures) {
           if (p.isStatic) continue;
           if (p.isSetter) continue;
           if (!members.remove(p)) {
-            throw "Didn't find ${p.name.text} in lazyDeclaredGettersAndCalls "
-                "for ${c.name} in ${library.importUri}";
+            return new Result<TestData>(
+                data,
+                ClassHierarchyError,
+                "Didn't find ${p.name.text} in lazyDeclaredGettersAndCalls "
+                "for ${c.name} in ${library.importUri}");
           }
         }
         if (members.isNotEmpty) {
-          throw "Still have ${members.map((m) => m.name.text)} left "
-              "for ${c.name} in ${library.importUri}";
+          return new Result<TestData>(
+              data,
+              ClassHierarchyError,
+              "Still have ${members.map((m) => m.name.text)} left "
+              "for ${c.name} in ${library.importUri}");
         }
       }
       if (info.lazyDeclaredSetters != null) {
@@ -1051,19 +1190,24 @@ void checkClassHierarchy(TestIncrementalCompiler compiler, Component component,
           if (f.isStatic) continue;
           if (!f.hasImplicitSetter) continue;
           if (!members.remove(f)) {
-            throw "Didn't find $f in lazyDeclaredSetters for $c";
+            return new Result<TestData>(data, ClassHierarchyError,
+                "Didn't find $f in lazyDeclaredSetters for $c");
           }
         }
         for (Procedure p in c.procedures) {
           if (p.isStatic) continue;
           if (!p.isSetter) continue;
           if (!members.remove(p)) {
-            throw "Didn't find $p in lazyDeclaredSetters for $c";
+            return new Result<TestData>(data, ClassHierarchyError,
+                "Didn't find $p in lazyDeclaredSetters for $c");
           }
         }
         if (members.isNotEmpty) {
-          throw "Still have ${members.map((m) => m.name.text)} left "
-              "for ${c.name} in ${library.importUri}";
+          return new Result<TestData>(
+              data,
+              ClassHierarchyError,
+              "Still have ${members.map((m) => m.name.text)} left "
+              "for ${c.name} in ${library.importUri}");
         }
       }
       if (info.lazyImplementedGettersAndCalls != null) {
@@ -1107,12 +1251,16 @@ void checkClassHierarchy(TestIncrementalCompiler compiler, Component component,
       } else {
         String extra = "";
         if (expected == null) extra = "Expect file did not exist.\n";
-        throw "${extra}Unexpected serialized representation. "
+        return new Result<TestData>(
+            data,
+            ClassHierarchyError,
+            "${extra}Unexpected serialized representation. "
             "Fix or update $uri to contain the below:\n\n"
-            "$actualClassHierarchy";
+            "$actualClassHierarchy");
       }
     }
   }
+  return null;
 }
 
 void checkErrorsAndWarnings(
@@ -1144,7 +1292,7 @@ void checkErrorsAndWarnings(
   }
 }
 
-List<int> checkIncrementalSerialization(
+Result<List<int>> checkIncrementalSerialization(
     bool incrementalSerialization,
     Component component,
     IncrementalSerializer incrementalSerializer,
@@ -1159,18 +1307,26 @@ List<int> checkIncrementalSerialization(
     incrementalSerializer.writePackagesToSinkAndTrimComponent(c, sink);
     int librariesAfter = c.libraries.length;
     if (librariesAfter > librariesBefore) {
-      throw "Incremental serialization added libraries!";
+      return new Result<List<int>>(null, IncrementalSerializationError,
+          "Incremental serialization added libraries!");
     }
     if (librariesBefore == librariesAfter &&
         world["incrementalSerializationDoesWork"] == true) {
-      throw "Incremental serialization didn't remove any libraries!";
+      return new Result<List<int>>(null, IncrementalSerializationError,
+          "Incremental serialization didn't remove any libraries!");
     }
     if (librariesAfter < librariesBefore && sink.builder.isEmpty) {
-      throw "Incremental serialization didn't output any bytes, "
-          "but did remove libraries";
+      return new Result<List<int>>(
+          null,
+          IncrementalSerializationError,
+          "Incremental serialization didn't output any bytes, "
+          "but did remove libraries");
     } else if (librariesAfter == librariesBefore && !sink.builder.isEmpty) {
-      throw "Incremental serialization did output bytes, "
-          "but didn't remove libraries";
+      return new Result<List<int>>(
+          null,
+          IncrementalSerializationError,
+          "Incremental serialization did output bytes, "
+          "but didn't remove libraries");
     }
     if (librariesAfter < librariesBefore) {
       // If we actually did incrementally serialize anything, check the output!
@@ -1191,8 +1347,11 @@ List<int> checkIncrementalSerialization(
         for (String uriString in serializationShouldNotInclude) {
           Uri uri = Uri.parse(uriString);
           if (includedImportUris.contains(uri)) {
-            throw "Incremental serialization shouldn't include "
-                "$uriString but did.";
+            return new Result<List<int>>(
+                null,
+                IncrementalSerializationError,
+                "Incremental serialization shouldn't include "
+                "$uriString but did.");
           }
         }
       }
@@ -1207,18 +1366,20 @@ List<int> checkIncrementalSerialization(
       for (String key in newKeys) {
         afterContent.remove(key);
       }
-      checkExpectedContentData(afterContent, originalContent);
+      Result result = checkExpectedContentData(afterContent, originalContent);
+      if (result != null) return result.copyWithOutput<List<int>>(null);
 
       // Check that the result is self-contained.
-      checkSelfContained(loadedComponent);
+      result = checkSelfContained(loadedComponent);
+      if (result != null) return result.copyWithOutput<List<int>>(null);
 
-      return bytes;
+      return new Result<List<int>>.pass(bytes);
     }
   }
-  return null;
+  return new Result<List<int>>.pass(null);
 }
 
-void checkSelfContained(Component component) {
+Result checkSelfContained(Component component) {
   Set<Library> got = new Set<Library>.from(component.libraries);
   for (Library lib in component.libraries) {
     for (LibraryDependency dependency in lib.dependencies) {
@@ -1229,11 +1390,15 @@ void checkSelfContained(Component component) {
             .startsWith("root::dart:")) {
           continue;
         }
-        throw "Component didn't contain ${dependency.importedLibraryReference} "
-            "and it should have.";
+        return Result(
+            null,
+            IncrementalSerializationError,
+            "Component didn't contain ${dependency.importedLibraryReference} "
+            "and it should have.");
       }
     }
   }
+  return null;
 }
 
 void computeAllReachableLibrariesFor(Library lib, Set<Library> allLibraries) {
@@ -1254,33 +1419,46 @@ void computeAllReachableLibrariesFor(Library lib, Set<Library> allLibraries) {
   }
 }
 
-void checkExpectedContent(YamlMap world, Component component) {
+Result checkExpectedContent(YamlMap world, Component component) {
   if (world["expectedContent"] != null) {
     Map<String, Set<String>> actualContent = buildMapOfContent(component);
     Map expectedContent = world["expectedContent"];
-    checkExpectedContentData(actualContent, expectedContent);
+    return checkExpectedContentData(actualContent, expectedContent);
   }
+  return null;
 }
 
-void checkExpectedContentData(
+Result checkExpectedContentData(
     Map<String, Set<String>> actualContent, Map expectedContent) {
-  doThrow() {
-    throw "Expected and actual content not the same.\n"
+  Result<TestData> createFailureResult() {
+    return new Result(
+        null,
+        ContentDataMismatch,
+        "Expected and actual content not the same.\n"
         "Expected $expectedContent.\n"
-        "Got $actualContent";
+        "Got $actualContent");
   }
 
-  if (actualContent.length != expectedContent.length) doThrow();
+  if (actualContent.length != expectedContent.length) {
+    return createFailureResult();
+  }
   Set<String> missingKeys = actualContent.keys.toSet()
     ..removeAll(expectedContent.keys);
-  if (missingKeys.isNotEmpty) doThrow();
+  if (missingKeys.isNotEmpty) {
+    return createFailureResult();
+  }
   for (String key in expectedContent.keys) {
     Set<String> expected = new Set<String>.from(expectedContent[key]);
     Set<String> actual = actualContent[key].toSet();
-    if (expected.length != actual.length) doThrow();
+    if (expected.length != actual.length) {
+      return createFailureResult();
+    }
     actual.removeAll(expected);
-    if (actual.isNotEmpty) doThrow();
+    if (actual.isNotEmpty) {
+      return createFailureResult();
+    }
   }
+  return null;
 }
 
 Map<String, Set<String>> buildMapOfContent(Component component) {
@@ -1301,8 +1479,8 @@ Map<String, Set<String>> buildMapOfContent(Component component) {
   return actualContent;
 }
 
-void checkNeededDillLibraries(
-    YamlMap world, Set<Library> neededDillLibraries, Uri base) {
+Result<TestData> checkNeededDillLibraries(
+    YamlMap world, TestData data, Set<Library> neededDillLibraries, Uri base) {
   if (world["neededDillLibraries"] != null) {
     List<Uri> actualContent = new List<Uri>();
     for (Library lib in neededDillLibraries) {
@@ -1315,20 +1493,30 @@ void checkNeededDillLibraries(
       expectedContent.add(base.resolve(entry));
     }
 
-    doThrow() {
-      throw "Expected and actual content not the same.\n"
+    Result<TestData> createFailureResult() {
+      return new Result<TestData>(
+          data,
+          NeededDillMismatch,
+          "Expected and actual content not the same.\n"
           "Expected $expectedContent.\n"
-          "Got $actualContent";
+          "Got $actualContent");
     }
 
-    if (actualContent.length != expectedContent.length) doThrow();
+    if (actualContent.length != expectedContent.length) {
+      return createFailureResult();
+    }
     Set<Uri> notInExpected =
         actualContent.toSet().difference(expectedContent.toSet());
     Set<Uri> notInActual =
         expectedContent.toSet().difference(actualContent.toSet());
-    if (notInExpected.isNotEmpty) doThrow();
-    if (notInActual.isNotEmpty) doThrow();
+    if (notInExpected.isNotEmpty) {
+      return createFailureResult();
+    }
+    if (notInActual.isNotEmpty) {
+      return createFailureResult();
+    }
   }
+  return null;
 }
 
 String nodeToString(TreeNode node) {
@@ -1386,22 +1574,28 @@ int countSyntheticLibraries(Component c) {
   return result;
 }
 
-void performErrorAndWarningCheck(
+Result<TestData> performErrorAndWarningCheck(
     YamlMap world,
+    TestData data,
     bool gotError,
     Set<String> formattedErrors,
     bool gotWarning,
     Set<String> formattedWarnings) {
   if (world["errors"] == true && !gotError) {
-    throw "Expected error, but didn't get any.";
+    return new Result<TestData>(
+        data, MissingErrors, "Expected error, but didn't get any.");
   } else if (world["errors"] != true && gotError) {
-    throw "Got unexpected error(s): $formattedErrors.";
+    return new Result<TestData>(
+        data, UnexpectedErrors, "Got unexpected error(s): $formattedErrors.");
   }
   if (world["warnings"] == true && !gotWarning) {
-    throw "Expected warning, but didn't get any.";
+    return new Result<TestData>(
+        data, MissingWarnings, "Expected warning, but didn't get any.");
   } else if (world["warnings"] != true && gotWarning) {
-    throw "Got unexpected warnings(s): $formattedWarnings.";
+    return new Result<TestData>(data, UnexpectedWarnings,
+        "Got unexpected warnings(s): $formattedWarnings.");
   }
+  return null;
 }
 
 void checkIsEqual(List<int> a, List<int> b) {

pkg/front_end/test/utils/kernel_chain.dart

@@ -94,8 +94,8 @@ abstract class MatchContext implements ChainContext {
         }
         String diff = await runDiff(expectedFile.uri, actual);
         onMismatch ??= expectationFileMismatch;
-        return new Result<O>(output, onMismatch,
-            "$uri doesn't match ${expectedFile.uri}\n$diff", null,
+        return new Result<O>(
+            output, onMismatch, "$uri doesn't match ${expectedFile.uri}\n$diff",
             autoFixCommand: onMismatch == expectationFileMismatch
                 ? updateExpectationsOption
                 : null);
@@ -113,7 +113,6 @@ abstract class MatchContext implements ChainContext {
           """
 Please create file ${expectedFile.path} with this content:
 $actual""",
-          null,
           autoFixCommand: updateExpectationsOption);
     }
   }
@@ -179,8 +178,7 @@ class TypeCheck extends Step<ComponentResult, ComponentResult, ChainContext> {
       return new Result<ComponentResult>(
           null,
           context.expectationSet["TypeCheckError"],
-          '${errorFormatter.numberOfFailures} type errors',
-          null);
+          '${errorFormatter.numberOfFailures} type errors');
     }
   }
 }
@@ -377,11 +375,8 @@ class KernelTextSerialization
       }
 
       if (failures.isNotEmpty) {
-        return new Result<ComponentResult>(
-            null,
-            context.expectationSet["TextSerializationFailure"],
-            "$messages",
-            null);
+        return new Result<ComponentResult>(null,
+            context.expectationSet["TextSerializationFailure"], "$messages");
       }
       return pass(result);
     });

pkg/testing/lib/src/chain.dart

@@ -358,16 +358,18 @@ class Result<O> {
   /// update the test to match new expectations.
   final String autoFixCommand;
 
-  Result(this.output, this.outcome, this.error, this.trace,
-      {this.autoFixCommand});
+  Result(this.output, this.outcome, this.error,
+      {this.trace, this.autoFixCommand});
 
-  Result.pass(O output) : this(output, Expectation.Pass, null, null);
+  Result.pass(O output) : this(output, Expectation.Pass, null);
 
   Result.crash(error, StackTrace trace)
-      : this(null, Expectation.Crash, error, trace);
+      : this(null, Expectation.Crash, error, trace: trace);
 
   Result.fail(O output, [error, StackTrace trace])
-      : this(output, Expectation.Fail, error, trace);
+      : this(output, Expectation.Fail, error, trace: trace);
+
+  bool get isPass => outcome == Expectation.Pass;
 
   String get log => logs.join();
 
@@ -376,7 +378,14 @@ class Result<O> {
   }
 
   Result<O> copyWithOutcome(Expectation outcome) {
-    return new Result<O>(output, outcome, error, trace)..logs.addAll(logs);
+    return new Result<O>(output, outcome, error, trace: trace)
+      ..logs.addAll(logs);
+  }
+
+  Result<O2> copyWithOutput<O2>(O2 output) {
+    return new Result<O2>(output, outcome, error,
+        trace: trace, autoFixCommand: autoFixCommand)
+      ..logs.addAll(logs);
   }
 }
 

pkg/testing/lib/src/expectation.dart

@@ -34,6 +34,8 @@ class Expectation {
 
   const Expectation(this.name, this.group);
 
+  const Expectation.fail(this.name) : group = ExpectationGroup.Fail;
+
   /// Returns the canonical expectation representing [group]. That is, one of
   /// the above expectations (except for `Meta` which returns `this`).
   Expectation get canonical => fromGroup(group) ?? this;

pkg/testing/lib/src/stdio_process.dart

@@ -28,7 +28,7 @@ class StdioProcess {
       return new Result<int>.pass(exitCode);
     } else {
       return new Result<int>(
-          exitCode, ExpectationSet.Default["RuntimeError"], output, null);
+          exitCode, ExpectationSet.Default["RuntimeError"], output);
     }
   }