[analyzer,cfe,dart2js] Add marker.options files to automate running id tests on all configurations

Change-Id: Ib5eb7f3967634721047f8dcc57b9e6c350b91a41 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/136226 Reviewed-by: Dmitry Stefantsov <dmitryas@google.com> Reviewed-by: Paul Berry <paulberry@google.com> Commit-Queue: Johnni Winther <johnniwinther@google.com>
2020-02-21 08:34:26 +00:00
parent d986ab830e
commit 5829fc7829
62 changed files with 168 additions and 40 deletions
@@ -628,20 +628,95 @@ typedef Future<Map<String, TestResult<T>>> RunTestFunction<T>(TestData testData,
    Map<String, List<String>> skipMap,
    Uri nullUri});

+class MarkerOptions {
+  final Map<String, Uri> markers;
+
+  MarkerOptions.internal(this.markers);
+
+  factory MarkerOptions.fromFile(File file, {bool isUsingShards: false}) {
+    File script = new File.fromUri(Platform.script);
+    if (!file.existsSync()) {
+      throw new ArgumentError("Marker option file '$file' doesn't exist.");
+    }
+
+    Map<String, Uri> markers = {};
+    String text = file.readAsStringSync();
+    bool isScriptFound = false;
+    for (String line in text.split('\n')) {
+      line = line.trim();
+      if (line.isEmpty || line.startsWith('#')) continue;
+      int eqPos = line.indexOf('=');
+      if (eqPos == -1) {
+        throw new ArgumentError(
+            "Unsupported marker option '$line' in ${file.uri}");
+      }
+      String marker = line.substring(0, eqPos);
+      String tester = line.substring(eqPos + 1);
+      File testerFile = new File(tester);
+      if (!testerFile.existsSync()) {
+        throw new ArgumentError(
+            "Tester '$tester' does not exist for marker '$marker' in "
+            "${file.uri}");
+      }
+      if (markers.containsKey(marker)) {
+        throw new ArgumentError("Duplicate marker '$marker' in ${file.uri}");
+      }
+      markers[marker] = testerFile.uri;
+      if (testerFile.absolute.uri == script.absolute.uri) {
+        isScriptFound = true;
+      }
+    }
+    if (!isUsingShards && !isScriptFound) {
+      throw new ArgumentError(
+          "Script '${script.uri}' not found in ${file.uri}");
+    }
+    return new MarkerOptions.internal(markers);
+  }
+
+  Iterable<String> get supportedMarkers => markers.keys;
+
+  Future<void> runAll(List<String> args) async {
+    Set<Uri> testers = markers.values.toSet();
+    bool allOk = true;
+    for (Uri tester in testers) {
+      print('================================================================');
+      print('Running tester: $tester ${args.join(' ')}');
+      print('================================================================');
+      Process process = await Process.start(
+          Platform.resolvedExecutable, [tester.toString(), ...args],
+          mode: ProcessStartMode.inheritStdio);
+      if (await process.exitCode != 0) {
+        allOk = false;
+      }
+    }
+    if (!allOk) {
+      throw "Error(s) occurred.";
+    }
+  }
+}
+
 /// Check code for all tests in [dataDir] using [runTest].
 Future<void> runTests<T>(Directory dataDir,
    {List<String> args: const <String>[],
    int shards: 1,
    int shardIndex: 0,
    void onTest(Uri uri),
-    Iterable<String> supportedMarkers,
    Uri createUriForFileName(String fileName),
    void onFailure(String message),
    RunTestFunction<T> runTest,
    List<String> skipList,
    Map<String, List<String>> skipMap}) async {
+  File markerOptionsFile =
+      new File.fromUri(dataDir.uri.resolve('marker.options'));
+  MarkerOptions markerOptions =
+      new MarkerOptions.fromFile(markerOptionsFile, isUsingShards: shards != 1);
  // TODO(johnniwinther): Support --show to show actual data for an input.
  args = args.toList();
+  bool runAll = args.remove('--run-all');
+  if (runAll) {
+    await markerOptions.runAll(args);
+    return;
+  }
  bool verbose = args.remove('-v');
  bool succinct = args.remove('-s');
  bool shouldContinue = args.remove('-c');
@@ -653,8 +728,11 @@ Future<void> runTests<T>(Directory dataDir,

  String relativeDir = dataDir.uri.path.replaceAll(Uri.base.path, '');
  print('Data dir: ${relativeDir}');
-  List<FileSystemEntity> entities =
-      dataDir.listSync().where((entity) => !entity.path.endsWith('~')).toList();
+  List<FileSystemEntity> entities = dataDir
+      .listSync()
+      .where((entity) =>
+          !entity.path.endsWith('~') && !entity.path.endsWith('marker.options'))
+      .toList();
  if (shards > 1) {
    int start = entities.length * shardIndex ~/ shards;
    int end = entities.length * (shardIndex + 1) ~/ shards;
@@ -688,7 +766,7 @@ Future<void> runTests<T>(Directory dataDir,
    }

    TestData testData = computeTestData(entity,
-        supportedMarkers: supportedMarkers,
+        supportedMarkers: markerOptions.supportedMarkers,
        createTestUri: createTestUri,
        onFailure: onFailure);
    print('Test: ${testData.testFileUri}');
@@ -0,0 +1,3 @@
+cfe=pkg/front_end/test/id_tests/constant_test.dart
+analyzer=pkg/analyzer/test/id_tests/constant_test.dart
+dart2js=tests/compiler/dart2js/model/cfe_constant_test.dart
@@ -0,0 +1,2 @@
+cfe=pkg/front_end/test/id_tests/assigned_variables_test.dart
+analyzer=pkg/analyzer/test/id_tests/assigned_variables_test.dart
@@ -0,0 +1,2 @@
+cfe=pkg/front_end/test/id_tests/definite_assignment_test.dart
+analyzer=pkg/analyzer/test/id_tests/definite_assignment_test.dart
@@ -0,0 +1,2 @@
+cfe=pkg/front_end/test/id_tests/nullability_test.dart
+analyzer=pkg/analyzer/test/id_tests/nullability_test.dart
@@ -0,0 +1,2 @@
+cfe=pkg/front_end/test/id_tests/reachability_test.dart
+analyzer=pkg/analyzer/test/id_tests/reachability_test.dart
@@ -0,0 +1,2 @@
+cfe=pkg/front_end/test/id_tests/type_promotion_test.dart
+analyzer=pkg/analyzer/test/id_tests/type_promotion_test.dart
@@ -0,0 +1,2 @@
+cfe=pkg/front_end/test/id_tests/inheritance_test.dart
+cfe:builder=pkg/front_end/test/id_tests/inheritance_test.dart
@@ -23,7 +23,6 @@ main(List<String> args) async {
  await NullSafetyUnderstandingFlag.enableNullSafetyTypes(() {
    return runTests<_Data>(dataDir,
        args: args,
-        supportedMarkers: cfeAnalyzerMarkers,
        createUriForFileName: createUriForFileName,
        onFailure: onFailure,
        runTest: runTestFor(
@@ -22,7 +22,6 @@ main(List<String> args) async {
  await NullSafetyUnderstandingFlag.enableNullSafetyTypes(() {
    return runTests<String>(dataDir,
        args: args,
-        supportedMarkers: sharedMarkers,
        createUriForFileName: createUriForFileName,
        onFailure: onFailure,
        runTest: runTestFor(
@@ -22,7 +22,6 @@ main(List<String> args) async {
  await NullSafetyUnderstandingFlag.enableNullSafetyTypes(() {
    return runTests<String>(dataDir,
        args: args,
-        supportedMarkers: cfeAnalyzerMarkers,
        createUriForFileName: createUriForFileName,
        onFailure: onFailure,
        runTest: runTestFor(
@@ -25,7 +25,6 @@ main(List<String> args) async {
  await NullSafetyUnderstandingFlag.enableNullSafetyTypes(() {
    return runTests<String>(dataDir,
        args: args,
-        supportedMarkers: cfeAnalyzerMarkers,
        createUriForFileName: createUriForFileName,
        onFailure: onFailure,
        runTest:
@@ -22,7 +22,6 @@ main(List<String> args) async {
  await NullSafetyUnderstandingFlag.enableNullSafetyTypes(() {
    return runTests<Set<_ReachabilityAssertion>>(dataDir,
        args: args,
-        supportedMarkers: cfeAnalyzerMarkers,
        createUriForFileName: createUriForFileName,
        onFailure: onFailure,
        runTest: runTestFor(
@@ -23,7 +23,6 @@ main(List<String> args) async {
  await NullSafetyUnderstandingFlag.enableNullSafetyTypes(() {
    return runTests<DartType>(dataDir,
        args: args,
-        supportedMarkers: sharedMarkers,
        createUriForFileName: createUriForFileName,
        onFailure: onFailure,
        runTest: runTestFor(
@@ -82,6 +82,13 @@ Future<Map<String, TestResult<T>>> runTest<T>(TestData testData,
    Iterable<Id> globalIds = const <Id>[],
    void Function(String message) onFailure,
    Map<String, List<String>> skipMap}) async {
+  for (TestConfig config in testedConfigs) {
+    if (!testData.expectedMaps.containsKey(config.marker)) {
+      throw ArgumentError("Unexpected test marker '${config.marker}'. "
+          "Supported markers: ${testData.expectedMaps.keys}.");
+    }
+  }
+
  Map<String, TestResult<T>> results = {};
  for (TestConfig config in testedConfigs) {
    if (skipForConfig(testData.name, config.marker, skipMap)) {
@@ -235,6 +235,13 @@ Future<Map<String, TestResult<T>>> runTest<T>(TestData testData,
    void onFailure(String message),
    Map<String, List<String>> skipMap,
    Uri nullUri}) async {
+  for (TestConfig config in testedConfigs) {
+    if (!testData.expectedMaps.containsKey(config.marker)) {
+      throw new ArgumentError("Unexpected test marker '${config.marker}'. "
+          "Supported markers: ${testData.expectedMaps.keys}.");
+    }
+  }
+
  Map<String, TestResult<T>> results = {};
  for (TestConfig config in testedConfigs) {
    if (skipForConfig(testData.name, config.marker, skipMap)) {
@@ -15,7 +15,6 @@ main(List<String> args) async {
  Directory dataDir = new Directory.fromUri(Platform.script.resolve('data'));
  await runTests<String>(dataDir,
      args: args,
-      supportedMarkers: [cfeMarker],
      createUriForFileName: createUriForFileName,
      onFailure: onFailure,
      runTest:
@@ -0,0 +1 @@
+cfe=pkg/front_end/test/covariance_check/covariance_check_test.dart
@@ -0,0 +1 @@
+cfe=pkg/front_end/test/extensions/extensions_test.dart
@@ -24,7 +24,6 @@ main(List<String> args) async {
  Directory dataDir = new Directory.fromUri(Platform.script.resolve('data'));
  await runTests<Features>(dataDir,
      args: args,
-      supportedMarkers: [cfeMarker],
      createUriForFileName: createUriForFileName,
      onFailure: onFailure,
      runTest: runTestFor(const ExtensionsDataComputer(), [
@@ -0,0 +1,2 @@
+cfe=pkg/front_end/test/id_testing/id_testing_test.dart
+dart2js=tests/compiler/dart2js/equivalence/id_testing_test.dart
@@ -32,7 +32,6 @@ main(List<String> args) async {
  Directory dataDir = new Directory.fromUri(Platform.script.resolve('data'));
  await runTests<String>(dataDir,
      args: args,
-      supportedMarkers: [cfeMarker, dart2jsMarker],
      createUriForFileName: createUriForFileName,
      onFailure: onFailure,
      runTest: runTestFor(const IdTestingDataComputer(), [defaultCfeConfig]));
@@ -23,7 +23,6 @@ main(List<String> args) async {
      'data'));
  await runTests<_Data>(dataDir,
      args: args,
-      supportedMarkers: cfeAnalyzerMarkers,
      createUriForFileName: createUriForFileName,
      onFailure: onFailure,
      runTest: runTestFor<_Data>(
@@ -27,7 +27,6 @@ main(List<String> args) async {
      .resolve('../../../_fe_analyzer_shared/test/constants/data'));
  await runTests<String>(dataDir,
      args: args,
-      supportedMarkers: sharedMarkers,
      createUriForFileName: createUriForFileName,
      onFailure: onFailure,
      runTest: runTestFor(const ConstantsDataComputer(), [defaultCfeConfig]));
@@ -20,7 +20,6 @@ main(List<String> args) async {
      'data'));
  await runTests<String>(dataDir,
      args: args,
-      supportedMarkers: cfeAnalyzerMarkers,
      createUriForFileName: createUriForFileName,
      onFailure: onFailure,
      runTest: runTestFor(
@@ -20,7 +20,6 @@ main(List<String> args) async {
      .resolve('../../../_fe_analyzer_shared/test/inheritance/data'));
  await runTests<String>(dataDir,
      args: args,
-      supportedMarkers: [cfeMarker, cfeFromBuilderMarker],
      createUriForFileName: createUriForFileName,
      onFailure: onFailure,
      runTest: runTestFor(const InheritanceDataComputer(), [
@@ -15,7 +15,6 @@ main(List<String> args) async {
      '../../../_fe_analyzer_shared/test/flow_analysis/nullability/data'));
  await runTests<String>(dataDir,
      args: args,
-      supportedMarkers: cfeAnalyzerMarkers,
      createUriForFileName: createUriForFileName,
      onFailure: onFailure,
      runTest: runTestFor(
@@ -19,7 +19,6 @@ main(List<String> args) async {
      '../../../_fe_analyzer_shared/test/flow_analysis/reachability/data'));
  await runTests<Set<_ReachabilityAssertion>>(dataDir,
      args: args,
-      supportedMarkers: cfeAnalyzerMarkers,
      createUriForFileName: createUriForFileName,
      onFailure: onFailure,
      runTest: runTestFor(
@@ -17,7 +17,6 @@ main(List<String> args) async {
          'data'));
  await runTests<DartType>(dataDir,
      args: args,
-      supportedMarkers: cfeAnalyzerMarkers,
      createUriForFileName: createUriForFileName,
      onFailure: onFailure,
      runTest: runTestFor(
@@ -0,0 +1 @@
+cfe=pkg/front_end/test/language_versioning/language_versioning_test.dart
@@ -29,7 +29,6 @@ main(List<String> args) async {
  Directory dataDir = new Directory.fromUri(Platform.script.resolve('data'));
  await runTests<String>(dataDir,
      args: args,
-      supportedMarkers: [cfeMarker],
      createUriForFileName: createUriForFileName,
      onFailure: onFailure,
      runTest: runTestFor(const LanguageVersioningDataComputer(), [cfeConfig]),
@@ -0,0 +1,2 @@
+cfe=pkg/front_end/test/patching/patching_test.dart
+cfe:nnbd=pkg/front_end/test/patching/patching_test.dart
@@ -22,7 +22,6 @@ main(List<String> args) async {
  Directory dataDir = new Directory.fromUri(Platform.script.resolve('data'));
  await runTests<Features>(dataDir,
      args: args,
-      supportedMarkers: [cfeMarker, cfeWithNnbdMarker],
      createUriForFileName: createUriForFileName,
      onFailure: onFailure,
      runTest: runTestFor(const PatchingDataComputer(), [
@@ -949,6 +949,7 @@ stale
 statics
 stderr
 stdin
+stdio
 stdout
 stmt
 str
@@ -1015,6 +1016,7 @@ termcap
 terminator
 test
 tester
+testers
 testing
 tex
 tflite
@@ -0,0 +1,2 @@
+cfe=pkg/front_end/test/static_types/static_type_test.dart
+cfe:nnbd=pkg/front_end/test/static_types/static_type_test.dart
@@ -17,7 +17,6 @@ main(List<String> args) async {
  Directory dataDir = new Directory.fromUri(Platform.script.resolve('data'));
  await runTests<String>(dataDir,
      args: args,
-      supportedMarkers: cfeMarkersWithNnbd,
      createUriForFileName: createUriForFileName,
      onFailure: onFailure,
      runTest: runTestFor(const StaticTypeDataComputer(),
@@ -0,0 +1,2 @@
+strong=tests/compiler/dart2js/annotations/annotations_test.dart
+omit=tests/compiler/dart2js/annotations/annotations_test.dart
@@ -0,0 +1,2 @@
+strong=tests/compiler/dart2js/closure/closure_test.dart
+omit=tests/compiler/dart2js/closure/closure_test.dart
@@ -0,0 +1,2 @@
+strong=tests/compiler/dart2js/codegen/model_test.dart
+omit=tests/compiler/dart2js/codegen/model_test.dart
@@ -0,0 +1 @@
+strong=tests/compiler/dart2js/deferred_loading/deferred_loading_test.dart
@@ -34,7 +34,7 @@ main(List<String> args) {
    await checkTests(dataDir, const OutputUnitDataComputer(),
        options: compilerOptions,
        args: args,
-        supportedMarkers: [strongMarker], setUpFunction: () {
+        setUpFunction: () {
      importPrefixes.clear();
    }, testedConfigs: allStrongConfigs);
  });
@@ -398,7 +398,6 @@ Future<void> checkTests<T>(Directory dataDir, DataComputer<T> dataComputer,
    int shards: 1,
    int shardIndex: 0,
    void onTest(Uri uri),
-    Iterable<String> supportedMarkers = allInternalMarkers,
    List<TestConfig> testedConfigs = defaultInternalConfigs}) async {
  Set<String> testedMarkers =
      testedConfigs.map((config) => config.marker).toSet();
@@ -406,12 +405,6 @@ Future<void> checkTests<T>(Directory dataDir, DataComputer<T> dataComputer,
      testedConfigs.length == testedMarkers.length,
      "Unexpected test markers $testedMarkers. "
      "Tested configs: $testedConfigs.");
-  Iterable<String> unknownMarkers =
-      testedMarkers.where((marker) => !supportedMarkers.contains(marker));
-  Expect.isTrue(
-      unknownMarkers.isEmpty,
-      "Unexpected test markers $unknownMarkers. "
-      "Supported markers: $supportedMarkers.");

  dataComputer.setup();

@@ -422,6 +415,13 @@ Future<void> checkTests<T>(Directory dataDir, DataComputer<T> dataComputer,
      bool printCode,
      Map<String, List<String>> skipMap,
      Uri nullUri}) async {
+    for (TestConfig testConfiguration in testedConfigs) {
+      Expect.isTrue(
+          testData.expectedMaps.containsKey(testConfiguration.marker),
+          "Unexpected test marker '${testConfiguration.marker}'. "
+          "Supported markers: ${testData.expectedMaps.keys}.");
+    }
+
    String name = testData.name;
    List<String> testOptions = options.toList();
    if (name.endsWith('_ea.dart')) {
@@ -457,7 +457,6 @@ Future<void> checkTests<T>(Directory dataDir, DataComputer<T> dataComputer,
      shards: shards,
      shardIndex: shardIndex,
      onTest: onTest,
-      supportedMarkers: supportedMarkers,
      createUriForFileName: createUriForFileName,
      onFailure: Expect.fail,
      runTest: checkTest);
@@ -24,9 +24,7 @@ main(List<String> args) {
    Directory dataDir = new Directory.fromUri(Platform.script
        .resolve('../../../../pkg/front_end/test/id_testing/data'));
    await checkTests(dataDir, new IdTestingDataComputer(),
-        args: args,
-        testedConfigs: [sharedConfig],
-        supportedMarkers: [cfeMarker, dart2jsMarker]);
+        args: args, testedConfigs: [sharedConfig]);
  });
 }

@@ -0,0 +1,2 @@
+strong=tests/compiler/dart2js/field_analysis/jfield_analysis_test.dart
+omit=tests/compiler/dart2js/field_analysis/jfield_analysis_test.dart
@@ -0,0 +1,2 @@
+strong=tests/compiler/dart2js/field_analysis/kfield_analysis_test.dart
+omit=tests/compiler/dart2js/field_analysis/kfield_analysis_test.dart
@@ -0,0 +1 @@
+strong=tests/compiler/dart2js/impact/impact_test.dart
@@ -24,7 +24,6 @@ main(List<String> args) {
    useImpactDataForTesting = false;
    await checkTests(dataDir, const ImpactDataComputer(),
        args: args,
-        supportedMarkers: [strongMarker],
        testedConfigs: [strongConfig]);

    print('Testing computation of ResolutionImpact through ImpactData');
@@ -32,7 +31,6 @@ main(List<String> args) {
    useImpactDataForTesting = true;
    await checkTests(dataDir, const ImpactDataComputer(),
        args: args,
-        supportedMarkers: [strongMarker],
        testedConfigs: [strongConfig]);
  });
 }
@@ -0,0 +1,2 @@
+strong=tests/compiler/dart2js/inference/callers_test.dart
+omit=tests/compiler/dart2js/inference/callers_test.dart
@@ -0,0 +1,2 @@
+strong=tests/compiler/dart2js/inference/inference_test_helper.dart
+omit=tests/compiler/dart2js/inference/inference_test_helper.dart
@@ -0,0 +1 @@
+strong=tests/compiler/dart2js/inference/inference_data_test.dart
@@ -24,7 +24,6 @@ main(List<String> args) {
    await checkTests(dataDir, const InferenceDataComputer(),
        args: args,
        testedConfigs: [strongConfig],
-        supportedMarkers: [strongMarker],
        options: [stopAfterTypeInference]);
  });
 }
@@ -0,0 +1,2 @@
+strong=tests/compiler/dart2js/inference/side_effects_test.dart
+omit=tests/compiler/dart2js/inference/side_effects_test.dart
@@ -0,0 +1,2 @@
+strong=tests/compiler/dart2js/inlining/inlining_test.dart
+omit=tests/compiler/dart2js/inlining/inlining_test.dart
@@ -0,0 +1,2 @@
+strong=tests/compiler/dart2js/jumps/jump_test.dart
+omit=tests/compiler/dart2js/jumps/jump_test.dart
@@ -0,0 +1,2 @@
+strong=tests/compiler/dart2js/member_usage/member_usage_test.dart
+omit=tests/compiler/dart2js/member_usage/member_usage_test.dart
@@ -25,8 +25,7 @@ main(List<String> args) {
        .resolve('../../../../pkg/_fe_analyzer_shared/test/constants/data'));
    await checkTests<String>(dataDir, new ConstantDataComputer(),
        args: args,
-        testedConfigs: [sharedConfig],
-        supportedMarkers: sharedMarkers);
+        testedConfigs: [sharedConfig]);
  });
 }

@@ -0,0 +1,2 @@
+strong=tests/compiler/dart2js/optimization/optimization_test.dart
+omit=tests/compiler/dart2js/optimization/optimization_test.dart
@@ -0,0 +1,2 @@
+strong=tests/compiler/dart2js/rti/rti_need_test_helper.dart
+omit=tests/compiler/dart2js/rti/rti_need_test_helper.dart
@@ -0,0 +1,2 @@
+strong=tests/compiler/dart2js/rti/rti_emission_test.dart
+omit=tests/compiler/dart2js/rti/rti_emission_test.dart
@@ -0,0 +1,2 @@
+strong=tests/compiler/dart2js/static_type/static_type_test.dart
+omit=tests/compiler/dart2js/static_type/static_type_test.dart
@@ -0,0 +1,2 @@
+strong=tests/compiler/dart2js/static_type/type_promotion_test.dart
+omit=tests/compiler/dart2js/static_type/type_promotion_test.dart
@@ -11,6 +11,7 @@
    "analyzer_unit_tests": [
      ".packages",
      "pkg/",
+      "tests/compiler/dart2js/",
      "third_party/pkg/",
      "third_party/pkg_tested/",
      "tools/",
				`@@ -0,0 +1 @@`
				`cfe=pkg/front_end/test/covariance_check/covariance_check_test.dart`
				`@@ -0,0 +1 @@`
				`cfe=pkg/front_end/test/extensions/extensions_test.dart`
				`@@ -0,0 +1 @@`
				`cfe=pkg/front_end/test/language_versioning/language_versioning_test.dart`
				`@@ -0,0 +1 @@`
				`strong=tests/compiler/dart2js/deferred_loading/deferred_loading_test.dart`
				`@@ -0,0 +1 @@`
				`strong=tests/compiler/dart2js/impact/impact_test.dart`
				`@@ -0,0 +1 @@`
				`strong=tests/compiler/dart2js/inference/inference_data_test.dart`