sdk/pkg/test_runner/lib/src/test_file.dart

// Copyright (c) 2019, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:io';

import 'feature.dart';
import 'path.dart';
import 'static_error.dart';

// TODO(rnystrom): Remove support for "///" once tests have been migrated.
// https://dart-review.googlesource.com/c/sdk/+/106201
// https://github.com/dart-lang/co19/issues/391
/// Require at least one non-space character before '//[/#]'.
final _multitestRegExp = RegExp(r"\S *//[#/] \w+:(.*)");

final _vmOptionsRegExp = RegExp(r"// VMOptions=(.*)");
final _environmentRegExp = RegExp(r"// Environment=(.*)");
final _packagesRegExp = RegExp(r"// Packages=(.*)");
final _experimentRegExp = RegExp(r"^--enable-experiment=([a-z,-]+)$");

List<String> _splitWords(String s) =>
    s.split(' ').where((e) => e != '').toList();

List<T> _parseOption<T>(
    String filePath, String contents, String name, T Function(String) convert,
    {bool allowMultiple = false}) {
  var matches = RegExp('// $name=(.*)').allMatches(contents);
  if (!allowMultiple && matches.length > 1) {
    throw Exception('More than one "// $name=" line in test $filePath');
  }

  var options = <T>[];
  for (var match in matches) {
    for (var option in _splitWords(match[1])) {
      options.add(convert(option));
    }
  }

  return options;
}

List<String> _parseStringOption(String filePath, String contents, String name,
        {bool allowMultiple = false}) =>
    _parseOption<String>(filePath, contents, name, (string) => string,
        allowMultiple: allowMultiple);

abstract class _TestFileBase {
  /// The test suite directory containing this test.
  final Path _suiteDirectory;

  /// The full path to the test file.
  final Path path;

  /// The path to the original multitest file this test was generated from.
  ///
  /// If this test was not generated from a multitest, just returns [path].
  Path get originPath;

  /// The parsed error expectation markers in this test, if it is a static
  /// error test.
  final List<StaticError> expectedErrors;

  /// The name of the multitest section this file corresponds to if it was
  /// generated from a multitest. Otherwise, returns an empty string.
  String get multitestKey;

  /// If the test contains static error expectations, it's a "static error
  /// test".
  ///
  /// These tests exist to validate that a front end reports the right static
  /// errors. Unless the expected errors are all warnings, a static error test
  /// is skipped on configurations that are not purely front end.
  bool get isStaticErrorTest => expectedErrors.isNotEmpty;

  /// If the test contains any web-specific (`[web]`) static error expectations,
  /// then it's a "web static error test".
  ///
  /// These tests exist to validate that a Dart web compiler reports the right
  /// expected errors.
  bool get isWebStaticErrorTest =>
      expectedErrors.any((error) => error.hasError(ErrorSource.web));

  /// If the tests has no static error expectations, or all of the expectations
  /// are warnings, then the test tests runtime semantics.
  ///
  /// Note that this is *not* the negation of [isStaticErrorTest]. A test that
  /// contains only warning expectations is both a static "error" test and a
  /// runtime test. The test runner will validate that the front ends produce
  /// the expected warnings *and* that a runtime also correctly executes the
  /// test.
  bool get isRuntimeTest => expectedErrors.every((error) => error.isWarning);

  /// A hash code used to spread tests across shards.
  int get shardHash {
    // The VM C++ unit tests have a special fake TestFile with no suite
    // directory or path. Don't crash in that case.
    // TODO(rnystrom): Is there a cleaner solution? Should we use the C++ file
    // as the path for the TestFile?
    if (originPath == null) return 0;

    return originPath.relativeTo(_suiteDirectory).toString().hashCode;
  }

  _TestFileBase(this._suiteDirectory, this.path, this.expectedErrors) {
    // The VM C++ unit tests have a special fake TestFile with no path.
    if (path != null) assert(path.isAbsolute);
  }

  /// The logical name of the test.
  ///
  /// This is its path relative to the test suite directory containing it,
  /// minus any file extension. If this test was split from a multitest,
  /// it contains the multitest key.
  String get name {
    var testNamePath = originPath.relativeTo(_suiteDirectory);
    var directory = testNamePath.directoryPath;
    var filenameWithoutExt = testNamePath.filenameWithoutExtension;

    String concat(String base, String part) {
      if (base == "") return part;
      if (part == "") return base;
      return "$base/$part";
    }

    var result = "$directory";
    result = concat(result, "$filenameWithoutExt");
    result = concat(result, multitestKey);
    return result;
  }
}

/// Represents a single ".dart" file used as a test and the parsed metadata it
/// contains.
///
/// Special options for individual tests are currently specified in various
/// ways: with comments directly in test files, by using certain imports, or
/// by creating additional files in the test directories.
///
/// Here is a list of options that are used by 'test.dart' today:
///
/// *   Flags can be passed to the VM process that runs the test by adding a
///     comment to the test file:
///
///         // VMOptions=--flag1 --flag2
///
/// *   Flags can be passed to dart2js, VM or dartdevc by adding a comment to
///     the test file:
///
///         // SharedOptions=--flag1 --flag2
///
/// *   Flags can be passed to dart2js by adding a comment to the test file:
///
///         // dart2jsOptions=--flag1 --flag2
///
/// *   Flags can be passed to the dart script that contains the test also
///     using comments, as follows:
///
///         // DartOptions=--flag1 --flag2
///
/// *   Extra environment variables can be passed to the process that runs
///     the test by adding comment(s) to the test file:
///
///         // Environment=ENV_VAR1=foo bar
///         // Environment=ENV_VAR2=bazz
///
/// *   Most tests are not web tests, but can (and will be) wrapped within an
///     HTML file and another script file to test them also on browser
///     environments (e.g. language and corelib tests are run this way). We
///     deduce that if a file with the same name as the test, but ending in
///     ".html" instead of ".dart" exists, the test was intended to be a web
///     test and no wrapping is necessary.
///
/// *   This test requires libfoobar.so, libfoobar.dylib or foobar.dll to be in
///     the system linker path of the VM.
///
///         // SharedObjects=foobar
///
/// *   'test.dart' assumes tests fail if the process returns a non-zero exit
///     code (in the case of web tests, we check for PASS/FAIL indications in
///     the test output).
class TestFile extends _TestFileBase {
  /// Read the test file from the given [filePath].
  factory TestFile.read(Path suiteDirectory, String filePath) => TestFile.parse(
      suiteDirectory, filePath, File(filePath).readAsStringSync());

  /// Parse a test file with [contents].
  factory TestFile.parse(
      Path suiteDirectory, String filePath, String contents) {
    if (filePath.endsWith('.dill')) {
      return TestFile._(suiteDirectory, Path(filePath), [],
          requirements: [],
          vmOptions: [[]],
          sharedOptions: [],
          dart2jsOptions: [],
          ddcOptions: [],
          dartOptions: [],
          packages: null,
          hasSyntaxError: false,
          hasCompileError: false,
          hasRuntimeError: false,
          hasStaticWarning: false,
          hasCrash: false,
          isMultitest: false,
          sharedObjects: [],
          otherResources: []);
    }

    // Required features.
    var requirements =
        _parseOption<Feature>(filePath, contents, 'Requirements', (name) {
      for (var feature in Feature.all) {
        if (feature.name == name) return feature;
      }

      throw FormatException('Unknown feature "$name" in test $filePath');
    });

    // VM options.
    var vmOptions = <List<String>>[];
    var matches = _vmOptionsRegExp.allMatches(contents);
    for (var match in matches) {
      vmOptions.add(_splitWords(match[1]));
    }
    if (vmOptions.isEmpty) vmOptions.add(<String>[]);

    // Other options.
    var dartOptions = _parseStringOption(filePath, contents, 'DartOptions');
    var sharedOptions = _parseStringOption(filePath, contents, 'SharedOptions');
    var dart2jsOptions =
        _parseStringOption(filePath, contents, 'dart2jsOptions');
    var ddcOptions = _parseStringOption(filePath, contents, 'dartdevcOptions');
    var otherResources = _parseStringOption(
        filePath, contents, 'OtherResources',
        allowMultiple: true);
    var sharedObjects = _parseStringOption(filePath, contents, 'SharedObjects',
        allowMultiple: true);

    // Extract the experiments from the shared options.
    // TODO(rnystrom): Either tests should stop specifying experiment flags
    // entirely and use "// Requirements=", or we should come up with a better
    // syntax. Parsing from "// SharedOptions=" for now since that's where they
    // are currently specified.
    var experiments = <String>[];
    for (var i = 0; i < sharedOptions.length; i++) {
      var sharedOption = sharedOptions[i];
      if (sharedOption.contains("--enable-experiment")) {
        var match = _experimentRegExp.firstMatch(sharedOption);
        if (match == null) {
          throw Exception(
              "SharedOptions marker cannot mix experiment flags with other "
              "flags. Was:\n$sharedOption");
        }

        experiments.addAll(match.group(1).split(","));
        sharedOptions.removeAt(i);
        i--;
      }
    }

    // Environment.
    Map<String, String> environment;
    matches = _environmentRegExp.allMatches(contents);
    for (var match in matches) {
      var envDef = match[1];
      var pos = envDef.indexOf('=');
      var name = (pos < 0) ? envDef : envDef.substring(0, pos);
      var value = (pos < 0) ? '' : envDef.substring(pos + 1);
      environment ??= {};
      environment[name] = value;
    }

    // Packages.
    String packages;

    matches = _packagesRegExp.allMatches(contents);
    for (var match in matches) {
      if (packages != null) {
        throw Exception('More than one "// Package..." line in test $filePath');
      }
      packages = match[1];
      if (packages != 'none') {
        // Packages=none means that no packages option should be given. Any
        // other value overrides packages.
        packages =
            Uri.file(filePath).resolveUri(Uri.file(packages)).toFilePath();
      }
    }

    var isMultitest = _multitestRegExp.hasMatch(contents);

    // TODO(rnystrom): During the migration of the existing tests to Dart 2.0,
    // we have a number of tests that used to both generate static type warnings
    // and also validate some runtime behavior in an implementation that
    // ignores those warnings. Those warnings are now errors. The test code
    // validates the runtime behavior can and should be removed, but the code
    // that causes the static warning should still be preserved since that is
    // part of our coverage of the static type system.
    //
    // The test needs to indicate that it should have a static error. We could
    // put that in the status file, but that makes it confusing because it
    // would look like implementations that *don't* report the error are more
    // correct. Eventually, we want to have a notation similar to what front_end
    // is using for the inference tests where we can put a comment inside the
    // test that says "This specific static error should be reported right by
    // this token."
    //
    // That system isn't in place yet, so we do a crude approximation here in
    // test.dart. If a test contains `/*@compile-error=`, which matches the
    // beginning of the tag syntax that front_end uses, then we assume that
    // this test must have a static error somewhere in it.
    //
    // Redo this code once we have a more precise test framework for detecting
    // and locating these errors.
    var hasSyntaxError = contents.contains("@syntax-error");
    var hasCompileError = hasSyntaxError || contents.contains("@compile-error");

    List<StaticError> errorExpectations;
    try {
      errorExpectations = StaticError.parseExpectations(contents);
    } on FormatException catch (error) {
      throw FormatException(
          "Invalid error expectation syntax in $filePath:\n$error");
    }

    return TestFile._(suiteDirectory, Path(filePath), errorExpectations,
        packages: packages,
        environment: environment,
        isMultitest: isMultitest,
        hasSyntaxError: hasSyntaxError,
        hasCompileError: hasCompileError,
        hasRuntimeError: contents.contains("@runtime-error"),
        hasStaticWarning: contents.contains("@static-warning"),
        hasCrash: false,
        requirements: requirements,
        sharedOptions: sharedOptions,
        dartOptions: dartOptions,
        dart2jsOptions: dart2jsOptions,
        ddcOptions: ddcOptions,
        vmOptions: vmOptions,
        sharedObjects: sharedObjects,
        otherResources: otherResources,
        experiments: experiments);
  }

  /// A special fake test file for representing a VM unit test written in C++.
  TestFile.vmUnitTest(
      {this.hasSyntaxError,
      this.hasCompileError,
      this.hasRuntimeError,
      this.hasStaticWarning,
      this.hasCrash})
      : packages = null,
        environment = null,
        isMultitest = false,
        requirements = [],
        sharedOptions = [],
        dartOptions = [],
        dart2jsOptions = [],
        ddcOptions = [],
        vmOptions = [],
        sharedObjects = [],
        otherResources = [],
        experiments = [],
        super(null, null, []);

  TestFile._(Path suiteDirectory, Path path, List<StaticError> expectedErrors,
      {this.packages,
      this.environment,
      this.isMultitest,
      this.hasSyntaxError,
      this.hasCompileError,
      this.hasRuntimeError,
      this.hasStaticWarning,
      this.hasCrash,
      this.requirements,
      this.sharedOptions,
      this.dartOptions,
      this.dart2jsOptions,
      this.ddcOptions,
      this.vmOptions,
      this.sharedObjects,
      this.otherResources,
      this.experiments})
      : super(suiteDirectory, path, expectedErrors) {
    assert(!isMultitest || dartOptions.isEmpty);
  }

  Path get originPath => path;

  String get multitestKey => "";

  final String packages;

  final Map<String, String> environment;

  final bool isMultitest;
  final bool hasSyntaxError;
  final bool hasCompileError;
  final bool hasRuntimeError;
  final bool hasStaticWarning;
  final bool hasCrash;

  /// The features that a test configuration must support in order to run this
  /// test.
  ///
  /// If the current configuration does not support one or more of these
  /// requirements, the test is implicitly skipped.
  final List<Feature> requirements;

  final List<String> sharedOptions;
  final List<String> dartOptions;
  final List<String> dart2jsOptions;
  final List<String> ddcOptions;
  final List<List<String>> vmOptions;
  final List<String> sharedObjects;
  final List<String> otherResources;

  /// The experiments this test enables.
  ///
  /// Parsed from a shared options line like:
  ///
  ///     // SharedOptions=--enable-experiment=flubber,gloop
  final List<String> experiments;

  /// Derive a multitest test section file from this multitest file with the
  /// given [multitestKey] and expectations.
  TestFile split(Path path, String multitestKey, String contents,
          {bool hasCompileError,
          bool hasRuntimeError,
          bool hasStaticWarning,
          bool hasSyntaxError}) =>
      _MultitestFile(
          this, path, multitestKey, StaticError.parseExpectations(contents),
          hasCompileError: hasCompileError ?? false,
          hasRuntimeError: hasRuntimeError ?? false,
          hasStaticWarning: hasStaticWarning ?? false,
          hasSyntaxError: hasSyntaxError ?? false);

  String toString() => """TestFile(
  packages: $packages
  environment: $environment
  isMultitest: $isMultitest
  hasSyntaxError: $hasSyntaxError
  hasCompileError: $hasCompileError
  hasRuntimeError: $hasRuntimeError
  hasStaticWarning: $hasStaticWarning
  hasCrash: $hasCrash
  requirements: $requirements
  sharedOptions: $sharedOptions
  dartOptions: $dartOptions
  dart2jsOptions: $dart2jsOptions
  ddcOptions: $ddcOptions
  vmOptions: $vmOptions
  sharedObjects: $sharedObjects
  otherResources: $otherResources
  experiments: $experiments
)""";
}

/// A [TestFile] for a single section file derived from a multitest.
///
/// This inherits most properties from the original test file, but overrides
/// the error flags based on the multitest section's expectation.
class _MultitestFile extends _TestFileBase implements TestFile {
  /// The authored test file that was split to generate this multitest.
  final TestFile _origin;

  final String multitestKey;

  final bool hasCompileError;
  final bool hasRuntimeError;
  final bool hasStaticWarning;
  final bool hasSyntaxError;
  bool get hasCrash => _origin.hasCrash;

  _MultitestFile(this._origin, Path path, this.multitestKey,
      List<StaticError> expectedErrors,
      {this.hasCompileError,
      this.hasRuntimeError,
      this.hasStaticWarning,
      this.hasSyntaxError})
      : super(_origin._suiteDirectory, path, expectedErrors);

  Path get originPath => _origin.path;

  String get packages => _origin.packages;

  List<Feature> get requirements => _origin.requirements;
  List<String> get dart2jsOptions => _origin.dart2jsOptions;
  List<String> get dartOptions => _origin.dartOptions;
  List<String> get ddcOptions => _origin.ddcOptions;
  Map<String, String> get environment => _origin.environment;

  bool get isMultitest => _origin.isMultitest;

  List<String> get otherResources => _origin.otherResources;
  List<String> get sharedObjects => _origin.sharedObjects;
  List<String> get experiments => _origin.experiments;
  List<String> get sharedOptions => _origin.sharedOptions;
  List<List<String>> get vmOptions => _origin.vmOptions;

  TestFile split(Path path, String multitestKey, String contents,
          {bool hasCompileError,
          bool hasRuntimeError,
          bool hasStaticWarning,
          bool hasSyntaxError}) =>
      throw UnsupportedError(
          "Can't derive a test from one already derived from a multitest.");
}