sdk/pkg/testing/lib/src/chain.dart

// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE.md file.

library testing.chain;

import 'dart:async' show Future, Stream;

import 'dart:convert' show json, JsonEncoder;

import 'dart:io' show Directory, File, FileSystemEntity, exitCode;

import 'suite.dart' show Suite;

import '../testing.dart' show FileBasedTestDescription, TestDescription;

import 'test_dart/status_file_parser.dart'
    show ReadTestExpectations, TestExpectations;

import 'zone_helper.dart' show runGuarded;

import 'error_handling.dart' show withErrorHandling;

import 'log.dart' show Logger, StdoutLogger, splitLines;

import 'multitest.dart' show MultitestTransformer, isError;

import 'expectation.dart' show Expectation, ExpectationSet;

typedef Future<ChainContext> CreateContext(
    Chain suite, Map<String, String> environment);

/// A test suite for tool chains, for example, a compiler.
class Chain extends Suite {
  final Uri source;

  final Uri uri;

  final List<RegExp> pattern;

  final List<RegExp> exclude;

  final bool processMultitests;

  Chain(String name, String kind, this.source, this.uri, Uri statusFile,
      this.pattern, this.exclude, this.processMultitests)
      : super(name, kind, statusFile);

  factory Chain.fromJsonMap(Uri base, Map json, String name, String kind) {
    Uri source = base.resolve(json["source"]);
    String path = json["path"];
    if (!path.endsWith("/")) {
      path += "/";
    }
    Uri uri = base.resolve(path);
    Uri statusFile = base.resolve(json["status"]);
    List<RegExp> pattern =
        json["pattern"].map<RegExp>((p) => new RegExp(p)).toList();
    List<RegExp> exclude =
        json["exclude"].map<RegExp>((p) => new RegExp(p)).toList();
    bool processMultitests = json["process-multitests"] ?? false;
    return new Chain(name, kind, source, uri, statusFile, pattern, exclude,
        processMultitests);
  }

  void writeImportOn(StringSink sink) {
    sink.write("import '");
    sink.write(source);
    sink.write("' as ");
    sink.write(name);
    sink.writeln(";");
  }

  void writeClosureOn(StringSink sink) {
    sink.write("await runChain(");
    sink.write(name);
    sink.writeln(".createContext, environment, selectors, r'''");
    const String jsonExtraIndent = "    ";
    sink.write(jsonExtraIndent);
    sink.writeAll(splitLines(new JsonEncoder.withIndent("  ").convert(this)),
        jsonExtraIndent);
    sink.writeln("''');");
  }

  Map toJson() {
    return {
      "name": name,
      "kind": kind,
      "source": "$source",
      "path": "$uri",
      "status": "$statusFile",
      "process-multitests": processMultitests,
      "pattern": []..addAll(pattern.map((RegExp r) => r.pattern)),
      "exclude": []..addAll(exclude.map((RegExp r) => r.pattern)),
    };
  }
}

abstract class ChainContext {
  const ChainContext();

  List<Step> get steps;

  ExpectationSet get expectationSet => ExpectationSet.Default;

  Future<Null> run(Chain suite, Set<String> selectors,
      {int shards = 1,
      int shard = 0,
      Logger logger: const StdoutLogger()}) async {
    assert(shards >= 1, "Invalid shards count: $shards");
    assert(0 <= shard && shard < shards,
        "Invalid shard index: $shard, not in range [0,$shards[.");
    List<String> partialSelectors = selectors
        .where((s) => s.endsWith('...'))
        .map((s) => s.substring(0, s.length - 3))
        .toList();
    TestExpectations expectations = await ReadTestExpectations(
        <String>[suite.statusFile.toFilePath()], {}, expectationSet);
    Stream<TestDescription> stream = list(suite);
    if (suite.processMultitests) {
      stream = stream.transform(new MultitestTransformer());
    }
    List<TestDescription> descriptions = await stream.toList();
    descriptions.sort();
    if (shards > 1) {
      List<TestDescription> shardDescriptions = [];
      for (int index = 0; index < descriptions.length; index++) {
        if (index % shards == shard) {
          shardDescriptions.add(descriptions[index]);
        }
      }
      descriptions = shardDescriptions;
    }
    Map<TestDescription, Result> unexpectedResults =
        <TestDescription, Result>{};
    Map<TestDescription, Set<Expectation>> unexpectedOutcomes =
        <TestDescription, Set<Expectation>>{};
    int completed = 0;
    logger.logSuiteStarted(suite);
    List<Future> futures = <Future>[];
    for (TestDescription description in descriptions) {
      String selector = "${suite.name}/${description.shortName}";
      if (selectors.isNotEmpty &&
          !selectors.contains(selector) &&
          !selectors.contains(suite.name) &&
          !partialSelectors.any((s) => selector.startsWith(s))) {
        continue;
      }
      final Set<Expectation> expectedOutcomes = processExpectedOutcomes(
          expectations.expectations(description.shortName), description);
      final StringBuffer sb = new StringBuffer();
      final Step lastStep = steps.isNotEmpty ? steps.last : null;
      final Iterator<Step> iterator = steps.iterator;

      Result result;
      // Records the outcome of the last step that was run.
      Step lastStepRun;

      /// Performs one step of [iterator].
      ///
      /// If `step.isAsync` is true, the corresponding step is said to be
      /// asynchronous.
      ///
      /// If a step is asynchronous the future returned from this function will
      /// complete after the first asynchronous step is scheduled.  This
      /// allows us to start processing the next test while an external process
      /// completes as steps can be interleaved. To ensure all steps are
      /// completed, wait for [futures].
      ///
      /// Otherwise, the future returned will complete when all steps are
      /// completed. This ensures that tests are run in sequence without
      /// interleaving steps.
      Future doStep(dynamic input) async {
        Future future;
        bool isAsync = false;
        if (iterator.moveNext()) {
          Step step = iterator.current;
          lastStepRun = step;
          isAsync = step.isAsync;
          logger.logStepStart(completed, unexpectedResults.length,
              descriptions.length, suite, description, step);
          // TODO(ahe): It's important to share the zone error reporting zone
          // between all the tasks. Otherwise, if a future completes with an
          // error in one zone, and gets stored, it becomes an uncaught error
          // in other zones (this happened in createPlatform).
          future = runGuarded(() async {
            try {
              return await step.run(input, this);
            } catch (error, trace) {
              return step.unhandledError(error, trace);
            }
          }, printLineOnStdout: sb.writeln);
        } else {
          future = new Future.value(null);
        }
        future = future.then((_currentResult) async {
          Result currentResult = _currentResult;
          if (currentResult != null) {
            logger.logStepComplete(completed, unexpectedResults.length,
                descriptions.length, suite, description, lastStepRun);
            result = currentResult;
            if (currentResult.outcome == Expectation.Pass) {
              // The input to the next step is the output of this step.
              return doStep(result.output);
            }
          }
          await cleanUp(description, result);
          result =
              processTestResult(description, result, lastStep == lastStepRun);
          if (!expectedOutcomes.contains(result.outcome) &&
              !expectedOutcomes.contains(result.outcome.canonical)) {
            result.addLog("$sb");
            unexpectedResults[description] = result;
            unexpectedOutcomes[description] = expectedOutcomes;
            logger.logUnexpectedResult(
                suite, description, result, expectedOutcomes);
            exitCode = 1;
          } else {
            logger.logExpectedResult(
                suite, description, result, expectedOutcomes);
            logger.logMessage(sb);
          }
          logger.logTestComplete(++completed, unexpectedResults.length,
              descriptions.length, suite, description);
        });
        if (isAsync) {
          futures.add(future);
          return null;
        } else {
          return future;
        }
      }

      logger.logTestStart(completed, unexpectedResults.length,
          descriptions.length, suite, description);
      // The input of the first step is [description].
      await doStep(description);
    }
    await Future.wait(futures);
    logger.logSuiteComplete(suite);
    if (unexpectedResults.isNotEmpty) {
      unexpectedResults.forEach((TestDescription description, Result result) {
        logger.logUnexpectedResult(
            suite, description, result, unexpectedOutcomes[description]);
      });
      print("${unexpectedResults.length} failed:");
      unexpectedResults.forEach((TestDescription description, Result result) {
        print("${suite.name}/${description.shortName}: ${result.outcome}");
      });
    }
    postRun();
  }

  Stream<TestDescription> list(Chain suite) async* {
    Directory testRoot = new Directory.fromUri(suite.uri);
    if (await testRoot.exists()) {
      Stream<FileSystemEntity> files =
          testRoot.list(recursive: true, followLinks: false);
      await for (FileSystemEntity entity in files) {
        if (entity is! File) continue;
        String path = entity.uri.path;
        if (suite.exclude.any((RegExp r) => path.contains(r))) continue;
        if (suite.pattern.any((RegExp r) => path.contains(r))) {
          yield new FileBasedTestDescription(suite.uri, entity);
        }
      }
    } else {
      throw "${suite.uri} isn't a directory";
    }
  }

  Set<Expectation> processExpectedOutcomes(
      Set<Expectation> outcomes, TestDescription description) {
    return outcomes;
  }

  Result processTestResult(
      TestDescription description, Result result, bool last) {
    if (description is FileBasedTestDescription &&
        description.multitestExpectations != null) {
      if (isError(description.multitestExpectations)) {
        result =
            toNegativeTestResult(result, description.multitestExpectations);
      }
    } else if (last && description.shortName.endsWith("negative_test")) {
      if (result.outcome == Expectation.Pass) {
        result.addLog("Negative test didn't report an error.\n");
      } else if (result.outcome == Expectation.Fail) {
        result.addLog("Negative test reported an error as expeceted.\n");
      }
      result = toNegativeTestResult(result);
    }
    return result;
  }

  Result toNegativeTestResult(Result result, [Set<String> expectations]) {
    Expectation outcome = result.outcome;
    if (outcome == Expectation.Pass) {
      if (expectations == null) {
        outcome = Expectation.Fail;
      } else if (expectations.contains("compile-time error")) {
        outcome = expectationSet["MissingCompileTimeError"];
      } else if (expectations.contains("runtime error") ||
          expectations.contains("dynamic type error")) {
        outcome = expectationSet["MissingRuntimeError"];
      } else {
        outcome = Expectation.Fail;
      }
    } else if (outcome == Expectation.Fail) {
      outcome = Expectation.Pass;
    }
    return result.copyWithOutcome(outcome);
  }

  Future<void> cleanUp(TestDescription description, Result result) => null;

  Future<void> postRun() => null;
}

abstract class Step<I, O, C extends ChainContext> {
  const Step();

  String get name;

  bool get isAsync => false;

  bool get isCompiler => false;

  bool get isRuntime => false;

  Future<Result<O>> run(I input, C context);

  Result<O> unhandledError(error, StackTrace trace) {
    return new Result<O>.crash(error, trace);
  }

  Result<O> pass(O output) => new Result<O>.pass(output);

  Result<O> crash(error, StackTrace trace) => new Result<O>.crash(error, trace);

  Result<O> fail(O output, [error, StackTrace trace]) {
    return new Result<O>.fail(output, error, trace);
  }
}

class Result<O> {
  final O output;

  final Expectation outcome;

  final error;

  final StackTrace trace;

  final List<String> logs = <String>[];

  /// If set, running the test with '-D$autoFixCommand' will automatically
  /// update the test to match new expectations.
  final String autoFixCommand;

  /// If set, the test can be fixed by running
  ///
  ///     dart pkg/front_end/tool/update_expectations.dart
  ///
  final bool canBeFixWithUpdateExpectations;

  Result(this.output, this.outcome, this.error,
      {this.trace,
      this.autoFixCommand,
      this.canBeFixWithUpdateExpectations: false});

  Result.pass(O output) : this(output, Expectation.Pass, null);

  Result.crash(error, StackTrace trace)
      : this(null, Expectation.Crash, error, trace: trace);

  Result.fail(O output, [error, StackTrace trace])
      : this(output, Expectation.Fail, error, trace: trace);

  bool get isPass => outcome == Expectation.Pass;

  String get log => logs.join();

  void addLog(String log) {
    logs.add(log);
  }

  Result<O> copyWithOutcome(Expectation outcome) {
    return new Result<O>(output, outcome, error, trace: trace)
      ..logs.addAll(logs);
  }

  Result<O2> copyWithOutput<O2>(O2 output) {
    return new Result<O2>(output, outcome, error,
        trace: trace,
        autoFixCommand: autoFixCommand,
        canBeFixWithUpdateExpectations: canBeFixWithUpdateExpectations)
      ..logs.addAll(logs);
  }
}

/// This is called from generated code.
Future<Null> runChain(CreateContext f, Map<String, String> environment,
    Set<String> selectors, String jsonText) {
  return withErrorHandling(() async {
    Chain suite = new Suite.fromJsonMap(Uri.base, json.decode(jsonText));
    print("Running ${suite.name}");
    ChainContext context = await f(suite, environment);
    return context.run(suite, selectors);
  });
}