sdk/pkg/analysis_server/lib/src/context_manager.dart

// Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:async';
import 'dart:collection';
import 'dart:core';

import 'package:analysis_server/src/services/correction/fix/data_driven/transform_set_parser.dart';
import 'package:analyzer/error/listener.dart';
import 'package:analyzer/file_system/file_system.dart';
import 'package:analyzer/instrumentation/instrumentation.dart';
import 'package:analyzer/src/dart/analysis/analysis_context_collection.dart';
import 'package:analyzer/src/dart/analysis/byte_store.dart';
import 'package:analyzer/src/dart/analysis/driver.dart';
import 'package:analyzer/src/dart/analysis/driver_based_analysis_context.dart';
import 'package:analyzer/src/dart/analysis/performance_logger.dart';
import 'package:analyzer/src/generated/java_engine.dart';
import 'package:analyzer/src/generated/sdk.dart';
import 'package:analyzer/src/generated/source.dart';
import 'package:analyzer/src/generated/source_io.dart';
import 'package:analyzer/src/lint/linter.dart';
import 'package:analyzer/src/lint/pub.dart';
import 'package:analyzer/src/manifest/manifest_validator.dart';
import 'package:analyzer/src/pubspec/pubspec_validator.dart';
import 'package:analyzer/src/task/options.dart';
import 'package:analyzer/src/util/file_paths.dart' as file_paths;
import 'package:analyzer/src/workspace/bazel.dart';
import 'package:analyzer/src/workspace/bazel_watcher.dart';
import 'package:analyzer_plugin/protocol/protocol_common.dart' as protocol;
import 'package:analyzer_plugin/utilities/analyzer_converter.dart';
import 'package:meta/meta.dart';
import 'package:path/path.dart' as pathos;
import 'package:watcher/watcher.dart';
import 'package:yaml/yaml.dart';

/// Enables watching of files generated by Bazel.
///
/// TODO(michalt): This is a temporary flag that we use to disable this
/// functionality due its performance issues. We plan to benchmark and optimize
/// it and re-enable it everywhere.
/// Not private to enable testing.
/// NB: If you set this to `false` remember to disable the
/// `test/integration/serve/bazel_changes_test.dart`.
var experimentalEnableBazelWatching = true;

/// Class that maintains a mapping from included/excluded paths to a set of
/// folders that should correspond to analysis contexts.
abstract class ContextManager {
  // TODO(brianwilkerson) Support:
  //   setting the default analysis options
  //   setting the default content cache
  //   setting the default SDK
  //   telling server when a context has been added or removed
  //       (see onContextsChanged)
  //   telling server when a context needs to be re-analyzed
  //   notifying the client when results should be flushed
  //   using analyzeFileFunctions to determine which files to analyze
  //
  // TODO(brianwilkerson) Move this class to a public library.

  /// Get the callback interface used to create, destroy, and update contexts.
  ContextManagerCallbacks get callbacks;

  /// Set the callback interface used to create, destroy, and update contexts.
  set callbacks(ContextManagerCallbacks value);

  /// A table mapping [Folder]s to the [AnalysisDriver]s associated with them.
  Map<Folder, AnalysisDriver> get driverMap;

  /// Return the list of excluded paths (folders and files) most recently passed
  /// to [setRoots].
  List<String> get excludedPaths;

  /// Return the list of included paths (folders and files) most recently passed
  /// to [setRoots].
  List<String> get includedPaths;

  /// Return the existing analysis context that should be used to analyze the
  /// given [path], or `null` if the [path] is not analyzed in any of the
  /// created analysis contexts.
  DriverBasedAnalysisContext getContextFor(String path);

  /// Return the [AnalysisDriver] for the "innermost" context whose associated
  /// folder is or contains the given path.  ("innermost" refers to the nesting
  /// of contexts, so if there is a context for path /foo and a context for
  /// path /foo/bar, then the innermost context containing /foo/bar/baz.dart is
  /// the context for /foo/bar.)
  ///
  /// If no driver contains the given path, `null` is returned.
  AnalysisDriver getDriverFor(String path);

  /// Return `true` if the file or directory with the given [path] will be
  /// analyzed in one of the analysis contexts.
  bool isAnalyzed(String path);

  /// Rebuild the set of contexts from scratch based on the data last sent to
  /// [setRoots].
  void refresh();

  /// Change the set of paths which should be used as starting points to
  /// determine the context directories.
  void setRoots(List<String> includedPaths, List<String> excludedPaths);
}

/// Callback interface used by [ContextManager] to (a) request that contexts be
/// created, destroyed or updated, (b) inform the client when "pub list"
/// operations are in progress, and (c) determine which files should be
/// analyzed.
///
/// TODO(paulberry): eliminate this interface, and instead have [ContextManager]
/// operations return data structures describing how context state should be
/// modified.
abstract class ContextManagerCallbacks {
  /// Called after analysis contexts are created, usually when new analysis
  /// roots are set, or after detecting a change that required rebuilding
  /// the set of analysis contexts.
  void afterContextsCreated();

  /// Called after analysis contexts are destroyed.
  void afterContextsDestroyed();

  /// An [event] was processed, so analysis state might be different now.
  void afterWatchEvent(WatchEvent event);

  /// The given [file] was removed.
  void applyFileRemoved(String file);

  /// Sent the given watch [event] to any interested plugins.
  void broadcastWatchEvent(WatchEvent event);

  /// Add listeners to the [driver]. This must be the only listener.
  ///
  /// TODO(scheglov) Just pass results in here?
  void listenAnalysisDriver(AnalysisDriver driver);

  /// Record error information for the file with the given [path].
  void recordAnalysisErrors(String path, List<protocol.AnalysisError> errors);
}

/// Class that maintains a mapping from included/excluded paths to a set of
/// folders that should correspond to analysis contexts.
class ContextManagerImpl implements ContextManager {
  /// The [ResourceProvider] using which paths are converted into [Resource]s.
  final ResourceProvider resourceProvider;

  /// The manager used to access the SDK that should be associated with a
  /// particular context.
  final DartSdkManager sdkManager;

  /// The storage for cached results.
  final ByteStore _byteStore;

  /// The logger used to create analysis contexts.
  final PerformanceLog _performanceLog;

  /// The scheduler used to create analysis contexts, and report status.
  final AnalysisDriverScheduler _scheduler;

  /// The current set of analysis contexts.
  AnalysisContextCollectionImpl _collection;

  /// The context used to work with file system paths.
  pathos.Context pathContext;

  /// The list of excluded paths (folders and files) most recently passed to
  /// [setRoots].
  @override
  List<String> excludedPaths = <String>[];

  /// The list of included paths (folders and files) most recently passed to
  /// [setRoots].
  @override
  List<String> includedPaths = <String>[];

  /// The instrumentation service used to report instrumentation data.
  final InstrumentationService _instrumentationService;

  @override
  ContextManagerCallbacks callbacks;

  @override
  final Map<Folder, AnalysisDriver> driverMap =
      HashMap<Folder, AnalysisDriver>();

  /// Stream subscription we are using to watch each analysis root directory for
  /// changes.
  final Map<Folder, StreamSubscription<WatchEvent>> changeSubscriptions =
      <Folder, StreamSubscription<WatchEvent>>{};

  /// For each folder, stores the subscription to the Bazel workspace so that we
  /// can establish watches for the generated files.
  final bazelSearchSubscriptions =
      <Folder, StreamSubscription<BazelSearchInfo>>{};

  /// The watcher service running in a separate isolate to watch for changes
  /// to files generated by Bazel.
  ///
  /// Might be `null` if watching Bazel files is not enabled.
  BazelFileWatcherService bazelWatcherService;

  /// The subscription to changes in the files watched by [bazelWatcherService].
  ///
  /// Might be `null` if watching Bazel files is not enabled.
  StreamSubscription<List<WatchEvent>> bazelWatcherSubscription;

  /// For each [Folder] store which files are being watched. This allows us to
  /// clean up when we destroy a context.
  final bazelWatchedPathsPerFolder = <Folder, _BazelWatchedFiles>{};

  ContextManagerImpl(this.resourceProvider, this.sdkManager, this._byteStore,
      this._performanceLog, this._scheduler, this._instrumentationService,
      {@required enableBazelWatcher}) {
    pathContext = resourceProvider.pathContext;
    if (enableBazelWatcher) {
      bazelWatcherService = BazelFileWatcherService(_instrumentationService);
      bazelWatcherSubscription = bazelWatcherService.events
          .listen((events) => _handleBazelWatchEvents(events));
    }
  }

  @override
  DriverBasedAnalysisContext getContextFor(String path) {
    try {
      return _collection?.contextFor(path);
    } on StateError {
      return null;
    }
  }

  @override
  AnalysisDriver getDriverFor(String path) {
    return getContextFor(path)?.driver;
  }

  @override
  bool isAnalyzed(String path) {
    var collection = _collection;
    if (collection == null) {
      return false;
    }

    return collection.contexts.any(
      (context) => context.contextRoot.isAnalyzed(path),
    );
  }

  @override
  void refresh() {
    _createAnalysisContexts();
  }

  @override
  void setRoots(List<String> includedPaths, List<String> excludedPaths) {
    this.includedPaths = includedPaths;
    this.excludedPaths = excludedPaths;

    _createAnalysisContexts();
  }

  /// Use the given analysis [driver] to analyze the content of the analysis
  /// options file at the given [path].
  void _analyzeAnalysisOptionsYaml(AnalysisDriver driver, String path) {
    var convertedErrors = const <protocol.AnalysisError>[];
    try {
      var content = _readFile(path);
      var lineInfo = _computeLineInfo(content);
      var errors = analyzeAnalysisOptions(
          resourceProvider.getFile(path).createSource(),
          content,
          driver.sourceFactory);
      var converter = AnalyzerConverter();
      convertedErrors = converter.convertAnalysisErrors(errors,
          lineInfo: lineInfo, options: driver.analysisOptions);
    } catch (exception) {
      // If the file cannot be analyzed, fall through to clear any previous
      // errors.
    }
    callbacks.recordAnalysisErrors(path, convertedErrors);
  }

  /// Use the given analysis [driver] to analyze the content of the
  /// AndroidManifest file at the given [path].
  void _analyzeAndroidManifestXml(AnalysisDriver driver, String path) {
    var convertedErrors = const <protocol.AnalysisError>[];
    try {
      var content = _readFile(path);
      var validator =
          ManifestValidator(resourceProvider.getFile(path).createSource());
      var lineInfo = _computeLineInfo(content);
      var errors = validator.validate(
          content, driver.analysisOptions.chromeOsManifestChecks);
      var converter = AnalyzerConverter();
      convertedErrors = converter.convertAnalysisErrors(errors,
          lineInfo: lineInfo, options: driver.analysisOptions);
    } catch (exception) {
      // If the file cannot be analyzed, fall through to clear any previous
      // errors.
    }
    callbacks.recordAnalysisErrors(path, convertedErrors);
  }

  /// Use the given analysis [driver] to analyze the content of the
  /// data file at the given [path].
  void _analyzeFixDataYaml(AnalysisDriver driver, String path) {
    var convertedErrors = const <protocol.AnalysisError>[];
    try {
      var file = resourceProvider.getFile(path);
      var packageName = file.parent2.parent2.shortName;
      var content = _readFile(path);
      var errorListener = RecordingErrorListener();
      var errorReporter = ErrorReporter(errorListener, file.createSource());
      var parser = TransformSetParser(errorReporter, packageName);
      parser.parse(content);
      var converter = AnalyzerConverter();
      convertedErrors = converter.convertAnalysisErrors(errorListener.errors,
          lineInfo: _computeLineInfo(content), options: driver.analysisOptions);
    } catch (exception) {
      // If the file cannot be analyzed, fall through to clear any previous
      // errors.
    }
    callbacks.recordAnalysisErrors(path, convertedErrors);
  }

  /// Use the given analysis [driver] to analyze the content of the pubspec file
  /// at the given [path].
  void _analyzePubspecYaml(AnalysisDriver driver, String path) {
    var convertedErrors = const <protocol.AnalysisError>[];
    try {
      var content = _readFile(path);
      var node = loadYamlNode(content);
      if (node is YamlMap) {
        var validator = PubspecValidator(
            resourceProvider, resourceProvider.getFile(path).createSource());
        var lineInfo = _computeLineInfo(content);
        var errors = validator.validate(node.nodes);
        var converter = AnalyzerConverter();
        convertedErrors = converter.convertAnalysisErrors(errors,
            lineInfo: lineInfo, options: driver.analysisOptions);

        if (driver.analysisOptions.lint) {
          var visitors = <LintRule, PubspecVisitor>{};
          for (var linter in driver.analysisOptions.lintRules) {
            if (linter is LintRule) {
              var visitor = linter.getPubspecVisitor();
              if (visitor != null) {
                visitors[linter] = visitor;
              }
            }
          }

          if (visitors.isNotEmpty) {
            var sourceUri = resourceProvider.pathContext.toUri(path);
            var pubspecAst = Pubspec.parse(content,
                sourceUrl: sourceUri, resourceProvider: resourceProvider);
            var listener = RecordingErrorListener();
            var reporter = ErrorReporter(listener,
                resourceProvider.getFile(path).createSource(sourceUri),
                isNonNullableByDefault: false);
            for (var entry in visitors.entries) {
              entry.key.reporter = reporter;
              pubspecAst.accept(entry.value);
            }
            if (listener.errors.isNotEmpty) {
              convertedErrors ??= <protocol.AnalysisError>[];
              convertedErrors.addAll(converter.convertAnalysisErrors(
                  listener.errors,
                  lineInfo: lineInfo,
                  options: driver.analysisOptions));
            }
          }
        }
      }
    } catch (exception) {
      // If the file cannot be analyzed, fall through to clear any previous
      // errors.
    }
    callbacks.recordAnalysisErrors(path, convertedErrors);
  }

  void _checkForAndroidManifestXmlUpdate(String path) {
    if (file_paths.isAndroidManifestXml(pathContext, path)) {
      var driver = getDriverFor(path);
      if (driver != null) {
        _analyzeAndroidManifestXml(driver, path);
      }
    }
  }

  void _checkForFixDataYamlUpdate(String path) {
    if (file_paths.isFixDataYaml(pathContext, path)) {
      var driver = getDriverFor(path);
      if (driver != null) {
        _analyzeFixDataYaml(driver, path);
      }
    }
  }

  /// Compute line information for the given [content].
  LineInfo _computeLineInfo(String content) {
    var lineStarts = StringUtilities.computeLineStarts(content);
    return LineInfo(lineStarts);
  }

  void _createAnalysisContexts() {
    _destroyAnalysisContexts();

    _collection = AnalysisContextCollectionImpl(
      includedPaths: includedPaths,
      excludedPaths: excludedPaths,
      byteStore: _byteStore,
      drainStreams: false,
      enableIndex: true,
      performanceLog: _performanceLog,
      resourceProvider: resourceProvider,
      scheduler: _scheduler,
      sdkPath: sdkManager.defaultSdkDirectory,
    );

    for (var analysisContext in _collection.contexts) {
      var driver = analysisContext.driver;

      callbacks.listenAnalysisDriver(driver);

      var rootFolder = analysisContext.contextRoot.root;
      driverMap[rootFolder] = driver;

      changeSubscriptions[rootFolder] = rootFolder.changes
          .listen(_handleWatchEvent, onError: _handleWatchInterruption);

      _watchBazelFilesIfNeeded(rootFolder, driver);

      for (var file in analysisContext.contextRoot.analyzedFiles()) {
        if (file_paths.isAndroidManifestXml(pathContext, file)) {
          _analyzeAndroidManifestXml(driver, file);
        } else if (file_paths.isDart(pathContext, file)) {
          driver.addFile(file);
        }
      }

      var optionsFile = analysisContext.contextRoot.optionsFile;
      if (optionsFile != null) {
        _analyzeAnalysisOptionsYaml(driver, optionsFile.path);
      }

      var fixDataYamlFile = rootFolder
          .getChildAssumingFolder('lib')
          .getChildAssumingFile(file_paths.fixDataYaml);
      if (fixDataYamlFile.exists) {
        _analyzeFixDataYaml(driver, fixDataYamlFile.path);
      }

      var pubspecFile = rootFolder.getChildAssumingFile(file_paths.pubspecYaml);
      if (pubspecFile.exists) {
        _analyzePubspecYaml(driver, pubspecFile.path);
      }
    }

    callbacks.afterContextsCreated();
  }

  /// Clean up and destroy the context associated with the given folder.
  void _destroyAnalysisContext(DriverBasedAnalysisContext context) {
    context.driver.dispose();
    var rootFolder = context.contextRoot.root;
    changeSubscriptions.remove(rootFolder)?.cancel();
    var watched = bazelWatchedPathsPerFolder.remove(rootFolder);
    if (watched != null) {
      for (var path in watched.paths) {
        bazelWatcherService.stopWatching(watched.workspace, path);
      }
    }
    driverMap.remove(rootFolder);
  }

  void _destroyAnalysisContexts() {
    if (_collection != null) {
      for (var analysisContext in _collection.contexts) {
        _destroyAnalysisContext(analysisContext);
      }
      callbacks.afterContextsDestroyed();
    }
  }

  /// Establishes watch(es) for the Bazel generated files provided in
  /// [notification].
  ///
  /// Whenever the files change, we trigger re-analysis. This allows us to react
  /// to creation/modification of files that were generated by Bazel.
  void _handleBazelSearchInfo(
      Folder folder, String workspace, BazelSearchInfo info) {
    var watched = bazelWatchedPathsPerFolder.putIfAbsent(
        folder, () => _BazelWatchedFiles(workspace));
    var added = watched.paths.add(info.requestedPath);
    if (added) bazelWatcherService.startWatching(workspace, info);
  }

  /// Notifies the drivers that a generated Bazel file has changed.
  void _handleBazelWatchEvents(List<WatchEvent> events) {
    for (var driver in driverMap.values) {
      var needsUriReset = false;
      for (var event in events) {
        if (event.type == ChangeType.ADD) {
          driver.addFile(event.path);
          needsUriReset = true;
        }
        if (event.type == ChangeType.MODIFY) driver.changeFile(event.path);
        if (event.type == ChangeType.REMOVE) driver.removeFile(event.path);
      }
      // Since the file has been created after we've searched for it, the
      // URI resolution is likely wrong, so we need to reset it.
      if (needsUriReset) driver.resetUriResolution();
    }
  }

  void _handleWatchEvent(WatchEvent event) {
    callbacks.broadcastWatchEvent(event);
    _handleWatchEventImpl(event);
    callbacks.afterWatchEvent(event);
  }

  void _handleWatchEventImpl(WatchEvent event) {
    // Figure out which context this event applies to.
    // TODO(brianwilkerson) If a file is explicitly included in one context
    // but implicitly referenced in another context, we will only send a
    // changeSet to the context that explicitly includes the file (because
    // that's the only context that's watching the file).
    var path = event.path;
    var type = event.type;

    _instrumentationService.logWatchEvent('<unknown>', path, type.toString());

    if (file_paths.isAnalysisOptionsYaml(pathContext, path) ||
        file_paths.isDotPackages(pathContext, path) ||
        file_paths.isPackageConfigJson(pathContext, path) ||
        file_paths.isPubspecYaml(pathContext, path) ||
        false) {
      _createAnalysisContexts();
      return;
    }

    if (file_paths.isDart(pathContext, path)) {
      for (var analysisContext in _collection.contexts) {
        switch (type) {
          case ChangeType.ADD:
            if (analysisContext.contextRoot.isAnalyzed(path)) {
              analysisContext.driver.addFile(path);
            } else {
              analysisContext.driver.changeFile(path);
            }
            break;
          case ChangeType.MODIFY:
            analysisContext.driver.changeFile(path);
            break;
          case ChangeType.REMOVE:
            analysisContext.driver.removeFile(path);
            break;
        }
      }
    }

    switch (type) {
      case ChangeType.ADD:
      case ChangeType.MODIFY:
        _checkForAndroidManifestXmlUpdate(path);
        _checkForFixDataYamlUpdate(path);
        break;
      case ChangeType.REMOVE:
        callbacks.applyFileRemoved(path);
        break;
    }
  }

  /// On windows, the directory watcher may overflow, and we must recover.
  void _handleWatchInterruption(dynamic error, StackTrace stackTrace) {
    // We've handled the error, so we only have to log it.
    _instrumentationService
        .logError('Watcher error; refreshing contexts.\n$error\n$stackTrace');
    // TODO(mfairhurst): Optimize this, or perhaps be less complete.
    refresh();
  }

  /// Read the contents of the file at the given [path], or throw an exception
  /// if the contents cannot be read.
  String _readFile(String path) {
    return resourceProvider.getFile(path).readAsStringSync();
  }

  /// Listens to files generated by Bazel that were found or searched for.
  ///
  /// This is handled specially because the files are outside the package
  /// folder, but we still want to watch for changes to them.
  ///
  /// Does nothing if the [driver] is not in a Bazel workspace.
  void _watchBazelFilesIfNeeded(Folder folder, AnalysisDriver analysisDriver) {
    if (!experimentalEnableBazelWatching) return;
    var workspace = analysisDriver.analysisContext.contextRoot.workspace;
    if (workspace is BazelWorkspace &&
        !bazelSearchSubscriptions.containsKey(folder)) {
      var searchSubscription = workspace.bazelCandidateFiles.listen(
          (notification) =>
              _handleBazelSearchInfo(folder, workspace.root, notification));
      bazelSearchSubscriptions[folder] = searchSubscription;
    }
  }
}

class _BazelWatchedFiles {
  final String workspace;
  final paths = <String>{};
  _BazelWatchedFiles(this.workspace);
}