Store exceptions with transitive files context into ByteStore.

The key under which the exception context is stored, is included into
ExceptionResult, so Analysis Server can include the key into the message.

R=brianwilkerson@google.com, paulberry@google.com
BUG=

Review-Url: https://codereview.chromium.org/2663903002 .

pkg/analysis_server/lib/src/analysis_server.dart

@@ -1868,8 +1868,11 @@ class ServerContextManagerCallbacks extends ContextManagerCallbacks {
       // IMPLEMENTED
     });
     analysisDriver.exceptions.listen((nd.ExceptionResult result) {
-      AnalysisEngine.instance.logger
-          .logError('Analysis failed: ${result.path}', result.exception);
+      String message = 'Analysis failed: ${result.path}';
+      if (result.contextKey != null) {
+        message += ' context: ${result.contextKey}';
+      }
+      AnalysisEngine.instance.logger.logError(message, result.exception);
     });
     analysisServer.driverMap[folder] = analysisDriver;
     return analysisDriver;

pkg/analyzer/lib/src/dart/analysis/driver.dart

@@ -76,6 +76,12 @@ class AnalysisDriver {
    */
   static const int DATA_VERSION = 15;
 
+  /**
+   * The number of exception contexts allowed to write. Once this field is
+   * zero, we stop writing any new exception contexts in this process.
+   */
+  static int allowedNumberOfContextsToWrite = 10;
+
   /**
    * The name of the driver, e.g. the name of the folder.
    */
@@ -732,44 +738,52 @@ class AnalysisDriver {
         return null;
       }
 
-      _LibraryContext libraryContext = _createLibraryContext(libraryFile);
-      AnalysisContext analysisContext = _createAnalysisContext(libraryContext);
       try {
-        CompilationUnit resolvedUnit = analysisContext.resolveCompilationUnit2(
-            file.source, libraryFile.source);
-        List<AnalysisError> errors = analysisContext.computeErrors(file.source);
-        AnalysisDriverUnitIndexBuilder index = indexUnit(resolvedUnit);
+        _LibraryContext libraryContext = _createLibraryContext(libraryFile);
+        AnalysisContext analysisContext =
+            _createAnalysisContext(libraryContext);
+        try {
+          CompilationUnit resolvedUnit = analysisContext
+              .resolveCompilationUnit2(file.source, libraryFile.source);
+          List<AnalysisError> errors =
+              analysisContext.computeErrors(file.source);
+          AnalysisDriverUnitIndexBuilder index = indexUnit(resolvedUnit);
 
-        // Store the result into the cache.
-        List<int> bytes;
-        {
-          bytes = new AnalysisDriverResolvedUnitBuilder(
-                  errors: errors
-                      .map((error) => new AnalysisDriverUnitErrorBuilder(
-                          offset: error.offset,
-                          length: error.length,
-                          uniqueName: error.errorCode.uniqueName,
-                          message: error.message,
-                          correction: error.correction))
-                      .toList(),
-                  index: index)
-              .toBuffer();
-          String key = _getResolvedUnitKey(libraryFile, file);
-          _byteStore.put(key, bytes);
-        }
+          // Store the result into the cache.
+          List<int> bytes;
+          {
+            bytes = new AnalysisDriverResolvedUnitBuilder(
+                    errors: errors
+                        .map((error) => new AnalysisDriverUnitErrorBuilder(
+                            offset: error.offset,
+                            length: error.length,
+                            uniqueName: error.errorCode.uniqueName,
+                            message: error.message,
+                            correction: error.correction))
+                        .toList(),
+                    index: index)
+                .toBuffer();
+            String key = _getResolvedUnitKey(libraryFile, file);
+            _byteStore.put(key, bytes);
+          }
 
-        // Return the result, full or partial.
-        _logger.writeln('Computed new analysis result.');
-        AnalysisResult result = _getAnalysisResultFromBytes(file, bytes,
-            content: withUnit ? file.content : null,
-            withErrors: _addedFiles.contains(path),
-            resolvedUnit: withUnit ? resolvedUnit : null);
-        if (withUnit && _priorityFiles.contains(path)) {
-          _priorityResults[path] = result;
+          // Return the result, full or partial.
+          _logger.writeln('Computed new analysis result.');
+          AnalysisResult result = _getAnalysisResultFromBytes(file, bytes,
+              content: withUnit ? file.content : null,
+              withErrors: _addedFiles.contains(path),
+              resolvedUnit: withUnit ? resolvedUnit : null);
+          if (withUnit && _priorityFiles.contains(path)) {
+            _priorityResults[path] = result;
+          }
+          return result;
+        } finally {
+          analysisContext.dispose();
         }
-        return result;
-      } finally {
-        analysisContext.dispose();
+      } catch (exception, stackTrace) {
+        String contextKey =
+            _storeExceptionContext(path, libraryFile, exception, stackTrace);
+        throw new _ExceptionState(exception, stackTrace, contextKey);
       }
     });
   }
@@ -1076,7 +1090,7 @@ class AnalysisDriver {
               _resultController.add(result);
             }
           } catch (exception, stackTrace) {
-            _reportError(path, exception, stackTrace);
+            _reportException(path, exception, stackTrace);
           }
           return;
         }
@@ -1094,7 +1108,7 @@ class AnalysisDriver {
           _resultController.add(result);
         }
       } catch (exception, stackTrace) {
-        _reportError(path, exception, stackTrace);
+        _reportException(path, exception, stackTrace);
       }
       return;
     }
@@ -1130,15 +1144,69 @@ class AnalysisDriver {
             asIsIfPartWithoutLibrary: true);
         _resultController.add(result);
       } catch (exception, stackTrace) {
-        _reportError(path, exception, stackTrace);
+        _reportException(path, exception, stackTrace);
       }
       return;
     }
   }
 
-  void _reportError(String path, exception, StackTrace stackTrace) {
+  void _reportException(String path, exception, StackTrace stackTrace) {
+    String contextKey = null;
+    if (exception is _ExceptionState) {
+      var state = exception as _ExceptionState;
+      exception = state.exception;
+      stackTrace = state.stackTrace;
+      contextKey = state.contextKey;
+    }
     CaughtException caught = new CaughtException(exception, stackTrace);
-    _exceptionController.add(new ExceptionResult(path, caught));
+    _exceptionController.add(new ExceptionResult(path, caught, contextKey));
+  }
+
+  String _storeExceptionContext(
+      String path, FileState libraryFile, exception, StackTrace stackTrace) {
+    if (allowedNumberOfContextsToWrite > 0) {
+      allowedNumberOfContextsToWrite--;
+    }
+    try {
+      List<AnalysisDriverExceptionFileBuilder> contextFiles = libraryFile
+          .transitiveFiles
+          .map((file) => new AnalysisDriverExceptionFileBuilder(
+              path: file.path, content: file.content))
+          .toList();
+      contextFiles.sort((a, b) => a.path.compareTo(b.path));
+      AnalysisDriverExceptionContextBuilder contextBuilder =
+          new AnalysisDriverExceptionContextBuilder(
+              path: path,
+              exception: exception.toString(),
+              stackTrace: stackTrace.toString(),
+              files: contextFiles);
+      List<int> bytes = contextBuilder.toBuffer();
+
+      String _twoDigits(int n) {
+        if (n >= 10) return '$n';
+        return '0$n';
+      }
+
+      String _threeDigits(int n) {
+        if (n >= 100) return '$n';
+        if (n >= 10) return '0$n';
+        return '00$n';
+      }
+
+      DateTime time = new DateTime.now();
+      String m = _twoDigits(time.month);
+      String d = _twoDigits(time.day);
+      String h = _twoDigits(time.hour);
+      String min = _twoDigits(time.minute);
+      String sec = _twoDigits(time.second);
+      String ms = _threeDigits(time.millisecond);
+      String key = 'exception_${time.year}$m$d' '_$h$min$sec' + '_$ms';
+
+      _byteStore.put(key, bytes);
+      return key;
+    } catch (_) {
+      return null;
+    }
   }
 
   /**
@@ -1487,7 +1555,15 @@ class ExceptionResult {
    */
   final CaughtException exception;
 
-  ExceptionResult(this.path, this.exception);
+  /**
+   * If the exception happened during a file analysis, and the context in which
+   * the exception happened was stored, this field is the key of the context
+   * in the byte store. May be `null` if the context is unknown, the maximum
+   * number of context to store was reached, etc.
+   */
+  final String contextKey;
+
+  ExceptionResult(this.path, this.exception, this.contextKey);
 }
 
 /**
@@ -1658,6 +1734,22 @@ class _ContentCacheWrapper implements ContentCache {
   }
 }
 
+/**
+ * Information about an exception and its context.
+ */
+class _ExceptionState {
+  final exception;
+  final StackTrace stackTrace;
+
+  /**
+   * The key under which the context of the exception was stored, or `null`
+   * if unknown, the maximum number of context to store was reached, etc.
+   */
+  final String contextKey;
+
+  _ExceptionState(this.exception, this.stackTrace, this.contextKey);
+}
+
 /**
  * Task that computes the list of files that were added to the driver and
  * have at least one reference to an identifier [name] defined outside of the

pkg/analyzer/lib/src/summary/format.dart

@@ -129,6 +129,302 @@ class _UnlinkedParamKindReader extends fb.Reader<idl.UnlinkedParamKind> {
   }
 }
 
+class AnalysisDriverExceptionContextBuilder extends Object with _AnalysisDriverExceptionContextMixin implements idl.AnalysisDriverExceptionContext {
+  String _exception;
+  List<AnalysisDriverExceptionFileBuilder> _files;
+  String _path;
+  String _stackTrace;
+
+  @override
+  String get exception => _exception ??= '';
+
+  /**
+   * The exception string.
+   */
+  void set exception(String value) {
+    this._exception = value;
+  }
+
+  @override
+  List<AnalysisDriverExceptionFileBuilder> get files => _files ??= <AnalysisDriverExceptionFileBuilder>[];
+
+  /**
+   * The state of files when the exception happened.
+   */
+  void set files(List<AnalysisDriverExceptionFileBuilder> value) {
+    this._files = value;
+  }
+
+  @override
+  String get path => _path ??= '';
+
+  /**
+   * The path of the file being analyzed when the exception happened.
+   */
+  void set path(String value) {
+    this._path = value;
+  }
+
+  @override
+  String get stackTrace => _stackTrace ??= '';
+
+  /**
+   * The exception stack trace string.
+   */
+  void set stackTrace(String value) {
+    this._stackTrace = value;
+  }
+
+  AnalysisDriverExceptionContextBuilder({String exception, List<AnalysisDriverExceptionFileBuilder> files, String path, String stackTrace})
+    : _exception = exception,
+      _files = files,
+      _path = path,
+      _stackTrace = stackTrace;
+
+  /**
+   * Flush [informative] data recursively.
+   */
+  void flushInformative() {
+    _files?.forEach((b) => b.flushInformative());
+  }
+
+  /**
+   * Accumulate non-[informative] data into [signature].
+   */
+  void collectApiSignature(api_sig.ApiSignature signature) {
+    signature.addString(this._path ?? '');
+    signature.addString(this._exception ?? '');
+    signature.addString(this._stackTrace ?? '');
+    if (this._files == null) {
+      signature.addInt(0);
+    } else {
+      signature.addInt(this._files.length);
+      for (var x in this._files) {
+        x?.collectApiSignature(signature);
+      }
+    }
+  }
+
+  List<int> toBuffer() {
+    fb.Builder fbBuilder = new fb.Builder();
+    return fbBuilder.finish(finish(fbBuilder), "ADEC");
+  }
+
+  fb.Offset finish(fb.Builder fbBuilder) {
+    fb.Offset offset_exception;
+    fb.Offset offset_files;
+    fb.Offset offset_path;
+    fb.Offset offset_stackTrace;
+    if (_exception != null) {
+      offset_exception = fbBuilder.writeString(_exception);
+    }
+    if (!(_files == null || _files.isEmpty)) {
+      offset_files = fbBuilder.writeList(_files.map((b) => b.finish(fbBuilder)).toList());
+    }
+    if (_path != null) {
+      offset_path = fbBuilder.writeString(_path);
+    }
+    if (_stackTrace != null) {
+      offset_stackTrace = fbBuilder.writeString(_stackTrace);
+    }
+    fbBuilder.startTable();
+    if (offset_exception != null) {
+      fbBuilder.addOffset(1, offset_exception);
+    }
+    if (offset_files != null) {
+      fbBuilder.addOffset(3, offset_files);
+    }
+    if (offset_path != null) {
+      fbBuilder.addOffset(0, offset_path);
+    }
+    if (offset_stackTrace != null) {
+      fbBuilder.addOffset(2, offset_stackTrace);
+    }
+    return fbBuilder.endTable();
+  }
+}
+
+idl.AnalysisDriverExceptionContext readAnalysisDriverExceptionContext(List<int> buffer) {
+  fb.BufferContext rootRef = new fb.BufferContext.fromBytes(buffer);
+  return const _AnalysisDriverExceptionContextReader().read(rootRef, 0);
+}
+
+class _AnalysisDriverExceptionContextReader extends fb.TableReader<_AnalysisDriverExceptionContextImpl> {
+  const _AnalysisDriverExceptionContextReader();
+
+  @override
+  _AnalysisDriverExceptionContextImpl createObject(fb.BufferContext bc, int offset) => new _AnalysisDriverExceptionContextImpl(bc, offset);
+}
+
+class _AnalysisDriverExceptionContextImpl extends Object with _AnalysisDriverExceptionContextMixin implements idl.AnalysisDriverExceptionContext {
+  final fb.BufferContext _bc;
+  final int _bcOffset;
+
+  _AnalysisDriverExceptionContextImpl(this._bc, this._bcOffset);
+
+  String _exception;
+  List<idl.AnalysisDriverExceptionFile> _files;
+  String _path;
+  String _stackTrace;
+
+  @override
+  String get exception {
+    _exception ??= const fb.StringReader().vTableGet(_bc, _bcOffset, 1, '');
+    return _exception;
+  }
+
+  @override
+  List<idl.AnalysisDriverExceptionFile> get files {
+    _files ??= const fb.ListReader<idl.AnalysisDriverExceptionFile>(const _AnalysisDriverExceptionFileReader()).vTableGet(_bc, _bcOffset, 3, const <idl.AnalysisDriverExceptionFile>[]);
+    return _files;
+  }
+
+  @override
+  String get path {
+    _path ??= const fb.StringReader().vTableGet(_bc, _bcOffset, 0, '');
+    return _path;
+  }
+
+  @override
+  String get stackTrace {
+    _stackTrace ??= const fb.StringReader().vTableGet(_bc, _bcOffset, 2, '');
+    return _stackTrace;
+  }
+}
+
+abstract class _AnalysisDriverExceptionContextMixin implements idl.AnalysisDriverExceptionContext {
+  @override
+  Map<String, Object> toJson() {
+    Map<String, Object> _result = <String, Object>{};
+    if (exception != '') _result["exception"] = exception;
+    if (files.isNotEmpty) _result["files"] = files.map((_value) => _value.toJson()).toList();
+    if (path != '') _result["path"] = path;
+    if (stackTrace != '') _result["stackTrace"] = stackTrace;
+    return _result;
+  }
+
+  @override
+  Map<String, Object> toMap() => {
+    "exception": exception,
+    "files": files,
+    "path": path,
+    "stackTrace": stackTrace,
+  };
+
+  @override
+  String toString() => convert.JSON.encode(toJson());
+}
+
+class AnalysisDriverExceptionFileBuilder extends Object with _AnalysisDriverExceptionFileMixin implements idl.AnalysisDriverExceptionFile {
+  String _content;
+  String _path;
+
+  @override
+  String get content => _content ??= '';
+
+  /**
+   * The content of the file.
+   */
+  void set content(String value) {
+    this._content = value;
+  }
+
+  @override
+  String get path => _path ??= '';
+
+  /**
+   * The path of the file.
+   */
+  void set path(String value) {
+    this._path = value;
+  }
+
+  AnalysisDriverExceptionFileBuilder({String content, String path})
+    : _content = content,
+      _path = path;
+
+  /**
+   * Flush [informative] data recursively.
+   */
+  void flushInformative() {
+  }
+
+  /**
+   * Accumulate non-[informative] data into [signature].
+   */
+  void collectApiSignature(api_sig.ApiSignature signature) {
+    signature.addString(this._path ?? '');
+    signature.addString(this._content ?? '');
+  }
+
+  fb.Offset finish(fb.Builder fbBuilder) {
+    fb.Offset offset_content;
+    fb.Offset offset_path;
+    if (_content != null) {
+      offset_content = fbBuilder.writeString(_content);
+    }
+    if (_path != null) {
+      offset_path = fbBuilder.writeString(_path);
+    }
+    fbBuilder.startTable();
+    if (offset_content != null) {
+      fbBuilder.addOffset(1, offset_content);
+    }
+    if (offset_path != null) {
+      fbBuilder.addOffset(0, offset_path);
+    }
+    return fbBuilder.endTable();
+  }
+}
+
+class _AnalysisDriverExceptionFileReader extends fb.TableReader<_AnalysisDriverExceptionFileImpl> {
+  const _AnalysisDriverExceptionFileReader();
+
+  @override
+  _AnalysisDriverExceptionFileImpl createObject(fb.BufferContext bc, int offset) => new _AnalysisDriverExceptionFileImpl(bc, offset);
+}
+
+class _AnalysisDriverExceptionFileImpl extends Object with _AnalysisDriverExceptionFileMixin implements idl.AnalysisDriverExceptionFile {
+  final fb.BufferContext _bc;
+  final int _bcOffset;
+
+  _AnalysisDriverExceptionFileImpl(this._bc, this._bcOffset);
+
+  String _content;
+  String _path;
+
+  @override
+  String get content {
+    _content ??= const fb.StringReader().vTableGet(_bc, _bcOffset, 1, '');
+    return _content;
+  }
+
+  @override
+  String get path {
+    _path ??= const fb.StringReader().vTableGet(_bc, _bcOffset, 0, '');
+    return _path;
+  }
+}
+
+abstract class _AnalysisDriverExceptionFileMixin implements idl.AnalysisDriverExceptionFile {
+  @override
+  Map<String, Object> toJson() {
+    Map<String, Object> _result = <String, Object>{};
+    if (content != '') _result["content"] = content;
+    if (path != '') _result["path"] = path;
+    return _result;
+  }
+
+  @override
+  Map<String, Object> toMap() => {
+    "content": content,
+    "path": path,
+  };
+
+  @override
+  String toString() => convert.JSON.encode(toJson());
+}
+
 class AnalysisDriverResolvedUnitBuilder extends Object with _AnalysisDriverResolvedUnitMixin implements idl.AnalysisDriverResolvedUnit {
   List<AnalysisDriverUnitErrorBuilder> _errors;
   AnalysisDriverUnitIndexBuilder _index;

pkg/analyzer/lib/src/summary/format.fbs

@@ -840,6 +840,46 @@ enum UnlinkedParamKind : byte {
   named
 }
 
+/**
+ * Information about the context of an exception in analysis driver.
+ */
+table AnalysisDriverExceptionContext {
+  /**
+   * The exception string.
+   */
+  exception:string (id: 1);
+
+  /**
+   * The state of files when the exception happened.
+   */
+  files:[AnalysisDriverExceptionFile] (id: 3);
+
+  /**
+   * The path of the file being analyzed when the exception happened.
+   */
+  path:string (id: 0);
+
+  /**
+   * The exception stack trace string.
+   */
+  stackTrace:string (id: 2);
+}
+
+/**
+ * Information about a single file in [AnalysisDriverExceptionContext].
+ */
+table AnalysisDriverExceptionFile {
+  /**
+   * The content of the file.
+   */
+  content:string (id: 1);
+
+  /**
+   * The path of the file.
+   */
+  path:string (id: 0);
+}
+
 /**
  * Information about a resolved unit.
  */

pkg/analyzer/lib/src/summary/idl.dart

@@ -57,6 +57,56 @@ import 'format.dart' as generated;
  */
 const informative = null;
 
+/**
+ * Information about the context of an exception in analysis driver.
+ */
+@TopLevel('ADEC')
+abstract class AnalysisDriverExceptionContext extends base.SummaryClass {
+  factory AnalysisDriverExceptionContext.fromBuffer(List<int> buffer) =>
+      generated.readAnalysisDriverExceptionContext(buffer);
+
+  /**
+   * The exception string.
+   */
+  @Id(1)
+  String get exception;
+
+  /**
+   * The state of files when the exception happened.
+   */
+  @Id(3)
+  List<AnalysisDriverExceptionFile> get files;
+
+  /**
+   * The path of the file being analyzed when the exception happened.
+   */
+  @Id(0)
+  String get path;
+
+  /**
+   * The exception stack trace string.
+   */
+  @Id(2)
+  String get stackTrace;
+}
+
+/**
+ * Information about a single file in [AnalysisDriverExceptionContext].
+ */
+abstract class AnalysisDriverExceptionFile extends base.SummaryClass {
+  /**
+   * The content of the file.
+   */
+  @Id(1)
+  String get content;
+
+  /**
+   * The path of the file.
+   */
+  @Id(0)
+  String get path;
+}
+
 /**
  * Information about a resolved unit.
  */