[analyzer] Fix handling of overlapping LSP SemanticTokens

Fixes https://github.com/Dart-Code/Dart-Code/issues/3289.

Change-Id: I15c41f1519a0c06b810be8e840e8ef7c13af45ea
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/195995
Reviewed-by: Brian Wilkerson <brianwilkerson@google.com>
Commit-Queue: Brian Wilkerson <brianwilkerson@google.com>

pkg/analysis_server/lib/src/lsp/semantic_tokens/encoder.dart

@@ -109,42 +109,60 @@ class SemanticTokenEncoder {
   /// Splits overlapping/nested tokens into descrete ranges for the "top-most"
   /// token.
   ///
-  /// Tokens must be pre-sorted by offset, with tokens having the same offset sorted
-  /// with the longest first.
+  /// Tokens must be pre-sorted by offset, with tokens having the same offset
+  /// sorted with the longest first.
   Iterable<SemanticTokenInfo> splitOverlappingTokens(
       Iterable<SemanticTokenInfo> sortedTokens) sync* {
     if (sortedTokens.isEmpty) {
       return;
     }
 
-    final firstToken = sortedTokens.first;
-    final stack = ListQueue<SemanticTokenInfo>()..add(firstToken);
-    var pos = firstToken.offset;
+    final stack = ListQueue<SemanticTokenInfo>();
 
-    for (final current in sortedTokens.skip(1)) {
-      final last = stack.last;
-      final newPos = current.offset;
-      if (newPos - pos > 0) {
-        // The previous region ends at either its original end or
-        // the position of this next region, whichever is shorter.
-        final end = math.min(last.offset + last.length, newPos);
-        final length = end - pos;
-        yield SemanticTokenInfo(pos, length, last.type, last.modifiers);
-        pos = newPos;
+    /// Yields tokens for anything on the stack from between [fromOffset]
+    /// and [toOffset].
+    Iterable<SemanticTokenInfo> processStack(
+        int fromOffset, int toOffset) sync* {
+      // Process each item on the stack to figure out if we need to send
+      // a token for it, and pop it off the stack if we've passed the end of it.
+      while (stack.isNotEmpty) {
+        final last = stack.last;
+        final lastEnd = last.offset + last.length;
+        final end = math.min(lastEnd, toOffset);
+        final length = end - fromOffset;
+        if (length > 0) {
+          yield SemanticTokenInfo(
+              fromOffset, length, last.type, last.modifiers);
+          fromOffset = end;
+        }
+
+        // If this token is completely done with, remove it and continue
+        // through the stack. Otherwise, if this token remains then we're done
+        // for now.
+        if (lastEnd <= toOffset) {
+          stack.removeLast();
+        } else {
+          return;
+        }
       }
+    }
 
+    var lastPos = sortedTokens.first.offset;
+    for (final current in sortedTokens) {
+      // Before processing each token, process the stack as there may be tokens
+      // on it that need filling in the gap up until this point.
+      yield* processStack(lastPos, current.offset);
+
+      // Add this token to the stack but don't process it, it will be done by
+      // the next iteration processing the stack since we don't know where this
+      // one should end until we see the start of the next one.
       stack.addLast(current);
+      lastPos = current.offset;
     }
 
     // Process any remaining stack after the last region.
-    while (stack.isNotEmpty) {
-      final last = stack.removeLast();
-      final newPos = last.offset + last.length;
-      final length = newPos - pos;
-      if (length > 0) {
-        yield SemanticTokenInfo(pos, length, last.type, last.modifiers);
-        pos = newPos;
-      }
+    if (stack.isNotEmpty) {
+      yield* processStack(lastPos, stack.first.offset + stack.first.length);
     }
   }
 }

pkg/analysis_server/test/lsp/semantic_tokens_test.dart

@@ -273,6 +273,51 @@ class SemanticTokensTest extends AbstractLspAnalysisServerTest {
     expect(decoded, equals(expected));
   }
 
+  Future<void> test_dartdoc() async {
+    final content = '''
+    /// before [aaa] after
+    class MyClass {
+      String aaa;
+    }
+      
+    /// before [bbb] after
+    int double(int bbb) => bbb * 2;
+    ''';
+
+    final expected = [
+      _Token('/// before [', SemanticTokenTypes.comment,
+          [SemanticTokenModifiers.documentation]),
+      _Token('aaa', SemanticTokenTypes.property),
+      _Token('] after', SemanticTokenTypes.comment,
+          [SemanticTokenModifiers.documentation]),
+      _Token('class', SemanticTokenTypes.keyword),
+      _Token('MyClass', SemanticTokenTypes.class_),
+      _Token('String', SemanticTokenTypes.class_),
+      _Token('aaa', SemanticTokenTypes.variable,
+          [SemanticTokenModifiers.declaration]),
+      _Token('/// before [', SemanticTokenTypes.comment,
+          [SemanticTokenModifiers.documentation]),
+      _Token('bbb', SemanticTokenTypes.parameter),
+      _Token('] after', SemanticTokenTypes.comment,
+          [SemanticTokenModifiers.documentation]),
+      _Token('int', SemanticTokenTypes.class_),
+      _Token('double', SemanticTokenTypes.function,
+          [SemanticTokenModifiers.declaration, SemanticTokenModifiers.static]),
+      _Token('int', SemanticTokenTypes.class_),
+      _Token('bbb', SemanticTokenTypes.parameter,
+          [SemanticTokenModifiers.declaration]),
+      _Token('bbb', SemanticTokenTypes.parameter),
+      _Token('2', SemanticTokenTypes.number)
+    ];
+
+    await initialize();
+    await openFile(mainFileUri, withoutMarkers(content));
+
+    final tokens = await getSemanticTokens(mainFileUri);
+    final decoded = decodeSemanticTokens(content, tokens);
+    expect(decoded, equals(expected));
+  }
+
   Future<void> test_directives() async {
     final content = '''
     import 'package:flutter/material.dart';
@@ -347,8 +392,8 @@ class SemanticTokensTest extends AbstractLspAnalysisServerTest {
     }
     ''';
 
-    // Expect toe correct tokens for the valid code before/after but don't
-    // check the the tokens for the invalid code as thre are no concrete
+    // Expect the correct tokens for the valid code before/after but don't
+    // check the the tokens for the invalid code as there are no concrete
     // expectations for them.
     final expected1 = [
       _Token('/// class docs', SemanticTokenTypes.comment,
@@ -841,6 +886,13 @@ class _Token {
           modifiers ?? <SemanticTokenModifiers>[],
           (SemanticTokenModifiers a, SemanticTokenModifiers b) => a == b);
 
+  /// Outputs a text representation of the token in the form of constructor
+  /// args for easy copy/pasting into tests to update expectations.
   @override
-  String toString() => '$content (${[type, ...?modifiers]})';
+  String toString() {
+    final modifiersString = modifiers.isEmpty
+        ? ''
+        : ', [${modifiers.map((m) => 'SemanticTokenModifiers.$m').join(', ')}]';
+    return "('$content', SemanticTokenTypes.$type$modifiersString)";
+  }
 }