diff --git a/pkg/analysis_server/lib/src/lsp/semantic_tokens/encoder.dart b/pkg/analysis_server/lib/src/lsp/semantic_tokens/encoder.dart index f2e88e1cf93..170ff223172 100644 --- a/pkg/analysis_server/lib/src/lsp/semantic_tokens/encoder.dart +++ b/pkg/analysis_server/lib/src/lsp/semantic_tokens/encoder.dart @@ -109,42 +109,60 @@ class SemanticTokenEncoder { /// Splits overlapping/nested tokens into descrete ranges for the "top-most" /// token. /// - /// Tokens must be pre-sorted by offset, with tokens having the same offset sorted - /// with the longest first. + /// Tokens must be pre-sorted by offset, with tokens having the same offset + /// sorted with the longest first. Iterable splitOverlappingTokens( Iterable sortedTokens) sync* { if (sortedTokens.isEmpty) { return; } - final firstToken = sortedTokens.first; - final stack = ListQueue()..add(firstToken); - var pos = firstToken.offset; + final stack = ListQueue(); - for (final current in sortedTokens.skip(1)) { - final last = stack.last; - final newPos = current.offset; - if (newPos - pos > 0) { - // The previous region ends at either its original end or - // the position of this next region, whichever is shorter. - final end = math.min(last.offset + last.length, newPos); - final length = end - pos; - yield SemanticTokenInfo(pos, length, last.type, last.modifiers); - pos = newPos; + /// Yields tokens for anything on the stack from between [fromOffset] + /// and [toOffset]. + Iterable processStack( + int fromOffset, int toOffset) sync* { + // Process each item on the stack to figure out if we need to send + // a token for it, and pop it off the stack if we've passed the end of it. + while (stack.isNotEmpty) { + final last = stack.last; + final lastEnd = last.offset + last.length; + final end = math.min(lastEnd, toOffset); + final length = end - fromOffset; + if (length > 0) { + yield SemanticTokenInfo( + fromOffset, length, last.type, last.modifiers); + fromOffset = end; + } + + // If this token is completely done with, remove it and continue + // through the stack. Otherwise, if this token remains then we're done + // for now. + if (lastEnd <= toOffset) { + stack.removeLast(); + } else { + return; + } } + } + var lastPos = sortedTokens.first.offset; + for (final current in sortedTokens) { + // Before processing each token, process the stack as there may be tokens + // on it that need filling in the gap up until this point. + yield* processStack(lastPos, current.offset); + + // Add this token to the stack but don't process it, it will be done by + // the next iteration processing the stack since we don't know where this + // one should end until we see the start of the next one. stack.addLast(current); + lastPos = current.offset; } // Process any remaining stack after the last region. - while (stack.isNotEmpty) { - final last = stack.removeLast(); - final newPos = last.offset + last.length; - final length = newPos - pos; - if (length > 0) { - yield SemanticTokenInfo(pos, length, last.type, last.modifiers); - pos = newPos; - } + if (stack.isNotEmpty) { + yield* processStack(lastPos, stack.first.offset + stack.first.length); } } } diff --git a/pkg/analysis_server/test/lsp/semantic_tokens_test.dart b/pkg/analysis_server/test/lsp/semantic_tokens_test.dart index b97795e1c15..bbfcb99638e 100644 --- a/pkg/analysis_server/test/lsp/semantic_tokens_test.dart +++ b/pkg/analysis_server/test/lsp/semantic_tokens_test.dart @@ -273,6 +273,51 @@ class SemanticTokensTest extends AbstractLspAnalysisServerTest { expect(decoded, equals(expected)); } + Future test_dartdoc() async { + final content = ''' + /// before [aaa] after + class MyClass { + String aaa; + } + + /// before [bbb] after + int double(int bbb) => bbb * 2; + '''; + + final expected = [ + _Token('/// before [', SemanticTokenTypes.comment, + [SemanticTokenModifiers.documentation]), + _Token('aaa', SemanticTokenTypes.property), + _Token('] after', SemanticTokenTypes.comment, + [SemanticTokenModifiers.documentation]), + _Token('class', SemanticTokenTypes.keyword), + _Token('MyClass', SemanticTokenTypes.class_), + _Token('String', SemanticTokenTypes.class_), + _Token('aaa', SemanticTokenTypes.variable, + [SemanticTokenModifiers.declaration]), + _Token('/// before [', SemanticTokenTypes.comment, + [SemanticTokenModifiers.documentation]), + _Token('bbb', SemanticTokenTypes.parameter), + _Token('] after', SemanticTokenTypes.comment, + [SemanticTokenModifiers.documentation]), + _Token('int', SemanticTokenTypes.class_), + _Token('double', SemanticTokenTypes.function, + [SemanticTokenModifiers.declaration, SemanticTokenModifiers.static]), + _Token('int', SemanticTokenTypes.class_), + _Token('bbb', SemanticTokenTypes.parameter, + [SemanticTokenModifiers.declaration]), + _Token('bbb', SemanticTokenTypes.parameter), + _Token('2', SemanticTokenTypes.number) + ]; + + await initialize(); + await openFile(mainFileUri, withoutMarkers(content)); + + final tokens = await getSemanticTokens(mainFileUri); + final decoded = decodeSemanticTokens(content, tokens); + expect(decoded, equals(expected)); + } + Future test_directives() async { final content = ''' import 'package:flutter/material.dart'; @@ -347,8 +392,8 @@ class SemanticTokensTest extends AbstractLspAnalysisServerTest { } '''; - // Expect toe correct tokens for the valid code before/after but don't - // check the the tokens for the invalid code as thre are no concrete + // Expect the correct tokens for the valid code before/after but don't + // check the the tokens for the invalid code as there are no concrete // expectations for them. final expected1 = [ _Token('/// class docs', SemanticTokenTypes.comment, @@ -841,6 +886,13 @@ class _Token { modifiers ?? [], (SemanticTokenModifiers a, SemanticTokenModifiers b) => a == b); + /// Outputs a text representation of the token in the form of constructor + /// args for easy copy/pasting into tests to update expectations. @override - String toString() => '$content (${[type, ...?modifiers]})'; + String toString() { + final modifiersString = modifiers.isEmpty + ? '' + : ', [${modifiers.map((m) => 'SemanticTokenModifiers.$m').join(', ')}]'; + return "('$content', SemanticTokenTypes.$type$modifiersString)"; + } }