From 921ec7f507bcfda02003bd6b6b8074550afd1503 Mon Sep 17 00:00:00 2001 From: Konstantin Shcheglov Date: Tue, 5 May 2026 14:04:00 -0700 Subject: [PATCH] CQ. Simplify Scanner constructors. Replace the positional Scanner constructors with a single constructor that takes the input text and error reporting callback as named required arguments. Remove the unused fasta forwarding constructor, reader offset handling, and stored first token. The scanner now keeps only the input text and returns the token stream directly from tokenize. Update analyzer, analysis server, plugin, CLI, scrape, and scanner test call sites to use the named constructor arguments. Change-Id: Ieabb7b28570d4a963563fb2d1c5ef5ac0c89032a Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/500340 Reviewed-by: Johnni Winther Reviewed-by: Bob Nystrom Reviewed-by: Brian Wilkerson Reviewed-by: Paul Berry --- pkg/analysis_server/lib/src/g3/utilities.dart | 11 ++-- .../lib/src/lsp/source_edits.dart | 2 +- .../correction/statement_analyzer.dart | 2 +- .../test/stress/replay/replay.dart | 2 +- .../lib/edit/correction_utils.dart | 2 +- pkg/analyzer/lib/dart/analysis/utilities.dart | 11 ++-- .../lib/src/dart/analysis/file_state.dart | 11 ++-- .../lib/src/dart/scanner/scanner.dart | 41 ++---------- pkg/analyzer/test/generated/scanner_test.dart | 66 ++++++++++--------- pkg/analyzer_cli/tool/perf.dart | 2 +- pkg/scrape/lib/scrape.dart | 5 +- 11 files changed, 68 insertions(+), 87 deletions(-) diff --git a/pkg/analysis_server/lib/src/g3/utilities.dart b/pkg/analysis_server/lib/src/g3/utilities.dart index 8d9c56f1b55..7fd4356753a 100644 --- a/pkg/analysis_server/lib/src/g3/utilities.dart +++ b/pkg/analysis_server/lib/src/g3/utilities.dart @@ -60,11 +60,12 @@ ParseStringResult sortDirectives(String contents, {String? fileName}) { flags: [], ); var diagnosticReporter = DiagnosticReporter(diagnosticListener, source); - var scanner = Scanner(contents, diagnosticReporter.report) - ..configureFeatures( - featureSetForOverriding: FeatureSet.latestLanguageVersion(), - featureSet: featureSet, - ); + var scanner = + Scanner(inputText: contents, reportError: diagnosticReporter.report) + ..configureFeatures( + featureSetForOverriding: FeatureSet.latestLanguageVersion(), + featureSet: featureSet, + ); var token = scanner.tokenize(reportScannerErrors: false); var lineInfo = LineInfo(scanner.lineStarts); var languageVersion = LibraryLanguageVersion( diff --git a/pkg/analysis_server/lib/src/lsp/source_edits.dart b/pkg/analysis_server/lib/src/lsp/source_edits.dart index 88c6494e643..2fbb3b2b814 100644 --- a/pkg/analysis_server/lib/src/lsp/source_edits.dart +++ b/pkg/analysis_server/lib/src/lsp/source_edits.dart @@ -667,7 +667,7 @@ class _MinimalEditComputer { /// be parsed. static Token? _parse(String s, FeatureSet featureSet) { try { - var scanner = Scanner(s, (_) {}) + var scanner = Scanner(inputText: s, reportError: (_) {}) ..configureFeatures( featureSetForOverriding: featureSet, featureSet: featureSet, diff --git a/pkg/analysis_server/lib/src/services/correction/statement_analyzer.dart b/pkg/analysis_server/lib/src/services/correction/statement_analyzer.dart index e7c5a19764e..f4fdc226973 100644 --- a/pkg/analysis_server/lib/src/services/correction/statement_analyzer.dart +++ b/pkg/analysis_server/lib/src/services/correction/statement_analyzer.dart @@ -20,7 +20,7 @@ import 'package:analyzer_plugin/utilities/range_factory.dart'; List _getTokens(String text, FeatureSet featureSet) { try { var tokens = []; - var scanner = Scanner(text, (_) {}) + var scanner = Scanner(inputText: text, reportError: (_) {}) ..configureFeatures( featureSetForOverriding: featureSet, featureSet: featureSet, diff --git a/pkg/analysis_server/test/stress/replay/replay.dart b/pkg/analysis_server/test/stress/replay/replay.dart index ba8357d9416..aae4c1ec16b 100644 --- a/pkg/analysis_server/test/stress/replay/replay.dart +++ b/pkg/analysis_server/test/stress/replay/replay.dart @@ -217,7 +217,7 @@ class Driver { List _getBreakOffsets(String text) { var breakOffsets = []; var featureSet = FeatureSet.latestLanguageVersion(); - var scanner = Scanner(text, (_) {}) + var scanner = Scanner(inputText: text, reportError: (_) {}) ..configureFeatures( featureSetForOverriding: featureSet, featureSet: featureSet, diff --git a/pkg/analysis_server_plugin/lib/edit/correction_utils.dart b/pkg/analysis_server_plugin/lib/edit/correction_utils.dart index 8171cdbd6cc..e977501d50e 100644 --- a/pkg/analysis_server_plugin/lib/edit/correction_utils.dart +++ b/pkg/analysis_server_plugin/lib/edit/correction_utils.dart @@ -454,7 +454,7 @@ class TokenUtils { static List getTokens(String s, FeatureSet featureSet) { try { var tokens = []; - var scanner = Scanner(s, (_) {}) + var scanner = Scanner(inputText: s, reportError: (_) {}) ..configureFeatures( featureSetForOverriding: featureSet, featureSet: featureSet, diff --git a/pkg/analyzer/lib/dart/analysis/utilities.dart b/pkg/analyzer/lib/dart/analysis/utilities.dart index 71896dec4e7..8e79058c916 100644 --- a/pkg/analyzer/lib/dart/analysis/utilities.dart +++ b/pkg/analyzer/lib/dart/analysis/utilities.dart @@ -84,11 +84,12 @@ ParseStringResult parseString({ var source = StringSource(content, path ?? ''); var diagnosticCollector = RecordingDiagnosticListener(); var diagnosticReporter = DiagnosticReporter(diagnosticCollector, source); - var scanner = Scanner(content, diagnosticReporter.report) - ..configureFeatures( - featureSetForOverriding: featureSet, - featureSet: featureSet, - ); + var scanner = + Scanner(inputText: content, reportError: diagnosticReporter.report) + ..configureFeatures( + featureSetForOverriding: featureSet, + featureSet: featureSet, + ); var token = scanner.tokenize(); var languageVersion = LibraryLanguageVersion( package: ExperimentStatus.currentVersion, diff --git a/pkg/analyzer/lib/src/dart/analysis/file_state.dart b/pkg/analyzer/lib/src/dart/analysis/file_state.dart index edeb8ec93ad..6069efbac16 100644 --- a/pkg/analyzer/lib/src/dart/analysis/file_state.dart +++ b/pkg/analyzer/lib/src/dart/analysis/file_state.dart @@ -620,11 +620,12 @@ class FileState { performance.getDataInt('length').add(code.length); var diagnosticReporter = DiagnosticReporter(diagnosticListener, source); - Scanner scanner = Scanner(code, diagnosticReporter.report) - ..configureFeatures( - featureSetForOverriding: featureSet, - featureSet: featureSet.restrictToVersion(packageLanguageVersion), - ); + Scanner scanner = + Scanner(inputText: code, reportError: diagnosticReporter.report) + ..configureFeatures( + featureSetForOverriding: featureSet, + featureSet: featureSet.restrictToVersion(packageLanguageVersion), + ); scanner.preserveComments = scanComments; Token token = scanner.tokenize(reportScannerErrors: false); LineInfo lineInfo = LineInfo(scanner.lineStarts); diff --git a/pkg/analyzer/lib/src/dart/scanner/scanner.dart b/pkg/analyzer/lib/src/dart/scanner/scanner.dart index 735e00e86fe..fc3a668e66c 100644 --- a/pkg/analyzer/lib/src/dart/scanner/scanner.dart +++ b/pkg/analyzer/lib/src/dart/scanner/scanner.dart @@ -33,11 +33,8 @@ class Scanner { @visibleForTesting static bool preserveCommentsDefaultForTesting = true; - /// The text to be scanned. - final String _contents; - - /// The offset of the first character from the reader. - final int _readerOffset; + /// The input text to be scanned. + final String _inputText; /// The callback to report diagnostics. final void Function(LocatedDiagnostic) reportError; @@ -50,31 +47,16 @@ class Scanner { /// The flag specifying whether documentation comments should be parsed. bool _preserveComments = preserveCommentsDefaultForTesting; List? _lineStarts; - late final Token firstToken; Version? _overrideVersion; late FeatureSet _featureSet; - /// Initializes a scanner to scan the given [contents]. + /// Initializes a scanner to scan the given [inputText]. /// /// The [reportError] callback will be informed of any errors that are found. - factory Scanner( - String contents, - void Function(LocatedDiagnostic) reportError, - ) { - return Scanner.fasta(reportError, contents: contents); - } - - factory Scanner.fasta( - void Function(LocatedDiagnostic) reportError, { - required String contents, - int offset = -1, - }) { - return Scanner._(contents, offset, reportError); - } - - Scanner._(this._contents, this._readerOffset, this.reportError); + Scanner({required String inputText, required this.reportError}) + : _inputText = inputText; /// The features associated with this scanner. /// @@ -111,7 +93,7 @@ class Scanner { /// Set [reportScannerErrors] `true` when using the old parser. Token tokenize({bool reportScannerErrors = true}) { fasta.ScannerResult result = fasta.scanString( - _contents, + _inputText, configuration: ExperimentalFeaturesStatus( _featureSet, ).buildScannerConfiguration(), @@ -142,16 +124,7 @@ class Scanner { } } - firstToken = token; - // Update all token offsets based upon the reader's starting offset - if (_readerOffset != -1) { - int delta = _readerOffset + 1; - do { - token.offset += delta; - token = token.next!; - } while (!token.isEof); - } - return firstToken; + return token; } void _languageVersionChanged( diff --git a/pkg/analyzer/test/generated/scanner_test.dart b/pkg/analyzer/test/generated/scanner_test.dart index bf77056a3f7..91fcd9d0718 100644 --- a/pkg/analyzer/test/generated/scanner_test.dart +++ b/pkg/analyzer/test/generated/scanner_test.dart @@ -5,12 +5,10 @@ import 'package:_fe_analyzer_shared/src/scanner/error_token.dart'; import 'package:analyzer/dart/analysis/features.dart'; import 'package:analyzer/dart/ast/token.dart'; -import 'package:analyzer/error/listener.dart'; import 'package:analyzer/source/file_source.dart'; import 'package:analyzer/source/line_info.dart'; import 'package:analyzer/src/dart/scanner/scanner.dart'; import 'package:analyzer/src/error/listener.dart'; -import 'package:analyzer/src/string_source.dart'; import 'package:analyzer_testing/resource_provider_mixin.dart'; import 'package:test/test.dart'; import 'package:test_reflective_loader/test_reflective_loader.dart'; @@ -73,18 +71,19 @@ class LineInfoTest with ResourceProviderMixin { ]); } - void test_linestarts() { + void test_lineStarts() { String source = "var\r\ni\n=\n1;\n"; GatheringDiagnosticListener listener = GatheringDiagnosticListener(); var diagnosticReporter = DiagnosticReporter( listener, FileSource(newFile('/test.dart', '')), ); - Scanner scanner = Scanner(source, diagnosticReporter.report) - ..configureFeatures( - featureSetForOverriding: featureSet, - featureSet: featureSet, - ); + Scanner scanner = + Scanner(inputText: source, reportError: diagnosticReporter.report) + ..configureFeatures( + featureSetForOverriding: featureSet, + featureSet: featureSet, + ); var token = scanner.tokenize(); expect(token.lexeme, 'var'); var lineStarts = scanner.lineStarts; @@ -101,11 +100,12 @@ class LineInfoTest with ResourceProviderMixin { listener, FileSource(newFile('/test.dart', '')), ); - Scanner scanner = Scanner(source, diagnosticReporter.report) - ..configureFeatures( - featureSetForOverriding: featureSet, - featureSet: featureSet, - ); + Scanner scanner = + Scanner(inputText: source, reportError: diagnosticReporter.report) + ..configureFeatures( + featureSetForOverriding: featureSet, + featureSet: featureSet, + ); Token token = scanner.tokenize(reportScannerErrors: false); expect(token, TypeMatcher()); token = token.next!; @@ -145,11 +145,12 @@ class LineInfoTest with ResourceProviderMixin { Token _scanWithListener(String source, GatheringDiagnosticListener listener) { var testSource = FileSource(newFile('/test.dart', '')); var diagnosticReporter = DiagnosticReporter(listener, testSource); - Scanner scanner = Scanner(source, diagnosticReporter.report) - ..configureFeatures( - featureSetForOverriding: featureSet, - featureSet: featureSet, - ); + Scanner scanner = + Scanner(inputText: source, reportError: diagnosticReporter.report) + ..configureFeatures( + featureSetForOverriding: featureSet, + featureSet: featureSet, + ); Token result = scanner.tokenize(); LineInfo lineInfo = LineInfo(scanner.lineStarts); listener.setLineInfo(testSource, lineInfo); @@ -160,9 +161,12 @@ class LineInfoTest with ResourceProviderMixin { @reflectiveTest class ScannerTest with ResourceProviderMixin { test_featureSet() { - var scanner = _createScanner(r''' + var scanner = Scanner( + inputText: r''' // @dart = 2.0 -'''); +''', + reportError: (_) {}, + ); var defaultFeatureSet = FeatureSet.latestLanguageVersion(); expect(defaultFeatureSet.isEnabled(Feature.extension_methods), isTrue); @@ -177,9 +181,12 @@ class ScannerTest with ResourceProviderMixin { } test_featureSet_majorOverflow() { - var scanner = _createScanner(r''' + var scanner = Scanner( + inputText: r''' // @dart = 99999999999999999999999999999999.0 -'''); +''', + reportError: (_) {}, + ); var featureSet = FeatureSet.latestLanguageVersion(); scanner.configureFeatures( featureSetForOverriding: featureSet, @@ -190,9 +197,12 @@ class ScannerTest with ResourceProviderMixin { } test_featureSet_minorOverflow() { - var scanner = _createScanner(r''' + var scanner = Scanner( + inputText: r''' // @dart = 3.99999999999999999999999999999999 -'''); +''', + reportError: (_) {}, + ); var featureSet = FeatureSet.latestLanguageVersion(); scanner.configureFeatures( featureSetForOverriding: featureSet, @@ -201,14 +211,6 @@ class ScannerTest with ResourceProviderMixin { scanner.tokenize(); // Don't check features, but should not crash. } - - Scanner _createScanner(String content) { - var path = convertPath('/test/lib/a.dart'); - var source = StringSource(content, path); - var diagnosticCollector = RecordingDiagnosticListener(); - var diagnosticReporter = DiagnosticReporter(diagnosticCollector, source); - return Scanner(content, diagnosticReporter.report); - } } /// An `ExpectedLocation` encodes information about the expected location of a diff --git a/pkg/analyzer_cli/tool/perf.dart b/pkg/analyzer_cli/tool/perf.dart index b12c0728c64..1ad79cb8234 100644 --- a/pkg/analyzer_cli/tool/perf.dart +++ b/pkg/analyzer_cli/tool/perf.dart @@ -231,7 +231,7 @@ ScannerResult tokenize(Source source) { var featureSet = FeatureSet.latestLanguageVersion(); // TODO(sigmund): is there a way to scan from a random-access-file without // first converting to String? - var scanner = Scanner(contents, (_) {}) + var scanner = Scanner(inputText: contents, reportError: (_) {}) ..configureFeatures( featureSetForOverriding: featureSet, featureSet: featureSet, diff --git a/pkg/scrape/lib/scrape.dart b/pkg/scrape/lib/scrape.dart index 37287ff2490..5eedb1fd7e8 100644 --- a/pkg/scrape/lib/scrape.dart +++ b/pkg/scrape/lib/scrape.dart @@ -277,7 +277,10 @@ class Scrape { var stringSource = StringSource(source, file.path); var diagnosticReporter = DiagnosticReporter(diagnosticListener, stringSource); - var scanner = Scanner(source, diagnosticReporter.report); + var scanner = Scanner( + inputText: source, + reportError: diagnosticReporter.report, + ); scanner.configureFeatures( featureSet: featureSet, featureSetForOverriding: featureSet); var startToken = scanner.tokenize();