CQ. Simplify Scanner constructors.

Replace the positional Scanner constructors with a single constructor
that takes the input text and error reporting callback as named required
arguments.

Remove the unused fasta forwarding constructor, reader offset handling,
and stored first token. The scanner now keeps only the input text and
returns the token stream directly from tokenize.

Update analyzer, analysis server, plugin, CLI, scrape, and scanner test
call sites to use the named constructor arguments.

Change-Id: Ieabb7b28570d4a963563fb2d1c5ef5ac0c89032a
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/500340
Reviewed-by: Johnni Winther <johnniwinther@google.com>
Reviewed-by: Bob Nystrom <rnystrom@google.com>
Reviewed-by: Brian Wilkerson <brianwilkerson@google.com>
Reviewed-by: Paul Berry <paulberry@google.com>

pkg/analysis_server/lib/src/g3/utilities.dart

@@ -60,11 +60,12 @@ ParseStringResult sortDirectives(String contents, {String? fileName}) {
     flags: [],
   );
   var diagnosticReporter = DiagnosticReporter(diagnosticListener, source);
-  var scanner = Scanner(contents, diagnosticReporter.report)
-    ..configureFeatures(
-      featureSetForOverriding: FeatureSet.latestLanguageVersion(),
-      featureSet: featureSet,
-    );
+  var scanner =
+      Scanner(inputText: contents, reportError: diagnosticReporter.report)
+        ..configureFeatures(
+          featureSetForOverriding: FeatureSet.latestLanguageVersion(),
+          featureSet: featureSet,
+        );
   var token = scanner.tokenize(reportScannerErrors: false);
   var lineInfo = LineInfo(scanner.lineStarts);
   var languageVersion = LibraryLanguageVersion(

pkg/analysis_server/lib/src/lsp/source_edits.dart

@@ -667,7 +667,7 @@ class _MinimalEditComputer {
   /// be parsed.
   static Token? _parse(String s, FeatureSet featureSet) {
     try {
-      var scanner = Scanner(s, (_) {})
+      var scanner = Scanner(inputText: s, reportError: (_) {})
         ..configureFeatures(
           featureSetForOverriding: featureSet,
           featureSet: featureSet,

pkg/analysis_server/lib/src/services/correction/statement_analyzer.dart

@@ -20,7 +20,7 @@ import 'package:analyzer_plugin/utilities/range_factory.dart';
 List<Token> _getTokens(String text, FeatureSet featureSet) {
   try {
     var tokens = <Token>[];
-    var scanner = Scanner(text, (_) {})
+    var scanner = Scanner(inputText: text, reportError: (_) {})
       ..configureFeatures(
         featureSetForOverriding: featureSet,
         featureSet: featureSet,

pkg/analysis_server/test/stress/replay/replay.dart

@@ -217,7 +217,7 @@ class Driver {
   List<int> _getBreakOffsets(String text) {
     var breakOffsets = <int>[];
     var featureSet = FeatureSet.latestLanguageVersion();
-    var scanner = Scanner(text, (_) {})
+    var scanner = Scanner(inputText: text, reportError: (_) {})
       ..configureFeatures(
         featureSetForOverriding: featureSet,
         featureSet: featureSet,

pkg/analysis_server_plugin/lib/edit/correction_utils.dart

@@ -454,7 +454,7 @@ class TokenUtils {
   static List<Token> getTokens(String s, FeatureSet featureSet) {
     try {
       var tokens = <Token>[];
-      var scanner = Scanner(s, (_) {})
+      var scanner = Scanner(inputText: s, reportError: (_) {})
         ..configureFeatures(
           featureSetForOverriding: featureSet,
           featureSet: featureSet,

pkg/analyzer/lib/dart/analysis/utilities.dart

@@ -84,11 +84,12 @@ ParseStringResult parseString({
   var source = StringSource(content, path ?? '');
   var diagnosticCollector = RecordingDiagnosticListener();
   var diagnosticReporter = DiagnosticReporter(diagnosticCollector, source);
-  var scanner = Scanner(content, diagnosticReporter.report)
-    ..configureFeatures(
-      featureSetForOverriding: featureSet,
-      featureSet: featureSet,
-    );
+  var scanner =
+      Scanner(inputText: content, reportError: diagnosticReporter.report)
+        ..configureFeatures(
+          featureSetForOverriding: featureSet,
+          featureSet: featureSet,
+        );
   var token = scanner.tokenize();
   var languageVersion = LibraryLanguageVersion(
     package: ExperimentStatus.currentVersion,

pkg/analyzer/lib/src/dart/analysis/file_state.dart

@@ -620,11 +620,12 @@ class FileState {
       performance.getDataInt('length').add(code.length);
 
       var diagnosticReporter = DiagnosticReporter(diagnosticListener, source);
-      Scanner scanner = Scanner(code, diagnosticReporter.report)
-        ..configureFeatures(
-          featureSetForOverriding: featureSet,
-          featureSet: featureSet.restrictToVersion(packageLanguageVersion),
-        );
+      Scanner scanner =
+          Scanner(inputText: code, reportError: diagnosticReporter.report)
+            ..configureFeatures(
+              featureSetForOverriding: featureSet,
+              featureSet: featureSet.restrictToVersion(packageLanguageVersion),
+            );
       scanner.preserveComments = scanComments;
       Token token = scanner.tokenize(reportScannerErrors: false);
       LineInfo lineInfo = LineInfo(scanner.lineStarts);

pkg/analyzer/lib/src/dart/scanner/scanner.dart

@@ -33,11 +33,8 @@ class Scanner {
   @visibleForTesting
   static bool preserveCommentsDefaultForTesting = true;
 
-  /// The text to be scanned.
-  final String _contents;
-
-  /// The offset of the first character from the reader.
-  final int _readerOffset;
+  /// The input text to be scanned.
+  final String _inputText;
 
   /// The callback to report diagnostics.
   final void Function(LocatedDiagnostic) reportError;
@@ -50,31 +47,16 @@ class Scanner {
   /// The flag specifying whether documentation comments should be parsed.
   bool _preserveComments = preserveCommentsDefaultForTesting;
   List<int>? _lineStarts;
-  late final Token firstToken;
 
   Version? _overrideVersion;
 
   late FeatureSet _featureSet;
 
-  /// Initializes a scanner to scan the given [contents].
+  /// Initializes a scanner to scan the given [inputText].
   ///
   /// The [reportError] callback will be informed of any errors that are found.
-  factory Scanner(
-    String contents,
-    void Function(LocatedDiagnostic) reportError,
-  ) {
-    return Scanner.fasta(reportError, contents: contents);
-  }
-
-  factory Scanner.fasta(
-    void Function(LocatedDiagnostic) reportError, {
-    required String contents,
-    int offset = -1,
-  }) {
-    return Scanner._(contents, offset, reportError);
-  }
-
-  Scanner._(this._contents, this._readerOffset, this.reportError);
+  Scanner({required String inputText, required this.reportError})
+    : _inputText = inputText;
 
   /// The features associated with this scanner.
   ///
@@ -111,7 +93,7 @@ class Scanner {
   /// Set [reportScannerErrors] `true` when using the old parser.
   Token tokenize({bool reportScannerErrors = true}) {
     fasta.ScannerResult result = fasta.scanString(
-      _contents,
+      _inputText,
       configuration: ExperimentalFeaturesStatus(
         _featureSet,
       ).buildScannerConfiguration(),
@@ -142,16 +124,7 @@ class Scanner {
       }
     }
 
-    firstToken = token;
-    // Update all token offsets based upon the reader's starting offset
-    if (_readerOffset != -1) {
-      int delta = _readerOffset + 1;
-      do {
-        token.offset += delta;
-        token = token.next!;
-      } while (!token.isEof);
-    }
-    return firstToken;
+    return token;
   }
 
   void _languageVersionChanged(

pkg/analyzer/test/generated/scanner_test.dart

@@ -5,12 +5,10 @@
 import 'package:_fe_analyzer_shared/src/scanner/error_token.dart';
 import 'package:analyzer/dart/analysis/features.dart';
 import 'package:analyzer/dart/ast/token.dart';
-import 'package:analyzer/error/listener.dart';
 import 'package:analyzer/source/file_source.dart';
 import 'package:analyzer/source/line_info.dart';
 import 'package:analyzer/src/dart/scanner/scanner.dart';
 import 'package:analyzer/src/error/listener.dart';
-import 'package:analyzer/src/string_source.dart';
 import 'package:analyzer_testing/resource_provider_mixin.dart';
 import 'package:test/test.dart';
 import 'package:test_reflective_loader/test_reflective_loader.dart';
@@ -73,18 +71,19 @@ class LineInfoTest with ResourceProviderMixin {
     ]);
   }
 
-  void test_linestarts() {
+  void test_lineStarts() {
     String source = "var\r\ni\n=\n1;\n";
     GatheringDiagnosticListener listener = GatheringDiagnosticListener();
     var diagnosticReporter = DiagnosticReporter(
       listener,
       FileSource(newFile('/test.dart', '')),
     );
-    Scanner scanner = Scanner(source, diagnosticReporter.report)
-      ..configureFeatures(
-        featureSetForOverriding: featureSet,
-        featureSet: featureSet,
-      );
+    Scanner scanner =
+        Scanner(inputText: source, reportError: diagnosticReporter.report)
+          ..configureFeatures(
+            featureSetForOverriding: featureSet,
+            featureSet: featureSet,
+          );
     var token = scanner.tokenize();
     expect(token.lexeme, 'var');
     var lineStarts = scanner.lineStarts;
@@ -101,11 +100,12 @@ class LineInfoTest with ResourceProviderMixin {
       listener,
       FileSource(newFile('/test.dart', '')),
     );
-    Scanner scanner = Scanner(source, diagnosticReporter.report)
-      ..configureFeatures(
-        featureSetForOverriding: featureSet,
-        featureSet: featureSet,
-      );
+    Scanner scanner =
+        Scanner(inputText: source, reportError: diagnosticReporter.report)
+          ..configureFeatures(
+            featureSetForOverriding: featureSet,
+            featureSet: featureSet,
+          );
     Token token = scanner.tokenize(reportScannerErrors: false);
     expect(token, TypeMatcher<UnmatchedToken>());
     token = token.next!;
@@ -145,11 +145,12 @@ class LineInfoTest with ResourceProviderMixin {
   Token _scanWithListener(String source, GatheringDiagnosticListener listener) {
     var testSource = FileSource(newFile('/test.dart', ''));
     var diagnosticReporter = DiagnosticReporter(listener, testSource);
-    Scanner scanner = Scanner(source, diagnosticReporter.report)
-      ..configureFeatures(
-        featureSetForOverriding: featureSet,
-        featureSet: featureSet,
-      );
+    Scanner scanner =
+        Scanner(inputText: source, reportError: diagnosticReporter.report)
+          ..configureFeatures(
+            featureSetForOverriding: featureSet,
+            featureSet: featureSet,
+          );
     Token result = scanner.tokenize();
     LineInfo lineInfo = LineInfo(scanner.lineStarts);
     listener.setLineInfo(testSource, lineInfo);
@@ -160,9 +161,12 @@ class LineInfoTest with ResourceProviderMixin {
 @reflectiveTest
 class ScannerTest with ResourceProviderMixin {
   test_featureSet() {
-    var scanner = _createScanner(r'''
+    var scanner = Scanner(
+      inputText: r'''
 // @dart = 2.0
-''');
+''',
+      reportError: (_) {},
+    );
     var defaultFeatureSet = FeatureSet.latestLanguageVersion();
     expect(defaultFeatureSet.isEnabled(Feature.extension_methods), isTrue);
 
@@ -177,9 +181,12 @@ class ScannerTest with ResourceProviderMixin {
   }
 
   test_featureSet_majorOverflow() {
-    var scanner = _createScanner(r'''
+    var scanner = Scanner(
+      inputText: r'''
 // @dart = 99999999999999999999999999999999.0
-''');
+''',
+      reportError: (_) {},
+    );
     var featureSet = FeatureSet.latestLanguageVersion();
     scanner.configureFeatures(
       featureSetForOverriding: featureSet,
@@ -190,9 +197,12 @@ class ScannerTest with ResourceProviderMixin {
   }
 
   test_featureSet_minorOverflow() {
-    var scanner = _createScanner(r'''
+    var scanner = Scanner(
+      inputText: r'''
 // @dart = 3.99999999999999999999999999999999
-''');
+''',
+      reportError: (_) {},
+    );
     var featureSet = FeatureSet.latestLanguageVersion();
     scanner.configureFeatures(
       featureSetForOverriding: featureSet,
@@ -201,14 +211,6 @@ class ScannerTest with ResourceProviderMixin {
     scanner.tokenize();
     // Don't check features, but should not crash.
   }
-
-  Scanner _createScanner(String content) {
-    var path = convertPath('/test/lib/a.dart');
-    var source = StringSource(content, path);
-    var diagnosticCollector = RecordingDiagnosticListener();
-    var diagnosticReporter = DiagnosticReporter(diagnosticCollector, source);
-    return Scanner(content, diagnosticReporter.report);
-  }
 }
 
 /// An `ExpectedLocation` encodes information about the expected location of a

pkg/analyzer_cli/tool/perf.dart

@@ -231,7 +231,7 @@ ScannerResult tokenize(Source source) {
   var featureSet = FeatureSet.latestLanguageVersion();
   // TODO(sigmund): is there a way to scan from a random-access-file without
   // first converting to String?
-  var scanner = Scanner(contents, (_) {})
+  var scanner = Scanner(inputText: contents, reportError: (_) {})
     ..configureFeatures(
       featureSetForOverriding: featureSet,
       featureSet: featureSet,

pkg/scrape/lib/scrape.dart

@@ -277,7 +277,10 @@ class Scrape {
     var stringSource = StringSource(source, file.path);
     var diagnosticReporter =
         DiagnosticReporter(diagnosticListener, stringSource);
-    var scanner = Scanner(source, diagnosticReporter.report);
+    var scanner = Scanner(
+        inputText: source,
+        reportError:  diagnosticReporter.report,
+    );
     scanner.configureFeatures(
         featureSet: featureSet, featureSetForOverriding: featureSet);
     var startToken = scanner.tokenize();