sdk/pkg/front_end/test/scanner_replacement_test.dart

// Copyright (c) 2017, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:convert' show utf8;

import 'package:front_end/src/fasta/scanner/recover.dart'
    show defaultRecoveryStrategy;
import 'package:front_end/src/fasta/scanner.dart' as fasta;
import 'package:front_end/src/fasta/scanner/token.dart' as fasta;
import 'package:front_end/src/fasta/scanner/error_token.dart' as fasta;
import 'package:front_end/src/scanner/token.dart' as analyzer;
import 'package:front_end/src/scanner/errors.dart'
    show ScannerErrorCode, translateErrorToken;
import 'package:test/test.dart';
import 'package:test_reflective_loader/test_reflective_loader.dart';

import 'scanner_test.dart';

main() {
  defineReflectiveSuite(() {
    defineReflectiveTests(ScannerTest_Replacement);
  });
}

/// Scanner tests that use the analyzer scanner, then convert the resulting
/// token stream into a Fasta token stream, then convert back to an analyzer
/// token stream before verifying assertions.
///
/// These tests help to validate the correctness of the analyzer->Fasta token
/// stream conversion.
@reflectiveTest
class ScannerTest_Replacement extends ScannerTestBase {
  ScannerTest_Replacement() {
    usingFasta = true;
  }

  @override
  analyzer.Token scanWithListener(String source, ErrorListener listener,
      {bool genericMethodComments: false,
      bool lazyAssignmentOperators: false}) {
    // Process the source similar to
    // pkg/analyzer/lib/src/dart/scanner/scanner.dart
    // to simulate replacing the analyzer scanner

    fasta.ScannerResult result = fasta.scanString(source,
        includeComments: true,
        scanGenericMethodComments: genericMethodComments,
        scanLazyAssignmentOperators: lazyAssignmentOperators,
        recover: ((List<int> bytes, fasta.Token tokens, List<int> lineStarts) {
      // perform recovery as a separate step
      // so that the token stream can be validated before and after recovery
      return tokens;
    }));

    fasta.Token tokens = result.tokens;
    assertValidTokenStream(tokens);
    assertValidBeginTokens(tokens);
    if (result.hasErrors) {
      List<int> bytes = utf8.encode(source);
      tokens = defaultRecoveryStrategy(bytes, tokens, result.lineStarts);
      assertValidTokenStream(tokens, errorsFirst: true);
    }

    // fasta pretends there is an additional line at EOF
    result.lineStarts.removeLast();

    return extractErrors(tokens, listener);
  }

  void _assertOpenClosePair(String source) {
    analyzer.BeginToken open = _scan(source);
    fasta.Token close = open.next;
    expect(close.next.isEof, isTrue);
    expect(open.endGroup, close);
    expect(open.isSynthetic, isFalse);
    expect(close.isSynthetic, isFalse);
  }

  void _assertOpenOnly(String source, String expectedCloser) {
    ErrorListener listener = new ErrorListener();
    analyzer.BeginToken open = scanWithListener(source, listener);
    fasta.Token close = open.next;
    expect(close.next.isEof, isTrue);
    expect(open.endGroup, close);
    expect(open.isSynthetic, isFalse);
    expect(close.isSynthetic, isTrue);
    listener.assertErrors([
      new TestError(1, ScannerErrorCode.EXPECTED_TOKEN, [expectedCloser]),
    ]);
  }

  void test_double_error() {
    String source = "3457e";
    ErrorListener listener = new ErrorListener();
    analyzer.Token token = scanWithListener(source, listener);
    expect(token.type, analyzer.TokenType.DOUBLE);
    expect(token.offset, 0);
    expect(token.isSynthetic, isTrue);
    // the invalid token is updated to be valid ...
    expect(token.lexeme, source + "0");
    // ... but the length does *not* include the additional character
    // so as to be true to the original source.
    expect(token.length, source.length);
    expect(token.next.isEof, isTrue);
    expect(listener.errors, hasLength(1));
    TestError error = listener.errors[0];
    expect(error.errorCode, ScannerErrorCode.MISSING_DIGIT);
    expect(error.offset, source.length - 1);
  }

  void test_lt() {
    // fasta does not automatically insert a closer for '<'
    // because it could be part of an expression rather than an opener
    analyzer.BeginToken lt = _scan('<');
    expect(lt.next.isEof, isTrue);
    expect(lt.isSynthetic, isFalse);
  }

  void test_lt_gt() {
    _assertOpenClosePair('< >');
  }

  @override
  void test_open_curly_bracket() {
    _assertOpenOnly('{', '}');
  }

  void test_open_curly_bracket_with_close() {
    _assertOpenClosePair('{ }');
  }

  void test_open_paren() {
    _assertOpenOnly('(', ')');
  }

  void test_open_paren_with_close() {
    _assertOpenClosePair('( )');
  }

  void test_open_square_bracket() {
    _assertOpenOnly('[', ']');
  }

  void test_open_square_bracket_with_close() {
    _assertOpenClosePair('[ ]');
  }

  @override
  void test_mismatched_opener_in_interpolation() {
    // When openers and closers are mismatched,
    // fasta favors considering the opener to be mismatched
    // and inserts synthetic closers as needed.
    // r'"${({(}}"' is parsed as r'"${({()})}"'
    // where both ')' are synthetic
    ErrorListener listener = new ErrorListener();
    var stringStart = scanWithListener(r'"${({(}}"', listener);
    analyzer.BeginToken interpolationStart = stringStart.next;
    analyzer.BeginToken openParen1 = interpolationStart.next;
    analyzer.BeginToken openBrace = openParen1.next;
    analyzer.BeginToken openParen2 = openBrace.next;
    var closeParen2 = openParen2.next;
    var closeBrace = closeParen2.next;
    var closeParen1 = closeBrace.next;
    var interpolationEnd = closeParen1.next;
    var stringEnd = interpolationEnd.next;
    var eof = stringEnd.next;

    expect(interpolationStart.endToken, same(interpolationEnd));
    expect(interpolationEnd.isSynthetic, isFalse);
    expect(openParen1.endToken, same(closeParen1));
    expect(closeParen1.isSynthetic, isTrue);
    expect(openBrace.endToken, same(closeBrace));
    expect(closeBrace.isSynthetic, isFalse);
    expect(openParen2.endToken, same(closeParen2));
    expect(closeParen2.isSynthetic, isTrue);
    expect(eof.isEof, isTrue);
    listener.assertErrors([
      new TestError(6, ScannerErrorCode.EXPECTED_TOKEN, [')']),
      new TestError(7, ScannerErrorCode.EXPECTED_TOKEN, [')']),
    ]);
  }

  @override
  void test_unmatched_openers() {
    ErrorListener listener = new ErrorListener();
    // fasta inserts missing closers except for '<'
    analyzer.BeginToken openBrace = scanWithListener('{[(<', listener);
    analyzer.BeginToken openBracket = openBrace.next;
    analyzer.BeginToken openParen = openBracket.next;
    analyzer.BeginToken openLT = openParen.next;
    var closeParen = openLT.next;
    var closeBracket = closeParen.next;
    var closeBrace = closeBracket.next;
    var eof = closeBrace.next;

    expect(openBrace.endGroup, same(closeBrace));
    expect(openBracket.endGroup, same(closeBracket));
    expect(openParen.endGroup, same(closeParen));
    expect(eof.isEof, true);

    listener.assertErrors([
      new TestError(4, ScannerErrorCode.EXPECTED_TOKEN, [')']),
      new TestError(4, ScannerErrorCode.EXPECTED_TOKEN, [']']),
      new TestError(4, ScannerErrorCode.EXPECTED_TOKEN, ['}']),
    ]);
  }

  analyzer.Token _scan(String source,
      {bool genericMethodComments: false,
      bool lazyAssignmentOperators: false}) {
    ErrorListener listener = new ErrorListener();
    analyzer.Token token = scanWithListener(source, listener,
        genericMethodComments: genericMethodComments,
        lazyAssignmentOperators: lazyAssignmentOperators);
    listener.assertNoErrors();
    return token;
  }

  analyzer.Token extractErrors(fasta.Token firstToken, ErrorListener listener) {
    var token = firstToken;
    // The default recovery strategy used by scanString
    // places all error tokens at the head of the stream.
    while (token.type == analyzer.TokenType.BAD_INPUT) {
      translateErrorToken(token,
          (ScannerErrorCode errorCode, int offset, List<Object> arguments) {
        listener.errors.add(new TestError(offset, errorCode, arguments));
      });
      token = token.next;
    }
    if (!token.previous.isEof) {
      new analyzer.Token.eof(-1).setNext(token);
    }
    return token;
  }

  /// Assert that the tokens in the stream are correctly connected prev/next.
  void assertValidTokenStream(fasta.Token firstToken,
      {bool errorsFirst: false}) {
    fasta.Token token = firstToken;
    fasta.Token previous = token.previous;
    expect(previous.isEof, isTrue, reason: 'Missing leading EOF');
    expect(previous.next, token, reason: 'Invalid leading EOF');
    expect(previous.previous, previous, reason: 'Invalid leading EOF');
    if (errorsFirst) {
      while (!token.isEof && token is fasta.ErrorToken) {
        token = token.next;
      }
    }
    var isNotErrorToken = isNot(new isInstanceOf<fasta.ErrorToken>());
    while (!token.isEof) {
      if (errorsFirst) expect(token, isNotErrorToken);
      previous = token;
      token = token.next;
      expect(token, isNotNull, reason: previous.toString());
      expect(token.previous, previous, reason: token.toString());
    }
    expect(token.next, token, reason: 'Invalid trailing EOF');
  }

  /// Assert that all [analyzer.BeginToken] has a valid `endGroup`
  /// that is in the stream.
  void assertValidBeginTokens(fasta.Token firstToken) {
    var openerStack = <analyzer.BeginToken>[];
    analyzer.BeginToken lastClosedGroup;
    fasta.Token token = firstToken;
    while (!token.isEof) {
      if (token is analyzer.BeginToken) {
        if (token.lexeme != '<')
          expect(token.endGroup, isNotNull, reason: token.lexeme);
        if (token.endGroup != null) openerStack.add(token);
      } else if (openerStack.isNotEmpty && openerStack.last.endGroup == token) {
        lastClosedGroup = openerStack.removeLast();
        expect(token.isSynthetic, token.next is fasta.UnmatchedToken,
            reason: 'Expect synthetic closer then error token, '
                'but found "$token" followed by "${token.next}"');
      } else if (token is fasta.UnmatchedToken) {
        expect(lastClosedGroup?.endGroup?.next, same(token),
            reason: 'Unexpected error token for group: $lastClosedGroup');
        expect(token.begin, lastClosedGroup);
      }
      token = token.next;
    }
    expect(openerStack, isEmpty, reason: 'Missing closers');
  }
}