jmesserly@google.com
9e5b240923
deleted files that duplicate those in the Dart repository (LICENSE, pubspec.yaml, codereview.settings, .gitignore) Otherwise, just changed pkg.status and pubspec R=terry@google.com Review URL: https://codereview.chromium.org//23168002 git-svn-id: https://dart.googlecode.com/svn/branches/bleeding_edge/dart@26155 260f80e4-7a28-3924-810f-c04153c831b5
410 lines
13 KiB
Dart
410 lines
13 KiB
Dart
// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
|
||
// for details. All rights reserved. Use of this source code is governed by a
|
||
// BSD-style license that can be found in the LICENSE file.
|
||
|
||
part of csslib.parser;
|
||
|
||
class Tokenizer extends TokenizerBase {
|
||
/** U+ prefix for unicode characters. */
|
||
final UNICODE_U = 'U'.codeUnitAt(0);
|
||
final UNICODE_LOWER_U = 'u'.codeUnitAt(0);
|
||
final UNICODE_PLUS = '+'.codeUnitAt(0);
|
||
|
||
final QUESTION_MARK = '?'.codeUnitAt(0);
|
||
|
||
/** CDATA keyword. */
|
||
final List CDATA_NAME = 'CDATA'.codeUnits;
|
||
|
||
Tokenizer(SourceFile file, String text, bool skipWhitespace,
|
||
[int index = 0])
|
||
: super(file, text, skipWhitespace, index);
|
||
|
||
Token next({unicodeRange: false}) {
|
||
// keep track of our starting position
|
||
_startIndex = _index;
|
||
|
||
int ch;
|
||
ch = _nextChar();
|
||
switch (ch) {
|
||
case TokenChar.NEWLINE:
|
||
case TokenChar.RETURN:
|
||
case TokenChar.SPACE:
|
||
case TokenChar.TAB:
|
||
return finishWhitespace();
|
||
case TokenChar.END_OF_FILE:
|
||
return _finishToken(TokenKind.END_OF_FILE);
|
||
case TokenChar.AT:
|
||
int peekCh = _peekChar();
|
||
if (TokenizerHelpers.isIdentifierStart(peekCh)) {
|
||
var oldIndex = _index;
|
||
var oldStartIndex = _startIndex;
|
||
|
||
_startIndex = _index;
|
||
ch = _nextChar();
|
||
Token ident = this.finishIdentifier(ch);
|
||
|
||
// Is it a directive?
|
||
int tokId = TokenKind.matchDirectives(_text, _startIndex,
|
||
_index - _startIndex);
|
||
if (tokId == -1) {
|
||
// No, is it a margin directive?
|
||
tokId = TokenKind.matchMarginDirectives(_text, _startIndex,
|
||
_index - _startIndex);
|
||
}
|
||
|
||
if (tokId != -1) {
|
||
return _finishToken(tokId);
|
||
} else {
|
||
// Didn't find a CSS directive or margin directive so the @name is
|
||
// probably the Less definition '@name: value_variable_definition'.
|
||
_startIndex = oldStartIndex;
|
||
_index = oldIndex;
|
||
}
|
||
}
|
||
return _finishToken(TokenKind.AT);
|
||
case TokenChar.DOT:
|
||
int start = _startIndex; // Start where the dot started.
|
||
if (maybeEatDigit()) {
|
||
// looks like a number dot followed by digit(s).
|
||
Token number = finishNumber();
|
||
if (number.kind == TokenKind.INTEGER) {
|
||
// It's a number but it's preceeded by a dot, so make it a double.
|
||
_startIndex = start;
|
||
return _finishToken(TokenKind.DOUBLE);
|
||
} else {
|
||
// Don't allow dot followed by a double (e.g, '..1').
|
||
return _errorToken();
|
||
}
|
||
}
|
||
// It's really a dot.
|
||
return _finishToken(TokenKind.DOT);
|
||
case TokenChar.LPAREN:
|
||
return _finishToken(TokenKind.LPAREN);
|
||
case TokenChar.RPAREN:
|
||
return _finishToken(TokenKind.RPAREN);
|
||
case TokenChar.LBRACE:
|
||
return _finishToken(TokenKind.LBRACE);
|
||
case TokenChar.RBRACE:
|
||
return _finishToken(TokenKind.RBRACE);
|
||
case TokenChar.LBRACK:
|
||
return _finishToken(TokenKind.LBRACK);
|
||
case TokenChar.RBRACK:
|
||
if (_maybeEatChar(TokenChar.RBRACK) &&
|
||
_maybeEatChar(TokenChar.GREATER)) {
|
||
// ]]>
|
||
return next();
|
||
}
|
||
return _finishToken(TokenKind.RBRACK);
|
||
case TokenChar.HASH:
|
||
return _finishToken(TokenKind.HASH);
|
||
case TokenChar.PLUS:
|
||
if (maybeEatDigit()) return finishNumber();
|
||
return _finishToken(TokenKind.PLUS);
|
||
case TokenChar.MINUS:
|
||
if (selectorExpression || unicodeRange) {
|
||
// If parsing in pseudo function expression then minus is an operator
|
||
// not part of identifier e.g., interval value range (e.g. U+400-4ff)
|
||
// or minus operator in selector expression.
|
||
return _finishToken(TokenKind.MINUS);
|
||
} else if (maybeEatDigit()) {
|
||
return finishNumber();
|
||
} else if (TokenizerHelpers.isIdentifierStart(ch)) {
|
||
return this.finishIdentifier(ch);
|
||
}
|
||
return _finishToken(TokenKind.MINUS);
|
||
case TokenChar.GREATER:
|
||
return _finishToken(TokenKind.GREATER);
|
||
case TokenChar.TILDE:
|
||
if (_maybeEatChar(TokenChar.EQUALS)) {
|
||
return _finishToken(TokenKind.INCLUDES); // ~=
|
||
}
|
||
return _finishToken(TokenKind.TILDE);
|
||
case TokenChar.ASTERISK:
|
||
if (_maybeEatChar(TokenChar.EQUALS)) {
|
||
return _finishToken(TokenKind.SUBSTRING_MATCH); // *=
|
||
}
|
||
return _finishToken(TokenKind.ASTERISK);
|
||
case TokenChar.AMPERSAND:
|
||
return _finishToken(TokenKind.AMPERSAND);
|
||
case TokenChar.NAMESPACE:
|
||
return _finishToken(TokenKind.NAMESPACE);
|
||
case TokenChar.COLON:
|
||
return _finishToken(TokenKind.COLON);
|
||
case TokenChar.COMMA:
|
||
return _finishToken(TokenKind.COMMA);
|
||
case TokenChar.SEMICOLON:
|
||
return _finishToken(TokenKind.SEMICOLON);
|
||
case TokenChar.PERCENT:
|
||
return _finishToken(TokenKind.PERCENT);
|
||
case TokenChar.SINGLE_QUOTE:
|
||
return _finishToken(TokenKind.SINGLE_QUOTE);
|
||
case TokenChar.DOUBLE_QUOTE:
|
||
return _finishToken(TokenKind.DOUBLE_QUOTE);
|
||
case TokenChar.SLASH:
|
||
if (_maybeEatChar(TokenChar.ASTERISK)) return finishMultiLineComment();
|
||
return _finishToken(TokenKind.SLASH);
|
||
case TokenChar.LESS: // <!--
|
||
if (_maybeEatChar(TokenChar.BANG)) {
|
||
if (_maybeEatChar(TokenChar.MINUS) &&
|
||
_maybeEatChar(TokenChar.MINUS)) {
|
||
return finishMultiLineComment();
|
||
} else if (_maybeEatChar(TokenChar.LBRACK) &&
|
||
_maybeEatChar(CDATA_NAME[0]) &&
|
||
_maybeEatChar(CDATA_NAME[1]) &&
|
||
_maybeEatChar(CDATA_NAME[2]) &&
|
||
_maybeEatChar(CDATA_NAME[3]) &&
|
||
_maybeEatChar(CDATA_NAME[4]) &&
|
||
_maybeEatChar(TokenChar.LBRACK)) {
|
||
// <![CDATA[
|
||
return next();
|
||
}
|
||
}
|
||
return _finishToken(TokenKind.LESS);
|
||
case TokenChar.EQUALS:
|
||
return _finishToken(TokenKind.EQUALS);
|
||
case TokenChar.OR:
|
||
if (_maybeEatChar(TokenChar.EQUALS)) {
|
||
return _finishToken(TokenKind.DASH_MATCH); // |=
|
||
}
|
||
return _finishToken(TokenKind.OR);
|
||
case TokenChar.CARET:
|
||
if (_maybeEatChar(TokenChar.EQUALS)) {
|
||
return _finishToken(TokenKind.PREFIX_MATCH); // ^=
|
||
}
|
||
return _finishToken(TokenKind.CARET);
|
||
case TokenChar.DOLLAR:
|
||
if (_maybeEatChar(TokenChar.EQUALS)) {
|
||
return _finishToken(TokenKind.SUFFIX_MATCH); // $=
|
||
}
|
||
return _finishToken(TokenKind.DOLLAR);
|
||
case TokenChar.BANG:
|
||
Token tok = finishIdentifier(ch);
|
||
return (tok == null) ? _finishToken(TokenKind.BANG) : tok;
|
||
case TokenChar.BACKSLASH:
|
||
return _finishToken(TokenKind.BACKSLASH);
|
||
default:
|
||
if (unicodeRange) {
|
||
// Three types of unicode ranges:
|
||
// - single code point (e.g. U+416)
|
||
// - interval value range (e.g. U+400-4ff)
|
||
// - range where trailing ‘?’ characters imply ‘any digit value’
|
||
// (e.g. U+4??)
|
||
if (maybeEatHexDigit()) {
|
||
var t = finishHexNumber();
|
||
// Any question marks then it's a HEX_RANGE not HEX_NUMBER.
|
||
if (maybeEatQuestionMark()) finishUnicodeRange();
|
||
return t;
|
||
} else if (maybeEatQuestionMark()) {
|
||
// HEX_RANGE U+N???
|
||
return finishUnicodeRange();
|
||
} else {
|
||
return _errorToken();
|
||
}
|
||
} else if ((ch == UNICODE_U || ch == UNICODE_LOWER_U) &&
|
||
(_peekChar() == UNICODE_PLUS)) {
|
||
// Unicode range: U+uNumber[-U+uNumber]
|
||
// uNumber = 0..10FFFF
|
||
_nextChar(); // Skip +
|
||
_startIndex = _index; // Starts at the number
|
||
return _finishToken(TokenKind.UNICODE_RANGE);
|
||
} else if (varDef(ch)) {
|
||
return _finishToken(TokenKind.VAR_DEFINITION);
|
||
} else if (varUsage(ch)) {
|
||
return _finishToken(TokenKind.VAR_USAGE);
|
||
} else if (TokenizerHelpers.isIdentifierStart(ch)) {
|
||
return finishIdentifier(ch);
|
||
} else if (TokenizerHelpers.isDigit(ch)) {
|
||
return finishNumber();
|
||
}
|
||
return _errorToken();
|
||
}
|
||
}
|
||
|
||
bool varDef(int ch) {
|
||
return ch == 'v'.codeUnitAt(0) && _maybeEatChar('a'.codeUnitAt(0)) &&
|
||
_maybeEatChar('r'.codeUnitAt(0)) && _maybeEatChar('-'.codeUnitAt(0));
|
||
}
|
||
|
||
bool varUsage(int ch) {
|
||
return ch == 'v'.codeUnitAt(0) && _maybeEatChar('a'.codeUnitAt(0)) &&
|
||
_maybeEatChar('r'.codeUnitAt(0)) && (_peekChar() == '-'.codeUnitAt(0));
|
||
}
|
||
|
||
Token _errorToken([String message = null]) {
|
||
return _finishToken(TokenKind.ERROR);
|
||
}
|
||
|
||
int getIdentifierKind() {
|
||
// Is the identifier a unit type?
|
||
int tokId = TokenKind.matchUnits(_text, _startIndex, _index - _startIndex);
|
||
if (tokId == -1) {
|
||
tokId = (_text.substring(_startIndex, _index) == '!important') ?
|
||
TokenKind.IMPORTANT : -1;
|
||
}
|
||
|
||
return tokId >= 0 ? tokId : TokenKind.IDENTIFIER;
|
||
}
|
||
|
||
// Need to override so CSS version of isIdentifierPart is used.
|
||
Token finishIdentifier(int ch) {
|
||
while (_index < _text.length) {
|
||
// If parsing in pseudo function expression then minus is an operator
|
||
// not part of identifier.
|
||
var isIdentifier = selectorExpression
|
||
? TokenizerHelpers.isIdentifierPartExpr(_text.codeUnitAt(_index))
|
||
: TokenizerHelpers.isIdentifierPart(_text.codeUnitAt(_index));
|
||
if (!isIdentifier) {
|
||
break;
|
||
} else {
|
||
_index += 1;
|
||
}
|
||
}
|
||
|
||
int kind = getIdentifierKind();
|
||
if (kind == TokenKind.IDENTIFIER) {
|
||
return _finishToken(TokenKind.IDENTIFIER);
|
||
} else {
|
||
return _finishToken(kind);
|
||
}
|
||
}
|
||
|
||
Token finishImportant() {
|
||
|
||
}
|
||
|
||
Token finishNumber() {
|
||
eatDigits();
|
||
|
||
if (_peekChar() == 46/*.*/) {
|
||
// Handle the case of 1.toString().
|
||
_nextChar();
|
||
if (TokenizerHelpers.isDigit(_peekChar())) {
|
||
eatDigits();
|
||
return _finishToken(TokenKind.DOUBLE);
|
||
} else {
|
||
_index -= 1;
|
||
}
|
||
}
|
||
|
||
return _finishToken(TokenKind.INTEGER);
|
||
}
|
||
|
||
bool maybeEatDigit() {
|
||
if (_index < _text.length
|
||
&& TokenizerHelpers.isDigit(_text.codeUnitAt(_index))) {
|
||
_index += 1;
|
||
return true;
|
||
}
|
||
return false;
|
||
}
|
||
|
||
Token finishHexNumber() {
|
||
eatHexDigits();
|
||
return _finishToken(TokenKind.HEX_INTEGER);
|
||
}
|
||
|
||
void eatHexDigits() {
|
||
while (_index < _text.length) {
|
||
if (TokenizerHelpers.isHexDigit(_text.codeUnitAt(_index))) {
|
||
_index += 1;
|
||
} else {
|
||
return;
|
||
}
|
||
}
|
||
}
|
||
|
||
bool maybeEatHexDigit() {
|
||
if (_index < _text.length
|
||
&& TokenizerHelpers.isHexDigit(_text.codeUnitAt(_index))) {
|
||
_index += 1;
|
||
return true;
|
||
}
|
||
return false;
|
||
}
|
||
|
||
bool maybeEatQuestionMark() {
|
||
if (_index < _text.length &&
|
||
_text.codeUnitAt(_index) == QUESTION_MARK) {
|
||
_index += 1;
|
||
return true;
|
||
}
|
||
return false;
|
||
}
|
||
|
||
void eatQuestionMarks() {
|
||
while (_index < _text.length) {
|
||
if (_text.codeUnitAt(_index) == QUESTION_MARK) {
|
||
_index += 1;
|
||
} else {
|
||
return;
|
||
}
|
||
}
|
||
}
|
||
|
||
Token finishUnicodeRange() {
|
||
eatQuestionMarks();
|
||
return _finishToken(TokenKind.HEX_RANGE);
|
||
}
|
||
|
||
Token finishMultiLineComment() {
|
||
while (true) {
|
||
int ch = _nextChar();
|
||
if (ch == 0) {
|
||
return _finishToken(TokenKind.INCOMPLETE_COMMENT);
|
||
} else if (ch == 42/*'*'*/) {
|
||
if (_maybeEatChar(47/*'/'*/)) {
|
||
if (_skipWhitespace) {
|
||
return next();
|
||
} else {
|
||
return _finishToken(TokenKind.COMMENT);
|
||
}
|
||
}
|
||
} else if (ch == TokenChar.MINUS) {
|
||
/* Check if close part of Comment Definition --> (CDC). */
|
||
if (_maybeEatChar(TokenChar.MINUS)) {
|
||
if (_maybeEatChar(TokenChar.GREATER)) {
|
||
if (_skipWhitespace) {
|
||
return next();
|
||
} else {
|
||
return _finishToken(TokenKind.HTML_COMMENT);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return _errorToken();
|
||
}
|
||
|
||
}
|
||
|
||
/** Static helper methods. */
|
||
class TokenizerHelpers {
|
||
static bool isIdentifierStart(int c) {
|
||
return isIdentifierStartExpr(c) || c == 45 /*-*/;
|
||
}
|
||
|
||
static bool isDigit(int c) {
|
||
return (c >= 48/*0*/ && c <= 57/*9*/);
|
||
}
|
||
|
||
static bool isHexDigit(int c) {
|
||
return (isDigit(c) || (c >= 97/*a*/ && c <= 102/*f*/)
|
||
|| (c >= 65/*A*/ && c <= 70/*F*/));
|
||
}
|
||
|
||
static bool isIdentifierPart(int c) {
|
||
return isIdentifierPartExpr(c) || c == 45 /*-*/;
|
||
}
|
||
|
||
/** Pseudo function expressions identifiers can't have a minus sign. */
|
||
static bool isIdentifierStartExpr(int c) {
|
||
return ((c >= 97/*a*/ && c <= 122/*z*/) || (c >= 65/*A*/ && c <= 90/*Z*/) ||
|
||
c == 95/*_*/);
|
||
}
|
||
|
||
/** Pseudo function expressions identifiers can't have a minus sign. */
|
||
static bool isIdentifierPartExpr(int c) {
|
||
return (isIdentifierStartExpr(c) || isDigit(c));
|
||
}
|
||
}
|