terry@google.com
91dd628bd8
Need unique name for library collision with template parser. Few more changes to work under VM. More changes for lang.dart seperation. Seperation from lang.dart (tokenizer can't access private fields of base class in another library). This was allowed in frog but not in the VM. VM is right so need to sever dependency. BUG= TEST= Review URL: https://chromiumcodereview.appspot.com//9695048 git-svn-id: https://dart.googlecode.com/svn/branches/bleeding_edge/dart@5544 260f80e4-7a28-3924-810f-c04153c831b5
320 lines
10 KiB
Dart
320 lines
10 KiB
Dart
// Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file
|
|
// for details. All rights reserved. Use of this source code is governed by a
|
|
// BSD-style license that can be found in the LICENSE file.
|
|
|
|
class Tokenizer extends CSSTokenizerBase {
|
|
TokenKind cssTokens;
|
|
|
|
bool _selectorParsing;
|
|
|
|
Tokenizer(SourceFile source, bool skipWhitespace, [int index = 0])
|
|
: super(source, skipWhitespace, index), _selectorParsing = false {
|
|
cssTokens = new TokenKind();
|
|
}
|
|
|
|
int get startIndex() => _startIndex;
|
|
|
|
Token next() {
|
|
// keep track of our starting position
|
|
_startIndex = _index;
|
|
|
|
if (_interpStack != null && _interpStack.depth == 0) {
|
|
var istack = _interpStack;
|
|
_interpStack = _interpStack.pop();
|
|
|
|
/* TODO(terry): Enable for variable and string interpolation.
|
|
* if (istack.isMultiline) {
|
|
* return finishMultilineStringBody(istack.quote);
|
|
* } else {
|
|
* return finishStringBody(istack.quote);
|
|
* }
|
|
*/
|
|
}
|
|
|
|
int ch;
|
|
ch = _nextChar();
|
|
switch(ch) {
|
|
case 0:
|
|
return _finishToken(TokenKind.END_OF_FILE);
|
|
case cssTokens.tokens[TokenKind.SPACE]:
|
|
case cssTokens.tokens[TokenKind.TAB]:
|
|
case cssTokens.tokens[TokenKind.NEWLINE]:
|
|
case cssTokens.tokens[TokenKind.RETURN]:
|
|
return finishWhitespace();
|
|
case cssTokens.tokens[TokenKind.END_OF_FILE]:
|
|
return _finishToken(TokenKind.END_OF_FILE);
|
|
case cssTokens.tokens[TokenKind.AT]:
|
|
return _finishToken(TokenKind.AT);
|
|
case cssTokens.tokens[TokenKind.DOT]:
|
|
int start = _startIndex; // Start where the dot started.
|
|
if (maybeEatDigit()) {
|
|
// looks like a number dot followed by digit(s).
|
|
Token number = finishNumber();
|
|
if (number.kind == TokenKind.INTEGER) {
|
|
// It's a number but it's preceeded by a dot, so make it a double.
|
|
_startIndex = start;
|
|
return _finishToken(TokenKind.DOUBLE);
|
|
} else {
|
|
// Don't allow dot followed by a double (e.g, '..1').
|
|
return _errorToken();
|
|
}
|
|
} else {
|
|
// It's really a dot.
|
|
return _finishToken(TokenKind.DOT);
|
|
}
|
|
case cssTokens.tokens[TokenKind.LPAREN]:
|
|
return _finishToken(TokenKind.LPAREN);
|
|
case cssTokens.tokens[TokenKind.RPAREN]:
|
|
return _finishToken(TokenKind.RPAREN);
|
|
case cssTokens.tokens[TokenKind.LBRACE]:
|
|
return _finishToken(TokenKind.LBRACE);
|
|
case cssTokens.tokens[TokenKind.RBRACE]:
|
|
return _finishToken(TokenKind.RBRACE);
|
|
case cssTokens.tokens[TokenKind.LBRACK]:
|
|
return _finishToken(TokenKind.LBRACK);
|
|
case cssTokens.tokens[TokenKind.RBRACK]:
|
|
return _finishToken(TokenKind.RBRACK);
|
|
case cssTokens.tokens[TokenKind.HASH]:
|
|
return _finishToken(TokenKind.HASH);
|
|
case cssTokens.tokens[TokenKind.PLUS]:
|
|
if (maybeEatDigit()) {
|
|
return finishNumber();
|
|
} else {
|
|
return _finishToken(TokenKind.PLUS);
|
|
}
|
|
case cssTokens.tokens[TokenKind.MINUS]:
|
|
if (maybeEatDigit()) {
|
|
return finishNumber();
|
|
} else if (TokenizerHelpers.isIdentifierStart(ch)) {
|
|
return this.finishIdentifier(ch);
|
|
} else {
|
|
return _finishToken(TokenKind.MINUS);
|
|
}
|
|
case cssTokens.tokens[TokenKind.GREATER]:
|
|
return _finishToken(TokenKind.GREATER);
|
|
case cssTokens.tokens[TokenKind.TILDE]:
|
|
if (_maybeEatChar(cssTokens.tokens[TokenKind.EQUALS])) {
|
|
return _finishToken(TokenKind.INCLUDES); // ~=
|
|
} else {
|
|
return _finishToken(TokenKind.TILDE);
|
|
}
|
|
case cssTokens.tokens[TokenKind.ASTERISK]:
|
|
if (_maybeEatChar(cssTokens.tokens[TokenKind.EQUALS])) {
|
|
return _finishToken(TokenKind.SUBSTRING_MATCH); // *=
|
|
} else {
|
|
return _finishToken(TokenKind.ASTERISK);
|
|
}
|
|
case cssTokens.tokens[TokenKind.NAMESPACE]:
|
|
return _finishToken(TokenKind.NAMESPACE);
|
|
case cssTokens.tokens[TokenKind.COLON]:
|
|
return _finishToken(TokenKind.COLON);
|
|
case cssTokens.tokens[TokenKind.COMMA]:
|
|
return _finishToken(TokenKind.COMMA);
|
|
case cssTokens.tokens[TokenKind.SEMICOLON]:
|
|
return _finishToken(TokenKind.SEMICOLON);
|
|
case cssTokens.tokens[TokenKind.PERCENT]:
|
|
return _finishToken(TokenKind.PERCENT);
|
|
case cssTokens.tokens[TokenKind.SINGLE_QUOTE]:
|
|
return _finishToken(TokenKind.SINGLE_QUOTE);
|
|
case cssTokens.tokens[TokenKind.DOUBLE_QUOTE]:
|
|
return _finishToken(TokenKind.DOUBLE_QUOTE);
|
|
case cssTokens.tokens[TokenKind.SLASH]:
|
|
if (_maybeEatChar(cssTokens.tokens[TokenKind.ASTERISK])) {
|
|
return finishMultiLineComment();
|
|
} else {
|
|
return _finishToken(TokenKind.SLASH);
|
|
}
|
|
case cssTokens.tokens[TokenKind.LESS]: // <!--
|
|
if (_maybeEatChar(cssTokens.tokens[TokenKind.BANG]) &&
|
|
_maybeEatChar(cssTokens.tokens[TokenKind.MINUS]) &&
|
|
_maybeEatChar(cssTokens.tokens[TokenKind.MINUS])) {
|
|
return finishMultiLineComment();
|
|
} else {
|
|
return _finishToken(TokenKind.LESS);
|
|
}
|
|
case cssTokens.tokens[TokenKind.EQUALS]:
|
|
return _finishToken(TokenKind.EQUALS);
|
|
case cssTokens.tokens[TokenKind.OR]:
|
|
if (_maybeEatChar(cssTokens.tokens[TokenKind.EQUALS])) {
|
|
return _finishToken(TokenKind.DASH_MATCH); // |=
|
|
} else {
|
|
return _finishToken(TokenKind.OR);
|
|
}
|
|
case cssTokens.tokens[TokenKind.CARET]:
|
|
if (_maybeEatChar(cssTokens.tokens[TokenKind.EQUALS])) {
|
|
return _finishToken(TokenKind.PREFIX_MATCH); // ^=
|
|
} else {
|
|
return _finishToken(TokenKind.CARET);
|
|
}
|
|
case cssTokens.tokens[TokenKind.DOLLAR]:
|
|
if (_maybeEatChar(cssTokens.tokens[TokenKind.EQUALS])) {
|
|
return _finishToken(TokenKind.SUFFIX_MATCH); // $=
|
|
} else {
|
|
return _finishToken(TokenKind.DOLLAR);
|
|
}
|
|
case cssTokens.tokens[TokenKind.BANG]:
|
|
Token tok = finishIdentifier(ch);
|
|
return (tok == null) ? _finishToken(TokenKind.BANG) : tok;
|
|
default:
|
|
if (TokenizerHelpers.isIdentifierStart(ch)) {
|
|
return this.finishIdentifier(ch);
|
|
} else if (isDigit(ch)) {
|
|
return this.finishNumber();
|
|
} else {
|
|
return _errorToken();
|
|
}
|
|
}
|
|
}
|
|
|
|
// TODO(jmesserly): we need a way to emit human readable error messages from
|
|
// the tokenizer.
|
|
Token _errorToken([String message = null]) {
|
|
return _finishToken(TokenKind.ERROR);
|
|
}
|
|
|
|
int getIdentifierKind() {
|
|
// Is the identifier a unit type?
|
|
int tokId = TokenKind.matchUnits(_text, _startIndex, _index - _startIndex);
|
|
if (tokId == -1) {
|
|
// No, is it a directive?
|
|
tokId = TokenKind.matchDirectives(_text, _startIndex, _index - _startIndex);
|
|
}
|
|
if (tokId == -1) {
|
|
tokId = (_text.substring(_startIndex, _index) == '!important') ?
|
|
TokenKind.IMPORTANT : -1;
|
|
}
|
|
|
|
return tokId >= 0 ? tokId : TokenKind.IDENTIFIER;
|
|
}
|
|
|
|
// Need to override so CSS version of isIdentifierPart is used.
|
|
Token finishIdentifier(int ch) {
|
|
while (_index < _text.length) {
|
|
// if (!TokenizerHelpers.isIdentifierPart(_text.charCodeAt(_index++))) {
|
|
if (!TokenizerHelpers.isIdentifierPart(_text.charCodeAt(_index))) {
|
|
// _index--;
|
|
break;
|
|
} else {
|
|
_index += 1;
|
|
}
|
|
}
|
|
if (_interpStack != null && _interpStack.depth == -1) {
|
|
_interpStack.depth = 0;
|
|
}
|
|
int kind = getIdentifierKind();
|
|
if (kind == TokenKind.IDENTIFIER) {
|
|
return _finishToken(TokenKind.IDENTIFIER);
|
|
} else {
|
|
return _finishToken(kind);
|
|
}
|
|
}
|
|
|
|
Token finishImportant() {
|
|
|
|
}
|
|
|
|
Token finishNumber() {
|
|
eatDigits();
|
|
|
|
if (_peekChar() == 46/*.*/) {
|
|
// Handle the case of 1.toString().
|
|
_nextChar();
|
|
if (isDigit(_peekChar())) {
|
|
eatDigits();
|
|
return _finishToken(TokenKind.DOUBLE);
|
|
} else {
|
|
_index -= 1;
|
|
}
|
|
}
|
|
|
|
return _finishToken(TokenKind.INTEGER);
|
|
}
|
|
|
|
bool maybeEatDigit() {
|
|
if (_index < _text.length && isDigit(_text.charCodeAt(_index))) {
|
|
_index += 1;
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
void eatHexDigits() {
|
|
while (_index < _text.length) {
|
|
if (isHexDigit(_text.charCodeAt(_index))) {
|
|
_index += 1;
|
|
} else {
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
bool maybeEatHexDigit() {
|
|
if (_index < _text.length && isHexDigit(_text.charCodeAt(_index))) {
|
|
_index += 1;
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
Token finishMultiLineComment() {
|
|
while (true) {
|
|
int ch = _nextChar();
|
|
if (ch == 0) {
|
|
return _finishToken(TokenKind.INCOMPLETE_COMMENT);
|
|
} else if (ch == 42/*'*'*/) {
|
|
if (_maybeEatChar(47/*'/'*/)) {
|
|
if (_skipWhitespace) {
|
|
return next();
|
|
} else {
|
|
return _finishToken(TokenKind.COMMENT);
|
|
}
|
|
}
|
|
} else if (ch == cssTokens.tokens[TokenKind.MINUS]) {
|
|
/* Check if close part of Comment Definition --> (CDC). */
|
|
if (_maybeEatChar(cssTokens.tokens[TokenKind.MINUS])) {
|
|
if (_maybeEatChar(cssTokens.tokens[TokenKind.GREATER])) {
|
|
if (_skipWhitespace) {
|
|
return next();
|
|
} else {
|
|
return _finishToken(TokenKind.HTML_COMMENT);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return _errorToken();
|
|
}
|
|
|
|
}
|
|
|
|
/** Static helper methods. */
|
|
/** Static helper methods. */
|
|
class TokenizerHelpers {
|
|
|
|
static bool isIdentifierStart(int c) {
|
|
return ((c >= 97/*a*/ && c <= 122/*z*/) || (c >= 65/*A*/ && c <= 90/*Z*/) ||
|
|
c == 95/*_*/ || c == 45 /*-*/);
|
|
}
|
|
|
|
static bool isDigit(int c) {
|
|
return (c >= 48/*0*/ && c <= 57/*9*/);
|
|
}
|
|
|
|
static bool isHexDigit(int c) {
|
|
return (isDigit(c) || (c >= 97/*a*/ && c <= 102/*f*/) || (c >= 65/*A*/ && c <= 70/*F*/));
|
|
}
|
|
|
|
static bool isWhitespace(int c) {
|
|
return (c == 32/*' '*/ || c == 9/*'\t'*/ || c == 10/*'\n'*/ || c == 13/*'\r'*/);
|
|
}
|
|
|
|
static bool isIdentifierPart(int c) {
|
|
return (isIdentifierStart(c) || isDigit(c) || c == 45 /*-*/);
|
|
}
|
|
|
|
static bool isInterpIdentifierPart(int c) {
|
|
return (isIdentifierStart(c) || isDigit(c));
|
|
}
|
|
}
|
|
|