lrn@google.com
3c0ac360e7
Make String.codeUnits return a List. Review URL: https://codereview.chromium.org//12282038 git-svn-id: https://dart.googlecode.com/svn/branches/bleeding_edge/dart@18960 260f80e4-7a28-3924-810f-c04153c831b5
316 lines
10 KiB
Dart
316 lines
10 KiB
Dart
// Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file
|
|
// for details. All rights reserved. Use of this source code is governed by a
|
|
// BSD-style license that can be found in the LICENSE file.
|
|
|
|
class Tokenizer extends CSSTokenizerBase {
|
|
TokenKind cssTokens;
|
|
|
|
bool _selectorParsing;
|
|
|
|
Tokenizer(SourceFile source, bool skipWhitespace, [int index = 0])
|
|
: super(source, skipWhitespace, index), _selectorParsing = false {
|
|
cssTokens = new TokenKind();
|
|
}
|
|
|
|
int get startIndex => _startIndex;
|
|
|
|
Token next() {
|
|
// keep track of our starting position
|
|
_startIndex = _index;
|
|
|
|
if (_interpStack != null && _interpStack.depth == 0) {
|
|
var istack = _interpStack;
|
|
_interpStack = _interpStack.pop();
|
|
|
|
/* TODO(terry): Enable for variable and string interpolation.
|
|
* if (istack.isMultiline) {
|
|
* return finishMultilineStringBody(istack.quote);
|
|
* } else {
|
|
* return finishStringBody(istack.quote);
|
|
* }
|
|
*/
|
|
}
|
|
|
|
int ch;
|
|
ch = _nextChar();
|
|
switch(ch) {
|
|
case 0:
|
|
return _finishToken(TokenKind.END_OF_FILE);
|
|
case cssTokens.tokens[TokenKind.SPACE]:
|
|
case cssTokens.tokens[TokenKind.TAB]:
|
|
case cssTokens.tokens[TokenKind.NEWLINE]:
|
|
case cssTokens.tokens[TokenKind.RETURN]:
|
|
return finishWhitespace();
|
|
case cssTokens.tokens[TokenKind.END_OF_FILE]:
|
|
return _finishToken(TokenKind.END_OF_FILE);
|
|
case cssTokens.tokens[TokenKind.AT]:
|
|
return _finishToken(TokenKind.AT);
|
|
case cssTokens.tokens[TokenKind.DOT]:
|
|
int start = _startIndex; // Start where the dot started.
|
|
if (maybeEatDigit()) {
|
|
// looks like a number dot followed by digit(s).
|
|
Token number = finishNumber();
|
|
if (number.kind == TokenKind.INTEGER) {
|
|
// It's a number but it's preceeded by a dot, so make it a double.
|
|
_startIndex = start;
|
|
return _finishToken(TokenKind.DOUBLE);
|
|
} else {
|
|
// Don't allow dot followed by a double (e.g, '..1').
|
|
return _errorToken();
|
|
}
|
|
} else {
|
|
// It's really a dot.
|
|
return _finishToken(TokenKind.DOT);
|
|
}
|
|
case cssTokens.tokens[TokenKind.LPAREN]:
|
|
return _finishToken(TokenKind.LPAREN);
|
|
case cssTokens.tokens[TokenKind.RPAREN]:
|
|
return _finishToken(TokenKind.RPAREN);
|
|
case cssTokens.tokens[TokenKind.LBRACE]:
|
|
return _finishToken(TokenKind.LBRACE);
|
|
case cssTokens.tokens[TokenKind.RBRACE]:
|
|
return _finishToken(TokenKind.RBRACE);
|
|
case cssTokens.tokens[TokenKind.LBRACK]:
|
|
return _finishToken(TokenKind.LBRACK);
|
|
case cssTokens.tokens[TokenKind.RBRACK]:
|
|
return _finishToken(TokenKind.RBRACK);
|
|
case cssTokens.tokens[TokenKind.HASH]:
|
|
return _finishToken(TokenKind.HASH);
|
|
case cssTokens.tokens[TokenKind.PLUS]:
|
|
if (maybeEatDigit()) {
|
|
return finishNumber();
|
|
} else {
|
|
return _finishToken(TokenKind.PLUS);
|
|
}
|
|
case cssTokens.tokens[TokenKind.MINUS]:
|
|
if (maybeEatDigit()) {
|
|
return finishNumber();
|
|
} else if (TokenizerHelpers.isIdentifierStart(ch)) {
|
|
return this.finishIdentifier(ch);
|
|
} else {
|
|
return _finishToken(TokenKind.MINUS);
|
|
}
|
|
case cssTokens.tokens[TokenKind.GREATER]:
|
|
return _finishToken(TokenKind.GREATER);
|
|
case cssTokens.tokens[TokenKind.TILDE]:
|
|
if (_maybeEatChar(cssTokens.tokens[TokenKind.EQUALS])) {
|
|
return _finishToken(TokenKind.INCLUDES); // ~=
|
|
} else {
|
|
return _finishToken(TokenKind.TILDE);
|
|
}
|
|
case cssTokens.tokens[TokenKind.ASTERISK]:
|
|
if (_maybeEatChar(cssTokens.tokens[TokenKind.EQUALS])) {
|
|
return _finishToken(TokenKind.SUBSTRING_MATCH); // *=
|
|
} else {
|
|
return _finishToken(TokenKind.ASTERISK);
|
|
}
|
|
case cssTokens.tokens[TokenKind.NAMESPACE]:
|
|
return _finishToken(TokenKind.NAMESPACE);
|
|
case cssTokens.tokens[TokenKind.COLON]:
|
|
return _finishToken(TokenKind.COLON);
|
|
case cssTokens.tokens[TokenKind.COMMA]:
|
|
return _finishToken(TokenKind.COMMA);
|
|
case cssTokens.tokens[TokenKind.SEMICOLON]:
|
|
return _finishToken(TokenKind.SEMICOLON);
|
|
case cssTokens.tokens[TokenKind.PERCENT]:
|
|
return _finishToken(TokenKind.PERCENT);
|
|
case cssTokens.tokens[TokenKind.SINGLE_QUOTE]:
|
|
return _finishToken(TokenKind.SINGLE_QUOTE);
|
|
case cssTokens.tokens[TokenKind.DOUBLE_QUOTE]:
|
|
return _finishToken(TokenKind.DOUBLE_QUOTE);
|
|
case cssTokens.tokens[TokenKind.SLASH]:
|
|
if (_maybeEatChar(cssTokens.tokens[TokenKind.ASTERISK])) {
|
|
return finishMultiLineComment();
|
|
} else {
|
|
return _finishToken(TokenKind.SLASH);
|
|
}
|
|
case cssTokens.tokens[TokenKind.LESS]: // <!--
|
|
if (_maybeEatChar(cssTokens.tokens[TokenKind.BANG]) &&
|
|
_maybeEatChar(cssTokens.tokens[TokenKind.MINUS]) &&
|
|
_maybeEatChar(cssTokens.tokens[TokenKind.MINUS])) {
|
|
return finishMultiLineComment();
|
|
} else {
|
|
return _finishToken(TokenKind.LESS);
|
|
}
|
|
case cssTokens.tokens[TokenKind.EQUALS]:
|
|
return _finishToken(TokenKind.EQUALS);
|
|
case cssTokens.tokens[TokenKind.OR]:
|
|
if (_maybeEatChar(cssTokens.tokens[TokenKind.EQUALS])) {
|
|
return _finishToken(TokenKind.DASH_MATCH); // |=
|
|
} else {
|
|
return _finishToken(TokenKind.OR);
|
|
}
|
|
case cssTokens.tokens[TokenKind.CARET]:
|
|
if (_maybeEatChar(cssTokens.tokens[TokenKind.EQUALS])) {
|
|
return _finishToken(TokenKind.PREFIX_MATCH); // ^=
|
|
} else {
|
|
return _finishToken(TokenKind.CARET);
|
|
}
|
|
case cssTokens.tokens[TokenKind.DOLLAR]:
|
|
if (_maybeEatChar(cssTokens.tokens[TokenKind.EQUALS])) {
|
|
return _finishToken(TokenKind.SUFFIX_MATCH); // $=
|
|
} else {
|
|
return _finishToken(TokenKind.DOLLAR);
|
|
}
|
|
case cssTokens.tokens[TokenKind.BANG]:
|
|
Token tok = finishIdentifier(ch);
|
|
return (tok == null) ? _finishToken(TokenKind.BANG) : tok;
|
|
default:
|
|
if (TokenizerHelpers.isIdentifierStart(ch)) {
|
|
return this.finishIdentifier(ch);
|
|
} else if (TokenizerHelpers.isDigit(ch)) {
|
|
return this.finishNumber();
|
|
} else {
|
|
return _errorToken();
|
|
}
|
|
}
|
|
}
|
|
|
|
// TODO(jmesserly): we need a way to emit human readable error messages from
|
|
// the tokenizer.
|
|
Token _errorToken([String message = null]) {
|
|
return _finishToken(TokenKind.ERROR);
|
|
}
|
|
|
|
int getIdentifierKind() {
|
|
// Is the identifier a unit type?
|
|
int tokId = TokenKind.matchUnits(_text, _startIndex, _index - _startIndex);
|
|
if (tokId == -1) {
|
|
// No, is it a directive?
|
|
tokId = TokenKind.matchDirectives(
|
|
_text, _startIndex, _index - _startIndex);
|
|
}
|
|
if (tokId == -1) {
|
|
tokId = (_text.substring(_startIndex, _index) == '!important') ?
|
|
TokenKind.IMPORTANT : -1;
|
|
}
|
|
|
|
return tokId >= 0 ? tokId : TokenKind.IDENTIFIER;
|
|
}
|
|
|
|
// Need to override so CSS version of isIdentifierPart is used.
|
|
Token finishIdentifier(int ch) {
|
|
while (_index < _text.length) {
|
|
// if (!TokenizerHelpers.isIdentifierPart(_text.codeUnitAt(_index++))) {
|
|
if (!TokenizerHelpers.isIdentifierPart(_text.codeUnitAt(_index))) {
|
|
// _index--;
|
|
break;
|
|
} else {
|
|
_index += 1;
|
|
}
|
|
}
|
|
if (_interpStack != null && _interpStack.depth == -1) {
|
|
_interpStack.depth = 0;
|
|
}
|
|
int kind = getIdentifierKind();
|
|
if (kind == TokenKind.IDENTIFIER) {
|
|
return _finishToken(TokenKind.IDENTIFIER);
|
|
} else {
|
|
return _finishToken(kind);
|
|
}
|
|
}
|
|
|
|
Token finishImportant() {
|
|
|
|
}
|
|
|
|
Token finishNumber() {
|
|
eatDigits();
|
|
|
|
if (_peekChar() == 46/*.*/) {
|
|
// Handle the case of 1.toString().
|
|
_nextChar();
|
|
if (TokenizerHelpers.isDigit(_peekChar())) {
|
|
eatDigits();
|
|
return _finishToken(TokenKind.DOUBLE);
|
|
} else {
|
|
_index -= 1;
|
|
}
|
|
}
|
|
|
|
return _finishToken(TokenKind.INTEGER);
|
|
}
|
|
|
|
bool maybeEatDigit() {
|
|
if (_index < _text.length
|
|
&& TokenizerHelpers.isDigit(_text.codeUnitAt(_index))) {
|
|
_index += 1;
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
void eatHexDigits() {
|
|
while (_index < _text.length) {
|
|
if (TokenizerHelpers.isHexDigit(_text.codeUnitAt(_index))) {
|
|
_index += 1;
|
|
} else {
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
bool maybeEatHexDigit() {
|
|
if (_index < _text.length
|
|
&& TokenizerHelpers.isHexDigit(_text.codeUnitAt(_index))) {
|
|
_index += 1;
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
Token finishMultiLineComment() {
|
|
while (true) {
|
|
int ch = _nextChar();
|
|
if (ch == 0) {
|
|
return _finishToken(TokenKind.INCOMPLETE_COMMENT);
|
|
} else if (ch == 42/*'*'*/) {
|
|
if (_maybeEatChar(47/*'/'*/)) {
|
|
if (_skipWhitespace) {
|
|
return next();
|
|
} else {
|
|
return _finishToken(TokenKind.COMMENT);
|
|
}
|
|
}
|
|
} else if (ch == cssTokens.tokens[TokenKind.MINUS]) {
|
|
/* Check if close part of Comment Definition --> (CDC). */
|
|
if (_maybeEatChar(cssTokens.tokens[TokenKind.MINUS])) {
|
|
if (_maybeEatChar(cssTokens.tokens[TokenKind.GREATER])) {
|
|
if (_skipWhitespace) {
|
|
return next();
|
|
} else {
|
|
return _finishToken(TokenKind.HTML_COMMENT);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return _errorToken();
|
|
}
|
|
|
|
}
|
|
|
|
/** Static helper methods. */
|
|
/** Static helper methods. */
|
|
class TokenizerHelpers {
|
|
|
|
static bool isIdentifierStart(int c) {
|
|
return ((c >= 97/*a*/ && c <= 122/*z*/) || (c >= 65/*A*/ && c <= 90/*Z*/) ||
|
|
c == 95/*_*/ || c == 45 /*-*/);
|
|
}
|
|
|
|
static bool isDigit(int c) {
|
|
return (c >= 48/*0*/ && c <= 57/*9*/);
|
|
}
|
|
|
|
static bool isHexDigit(int c) {
|
|
return (isDigit(c) || (c >= 97/*a*/ && c <= 102/*f*/)
|
|
|| (c >= 65/*A*/ && c <= 70/*F*/));
|
|
}
|
|
|
|
static bool isIdentifierPart(int c) {
|
|
return (isIdentifierStart(c) || isDigit(c) || c == 45 /*-*/);
|
|
}
|
|
}
|
|
|