12da489cd246702bee5938545b18a6f710ed214bcJamie Gennis#!/usr/bin/env python 22da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# 32da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# Copyright 2008 The Closure Linter Authors. All Rights Reserved. 42da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# 52da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# Licensed under the Apache License, Version 2.0 (the "License"); 62da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# you may not use this file except in compliance with the License. 72da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# You may obtain a copy of the License at 82da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# 92da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# http://www.apache.org/licenses/LICENSE-2.0 102da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# 112da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# Unless required by applicable law or agreed to in writing, software 122da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# distributed under the License is distributed on an "AS-IS" BASIS, 132da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 142da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# See the License for the specific language governing permissions and 152da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# limitations under the License. 162da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 172da489cd246702bee5938545b18a6f710ed214bcJamie Gennis"""Classes to represent JavaScript tokens.""" 182da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 192da489cd246702bee5938545b18a6f710ed214bcJamie Gennis__author__ = ('robbyw@google.com (Robert Walker)', 202da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 'ajp@google.com (Andy Perelson)') 212da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 222da489cd246702bee5938545b18a6f710ed214bcJamie Gennisfrom closure_linter.common import tokens 232da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 242da489cd246702bee5938545b18a6f710ed214bcJamie Gennisclass JavaScriptTokenType(tokens.TokenType): 252da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """Enumeration of JavaScript token types, and useful sets of token types.""" 262da489cd246702bee5938545b18a6f710ed214bcJamie Gennis NUMBER = 'number' 272da489cd246702bee5938545b18a6f710ed214bcJamie Gennis START_SINGLE_LINE_COMMENT = '//' 282da489cd246702bee5938545b18a6f710ed214bcJamie Gennis START_BLOCK_COMMENT = '/*' 292da489cd246702bee5938545b18a6f710ed214bcJamie Gennis START_DOC_COMMENT = '/**' 302da489cd246702bee5938545b18a6f710ed214bcJamie Gennis END_BLOCK_COMMENT = '*/' 312da489cd246702bee5938545b18a6f710ed214bcJamie Gennis END_DOC_COMMENT = 'doc */' 322da489cd246702bee5938545b18a6f710ed214bcJamie Gennis COMMENT = 'comment' 332da489cd246702bee5938545b18a6f710ed214bcJamie Gennis SINGLE_QUOTE_STRING_START = "'string" 342da489cd246702bee5938545b18a6f710ed214bcJamie Gennis SINGLE_QUOTE_STRING_END = "string'" 352da489cd246702bee5938545b18a6f710ed214bcJamie Gennis DOUBLE_QUOTE_STRING_START = '"string' 362da489cd246702bee5938545b18a6f710ed214bcJamie Gennis DOUBLE_QUOTE_STRING_END = 'string"' 372da489cd246702bee5938545b18a6f710ed214bcJamie Gennis STRING_TEXT = 'string' 382da489cd246702bee5938545b18a6f710ed214bcJamie Gennis START_BLOCK = '{' 392da489cd246702bee5938545b18a6f710ed214bcJamie Gennis END_BLOCK = '}' 402da489cd246702bee5938545b18a6f710ed214bcJamie Gennis START_PAREN = '(' 412da489cd246702bee5938545b18a6f710ed214bcJamie Gennis END_PAREN = ')' 422da489cd246702bee5938545b18a6f710ed214bcJamie Gennis START_BRACKET = '[' 432da489cd246702bee5938545b18a6f710ed214bcJamie Gennis END_BRACKET = ']' 442da489cd246702bee5938545b18a6f710ed214bcJamie Gennis REGEX = '/regex/' 452da489cd246702bee5938545b18a6f710ed214bcJamie Gennis FUNCTION_DECLARATION = 'function(...)' 462da489cd246702bee5938545b18a6f710ed214bcJamie Gennis FUNCTION_NAME = 'function functionName(...)' 472da489cd246702bee5938545b18a6f710ed214bcJamie Gennis START_PARAMETERS = 'startparams(' 482da489cd246702bee5938545b18a6f710ed214bcJamie Gennis PARAMETERS = 'pa,ra,ms' 492da489cd246702bee5938545b18a6f710ed214bcJamie Gennis END_PARAMETERS = ')endparams' 502da489cd246702bee5938545b18a6f710ed214bcJamie Gennis SEMICOLON = ';' 512da489cd246702bee5938545b18a6f710ed214bcJamie Gennis DOC_FLAG = '@flag' 522da489cd246702bee5938545b18a6f710ed214bcJamie Gennis DOC_INLINE_FLAG = '{@flag ...}' 532da489cd246702bee5938545b18a6f710ed214bcJamie Gennis DOC_START_BRACE = 'doc {' 542da489cd246702bee5938545b18a6f710ed214bcJamie Gennis DOC_END_BRACE = 'doc }' 552da489cd246702bee5938545b18a6f710ed214bcJamie Gennis DOC_PREFIX = 'comment prefix: * ' 562da489cd246702bee5938545b18a6f710ed214bcJamie Gennis SIMPLE_LVALUE = 'lvalue=' 572da489cd246702bee5938545b18a6f710ed214bcJamie Gennis KEYWORD = 'keyword' 582da489cd246702bee5938545b18a6f710ed214bcJamie Gennis OPERATOR = 'operator' 592da489cd246702bee5938545b18a6f710ed214bcJamie Gennis IDENTIFIER = 'identifier' 602da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 612da489cd246702bee5938545b18a6f710ed214bcJamie Gennis STRING_TYPES = frozenset([ 622da489cd246702bee5938545b18a6f710ed214bcJamie Gennis SINGLE_QUOTE_STRING_START, SINGLE_QUOTE_STRING_END, 632da489cd246702bee5938545b18a6f710ed214bcJamie Gennis DOUBLE_QUOTE_STRING_START, DOUBLE_QUOTE_STRING_END, STRING_TEXT]) 642da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 652da489cd246702bee5938545b18a6f710ed214bcJamie Gennis COMMENT_TYPES = frozenset([START_SINGLE_LINE_COMMENT, COMMENT, 662da489cd246702bee5938545b18a6f710ed214bcJamie Gennis START_BLOCK_COMMENT, START_DOC_COMMENT, 672da489cd246702bee5938545b18a6f710ed214bcJamie Gennis END_BLOCK_COMMENT, END_DOC_COMMENT, 682da489cd246702bee5938545b18a6f710ed214bcJamie Gennis DOC_START_BRACE, DOC_END_BRACE, 692da489cd246702bee5938545b18a6f710ed214bcJamie Gennis DOC_FLAG, DOC_INLINE_FLAG, DOC_PREFIX]) 702da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 712da489cd246702bee5938545b18a6f710ed214bcJamie Gennis FLAG_DESCRIPTION_TYPES = frozenset([ 722da489cd246702bee5938545b18a6f710ed214bcJamie Gennis DOC_INLINE_FLAG, COMMENT, DOC_START_BRACE, DOC_END_BRACE]) 732da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 742da489cd246702bee5938545b18a6f710ed214bcJamie Gennis FLAG_ENDING_TYPES = frozenset([DOC_FLAG, END_DOC_COMMENT]) 752da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 762da489cd246702bee5938545b18a6f710ed214bcJamie Gennis NON_CODE_TYPES = COMMENT_TYPES | frozenset([ 772da489cd246702bee5938545b18a6f710ed214bcJamie Gennis tokens.TokenType.WHITESPACE, tokens.TokenType.BLANK_LINE]) 782da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 792da489cd246702bee5938545b18a6f710ed214bcJamie Gennis UNARY_OPERATORS = ['!', 'new', 'delete', 'typeof', 'void'] 802da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 812da489cd246702bee5938545b18a6f710ed214bcJamie Gennis UNARY_OK_OPERATORS = ['--', '++', '-', '+'] + UNARY_OPERATORS 822da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 832da489cd246702bee5938545b18a6f710ed214bcJamie Gennis UNARY_POST_OPERATORS = ['--', '++'] 842da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 852da489cd246702bee5938545b18a6f710ed214bcJamie Gennis # An expression ender is any token that can end an object - i.e. we could have 862da489cd246702bee5938545b18a6f710ed214bcJamie Gennis # x.y or [1, 2], or (10 + 9) or {a: 10}. 872da489cd246702bee5938545b18a6f710ed214bcJamie Gennis EXPRESSION_ENDER_TYPES = [tokens.TokenType.NORMAL, IDENTIFIER, NUMBER, 882da489cd246702bee5938545b18a6f710ed214bcJamie Gennis SIMPLE_LVALUE, END_BRACKET, END_PAREN, END_BLOCK, 892da489cd246702bee5938545b18a6f710ed214bcJamie Gennis SINGLE_QUOTE_STRING_END, DOUBLE_QUOTE_STRING_END] 902da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 912da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 922da489cd246702bee5938545b18a6f710ed214bcJamie Gennisclass JavaScriptToken(tokens.Token): 932da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """JavaScript token subclass of Token, provides extra instance checks. 942da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 952da489cd246702bee5938545b18a6f710ed214bcJamie Gennis The following token types have data in attached_object: 962da489cd246702bee5938545b18a6f710ed214bcJamie Gennis - All JsDoc flags: a parser.JsDocFlag object. 972da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """ 982da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 992da489cd246702bee5938545b18a6f710ed214bcJamie Gennis def IsKeyword(self, keyword): 1002da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """Tests if this token is the given keyword. 1012da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 1022da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Args: 1032da489cd246702bee5938545b18a6f710ed214bcJamie Gennis keyword: The keyword to compare to. 1042da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 1052da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Returns: 1062da489cd246702bee5938545b18a6f710ed214bcJamie Gennis True if this token is a keyword token with the given name. 1072da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """ 1082da489cd246702bee5938545b18a6f710ed214bcJamie Gennis return self.type == JavaScriptTokenType.KEYWORD and self.string == keyword 1092da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 1102da489cd246702bee5938545b18a6f710ed214bcJamie Gennis def IsOperator(self, operator): 1112da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """Tests if this token is the given operator. 1122da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 1132da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Args: 1142da489cd246702bee5938545b18a6f710ed214bcJamie Gennis operator: The operator to compare to. 1152da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 1162da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Returns: 1172da489cd246702bee5938545b18a6f710ed214bcJamie Gennis True if this token is a operator token with the given name. 1182da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """ 1192da489cd246702bee5938545b18a6f710ed214bcJamie Gennis return self.type == JavaScriptTokenType.OPERATOR and self.string == operator 1202da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 1212da489cd246702bee5938545b18a6f710ed214bcJamie Gennis def IsAssignment(self): 1222da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """Tests if this token is an assignment operator. 1232da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 1242da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Returns: 1252da489cd246702bee5938545b18a6f710ed214bcJamie Gennis True if this token is an assignment operator. 1262da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """ 1272da489cd246702bee5938545b18a6f710ed214bcJamie Gennis return (self.type == JavaScriptTokenType.OPERATOR and 1282da489cd246702bee5938545b18a6f710ed214bcJamie Gennis self.string.endswith('=') and 1292da489cd246702bee5938545b18a6f710ed214bcJamie Gennis self.string not in ('==', '!=', '>=', '<=', '===', '!==')) 1302da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 1312da489cd246702bee5938545b18a6f710ed214bcJamie Gennis def IsComment(self): 1322da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """Tests if this token is any part of a comment. 1332da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 1342da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Returns: 1352da489cd246702bee5938545b18a6f710ed214bcJamie Gennis True if this token is any part of a comment. 1362da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """ 1372da489cd246702bee5938545b18a6f710ed214bcJamie Gennis return self.type in JavaScriptTokenType.COMMENT_TYPES 1382da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 1392da489cd246702bee5938545b18a6f710ed214bcJamie Gennis def IsCode(self): 1402da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """Tests if this token is code, as opposed to a comment or whitespace.""" 1412da489cd246702bee5938545b18a6f710ed214bcJamie Gennis return self.type not in JavaScriptTokenType.NON_CODE_TYPES 1422da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 1432da489cd246702bee5938545b18a6f710ed214bcJamie Gennis def __repr__(self): 1442da489cd246702bee5938545b18a6f710ed214bcJamie Gennis return '<JavaScriptToken: %d, %s, "%s", %r, %r>' % (self.line_number, 1452da489cd246702bee5938545b18a6f710ed214bcJamie Gennis self.type, self.string, 1462da489cd246702bee5938545b18a6f710ed214bcJamie Gennis self.values, 1472da489cd246702bee5938545b18a6f710ed214bcJamie Gennis self.metadata) 148