15821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)#!/usr/bin/env python 25821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# 35821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Copyright 2007 The Closure Linter Authors. All Rights Reserved. 45821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# 55821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Licensed under the Apache License, Version 2.0 (the "License"); 65821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# you may not use this file except in compliance with the License. 75821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# You may obtain a copy of the License at 85821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# 95821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# http://www.apache.org/licenses/LICENSE-2.0 105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# 115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Unless required by applicable law or agreed to in writing, software 125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# distributed under the License is distributed on an "AS-IS" BASIS, 135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# See the License for the specific language governing permissions and 155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# limitations under the License. 165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)"""Token utility functions.""" 185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)__author__ = ('robbyw@google.com (Robert Walker)', 205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 'ajp@google.com (Andy Perelson)') 215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import copy 235f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)import StringIO 245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)from closure_linter.common import tokens 265f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)from closure_linter.javascripttokens import JavaScriptToken 275f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)from closure_linter.javascripttokens import JavaScriptTokenType 285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Shorthand 305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)Type = tokens.TokenType 315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def GetFirstTokenInSameLine(token): 345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Returns the first token in the same line as token. 355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token: Any token in the line. 385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) The first token in the same line as token. 415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while not token.IsFirstInLine(): 435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token = token.previous 445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return token 455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def GetFirstTokenInPreviousLine(token): 485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Returns the first token in the previous line as token. 495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token: Any token in the line. 525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) The first token in the previous line as token, or None if token is on the 555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) first line. 565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) first_in_line = GetFirstTokenInSameLine(token) 585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if first_in_line.previous: 595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return GetFirstTokenInSameLine(first_in_line.previous) 605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return None 625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def GetLastTokenInSameLine(token): 655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Returns the last token in the same line as token. 665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token: Any token in the line. 695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) The last token in the same line as token. 725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while not token.IsLastInLine(): 745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token = token.next 755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return token 765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def GetAllTokensInSameLine(token): 795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Returns all tokens in the same line as the given token. 805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token: Any token in the line. 835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) All tokens on the same line as the given token. 865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) first_token = GetFirstTokenInSameLine(token) 885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) last_token = GetLastTokenInSameLine(token) 895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tokens_in_line = [] 915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while first_token != last_token: 925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tokens_in_line.append(first_token) 935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) first_token = first_token.next 945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) tokens_in_line.append(last_token) 955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return tokens_in_line 975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def CustomSearch(start_token, func, end_func=None, distance=None, 1005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reverse=False): 1015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Returns the first token where func is True within distance of this token. 1025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 1045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) start_token: The token to start searching from 1055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) func: The function to call to test a token for applicability 1065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) end_func: The function to call to test a token to determine whether to abort 1075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) the search. 1085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) distance: The number of tokens to look through before failing search. Must 1095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) be positive. If unspecified, will search until the end of the token 1105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) chain 1115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reverse: When true, search the tokens before this one instead of the tokens 1125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) after it 1135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 1155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) The first token matching func within distance of this token, or None if no 1165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) such token is found. 1175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 1185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token = start_token 1195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if reverse: 1205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while token and (distance is None or distance > 0): 1215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) previous = token.previous 1225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if previous: 1235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if func(previous): 1245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return previous 1255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if end_func and end_func(previous): 1265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return None 1275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token = previous 1295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if distance is not None: 1305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) distance -= 1 1315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 1335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while token and (distance is None or distance > 0): 1345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) next_token = token.next 1355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if next_token: 1365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if func(next_token): 1375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return next_token 1385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if end_func and end_func(next_token): 1395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return None 1405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token = next_token 1425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if distance is not None: 1435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) distance -= 1 1445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return None 1465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def Search(start_token, token_types, distance=None, reverse=False): 1495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Returns the first token of type in token_types within distance. 1505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 1525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) start_token: The token to start searching from 1535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token_types: The allowable types of the token being searched for 1545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) distance: The number of tokens to look through before failing search. Must 1555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) be positive. If unspecified, will search until the end of the token 1565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) chain 1575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reverse: When true, search the tokens before this one instead of the tokens 1585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) after it 1595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 1615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) The first token of any type in token_types within distance of this token, or 1625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) None if no such token is found. 1635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 1645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return CustomSearch(start_token, lambda token: token.IsAnyType(token_types), 1655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) None, distance, reverse) 1665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def SearchExcept(start_token, token_types, distance=None, reverse=False): 1695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Returns the first token not of any type in token_types within distance. 1705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 1725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) start_token: The token to start searching from 1735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token_types: The unallowable types of the token being searched for 1745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) distance: The number of tokens to look through before failing search. Must 1755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) be positive. If unspecified, will search until the end of the token 1765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) chain 1775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reverse: When true, search the tokens before this one instead of the tokens 1785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) after it 1795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 1815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) The first token of any type in token_types within distance of this token, or 1825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) None if no such token is found. 1835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 1845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return CustomSearch(start_token, 1855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) lambda token: not token.IsAnyType(token_types), 1865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) None, distance, reverse) 1875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def SearchUntil(start_token, token_types, end_types, distance=None, 1905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reverse=False): 1915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Returns the first token of type in token_types before a token of end_type. 1925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 1945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) start_token: The token to start searching from. 1955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token_types: The allowable types of the token being searched for. 1965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) end_types: Types of tokens to abort search if we find. 1975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) distance: The number of tokens to look through before failing search. Must 1985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) be positive. If unspecified, will search until the end of the token 1995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) chain 2005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) reverse: When true, search the tokens before this one instead of the tokens 2015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) after it 2025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 2045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) The first token of any type in token_types within distance of this token 2055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) before any tokens of type in end_type, or None if no such token is found. 2065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 2075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return CustomSearch(start_token, lambda token: token.IsAnyType(token_types), 2085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) lambda token: token.IsAnyType(end_types), 2095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) distance, reverse) 2105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def DeleteToken(token): 2135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Deletes the given token from the linked list. 2145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 2165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token: The token to delete 2175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 2185f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # When deleting a token, we do not update the deleted token itself to make 2195f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # sure the previous and next pointers are still pointing to tokens which are 2205f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # not deleted. Also it is very hard to keep track of all previously deleted 2215f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # tokens to update them when their pointers become invalid. So we add this 2225f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # flag that any token linked list iteration logic can skip deleted node safely 2235f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # when its current token is deleted. 2245f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) token.is_deleted = True 2255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if token.previous: 2265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token.previous.next = token.next 2275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if token.next: 2295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token.next.previous = token.previous 2305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) following_token = token.next 2325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while following_token and following_token.metadata.last_code == token: 2335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) following_token.metadata.last_code = token.metadata.last_code 2345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) following_token = following_token.next 2355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def DeleteTokens(token, token_count): 2385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Deletes the given number of tokens starting with the given token. 2395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 2415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token: The token to start deleting at. 2425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token_count: The total number of tokens to delete. 2435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 2445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for i in xrange(1, token_count): 2455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) DeleteToken(token.next) 2465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) DeleteToken(token) 2475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2495f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def InsertTokenBefore(new_token, token): 2505f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """Insert new_token before token. 2515f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 2525f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Args: 2535f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) new_token: A token to be added to the stream 2545f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) token: A token already in the stream 2555f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """ 2565f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) new_token.next = token 2575f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) new_token.previous = token.previous 2585f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 2595f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) new_token.metadata = copy.copy(token.metadata) 2605f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 2615f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if new_token.IsCode(): 2625f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) old_last_code = token.metadata.last_code 2635f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) following_token = token 2645f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) while (following_token and 2655f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) following_token.metadata.last_code == old_last_code): 2665f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) following_token.metadata.last_code = new_token 2675f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) following_token = following_token.next 2685f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 2695f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) token.previous = new_token 2705f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if new_token.previous: 2715f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) new_token.previous.next = new_token 2725f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 2735f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if new_token.start_index is None: 2745f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if new_token.line_number == token.line_number: 2755f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) new_token.start_index = token.start_index 2765f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) else: 2775f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) previous_token = new_token.previous 2785f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if previous_token: 2795f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) new_token.start_index = (previous_token.start_index + 2805f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) len(previous_token.string)) 2815f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) else: 2825f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) new_token.start_index = 0 2835f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 2845f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) iterator = new_token.next 2855f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) while iterator and iterator.line_number == new_token.line_number: 2865f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) iterator.start_index += len(new_token.string) 2875f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) iterator = iterator.next 2885f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 2895f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 2905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def InsertTokenAfter(new_token, token): 2915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Insert new_token after token. 2925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 2935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 2945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_token: A token to be added to the stream 2955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token: A token already in the stream 2965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 2975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_token.previous = token 2985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_token.next = token.next 2995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_token.metadata = copy.copy(token.metadata) 3015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if token.IsCode(): 3035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_token.metadata.last_code = token 3045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if new_token.IsCode(): 3065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) following_token = token.next 3075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while following_token and following_token.metadata.last_code == token: 3085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) following_token.metadata.last_code = new_token 3095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) following_token = following_token.next 3105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token.next = new_token 3125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if new_token.next: 3135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_token.next.previous = new_token 3145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if new_token.start_index is None: 3165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if new_token.line_number == token.line_number: 3175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_token.start_index = token.start_index + len(token.string) 3185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 3195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_token.start_index = 0 3205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) iterator = new_token.next 3225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while iterator and iterator.line_number == new_token.line_number: 3235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) iterator.start_index += len(new_token.string) 3245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) iterator = iterator.next 3255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def InsertTokensAfter(new_tokens, token): 3285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Insert multiple tokens after token. 3295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 3315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_tokens: An array of tokens to be added to the stream 3325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token: A token already in the stream 3335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 3345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # TODO(user): It would be nicer to have InsertTokenAfter defer to here 3355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # instead of vice-versa. 3365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) current_token = token 3375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for new_token in new_tokens: 3385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) InsertTokenAfter(new_token, current_token) 3395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) current_token = new_token 3405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def InsertSpaceTokenAfter(token): 3435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Inserts a space token after the given token. 3445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 3465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token: The token to insert a space token after 3475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 3495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) A single space token 3505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 3515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) space_token = JavaScriptToken(' ', Type.WHITESPACE, token.line, 3525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token.line_number) 3535821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) InsertTokenAfter(space_token, token) 3545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def InsertBlankLineAfter(token): 3575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Inserts a blank line after the given token. 3585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 3605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token: The token to insert a blank line after 3615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 3635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) A single space token 3645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 3655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) blank_token = JavaScriptToken('', Type.BLANK_LINE, '', 3665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token.line_number + 1) 3675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) InsertLineAfter(token, [blank_token]) 3685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def InsertLineAfter(token, new_tokens): 3715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Inserts a new line consisting of new_tokens after the given token. 3725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 3745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token: The token to insert after. 3755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_tokens: The tokens that will make up the new line. 3765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 3775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) insert_location = token 3785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for new_token in new_tokens: 3795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) InsertTokenAfter(new_token, insert_location) 3805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) insert_location = new_token 3815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Update all subsequent line numbers. 3835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) next_token = new_tokens[-1].next 3845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) while next_token: 3855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) next_token.line_number += 1 3865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) next_token = next_token.next 3875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def SplitToken(token, position): 3905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Splits the token into two tokens at position. 3915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 3935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token: The token to split 3945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) position: The position to split at. Will be the beginning of second token. 3955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 3965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 3975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) The new second token. 3985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 3995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_string = token.string[position:] 4005821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token.string = token.string[:position] 4015821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) new_token = JavaScriptToken(new_string, token.type, token.line, 4035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token.line_number) 4045821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) InsertTokenAfter(new_token, token) 4055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return new_token 4075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4085821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4095821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)def Compare(token1, token2): 4105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Compares two tokens and determines their relative order. 4115821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 4135821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token1: The first token to compare. 4145821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) token2: The second token to compare. 4155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 4165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 4175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) A negative integer, zero, or a positive integer as the first token is 4185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) before, equal, or after the second in the token stream. 4195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 4205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if token2.line_number != token1.line_number: 4215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return token1.line_number - token2.line_number 4225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 4235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return token1.start_index - token2.start_index 4245f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 4255f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 4265f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def GoogScopeOrNoneFromStartBlock(token): 4275f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """Determines if the given START_BLOCK is part of a goog.scope statement. 4285f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 4295f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Args: 4305f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) token: A token of type START_BLOCK. 4315f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 4325f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Returns: 4335f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) The goog.scope function call token, or None if such call doesn't exist. 4345f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """ 4355f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if token.type != JavaScriptTokenType.START_BLOCK: 4365f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return None 4375f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 4385f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # Search for a goog.scope statement, which will be 5 tokens before the 4395f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # block. Illustration of the tokens found prior to the start block: 4405f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # goog.scope(function() { 4415f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # 5 4 3 21 ^ 4425f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 4435f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) maybe_goog_scope = token 4445f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) for unused_i in xrange(5): 4455f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) maybe_goog_scope = (maybe_goog_scope.previous if maybe_goog_scope and 4465f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) maybe_goog_scope.previous else None) 4475f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if maybe_goog_scope and maybe_goog_scope.string == 'goog.scope': 4485f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return maybe_goog_scope 4495f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 4505f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 4515f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def GetTokenRange(start_token, end_token): 4525f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """Returns a list of tokens between the two given, inclusive. 4535f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 4545f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Args: 4555f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) start_token: Start token in the range. 4565f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) end_token: End token in the range. 4575f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 4585f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Returns: 4595f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) A list of tokens, in order, from start_token to end_token (including start 4605f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) and end). Returns none if the tokens do not describe a valid range. 4615f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """ 4625f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 4635f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) token_range = [] 4645f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) token = start_token 4655f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 4665f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) while token: 4675f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) token_range.append(token) 4685f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 4695f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if token == end_token: 4705f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return token_range 4715f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 4725f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) token = token.next 4735f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 4745f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 4755f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def TokensToString(token_iterable): 4765f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """Convert a number of tokens into a string. 4775f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 4785f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Newlines will be inserted whenever the line_number of two neighboring 4795f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) strings differ. 4805f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 4815f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Args: 4825f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) token_iterable: The tokens to turn to a string. 4835f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 4845f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Returns: 4855f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) A string representation of the given tokens. 4865f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """ 4875f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 4885f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) buf = StringIO.StringIO() 4895f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) token_list = list(token_iterable) 4905f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if not token_list: 4915f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return '' 4925f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 4935f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) line_number = token_list[0].line_number 4945f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 4955f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) for token in token_list: 4965f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 4975f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) while line_number < token.line_number: 4985f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) line_number += 1 4995f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) buf.write('\n') 5005f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5015f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if line_number > token.line_number: 5025f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) line_number = token.line_number 5035f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) buf.write('\n') 5045f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5055f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) buf.write(token.string) 5065f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5075f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return buf.getvalue() 5085f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5095f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5105f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def GetPreviousCodeToken(token): 5115f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """Returns the code token before the specified token. 5125f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5135f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Args: 5145f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) token: A token. 5155f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5165f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Returns: 5175f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) The code token before the specified token or None if no such token 5185f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) exists. 5195f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """ 5205f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5215f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return CustomSearch( 5225f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) token, 5235f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) lambda t: t and t.type not in JavaScriptTokenType.NON_CODE_TYPES, 5245f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) reverse=True) 5255f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5265f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5275f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def GetNextCodeToken(token): 5285f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """Returns the next code token after the specified token. 5295f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5305f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Args: 5315f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) token: A token. 5325f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5335f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Returns: 5345f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) The next code token after the specified token or None if no such token 5355f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) exists. 5365f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """ 5375f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5385f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return CustomSearch( 5395f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) token, 5405f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) lambda t: t and t.type not in JavaScriptTokenType.NON_CODE_TYPES, 5415f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) reverse=False) 5425f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5435f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5445f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def GetIdentifierStart(token): 5455f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """Returns the first token in an identifier. 5465f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5475f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Given a token which is part of an identifier, returns the token at the start 5485f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) of the identifier. 5495f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5505f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Args: 5515f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) token: A token which is part of an identifier. 5525f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5535f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Returns: 5545f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) The token at the start of the identifier or None if the identifier was not 5555f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) of the form 'a.b.c' (e.g. "['a']['b'].c"). 5565f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """ 5575f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5585f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) start_token = token 5595f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) previous_code_token = GetPreviousCodeToken(token) 5605f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5615f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) while (previous_code_token and ( 5625f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) previous_code_token.IsType(JavaScriptTokenType.IDENTIFIER) or 5635f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) _IsDot(previous_code_token))): 5645f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) start_token = previous_code_token 5655f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) previous_code_token = GetPreviousCodeToken(previous_code_token) 5665f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5675f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if _IsDot(start_token): 5685f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return None 5695f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5705f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return start_token 5715f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5725f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5735f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def GetIdentifierForToken(token): 5745f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """Get the symbol specified by a token. 5755f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5765f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Given a token, this function additionally concatenates any parts of an 5775f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) identifying symbol being identified that are split by whitespace or a 5785f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) newline. 5795f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5805f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) The function will return None if the token is not the first token of an 5815f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) identifier. 5825f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5835f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Args: 5845f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) token: The first token of a symbol. 5855f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5865f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Returns: 5875f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) The whole symbol, as a string. 5885f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """ 5895f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5905f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # Search backward to determine if this token is the first token of the 5915f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # identifier. If it is not the first token, return None to signal that this 5925f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # token should be ignored. 5935f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) prev_token = token.previous 5945f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) while prev_token: 5955f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if (prev_token.IsType(JavaScriptTokenType.IDENTIFIER) or 5965f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) _IsDot(prev_token)): 5975f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return None 5985f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 5995f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if (prev_token.IsType(tokens.TokenType.WHITESPACE) or 6005f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) prev_token.IsAnyType(JavaScriptTokenType.COMMENT_TYPES)): 6015f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) prev_token = prev_token.previous 6025f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) else: 6035f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) break 6045f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 6055f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # A "function foo()" declaration. 6065f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if token.type is JavaScriptTokenType.FUNCTION_NAME: 6075f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return token.string 6085f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 6095f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # A "var foo" declaration (if the previous token is 'var') 6105f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) previous_code_token = GetPreviousCodeToken(token) 6115f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 6125f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if previous_code_token and previous_code_token.IsKeyword('var'): 6135f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return token.string 6145f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 6155f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # Otherwise, this is potentially a namespaced (goog.foo.bar) identifier that 6165f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # could span multiple lines or be broken up by whitespace. We need 6175f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # to concatenate. 6185f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) identifier_types = set([ 6195f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) JavaScriptTokenType.IDENTIFIER, 6205f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) JavaScriptTokenType.SIMPLE_LVALUE 6215f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) ]) 6225f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 6235f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) assert token.type in identifier_types 6245f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 6255f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # Start with the first token 6265f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) symbol_tokens = [token] 6275f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 6285f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if token.next: 6295f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) for t in token.next: 6305f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) last_symbol_token = symbol_tokens[-1] 6315f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 6325f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # An identifier is part of the previous symbol if it has a trailing 6335f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # dot. 6345f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if t.type in identifier_types: 6355f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if last_symbol_token.string.endswith('.'): 6365f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) symbol_tokens.append(t) 6375f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) continue 6385f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) else: 6395f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) break 6405f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 6415f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # A dot is part of the previous symbol if it does not have a trailing 6425f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # dot. 6435f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if _IsDot(t): 6445f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if not last_symbol_token.string.endswith('.'): 6455f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) symbol_tokens.append(t) 6465f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) continue 6475f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) else: 6485f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) break 6495f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 6505f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # Skip any whitespace 6515f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if t.type in JavaScriptTokenType.NON_CODE_TYPES: 6525f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) continue 6535f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 6545f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) # This is the end of the identifier. Stop iterating. 6555f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) break 6565f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 6575f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if symbol_tokens: 6585f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return ''.join([t.string for t in symbol_tokens]) 6595f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 6605f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 6615f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def GetStringAfterToken(token): 6625f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """Get string after token. 6635f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 6645f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Args: 6655f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) token: Search will be done after this token. 6665f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 6675f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Returns: 6685f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) String if found after token else None (empty string will also 6695f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return None). 6705f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 6715f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Search until end of string as in case of empty string Type.STRING_TEXT is not 6725f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) present/found and don't want to return next string. 6735f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) E.g. 6745f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) a = ''; 6755f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) b = 'test'; 6765f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) When searching for string after 'a' if search is not limited by end of string 6775f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) then it will return 'test' which is not desirable as there is a empty string 6785f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) before that. 6795f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 6805f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) This will return None for cases where string is empty or no string found 6815f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) as in both cases there is no Type.STRING_TEXT. 6825f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """ 6835f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) string_token = SearchUntil(token, JavaScriptTokenType.STRING_TEXT, 6845f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) [JavaScriptTokenType.SINGLE_QUOTE_STRING_END, 6855f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) JavaScriptTokenType.DOUBLE_QUOTE_STRING_END]) 6865f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if string_token: 6875f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return string_token.string 6885f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) else: 6895f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return None 6905f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 6915f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 6925f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def _IsDot(token): 6935f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """Whether the token represents a "dot" operator (foo.bar).""" 6945f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) return token.type is tokens.TokenType.NORMAL and token.string == '.' 695