12da489cd246702bee5938545b18a6f710ed214bcJamie Gennis#!/usr/bin/env python 22da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# 32da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# Copyright 2007 The Closure Linter Authors. All Rights Reserved. 42da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# 52da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# Licensed under the Apache License, Version 2.0 (the "License"); 62da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# you may not use this file except in compliance with the License. 72da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# You may obtain a copy of the License at 82da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# 92da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# http://www.apache.org/licenses/LICENSE-2.0 102da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# 112da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# Unless required by applicable law or agreed to in writing, software 122da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# distributed under the License is distributed on an "AS-IS" BASIS, 132da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 142da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# See the License for the specific language governing permissions and 152da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# limitations under the License. 162da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 172da489cd246702bee5938545b18a6f710ed214bcJamie Gennis"""Token utility functions.""" 182da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 192da489cd246702bee5938545b18a6f710ed214bcJamie Gennis__author__ = ('robbyw@google.com (Robert Walker)', 202da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 'ajp@google.com (Andy Perelson)') 212da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 222da489cd246702bee5938545b18a6f710ed214bcJamie Gennisimport copy 232da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 242da489cd246702bee5938545b18a6f710ed214bcJamie Gennisfrom closure_linter import javascripttokens 252da489cd246702bee5938545b18a6f710ed214bcJamie Gennisfrom closure_linter.common import tokens 262da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 272da489cd246702bee5938545b18a6f710ed214bcJamie Gennis# Shorthand 282da489cd246702bee5938545b18a6f710ed214bcJamie GennisJavaScriptToken = javascripttokens.JavaScriptToken 292da489cd246702bee5938545b18a6f710ed214bcJamie GennisType = tokens.TokenType 302da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 312da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 322da489cd246702bee5938545b18a6f710ed214bcJamie Gennisdef GetFirstTokenInSameLine(token): 332da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """Returns the first token in the same line as token. 342da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 352da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Args: 362da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token: Any token in the line. 372da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 382da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Returns: 392da489cd246702bee5938545b18a6f710ed214bcJamie Gennis The first token in the same line as token. 402da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """ 412da489cd246702bee5938545b18a6f710ed214bcJamie Gennis while not token.IsFirstInLine(): 422da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token = token.previous 432da489cd246702bee5938545b18a6f710ed214bcJamie Gennis return token 442da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 452da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 462da489cd246702bee5938545b18a6f710ed214bcJamie Gennisdef GetFirstTokenInPreviousLine(token): 472da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """Returns the first token in the previous line as token. 482da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 492da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Args: 502da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token: Any token in the line. 512da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 522da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Returns: 532da489cd246702bee5938545b18a6f710ed214bcJamie Gennis The first token in the previous line as token, or None if token is on the 542da489cd246702bee5938545b18a6f710ed214bcJamie Gennis first line. 552da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """ 562da489cd246702bee5938545b18a6f710ed214bcJamie Gennis first_in_line = GetFirstTokenInSameLine(token) 572da489cd246702bee5938545b18a6f710ed214bcJamie Gennis if first_in_line.previous: 582da489cd246702bee5938545b18a6f710ed214bcJamie Gennis return GetFirstTokenInSameLine(first_in_line.previous) 592da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 602da489cd246702bee5938545b18a6f710ed214bcJamie Gennis return None 612da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 622da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 632da489cd246702bee5938545b18a6f710ed214bcJamie Gennisdef GetLastTokenInSameLine(token): 642da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """Returns the last token in the same line as token. 652da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 662da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Args: 672da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token: Any token in the line. 682da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 692da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Returns: 702da489cd246702bee5938545b18a6f710ed214bcJamie Gennis The last token in the same line as token. 712da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """ 722da489cd246702bee5938545b18a6f710ed214bcJamie Gennis while not token.IsLastInLine(): 732da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token = token.next 742da489cd246702bee5938545b18a6f710ed214bcJamie Gennis return token 752da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 762da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 772da489cd246702bee5938545b18a6f710ed214bcJamie Gennisdef GetAllTokensInSameLine(token): 782da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """Returns all tokens in the same line as the given token. 792da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 802da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Args: 812da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token: Any token in the line. 822da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 832da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Returns: 842da489cd246702bee5938545b18a6f710ed214bcJamie Gennis All tokens on the same line as the given token. 852da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """ 862da489cd246702bee5938545b18a6f710ed214bcJamie Gennis first_token = GetFirstTokenInSameLine(token) 872da489cd246702bee5938545b18a6f710ed214bcJamie Gennis last_token = GetLastTokenInSameLine(token) 882da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 892da489cd246702bee5938545b18a6f710ed214bcJamie Gennis tokens_in_line = [] 902da489cd246702bee5938545b18a6f710ed214bcJamie Gennis while first_token != last_token: 912da489cd246702bee5938545b18a6f710ed214bcJamie Gennis tokens_in_line.append(first_token) 922da489cd246702bee5938545b18a6f710ed214bcJamie Gennis first_token = first_token.next 932da489cd246702bee5938545b18a6f710ed214bcJamie Gennis tokens_in_line.append(last_token) 942da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 952da489cd246702bee5938545b18a6f710ed214bcJamie Gennis return tokens_in_line 962da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 972da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 982da489cd246702bee5938545b18a6f710ed214bcJamie Gennisdef CustomSearch(start_token, func, end_func=None, distance=None, 992da489cd246702bee5938545b18a6f710ed214bcJamie Gennis reverse=False): 1002da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """Returns the first token where func is True within distance of this token. 1012da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 1022da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Args: 1032da489cd246702bee5938545b18a6f710ed214bcJamie Gennis start_token: The token to start searching from 1042da489cd246702bee5938545b18a6f710ed214bcJamie Gennis func: The function to call to test a token for applicability 1052da489cd246702bee5938545b18a6f710ed214bcJamie Gennis end_func: The function to call to test a token to determine whether to abort 1062da489cd246702bee5938545b18a6f710ed214bcJamie Gennis the search. 1072da489cd246702bee5938545b18a6f710ed214bcJamie Gennis distance: The number of tokens to look through before failing search. Must 1082da489cd246702bee5938545b18a6f710ed214bcJamie Gennis be positive. If unspecified, will search until the end of the token 1092da489cd246702bee5938545b18a6f710ed214bcJamie Gennis chain 1102da489cd246702bee5938545b18a6f710ed214bcJamie Gennis reverse: When true, search the tokens before this one instead of the tokens 1112da489cd246702bee5938545b18a6f710ed214bcJamie Gennis after it 1122da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 1132da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Returns: 1142da489cd246702bee5938545b18a6f710ed214bcJamie Gennis The first token matching func within distance of this token, or None if no 1152da489cd246702bee5938545b18a6f710ed214bcJamie Gennis such token is found. 1162da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """ 1172da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token = start_token 1182da489cd246702bee5938545b18a6f710ed214bcJamie Gennis if reverse: 1192da489cd246702bee5938545b18a6f710ed214bcJamie Gennis while token and (distance is None or distance > 0): 1202da489cd246702bee5938545b18a6f710ed214bcJamie Gennis previous = token.previous 1212da489cd246702bee5938545b18a6f710ed214bcJamie Gennis if previous: 1222da489cd246702bee5938545b18a6f710ed214bcJamie Gennis if func(previous): 1232da489cd246702bee5938545b18a6f710ed214bcJamie Gennis return previous 1242da489cd246702bee5938545b18a6f710ed214bcJamie Gennis if end_func and end_func(previous): 1252da489cd246702bee5938545b18a6f710ed214bcJamie Gennis return None 1262da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 1272da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token = previous 1282da489cd246702bee5938545b18a6f710ed214bcJamie Gennis if distance is not None: 1292da489cd246702bee5938545b18a6f710ed214bcJamie Gennis distance -= 1 1302da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 1312da489cd246702bee5938545b18a6f710ed214bcJamie Gennis else: 1322da489cd246702bee5938545b18a6f710ed214bcJamie Gennis while token and (distance is None or distance > 0): 1332da489cd246702bee5938545b18a6f710ed214bcJamie Gennis next_token = token.next 1342da489cd246702bee5938545b18a6f710ed214bcJamie Gennis if next_token: 1352da489cd246702bee5938545b18a6f710ed214bcJamie Gennis if func(next_token): 1362da489cd246702bee5938545b18a6f710ed214bcJamie Gennis return next_token 1372da489cd246702bee5938545b18a6f710ed214bcJamie Gennis if end_func and end_func(next_token): 1382da489cd246702bee5938545b18a6f710ed214bcJamie Gennis return None 1392da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 1402da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token = next_token 1412da489cd246702bee5938545b18a6f710ed214bcJamie Gennis if distance is not None: 1422da489cd246702bee5938545b18a6f710ed214bcJamie Gennis distance -= 1 1432da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 1442da489cd246702bee5938545b18a6f710ed214bcJamie Gennis return None 1452da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 1462da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 1472da489cd246702bee5938545b18a6f710ed214bcJamie Gennisdef Search(start_token, token_types, distance=None, reverse=False): 1482da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """Returns the first token of type in token_types within distance. 1492da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 1502da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Args: 1512da489cd246702bee5938545b18a6f710ed214bcJamie Gennis start_token: The token to start searching from 1522da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token_types: The allowable types of the token being searched for 1532da489cd246702bee5938545b18a6f710ed214bcJamie Gennis distance: The number of tokens to look through before failing search. Must 1542da489cd246702bee5938545b18a6f710ed214bcJamie Gennis be positive. If unspecified, will search until the end of the token 1552da489cd246702bee5938545b18a6f710ed214bcJamie Gennis chain 1562da489cd246702bee5938545b18a6f710ed214bcJamie Gennis reverse: When true, search the tokens before this one instead of the tokens 1572da489cd246702bee5938545b18a6f710ed214bcJamie Gennis after it 1582da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 1592da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Returns: 1602da489cd246702bee5938545b18a6f710ed214bcJamie Gennis The first token of any type in token_types within distance of this token, or 1612da489cd246702bee5938545b18a6f710ed214bcJamie Gennis None if no such token is found. 1622da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """ 1632da489cd246702bee5938545b18a6f710ed214bcJamie Gennis return CustomSearch(start_token, lambda token: token.IsAnyType(token_types), 1642da489cd246702bee5938545b18a6f710ed214bcJamie Gennis None, distance, reverse) 1652da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 1662da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 1672da489cd246702bee5938545b18a6f710ed214bcJamie Gennisdef SearchExcept(start_token, token_types, distance=None, reverse=False): 1682da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """Returns the first token not of any type in token_types within distance. 1692da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 1702da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Args: 1712da489cd246702bee5938545b18a6f710ed214bcJamie Gennis start_token: The token to start searching from 1722da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token_types: The unallowable types of the token being searched for 1732da489cd246702bee5938545b18a6f710ed214bcJamie Gennis distance: The number of tokens to look through before failing search. Must 1742da489cd246702bee5938545b18a6f710ed214bcJamie Gennis be positive. If unspecified, will search until the end of the token 1752da489cd246702bee5938545b18a6f710ed214bcJamie Gennis chain 1762da489cd246702bee5938545b18a6f710ed214bcJamie Gennis reverse: When true, search the tokens before this one instead of the tokens 1772da489cd246702bee5938545b18a6f710ed214bcJamie Gennis after it 1782da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 1792da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Returns: 1802da489cd246702bee5938545b18a6f710ed214bcJamie Gennis The first token of any type in token_types within distance of this token, or 1812da489cd246702bee5938545b18a6f710ed214bcJamie Gennis None if no such token is found. 1822da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """ 1832da489cd246702bee5938545b18a6f710ed214bcJamie Gennis return CustomSearch(start_token, 1842da489cd246702bee5938545b18a6f710ed214bcJamie Gennis lambda token: not token.IsAnyType(token_types), 1852da489cd246702bee5938545b18a6f710ed214bcJamie Gennis None, distance, reverse) 1862da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 1872da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 1882da489cd246702bee5938545b18a6f710ed214bcJamie Gennisdef SearchUntil(start_token, token_types, end_types, distance=None, 1892da489cd246702bee5938545b18a6f710ed214bcJamie Gennis reverse=False): 1902da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """Returns the first token of type in token_types before a token of end_type. 1912da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 1922da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Args: 1932da489cd246702bee5938545b18a6f710ed214bcJamie Gennis start_token: The token to start searching from. 1942da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token_types: The allowable types of the token being searched for. 1952da489cd246702bee5938545b18a6f710ed214bcJamie Gennis end_types: Types of tokens to abort search if we find. 1962da489cd246702bee5938545b18a6f710ed214bcJamie Gennis distance: The number of tokens to look through before failing search. Must 1972da489cd246702bee5938545b18a6f710ed214bcJamie Gennis be positive. If unspecified, will search until the end of the token 1982da489cd246702bee5938545b18a6f710ed214bcJamie Gennis chain 1992da489cd246702bee5938545b18a6f710ed214bcJamie Gennis reverse: When true, search the tokens before this one instead of the tokens 2002da489cd246702bee5938545b18a6f710ed214bcJamie Gennis after it 2012da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 2022da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Returns: 2032da489cd246702bee5938545b18a6f710ed214bcJamie Gennis The first token of any type in token_types within distance of this token 2042da489cd246702bee5938545b18a6f710ed214bcJamie Gennis before any tokens of type in end_type, or None if no such token is found. 2052da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """ 2062da489cd246702bee5938545b18a6f710ed214bcJamie Gennis return CustomSearch(start_token, lambda token: token.IsAnyType(token_types), 2072da489cd246702bee5938545b18a6f710ed214bcJamie Gennis lambda token: token.IsAnyType(end_types), 2082da489cd246702bee5938545b18a6f710ed214bcJamie Gennis distance, reverse) 2092da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 2102da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 2112da489cd246702bee5938545b18a6f710ed214bcJamie Gennisdef DeleteToken(token): 2122da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """Deletes the given token from the linked list. 2132da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 2142da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Args: 2152da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token: The token to delete 2162da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """ 2172da489cd246702bee5938545b18a6f710ed214bcJamie Gennis if token.previous: 2182da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token.previous.next = token.next 2192da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 2202da489cd246702bee5938545b18a6f710ed214bcJamie Gennis if token.next: 2212da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token.next.previous = token.previous 2222da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 2232da489cd246702bee5938545b18a6f710ed214bcJamie Gennis following_token = token.next 2242da489cd246702bee5938545b18a6f710ed214bcJamie Gennis while following_token and following_token.metadata.last_code == token: 2252da489cd246702bee5938545b18a6f710ed214bcJamie Gennis following_token.metadata.last_code = token.metadata.last_code 2262da489cd246702bee5938545b18a6f710ed214bcJamie Gennis following_token = following_token.next 2272da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 2282da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 2292da489cd246702bee5938545b18a6f710ed214bcJamie Gennisdef DeleteTokens(token, token_count): 2302da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """Deletes the given number of tokens starting with the given token. 2312da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 2322da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Args: 2332da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token: The token to start deleting at. 2342da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token_count: The total number of tokens to delete. 2352da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """ 2362da489cd246702bee5938545b18a6f710ed214bcJamie Gennis for i in xrange(1, token_count): 2372da489cd246702bee5938545b18a6f710ed214bcJamie Gennis DeleteToken(token.next) 2382da489cd246702bee5938545b18a6f710ed214bcJamie Gennis DeleteToken(token) 2392da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 2402da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 2412da489cd246702bee5938545b18a6f710ed214bcJamie Gennisdef InsertTokenAfter(new_token, token): 2422da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """Insert new_token after token. 2432da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 2442da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Args: 2452da489cd246702bee5938545b18a6f710ed214bcJamie Gennis new_token: A token to be added to the stream 2462da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token: A token already in the stream 2472da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """ 2482da489cd246702bee5938545b18a6f710ed214bcJamie Gennis new_token.previous = token 2492da489cd246702bee5938545b18a6f710ed214bcJamie Gennis new_token.next = token.next 2502da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 2512da489cd246702bee5938545b18a6f710ed214bcJamie Gennis new_token.metadata = copy.copy(token.metadata) 2522da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 2532da489cd246702bee5938545b18a6f710ed214bcJamie Gennis if token.IsCode(): 2542da489cd246702bee5938545b18a6f710ed214bcJamie Gennis new_token.metadata.last_code = token 2552da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 2562da489cd246702bee5938545b18a6f710ed214bcJamie Gennis if new_token.IsCode(): 2572da489cd246702bee5938545b18a6f710ed214bcJamie Gennis following_token = token.next 2582da489cd246702bee5938545b18a6f710ed214bcJamie Gennis while following_token and following_token.metadata.last_code == token: 2592da489cd246702bee5938545b18a6f710ed214bcJamie Gennis following_token.metadata.last_code = new_token 2602da489cd246702bee5938545b18a6f710ed214bcJamie Gennis following_token = following_token.next 2612da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 2622da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token.next = new_token 2632da489cd246702bee5938545b18a6f710ed214bcJamie Gennis if new_token.next: 2642da489cd246702bee5938545b18a6f710ed214bcJamie Gennis new_token.next.previous = new_token 2652da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 2662da489cd246702bee5938545b18a6f710ed214bcJamie Gennis if new_token.start_index is None: 2672da489cd246702bee5938545b18a6f710ed214bcJamie Gennis if new_token.line_number == token.line_number: 2682da489cd246702bee5938545b18a6f710ed214bcJamie Gennis new_token.start_index = token.start_index + len(token.string) 2692da489cd246702bee5938545b18a6f710ed214bcJamie Gennis else: 2702da489cd246702bee5938545b18a6f710ed214bcJamie Gennis new_token.start_index = 0 2712da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 2722da489cd246702bee5938545b18a6f710ed214bcJamie Gennis iterator = new_token.next 2732da489cd246702bee5938545b18a6f710ed214bcJamie Gennis while iterator and iterator.line_number == new_token.line_number: 2742da489cd246702bee5938545b18a6f710ed214bcJamie Gennis iterator.start_index += len(new_token.string) 2752da489cd246702bee5938545b18a6f710ed214bcJamie Gennis iterator = iterator.next 2762da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 2772da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 2782da489cd246702bee5938545b18a6f710ed214bcJamie Gennisdef InsertTokensAfter(new_tokens, token): 2792da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """Insert multiple tokens after token. 2802da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 2812da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Args: 2822da489cd246702bee5938545b18a6f710ed214bcJamie Gennis new_tokens: An array of tokens to be added to the stream 2832da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token: A token already in the stream 2842da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """ 2852da489cd246702bee5938545b18a6f710ed214bcJamie Gennis # TODO(user): It would be nicer to have InsertTokenAfter defer to here 2862da489cd246702bee5938545b18a6f710ed214bcJamie Gennis # instead of vice-versa. 2872da489cd246702bee5938545b18a6f710ed214bcJamie Gennis current_token = token 2882da489cd246702bee5938545b18a6f710ed214bcJamie Gennis for new_token in new_tokens: 2892da489cd246702bee5938545b18a6f710ed214bcJamie Gennis InsertTokenAfter(new_token, current_token) 2902da489cd246702bee5938545b18a6f710ed214bcJamie Gennis current_token = new_token 2912da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 2922da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 2932da489cd246702bee5938545b18a6f710ed214bcJamie Gennisdef InsertSpaceTokenAfter(token): 2942da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """Inserts a space token after the given token. 2952da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 2962da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Args: 2972da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token: The token to insert a space token after 2982da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 2992da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Returns: 3002da489cd246702bee5938545b18a6f710ed214bcJamie Gennis A single space token 3012da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """ 3022da489cd246702bee5938545b18a6f710ed214bcJamie Gennis space_token = JavaScriptToken(' ', Type.WHITESPACE, token.line, 3032da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token.line_number) 3042da489cd246702bee5938545b18a6f710ed214bcJamie Gennis InsertTokenAfter(space_token, token) 3052da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 3062da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 3072da489cd246702bee5938545b18a6f710ed214bcJamie Gennisdef InsertBlankLineAfter(token): 3082da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """Inserts a blank line after the given token. 3092da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 3102da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Args: 3112da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token: The token to insert a blank line after 3122da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 3132da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Returns: 3142da489cd246702bee5938545b18a6f710ed214bcJamie Gennis A single space token 3152da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """ 3162da489cd246702bee5938545b18a6f710ed214bcJamie Gennis blank_token = JavaScriptToken('', Type.BLANK_LINE, '', 3172da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token.line_number + 1) 3182da489cd246702bee5938545b18a6f710ed214bcJamie Gennis InsertLineAfter(token, [blank_token]) 3192da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 3202da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 3212da489cd246702bee5938545b18a6f710ed214bcJamie Gennisdef InsertLineAfter(token, new_tokens): 3222da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """Inserts a new line consisting of new_tokens after the given token. 3232da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 3242da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Args: 3252da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token: The token to insert after. 3262da489cd246702bee5938545b18a6f710ed214bcJamie Gennis new_tokens: The tokens that will make up the new line. 3272da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """ 3282da489cd246702bee5938545b18a6f710ed214bcJamie Gennis insert_location = token 3292da489cd246702bee5938545b18a6f710ed214bcJamie Gennis for new_token in new_tokens: 3302da489cd246702bee5938545b18a6f710ed214bcJamie Gennis InsertTokenAfter(new_token, insert_location) 3312da489cd246702bee5938545b18a6f710ed214bcJamie Gennis insert_location = new_token 3322da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 3332da489cd246702bee5938545b18a6f710ed214bcJamie Gennis # Update all subsequent line numbers. 3342da489cd246702bee5938545b18a6f710ed214bcJamie Gennis next_token = new_tokens[-1].next 3352da489cd246702bee5938545b18a6f710ed214bcJamie Gennis while next_token: 3362da489cd246702bee5938545b18a6f710ed214bcJamie Gennis next_token.line_number += 1 3372da489cd246702bee5938545b18a6f710ed214bcJamie Gennis next_token = next_token.next 3382da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 3392da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 3402da489cd246702bee5938545b18a6f710ed214bcJamie Gennisdef SplitToken(token, position): 3412da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """Splits the token into two tokens at position. 3422da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 3432da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Args: 3442da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token: The token to split 3452da489cd246702bee5938545b18a6f710ed214bcJamie Gennis position: The position to split at. Will be the beginning of second token. 3462da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 3472da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Returns: 3482da489cd246702bee5938545b18a6f710ed214bcJamie Gennis The new second token. 3492da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """ 3502da489cd246702bee5938545b18a6f710ed214bcJamie Gennis new_string = token.string[position:] 3512da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token.string = token.string[:position] 3522da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 3532da489cd246702bee5938545b18a6f710ed214bcJamie Gennis new_token = JavaScriptToken(new_string, token.type, token.line, 3542da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token.line_number) 3552da489cd246702bee5938545b18a6f710ed214bcJamie Gennis InsertTokenAfter(new_token, token) 3562da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 3572da489cd246702bee5938545b18a6f710ed214bcJamie Gennis return new_token 3582da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 3592da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 3602da489cd246702bee5938545b18a6f710ed214bcJamie Gennisdef Compare(token1, token2): 3612da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """Compares two tokens and determines their relative order. 3622da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 3632da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Args: 3642da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token1: The first token to compare. 3652da489cd246702bee5938545b18a6f710ed214bcJamie Gennis token2: The second token to compare. 3662da489cd246702bee5938545b18a6f710ed214bcJamie Gennis 3672da489cd246702bee5938545b18a6f710ed214bcJamie Gennis Returns: 3682da489cd246702bee5938545b18a6f710ed214bcJamie Gennis A negative integer, zero, or a positive integer as the first token is 3692da489cd246702bee5938545b18a6f710ed214bcJamie Gennis before, equal, or after the second in the token stream. 3702da489cd246702bee5938545b18a6f710ed214bcJamie Gennis """ 3712da489cd246702bee5938545b18a6f710ed214bcJamie Gennis if token2.line_number != token1.line_number: 3722da489cd246702bee5938545b18a6f710ed214bcJamie Gennis return token1.line_number - token2.line_number 3732da489cd246702bee5938545b18a6f710ed214bcJamie Gennis else: 3742da489cd246702bee5938545b18a6f710ed214bcJamie Gennis return token1.start_index - token2.start_index 375