TokenConcatenation.cpp revision 1eb4433ac451dc16f4133a88af2d002ac26c58ef
1d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner//===--- TokenConcatenation.cpp - Token Concatenation Avoidance -----------===// 2d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner// 3d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner// The LLVM Compiler Infrastructure 4d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner// 5d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner// This file is distributed under the University of Illinois Open Source 6d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner// License. See LICENSE.TXT for details. 7d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner// 8d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner//===----------------------------------------------------------------------===// 9d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner// 10d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner// This file implements the TokenConcatenation class. 11d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner// 12d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner//===----------------------------------------------------------------------===// 13d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner 14d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner#include "clang/Lex/TokenConcatenation.h" 15d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner#include "clang/Lex/Preprocessor.h" 161eb4433ac451dc16f4133a88af2d002ac26c58efMike Stumpusing namespace clang; 17d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner 18d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner 19d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner/// StartsWithL - Return true if the spelling of this token starts with 'L'. 20d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattnerbool TokenConcatenation::StartsWithL(const Token &Tok) const { 21d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner if (!Tok.needsCleaning()) { 22d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner SourceManager &SM = PP.getSourceManager(); 23d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner return *SM.getCharacterData(SM.getSpellingLoc(Tok.getLocation())) == 'L'; 24d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner } 251eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 26d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner if (Tok.getLength() < 256) { 27d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner char Buffer[256]; 28d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner const char *TokPtr = Buffer; 29d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner PP.getSpelling(Tok, TokPtr); 30d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner return TokPtr[0] == 'L'; 31d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner } 321eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 33d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner return PP.getSpelling(Tok)[0] == 'L'; 34d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner} 35d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner 36d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner/// IsIdentifierL - Return true if the spelling of this token is literally 37d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner/// 'L'. 38d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattnerbool TokenConcatenation::IsIdentifierL(const Token &Tok) const { 39d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner if (!Tok.needsCleaning()) { 40d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner if (Tok.getLength() != 1) 41d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner return false; 42d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner SourceManager &SM = PP.getSourceManager(); 43d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner return *SM.getCharacterData(SM.getSpellingLoc(Tok.getLocation())) == 'L'; 44d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner } 451eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 46d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner if (Tok.getLength() < 256) { 47d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner char Buffer[256]; 48d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner const char *TokPtr = Buffer; 491eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump if (PP.getSpelling(Tok, TokPtr) != 1) 50d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner return false; 51d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner return TokPtr[0] == 'L'; 52d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner } 531eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 54d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner return PP.getSpelling(Tok) == "L"; 55d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner} 56d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner 57d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris LattnerTokenConcatenation::TokenConcatenation(Preprocessor &pp) : PP(pp) { 58d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner memset(TokenInfo, 0, sizeof(TokenInfo)); 591eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 60d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner // These tokens have custom code in AvoidConcat. 61d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner TokenInfo[tok::identifier ] |= aci_custom; 62d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner TokenInfo[tok::numeric_constant] |= aci_custom_firstchar; 63d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner TokenInfo[tok::period ] |= aci_custom_firstchar; 64d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner TokenInfo[tok::amp ] |= aci_custom_firstchar; 65d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner TokenInfo[tok::plus ] |= aci_custom_firstchar; 66d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner TokenInfo[tok::minus ] |= aci_custom_firstchar; 67d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner TokenInfo[tok::slash ] |= aci_custom_firstchar; 68d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner TokenInfo[tok::less ] |= aci_custom_firstchar; 69d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner TokenInfo[tok::greater ] |= aci_custom_firstchar; 70d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner TokenInfo[tok::pipe ] |= aci_custom_firstchar; 71d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner TokenInfo[tok::percent ] |= aci_custom_firstchar; 72d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner TokenInfo[tok::colon ] |= aci_custom_firstchar; 73d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner TokenInfo[tok::hash ] |= aci_custom_firstchar; 74d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner TokenInfo[tok::arrow ] |= aci_custom_firstchar; 751eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 76d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner // These tokens change behavior if followed by an '='. 77d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner TokenInfo[tok::amp ] |= aci_avoid_equal; // &= 78d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner TokenInfo[tok::plus ] |= aci_avoid_equal; // += 79d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner TokenInfo[tok::minus ] |= aci_avoid_equal; // -= 80d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner TokenInfo[tok::slash ] |= aci_avoid_equal; // /= 81d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner TokenInfo[tok::less ] |= aci_avoid_equal; // <= 82d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner TokenInfo[tok::greater ] |= aci_avoid_equal; // >= 83d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner TokenInfo[tok::pipe ] |= aci_avoid_equal; // |= 84d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner TokenInfo[tok::percent ] |= aci_avoid_equal; // %= 85d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner TokenInfo[tok::star ] |= aci_avoid_equal; // *= 86d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner TokenInfo[tok::exclaim ] |= aci_avoid_equal; // != 87d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner TokenInfo[tok::lessless ] |= aci_avoid_equal; // <<= 88d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner TokenInfo[tok::greaterequal] |= aci_avoid_equal; // >>= 89d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner TokenInfo[tok::caret ] |= aci_avoid_equal; // ^= 90d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner TokenInfo[tok::equal ] |= aci_avoid_equal; // == 91d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner} 92d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner 9399c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbar/// GetFirstChar - Get the first character of the token \arg Tok, 9499c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbar/// avoiding calls to getSpelling where possible. 9599c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbarstatic char GetFirstChar(Preprocessor &PP, const Token &Tok) { 9699c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbar if (IdentifierInfo *II = Tok.getIdentifierInfo()) { 9799c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbar // Avoid spelling identifiers, the most common form of token. 9899c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbar return II->getName()[0]; 9999c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbar } else if (!Tok.needsCleaning()) { 10099c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbar if (Tok.isLiteral() && Tok.getLiteralData()) { 10199c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbar return *Tok.getLiteralData(); 10299c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbar } else { 10399c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbar SourceManager &SM = PP.getSourceManager(); 10499c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbar return *SM.getCharacterData(SM.getSpellingLoc(Tok.getLocation())); 10599c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbar } 10699c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbar } else if (Tok.getLength() < 256) { 10799c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbar char Buffer[256]; 10899c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbar const char *TokPtr = Buffer; 10999c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbar PP.getSpelling(Tok, TokPtr); 11099c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbar return TokPtr[0]; 11199c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbar } else { 11299c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbar return PP.getSpelling(Tok)[0]; 11399c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbar } 11499c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbar} 11599c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbar 116d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner/// AvoidConcat - If printing PrevTok immediately followed by Tok would cause 117d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner/// the two individual tokens to be lexed as a single token, return true 118d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner/// (which causes a space to be printed between them). This allows the output 119d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner/// of -E mode to be lexed to the same token stream as lexing the input 120d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner/// directly would. 121d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner/// 122d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner/// This code must conservatively return true if it doesn't want to be 100% 123d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner/// accurate. This will cause the output to include extra space characters, 124d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner/// but the resulting output won't have incorrect concatenations going on. 125d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner/// Examples include "..", which we print with a space between, because we 126d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner/// don't want to track enough to tell "x.." from "...". 127d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattnerbool TokenConcatenation::AvoidConcat(const Token &PrevTok, 128d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner const Token &Tok) const { 129e1614bb01cc429658b414a9e00603c66ae96d8f5Chris Lattner // First, check to see if the tokens were directly adjacent in the original 130e1614bb01cc429658b414a9e00603c66ae96d8f5Chris Lattner // source. If they were, it must be okay to stick them together: if there 131e1614bb01cc429658b414a9e00603c66ae96d8f5Chris Lattner // were an issue, the tokens would have been lexed differently. 132e1614bb01cc429658b414a9e00603c66ae96d8f5Chris Lattner if (PrevTok.getLocation().isFileID() && Tok.getLocation().isFileID() && 1331eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump PrevTok.getLocation().getFileLocWithOffset(PrevTok.getLength()) == 134e1614bb01cc429658b414a9e00603c66ae96d8f5Chris Lattner Tok.getLocation()) 135e1614bb01cc429658b414a9e00603c66ae96d8f5Chris Lattner return false; 1361eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 137d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner tok::TokenKind PrevKind = PrevTok.getKind(); 138d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner if (PrevTok.getIdentifierInfo()) // Language keyword or named operator. 139d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner PrevKind = tok::identifier; 1401eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 141d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner // Look up information on when we should avoid concatenation with prevtok. 142d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner unsigned ConcatInfo = TokenInfo[PrevKind]; 1431eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 144d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner // If prevtok never causes a problem for anything after it, return quickly. 145d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner if (ConcatInfo == 0) return false; 1461eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 147d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner if (ConcatInfo & aci_avoid_equal) { 148d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner // If the next token is '=' or '==', avoid concatenation. 149d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner if (Tok.is(tok::equal) || Tok.is(tok::equalequal)) 150d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner return true; 151d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner ConcatInfo &= ~aci_avoid_equal; 152d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner } 1531eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 154d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner if (ConcatInfo == 0) return false; 1551eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 156d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner // Basic algorithm: we look at the first character of the second token, and 157d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner // determine whether it, if appended to the first token, would form (or 158d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner // would contribute) to a larger token if concatenated. 159d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner char FirstChar = 0; 160d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner if (ConcatInfo & aci_custom) { 161d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner // If the token does not need to know the first character, don't get it. 162d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner } else { 16399c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbar FirstChar = GetFirstChar(PP, Tok); 164d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner } 1651eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 166d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner switch (PrevKind) { 167d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner default: assert(0 && "InitAvoidConcatTokenInfo built wrong"); 1681eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump case tok::identifier: // id+id or id+number or id+L"foo". 16999c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbar // id+'.'... will not append. 17099c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbar if (Tok.is(tok::numeric_constant)) 17199c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbar return GetFirstChar(PP, Tok) != '.'; 17299c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbar 17399c7622d1f673e8929196cc6eec7825a42622d5fDaniel Dunbar if (Tok.getIdentifierInfo() || Tok.is(tok::wide_string_literal) /* || 174d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner Tok.is(tok::wide_char_literal)*/) 175d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner return true; 1761eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 177d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner // If this isn't identifier + string, we're done. 178d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner if (Tok.isNot(tok::char_constant) && Tok.isNot(tok::string_literal)) 179d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner return false; 1801eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 181d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner // FIXME: need a wide_char_constant! 1821eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 183d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner // If the string was a wide string L"foo" or wide char L'f', it would 184d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner // concat with the previous identifier into fooL"bar". Avoid this. 185d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner if (StartsWithL(Tok)) 186d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner return true; 1871eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 188d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner // Otherwise, this is a narrow character or string. If the *identifier* 189d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner // is a literal 'L', avoid pasting L "foo" -> L"foo". 190d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner return IsIdentifierL(PrevTok); 191d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner case tok::numeric_constant: 192d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner return isalnum(FirstChar) || Tok.is(tok::numeric_constant) || 193d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner FirstChar == '+' || FirstChar == '-' || FirstChar == '.'; 194d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner case tok::period: // ..., .*, .1234 1958849f1190330c493a89b0088557d1a2333465847Eli Friedman return FirstChar == '.' || isdigit(FirstChar) || 1968849f1190330c493a89b0088557d1a2333465847Eli Friedman (PP.getLangOptions().CPlusPlus && FirstChar == '*'); 197d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner case tok::amp: // && 198d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner return FirstChar == '&'; 199d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner case tok::plus: // ++ 200d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner return FirstChar == '+'; 201d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner case tok::minus: // --, ->, ->* 202d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner return FirstChar == '-' || FirstChar == '>'; 203d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner case tok::slash: //, /*, // 204d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner return FirstChar == '*' || FirstChar == '/'; 205d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner case tok::less: // <<, <<=, <:, <% 206d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner return FirstChar == '<' || FirstChar == ':' || FirstChar == '%'; 207d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner case tok::greater: // >>, >>= 208d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner return FirstChar == '>'; 209d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner case tok::pipe: // || 210d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner return FirstChar == '|'; 211d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner case tok::percent: // %>, %: 212896ccf83eb1a575d596fe757a9b259429ce7ab16Eli Friedman return FirstChar == '>' || FirstChar == ':'; 213d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner case tok::colon: // ::, :> 2148849f1190330c493a89b0088557d1a2333465847Eli Friedman return FirstChar == '>' || 2158849f1190330c493a89b0088557d1a2333465847Eli Friedman (PP.getLangOptions().CPlusPlus && FirstChar == ':'); 216d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner case tok::hash: // ##, #@, %:%: 217d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner return FirstChar == '#' || FirstChar == '@' || FirstChar == '%'; 218d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner case tok::arrow: // ->* 2198849f1190330c493a89b0088557d1a2333465847Eli Friedman return PP.getLangOptions().CPlusPlus && FirstChar == '*'; 220d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner } 221d7038e18ef540a78fe6ce2a43125ce9b08fdbbc5Chris Lattner} 222