TokenLexer.cpp revision 32fca722dd974b8202d0fb9c71b6c185c0767da6
11543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner//===--- TokenLexer.cpp - Lex from a token stream -------------------------===// 25f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 35f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// The LLVM Compiler Infrastructure 45f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 50bc735ffcfb223c0186419547abaa5c84482663eChris Lattner// This file is distributed under the University of Illinois Open Source 60bc735ffcfb223c0186419547abaa5c84482663eChris Lattner// License. See LICENSE.TXT for details. 75f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 85f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 95f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 101543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner// This file implements the TokenLexer interface. 115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 145d75de0f821023f4ed4815825bf3aea8a0b5e40dChris Lattner#include "clang/Lex/TokenLexer.h" 15e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner#include "MacroArgs.h" 165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "clang/Lex/MacroInfo.h" 175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "clang/Lex/Preprocessor.h" 185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "clang/Basic/SourceManager.h" 19500d3297d2a21edeac4d46cbcbe21bc2352c2a28Chris Lattner#include "clang/Lex/LexDiagnostic.h" 205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "llvm/ADT/SmallVector.h" 215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerusing namespace clang; 225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 241543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner/// Create a TokenLexer for the specified macro with the specified actual 255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// arguments. Note that this ctor takes ownership of the ActualArgs pointer. 26e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattnervoid TokenLexer::Init(Token &Tok, SourceLocation ILEnd, MacroArgs *Actuals) { 271543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner // If the client is reusing a TokenLexer, make sure to free any memory 289594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner // associated with it. 299594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner destroy(); 309594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner 31cc1a875f94630e58d24a55577ffbf0e89b7da8c7Chris Lattner Macro = PP.getMacroInfo(Tok.getIdentifierInfo()); 329594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner ActualArgs = Actuals; 339594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner CurToken = 0; 34b99bdadf9ace3e8c793e12e9bb447501f86b4f72Chris Lattner 35e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner InstantiateLocStart = Tok.getLocation(); 36e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner InstantiateLocEnd = ILEnd; 379594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner AtStartOfLine = Tok.isAtStartOfLine(); 389594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner HasLeadingSpace = Tok.hasLeadingSpace(); 398d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner Tokens = &*Macro->tokens_begin(); 408d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner OwnsTokens = false; 416b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattner DisableMacroExpansion = false; 428d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner NumTokens = Macro->tokens_end()-Macro->tokens_begin(); 435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is a function-like macro, expand the arguments and change 458d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner // Tokens to point to the expanded tokens. 465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Macro->isFunctionLike() && Macro->getNumArgs()) 475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ExpandFunctionArguments(); 485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Mark the macro as currently disabled, so that it is not recursively 505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // expanded. The macro must be disabled only after argument pre-expansion of 515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // function-like macro arguments occurs. 525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Macro->DisableMacro(); 535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 559594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner 569594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner 571543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner/// Create a TokenLexer for the specified token stream. This does not 585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// take ownership of the specified token vector. 596b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattnervoid TokenLexer::Init(const Token *TokArray, unsigned NumToks, 606b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattner bool disableMacroExpansion, bool ownsTokens) { 611543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner // If the client is reusing a TokenLexer, make sure to free any memory 629594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner // associated with it. 639594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner destroy(); 649594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner 659594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner Macro = 0; 669594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner ActualArgs = 0; 678d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner Tokens = TokArray; 686b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattner OwnsTokens = ownsTokens; 696b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattner DisableMacroExpansion = disableMacroExpansion; 708d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner NumTokens = NumToks; 719594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner CurToken = 0; 72e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner InstantiateLocStart = InstantiateLocEnd = SourceLocation(); 739594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner AtStartOfLine = false; 749594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner HasLeadingSpace = false; 755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Set HasLeadingSpace/AtStartOfLine so that the first token will be 775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // returned unmodified. 785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (NumToks != 0) { 795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer AtStartOfLine = TokArray[0].isAtStartOfLine(); 805f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer HasLeadingSpace = TokArray[0].hasLeadingSpace(); 815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 851543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattnervoid TokenLexer::destroy() { 865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this was a function-like macro that actually uses its arguments, delete 875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // the expanded tokens. 888d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner if (OwnsTokens) { 898d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner delete [] Tokens; 908d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner Tokens = 0; 9132fca722dd974b8202d0fb9c71b6c185c0767da6Chris Lattner OwnsTokens = false; 929c683062752a26014197df1c8792a19efa9e93cfChris Lattner } 935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 941543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner // TokenLexer owns its formal arguments. 955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (ActualArgs) ActualArgs->destroy(); 965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// Expand the arguments of a function-like macro so that we can quickly 998d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner/// return preexpanded tokens from Tokens. 1001543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattnervoid TokenLexer::ExpandFunctionArguments() { 101d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner llvm::SmallVector<Token, 128> ResultToks; 1025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1038d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner // Loop through 'Tokens', expanding them into ResultToks. Keep 1045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // track of whether we change anything. If not, no need to keep them. If so, 1058d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner // we install the newly expanded sequence as the new 'Tokens' list. 1065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer bool MadeChange = false; 1075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // NextTokGetsSpace - When this is true, the next token appended to the 1095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // output list will get a leading space, regardless of whether it had one to 1105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // begin with or not. This is used for placemarker support. 1115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer bool NextTokGetsSpace = false; 1125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1138d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner for (unsigned i = 0, e = NumTokens; i != e; ++i) { 1145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we found the stringify operator, get the argument stringified. The 1155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // preprocessor already verified that the following token is a macro name 1165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // when the #define was parsed. 1178d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner const Token &CurTok = Tokens[i]; 11822f6bbcafa8871f4f20c4402d9cbc5c024fee99aChris Lattner if (CurTok.is(tok::hash) || CurTok.is(tok::hashat)) { 1198d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner int ArgNo = Macro->getArgumentNum(Tokens[i+1].getIdentifierInfo()); 1205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer assert(ArgNo != -1 && "Token following # is not an argument?"); 1215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 122d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Token Res; 12322f6bbcafa8871f4f20c4402d9cbc5c024fee99aChris Lattner if (CurTok.is(tok::hash)) // Stringify 1245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Res = ActualArgs->getStringifiedArgument(ArgNo, PP); 1255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer else { 1265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // 'charify': don't bother caching these. 127e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner Res = MacroArgs::StringifyArgument(ActualArgs->getUnexpArgument(ArgNo), 128e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner PP, true); 1295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 1305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // The stringified/charified string leading space flag gets set to match 1325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // the #/#@ operator. 1335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurTok.hasLeadingSpace() || NextTokGetsSpace) 134d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Res.setFlag(Token::LeadingSpace); 1355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ResultToks.push_back(Res); 1375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer MadeChange = true; 1385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++i; // Skip arg name. 1395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NextTokGetsSpace = false; 1405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer continue; 1415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 1425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, if this is not an argument token, just add the token to the 1445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // output buffer. 1455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer IdentifierInfo *II = CurTok.getIdentifierInfo(); 1465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer int ArgNo = II ? Macro->getArgumentNum(II) : -1; 1475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (ArgNo == -1) { 1485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // This isn't an argument, just add it. 1495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ResultToks.push_back(CurTok); 1505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (NextTokGetsSpace) { 152d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner ResultToks.back().setFlag(Token::LeadingSpace); 1535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NextTokGetsSpace = false; 1545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 1555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer continue; 1565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 1575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // An argument is expanded somehow, the result is different than the 1595f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // input. 1605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer MadeChange = true; 1615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, this is a use of the argument. Find out if there is a paste 1635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // (##) operator before or after the argument. 1645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer bool PasteBefore = 16522f6bbcafa8871f4f20c4402d9cbc5c024fee99aChris Lattner !ResultToks.empty() && ResultToks.back().is(tok::hashhash); 1668d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner bool PasteAfter = i+1 != e && Tokens[i+1].is(tok::hashhash); 1675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If it is not the LHS/RHS of a ## operator, we must pre-expand the 1695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // argument and substitute the expanded tokens into the result. This is 1705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // C99 6.10.3.1p1. 1715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (!PasteBefore && !PasteAfter) { 172d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner const Token *ResultArgToks; 1735f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1745f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Only preexpand the argument if it could possibly need it. This 1755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // avoids some work in common cases. 176d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo); 177cc1a875f94630e58d24a55577ffbf0e89b7da8c7Chris Lattner if (ActualArgs->ArgNeedsPreexpansion(ArgTok, PP)) 1785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, PP)[0]; 1795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer else 1805f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ResultArgToks = ArgTok; // Use non-preexpanded tokens. 1815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If the arg token expanded into anything, append it. 18322f6bbcafa8871f4f20c4402d9cbc5c024fee99aChris Lattner if (ResultArgToks->isNot(tok::eof)) { 1845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned FirstResult = ResultToks.size(); 1855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned NumToks = MacroArgs::getArgLength(ResultArgToks); 1865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ResultToks.append(ResultArgToks, ResultArgToks+NumToks); 1875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If any tokens were substituted from the argument, the whitespace 1895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // before the first token should match the whitespace of the arg 1905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // identifier. 191d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner ResultToks[FirstResult].setFlagValue(Token::LeadingSpace, 1925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurTok.hasLeadingSpace() || 1935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NextTokGetsSpace); 1945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NextTokGetsSpace = false; 1955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 1965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is an empty argument, and if there was whitespace before the 1975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // formal token, make sure the next token gets whitespace before it. 1985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NextTokGetsSpace = CurTok.hasLeadingSpace(); 1995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 2005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer continue; 2015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 2025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 2035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Okay, we have a token that is either the LHS or RHS of a paste (##) 2045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // argument. It gets substituted as its non-pre-expanded tokens. 205d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo); 2065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned NumToks = MacroArgs::getArgLength(ArgToks); 2075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (NumToks) { // Not an empty argument? 20871a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner // If this is the GNU ", ## __VA_ARG__" extension, and we just learned 20971a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner // that __VA_ARG__ expands to multiple tokens, avoid a pasting error when 21071a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner // the expander trys to paste ',' with the first token of the __VA_ARG__ 21171a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner // expansion. 21271a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner if (PasteBefore && ResultToks.size() >= 2 && 21371a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner ResultToks[ResultToks.size()-2].is(tok::comma) && 21471a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner (unsigned)ArgNo == Macro->getNumArgs()-1 && 21571a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner Macro->isVariadic()) { 21671a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner // Remove the paste operator, report use of the extension. 21771a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma); 21871a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner ResultToks.pop_back(); 21971a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner } 22071a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner 2215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ResultToks.append(ArgToks, ArgToks+NumToks); 2225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 2235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If the next token was supposed to get leading whitespace, ensure it has 2245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // it now. 2255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (NextTokGetsSpace) { 226d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner ResultToks[ResultToks.size()-NumToks].setFlag(Token::LeadingSpace); 2275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NextTokGetsSpace = false; 2285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 2295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer continue; 2305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 2315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 2325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If an empty argument is on the LHS or RHS of a paste, the standard (C99 2335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // 6.10.3.3p2,3) calls for a bunch of placemarker stuff to occur. We 2345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // implement this by eating ## operators when a LHS or RHS expands to 2355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // empty. 2365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NextTokGetsSpace |= CurTok.hasLeadingSpace(); 2375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (PasteAfter) { 2385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Discard the argument token and skip (don't copy to the expansion 2395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // buffer) the paste operator after it. 2408d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner NextTokGetsSpace |= Tokens[i+1].hasLeadingSpace(); 2415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++i; 2425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer continue; 2435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 2445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 2455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is on the RHS of a paste operator, we've already copied the 2465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // paste operator to the ResultToks list. Remove it. 24722f6bbcafa8871f4f20c4402d9cbc5c024fee99aChris Lattner assert(PasteBefore && ResultToks.back().is(tok::hashhash)); 2485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NextTokGetsSpace |= ResultToks.back().hasLeadingSpace(); 2495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ResultToks.pop_back(); 2505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 2515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is the __VA_ARGS__ token, and if the argument wasn't provided, 2525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // and if the macro had at least one real argument, and if the token before 2535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // the ## was a comma, remove the comma. 2545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if ((unsigned)ArgNo == Macro->getNumArgs()-1 && // is __VA_ARGS__ 2555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ActualArgs->isVarargsElidedUse() && // Argument elided. 25622f6bbcafa8871f4f20c4402d9cbc5c024fee99aChris Lattner !ResultToks.empty() && ResultToks.back().is(tok::comma)) { 2575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Never add a space, even if the comma, ##, or arg had a space. 2585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NextTokGetsSpace = false; 25971a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner // Remove the paste operator, report use of the extension. 26071a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma); 2615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ResultToks.pop_back(); 2625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 2635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer continue; 2645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 2655f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 2668d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner // If anything changed, install this as the new Tokens list. 2675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (MadeChange) { 26832fca722dd974b8202d0fb9c71b6c185c0767da6Chris Lattner assert(!OwnsTokens && "This would leak if we already own the token list"); 2695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // This is deleted in the dtor. 2708d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner NumTokens = ResultToks.size(); 27132fca722dd974b8202d0fb9c71b6c185c0767da6Chris Lattner llvm::BumpPtrAllocator &Alloc = PP.getPreprocessorAllocator(); 27232fca722dd974b8202d0fb9c71b6c185c0767da6Chris Lattner Token *Res = 27332fca722dd974b8202d0fb9c71b6c185c0767da6Chris Lattner static_cast<Token *>(Alloc.Allocate(sizeof(Token)*ResultToks.size(), 27432fca722dd974b8202d0fb9c71b6c185c0767da6Chris Lattner llvm::alignof<Token>())); 2758d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner if (NumTokens) 2768d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner memcpy(Res, &ResultToks[0], NumTokens*sizeof(Token)); 2778d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner Tokens = Res; 27832fca722dd974b8202d0fb9c71b6c185c0767da6Chris Lattner 27932fca722dd974b8202d0fb9c71b6c185c0767da6Chris Lattner // The preprocessor bump pointer owns these tokens, not us. 28032fca722dd974b8202d0fb9c71b6c185c0767da6Chris Lattner OwnsTokens = false; 2815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 2825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 2835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 2845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// Lex - Lex and return a token from this macro stream. 2855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// 2861543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattnervoid TokenLexer::Lex(Token &Tok) { 2875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Lexing off the end of the macro, pop this macro off the expansion stack. 2885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (isAtEnd()) { 2895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is a macro (not a token stream), mark the macro enabled now 2905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // that it is no longer being expanded. 2915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Macro) Macro->EnableMacro(); 2925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 2935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Pop this context off the preprocessors lexer stack and get the next 2945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // token. This will delete "this" so remember the PP instance var. 2955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Preprocessor &PPCache = PP; 296fde2bf9befede63e3f01f84519784c17b4c81480Chris Lattner if (PP.HandleEndOfTokenLexer(Tok)) 2975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return; 2985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 299fde2bf9befede63e3f01f84519784c17b4c81480Chris Lattner // HandleEndOfTokenLexer may not return a token. If it doesn't, lex 300fde2bf9befede63e3f01f84519784c17b4c81480Chris Lattner // whatever is next. 3015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return PPCache.Lex(Tok); 3025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 3035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is the first token of the expanded result, we inherit spacing 3055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // properties later. 3065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer bool isFirstToken = CurToken == 0; 3075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Get the next token to return. 3098d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner Tok = Tokens[CurToken++]; 3105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this token is followed by a token paste (##) operator, paste the tokens! 3128d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner if (!isAtEnd() && Tokens[CurToken].is(tok::hashhash)) 3133f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner if (PasteTokens(Tok)) { 3143f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner // When handling the microsoft /##/ extension, the final token is 3153f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner // returned by PasteTokens, not the pasted token. 3163f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner return; 3173f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner } 3185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // The token's current location indicate where the token was lexed from. We 3205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // need this information to compute the spelling of the token, but any 3215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // diagnostics for the expanded token should appear as if they came from 3225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // InstantiationLoc. Pull this information together into a new SourceLocation 3235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // that captures all of this. 324e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner if (InstantiateLocStart.isValid()) { // Don't do this for token streams. 325e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner SourceManager &SM = PP.getSourceManager(); 326e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner Tok.setLocation(SM.createInstantiationLoc(Tok.getLocation(), 327e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner InstantiateLocStart, 328e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner InstantiateLocEnd, 329e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner Tok.getLength())); 3305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 3315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is the first token, set the lexical properties of the token to 3335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // match the lexical properties of the macro identifier. 3345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (isFirstToken) { 335d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Tok.setFlagValue(Token::StartOfLine , AtStartOfLine); 336d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace); 3375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 3385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Handle recursive expansion! 340863c486fcb6162495a94fddf7ac8409de2638995Chris Lattner if (IdentifierInfo *II = Tok.getIdentifierInfo()) { 341863c486fcb6162495a94fddf7ac8409de2638995Chris Lattner // Change the kind of this identifier to the appropriate token kind, e.g. 342863c486fcb6162495a94fddf7ac8409de2638995Chris Lattner // turning "for" into a keyword. 343863c486fcb6162495a94fddf7ac8409de2638995Chris Lattner Tok.setKind(II->getTokenID()); 344863c486fcb6162495a94fddf7ac8409de2638995Chris Lattner 345863c486fcb6162495a94fddf7ac8409de2638995Chris Lattner if (!DisableMacroExpansion && II->isHandleIdentifierCase()) 346863c486fcb6162495a94fddf7ac8409de2638995Chris Lattner PP.HandleIdentifier(Tok); 347863c486fcb6162495a94fddf7ac8409de2638995Chris Lattner } 3485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, return a normal token. 3505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 3515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// PasteTokens - Tok is the LHS of a ## operator, and CurToken is the ## 3535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// operator. Read the ## and RHS, and paste the LHS/RHS together. If there 35448002c89578eaeb008a0bb5a10d8216e61906459Nico Weber/// are more ## after it, chomp them iteratively. Return the result as Tok. 3553f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner/// If this returns true, the caller should immediately return the token. 3561543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattnerbool TokenLexer::PasteTokens(Token &Tok) { 3575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer llvm::SmallVector<char, 128> Buffer; 35847246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner const char *ResultTokStrPtr = 0; 3595f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer do { 3605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Consume the ## operator. 3618d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner SourceLocation PasteOpLoc = Tokens[CurToken].getLocation(); 3625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++CurToken; 3635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer assert(!isAtEnd() && "No token on the RHS of a paste operator!"); 3645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3655f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Get the RHS token. 3668d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner const Token &RHS = Tokens[CurToken]; 3675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer bool isInvalid = false; 3695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Allocate space for the result token. This is guaranteed to be enough for 3715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // the two tokens and a null terminator. 3725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Buffer.resize(Tok.getLength() + RHS.getLength() + 1); 3735f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3745f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Get the spelling of the LHS token in Buffer. 3755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *BufPtr = &Buffer[0]; 3765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned LHSLen = PP.getSpelling(Tok, BufPtr); 3775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (BufPtr != &Buffer[0]) // Really, we want the chars in Buffer! 3785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer memcpy(&Buffer[0], BufPtr, LHSLen); 3795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3805f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufPtr = &Buffer[LHSLen]; 3815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned RHSLen = PP.getSpelling(RHS, BufPtr); 3825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (BufPtr != &Buffer[LHSLen]) // Really, we want the chars in Buffer! 3835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer memcpy(&Buffer[LHSLen], BufPtr, RHSLen); 3845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Add null terminator. 3865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Buffer[LHSLen+RHSLen] = '\0'; 3875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Trim excess space. 3895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Buffer.resize(LHSLen+RHSLen+1); 3905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Plop the pasted result (including the trailing newline and null) into a 3925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // scratch buffer where we can lex it. 39347246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner Token ResultTokTmp; 39447246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner ResultTokTmp.startToken(); 3955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 39647246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner // Claim that the tmp token is a string_literal so that we can get the 39747246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner // character pointer back from CreateString. 39847246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner ResultTokTmp.setKind(tok::string_literal); 39947246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner PP.CreateString(&Buffer[0], Buffer.size(), ResultTokTmp); 40047246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner SourceLocation ResultTokLoc = ResultTokTmp.getLocation(); 40147246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner ResultTokStrPtr = ResultTokTmp.getLiteralData(); 40247246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner 4035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Lex the resultant pasted token into Result. 404d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Token Result; 4055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 4060af574270d3be2b0e73a3379dfaa633746f8fc6fChris Lattner if (Tok.is(tok::identifier) && RHS.is(tok::identifier)) { 4075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Common paste case: identifier+identifier = identifier. Avoid creating 4085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // a lexer and other overhead. 4095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer PP.IncrementPasteCounter(true); 4105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Result.startToken(); 4115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Result.setKind(tok::identifier); 4125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Result.setLocation(ResultTokLoc); 4135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Result.setLength(LHSLen+RHSLen); 4145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 4155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer PP.IncrementPasteCounter(false); 4165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 417bcc2a67e5180612417727cbdd8afd0f79fdf726dChris Lattner assert(ResultTokLoc.isFileID() && 418bcc2a67e5180612417727cbdd8afd0f79fdf726dChris Lattner "Should be a raw location into scratch buffer"); 4195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer SourceManager &SourceMgr = PP.getSourceManager(); 42047246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner FileID LocFileID = SourceMgr.getFileID(ResultTokLoc); 421bcc2a67e5180612417727cbdd8afd0f79fdf726dChris Lattner 42247246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner const char *ScratchBufStart = SourceMgr.getBufferData(LocFileID).first; 423bcc2a67e5180612417727cbdd8afd0f79fdf726dChris Lattner 424bcc2a67e5180612417727cbdd8afd0f79fdf726dChris Lattner // Make a lexer to lex this string from. Lex just this one token. 4255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Make a lexer object so that we lex and expand the paste result. 42647246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner Lexer TL(SourceMgr.getLocForStartOfFile(LocFileID), 42747246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner PP.getLangOptions(), ScratchBufStart, 42847246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner ResultTokStrPtr, 42947246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner ResultTokStrPtr+LHSLen+RHSLen /*don't include null*/); 4305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 4315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Lex a token in raw mode. This way it won't look up identifiers 4325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // automatically, lexing off the end will return an eof token, and 4335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // warnings are disabled. This returns true if the result token is the 4345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // entire buffer. 435590f0cc643274267d4d41125b62557e1d87886c3Chris Lattner bool IsComplete = TL.LexFromRawLexer(Result); 4365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 4375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we got an EOF token, we didn't form even ONE token. For example, we 4385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // did "/ ## /" to get "//". 43922f6bbcafa8871f4f20c4402d9cbc5c024fee99aChris Lattner IsComplete &= Result.isNot(tok::eof); 4405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer isInvalid = !IsComplete; 4415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 4425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 4435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If pasting the two tokens didn't form a full new token, this is an error. 4445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // This occurs with "x ## +" and other stuff. Return with Tok unmodified 4455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // and with RHS as the next token to lex. 4465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (isInvalid) { 4473f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner // Test for the Microsoft extension of /##/ turning into // here on the 4483f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner // error path. 4493f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner if (PP.getLangOptions().Microsoft && Tok.is(tok::slash) && 4503f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner RHS.is(tok::slash)) { 4513f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner HandleMicrosoftCommentPaste(Tok); 4523f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner return true; 4533f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner } 45447246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner 45547246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner // TODO: If not in assembler language mode. 45647246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner PP.Diag(PasteOpLoc, diag::err_pp_bad_paste) 45747246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner << std::string(Buffer.begin(), Buffer.end()-1); 45847246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner return false; 4595f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 4605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 4611543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner // Turn ## into 'unknown' to avoid # ## # from looking like a paste 4621543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattner // operator. 46322f6bbcafa8871f4f20c4402d9cbc5c024fee99aChris Lattner if (Result.is(tok::hashhash)) 4645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Result.setKind(tok::unknown); 46571a3a8d50643c450823d3fbcf3307a3d7db1a265Chris Lattner // FIXME: Turn __VA_ARGS__ into "not a token"? 4665f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 4675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Transfer properties of the LHS over the the Result. 468d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlagValue(Token::StartOfLine , Tok.isAtStartOfLine()); 469d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlagValue(Token::LeadingSpace, Tok.hasLeadingSpace()); 4705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 4715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Finally, replace LHS with the result, consume the RHS, and iterate. 4725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++CurToken; 4735f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Tok = Result; 4748d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner } while (!isAtEnd() && Tokens[CurToken].is(tok::hashhash)); 4755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 4765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Now that we got the result token, it will be subject to expansion. Since 4775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // token pasting re-lexes the result token in raw mode, identifier information 4785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // isn't looked up. As such, if the result is an identifier, look up id info. 47922f6bbcafa8871f4f20c4402d9cbc5c024fee99aChris Lattner if (Tok.is(tok::identifier)) { 4805f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Look up the identifier info for the token. We disabled identifier lookup 4815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // by saying we're skipping contents, so we need to do this manually. 48247246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner Tok.setIdentifierInfo(PP.LookUpIdentifierInfo(Tok, ResultTokStrPtr)); 4835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 4843f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner return false; 4855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 4865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 4875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// isNextTokenLParen - If the next token lexed will pop this macro off the 4885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// expansion stack, return 2. If the next unexpanded token is a '(', return 4895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// 1, otherwise return 0. 4901543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattnerunsigned TokenLexer::isNextTokenLParen() const { 4915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Out of tokens? 4925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (isAtEnd()) 4935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return 2; 4948d896438031f5990cb42eb95a7bae233db2eb333Chris Lattner return Tokens[CurToken].is(tok::l_paren); 4955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 4963f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner 4973f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner 4983f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner/// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes 4993f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner/// together to form a comment that comments out everything in the current 5003f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner/// macro, other active macros, and anything left on the current physical 5013f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner/// source line of the instantiated buffer. Handle this by returning the 5023f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner/// first token on the next line. 5031543e9c69202b8e128c7fe34784ae7aa90964889Chris Lattnervoid TokenLexer::HandleMicrosoftCommentPaste(Token &Tok) { 5043f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner // We 'comment out' the rest of this macro by just ignoring the rest of the 5053f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner // tokens that have not been lexed yet, if any. 5063f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner 5073f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner // Since this must be a macro, mark the macro enabled now that it is no longer 5083f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner // being expanded. 5093f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner assert(Macro && "Token streams can't paste comments"); 5103f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner Macro->EnableMacro(); 5113f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner 5123f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner PP.HandleMicrosoftCommentPaste(Tok); 5133f1cc838f9caf469990f23fccd0940263c0c61ffChris Lattner} 514