1//===--- TokenLexer.cpp - Lex from a token stream -------------------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the TokenLexer interface. 11// 12//===----------------------------------------------------------------------===// 13 14#include "clang/Lex/TokenLexer.h" 15#include "clang/Basic/SourceManager.h" 16#include "clang/Lex/LexDiagnostic.h" 17#include "clang/Lex/MacroArgs.h" 18#include "clang/Lex/MacroInfo.h" 19#include "clang/Lex/Preprocessor.h" 20#include "llvm/ADT/SmallString.h" 21using namespace clang; 22 23 24/// Create a TokenLexer for the specified macro with the specified actual 25/// arguments. Note that this ctor takes ownership of the ActualArgs pointer. 26void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI, 27 MacroArgs *Actuals) { 28 // If the client is reusing a TokenLexer, make sure to free any memory 29 // associated with it. 30 destroy(); 31 32 Macro = MI; 33 ActualArgs = Actuals; 34 CurToken = 0; 35 36 ExpandLocStart = Tok.getLocation(); 37 ExpandLocEnd = ELEnd; 38 AtStartOfLine = Tok.isAtStartOfLine(); 39 HasLeadingSpace = Tok.hasLeadingSpace(); 40 NextTokGetsSpace = false; 41 Tokens = &*Macro->tokens_begin(); 42 OwnsTokens = false; 43 DisableMacroExpansion = false; 44 NumTokens = Macro->tokens_end()-Macro->tokens_begin(); 45 MacroExpansionStart = SourceLocation(); 46 47 SourceManager &SM = PP.getSourceManager(); 48 MacroStartSLocOffset = SM.getNextLocalOffset(); 49 50 if (NumTokens > 0) { 51 assert(Tokens[0].getLocation().isValid()); 52 assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) && 53 "Macro defined in macro?"); 54 assert(ExpandLocStart.isValid()); 55 56 // Reserve a source location entry chunk for the length of the macro 57 // definition. Tokens that get lexed directly from the definition will 58 // have their locations pointing inside this chunk. This is to avoid 59 // creating separate source location entries for each token. 60 MacroDefStart = SM.getExpansionLoc(Tokens[0].getLocation()); 61 MacroDefLength = Macro->getDefinitionLength(SM); 62 MacroExpansionStart = SM.createExpansionLoc(MacroDefStart, 63 ExpandLocStart, 64 ExpandLocEnd, 65 MacroDefLength); 66 } 67 68 // If this is a function-like macro, expand the arguments and change 69 // Tokens to point to the expanded tokens. 70 if (Macro->isFunctionLike() && Macro->getNumArgs()) 71 ExpandFunctionArguments(); 72 73 // Mark the macro as currently disabled, so that it is not recursively 74 // expanded. The macro must be disabled only after argument pre-expansion of 75 // function-like macro arguments occurs. 76 Macro->DisableMacro(); 77} 78 79 80 81/// Create a TokenLexer for the specified token stream. This does not 82/// take ownership of the specified token vector. 83void TokenLexer::Init(const Token *TokArray, unsigned NumToks, 84 bool disableMacroExpansion, bool ownsTokens) { 85 // If the client is reusing a TokenLexer, make sure to free any memory 86 // associated with it. 87 destroy(); 88 89 Macro = nullptr; 90 ActualArgs = nullptr; 91 Tokens = TokArray; 92 OwnsTokens = ownsTokens; 93 DisableMacroExpansion = disableMacroExpansion; 94 NumTokens = NumToks; 95 CurToken = 0; 96 ExpandLocStart = ExpandLocEnd = SourceLocation(); 97 AtStartOfLine = false; 98 HasLeadingSpace = false; 99 NextTokGetsSpace = false; 100 MacroExpansionStart = SourceLocation(); 101 102 // Set HasLeadingSpace/AtStartOfLine so that the first token will be 103 // returned unmodified. 104 if (NumToks != 0) { 105 AtStartOfLine = TokArray[0].isAtStartOfLine(); 106 HasLeadingSpace = TokArray[0].hasLeadingSpace(); 107 } 108} 109 110 111void TokenLexer::destroy() { 112 // If this was a function-like macro that actually uses its arguments, delete 113 // the expanded tokens. 114 if (OwnsTokens) { 115 delete [] Tokens; 116 Tokens = nullptr; 117 OwnsTokens = false; 118 } 119 120 // TokenLexer owns its formal arguments. 121 if (ActualArgs) ActualArgs->destroy(PP); 122} 123 124bool TokenLexer::MaybeRemoveCommaBeforeVaArgs( 125 SmallVectorImpl<Token> &ResultToks, bool HasPasteOperator, MacroInfo *Macro, 126 unsigned MacroArgNo, Preprocessor &PP) { 127 // Is the macro argument __VA_ARGS__? 128 if (!Macro->isVariadic() || MacroArgNo != Macro->getNumArgs()-1) 129 return false; 130 131 // In Microsoft-compatibility mode, a comma is removed in the expansion 132 // of " ... , __VA_ARGS__ " if __VA_ARGS__ is empty. This extension is 133 // not supported by gcc. 134 if (!HasPasteOperator && !PP.getLangOpts().MSVCCompat) 135 return false; 136 137 // GCC removes the comma in the expansion of " ... , ## __VA_ARGS__ " if 138 // __VA_ARGS__ is empty, but not in strict C99 mode where there are no 139 // named arguments, where it remains. In all other modes, including C99 140 // with GNU extensions, it is removed regardless of named arguments. 141 // Microsoft also appears to support this extension, unofficially. 142 if (PP.getLangOpts().C99 && !PP.getLangOpts().GNUMode 143 && Macro->getNumArgs() < 2) 144 return false; 145 146 // Is a comma available to be removed? 147 if (ResultToks.empty() || !ResultToks.back().is(tok::comma)) 148 return false; 149 150 // Issue an extension diagnostic for the paste operator. 151 if (HasPasteOperator) 152 PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma); 153 154 // Remove the comma. 155 ResultToks.pop_back(); 156 157 // If the comma was right after another paste (e.g. "X##,##__VA_ARGS__"), 158 // then removal of the comma should produce a placemarker token (in C99 159 // terms) which we model by popping off the previous ##, giving us a plain 160 // "X" when __VA_ARGS__ is empty. 161 if (!ResultToks.empty() && ResultToks.back().is(tok::hashhash)) 162 ResultToks.pop_back(); 163 164 // Never add a space, even if the comma, ##, or arg had a space. 165 NextTokGetsSpace = false; 166 return true; 167} 168 169/// Expand the arguments of a function-like macro so that we can quickly 170/// return preexpanded tokens from Tokens. 171void TokenLexer::ExpandFunctionArguments() { 172 173 SmallVector<Token, 128> ResultToks; 174 175 // Loop through 'Tokens', expanding them into ResultToks. Keep 176 // track of whether we change anything. If not, no need to keep them. If so, 177 // we install the newly expanded sequence as the new 'Tokens' list. 178 bool MadeChange = false; 179 180 for (unsigned i = 0, e = NumTokens; i != e; ++i) { 181 // If we found the stringify operator, get the argument stringified. The 182 // preprocessor already verified that the following token is a macro name 183 // when the #define was parsed. 184 const Token &CurTok = Tokens[i]; 185 if (i != 0 && !Tokens[i-1].is(tok::hashhash) && CurTok.hasLeadingSpace()) 186 NextTokGetsSpace = true; 187 188 if (CurTok.isOneOf(tok::hash, tok::hashat)) { 189 int ArgNo = Macro->getArgumentNum(Tokens[i+1].getIdentifierInfo()); 190 assert(ArgNo != -1 && "Token following # is not an argument?"); 191 192 SourceLocation ExpansionLocStart = 193 getExpansionLocForMacroDefLoc(CurTok.getLocation()); 194 SourceLocation ExpansionLocEnd = 195 getExpansionLocForMacroDefLoc(Tokens[i+1].getLocation()); 196 197 Token Res; 198 if (CurTok.is(tok::hash)) // Stringify 199 Res = ActualArgs->getStringifiedArgument(ArgNo, PP, 200 ExpansionLocStart, 201 ExpansionLocEnd); 202 else { 203 // 'charify': don't bother caching these. 204 Res = MacroArgs::StringifyArgument(ActualArgs->getUnexpArgument(ArgNo), 205 PP, true, 206 ExpansionLocStart, 207 ExpansionLocEnd); 208 } 209 Res.setFlag(Token::StringifiedInMacro); 210 211 // The stringified/charified string leading space flag gets set to match 212 // the #/#@ operator. 213 if (NextTokGetsSpace) 214 Res.setFlag(Token::LeadingSpace); 215 216 ResultToks.push_back(Res); 217 MadeChange = true; 218 ++i; // Skip arg name. 219 NextTokGetsSpace = false; 220 continue; 221 } 222 223 // Find out if there is a paste (##) operator before or after the token. 224 bool NonEmptyPasteBefore = 225 !ResultToks.empty() && ResultToks.back().is(tok::hashhash); 226 bool PasteBefore = i != 0 && Tokens[i-1].is(tok::hashhash); 227 bool PasteAfter = i+1 != e && Tokens[i+1].is(tok::hashhash); 228 assert(!NonEmptyPasteBefore || PasteBefore); 229 230 // Otherwise, if this is not an argument token, just add the token to the 231 // output buffer. 232 IdentifierInfo *II = CurTok.getIdentifierInfo(); 233 int ArgNo = II ? Macro->getArgumentNum(II) : -1; 234 if (ArgNo == -1) { 235 // This isn't an argument, just add it. 236 ResultToks.push_back(CurTok); 237 238 if (NextTokGetsSpace) { 239 ResultToks.back().setFlag(Token::LeadingSpace); 240 NextTokGetsSpace = false; 241 } else if (PasteBefore && !NonEmptyPasteBefore) 242 ResultToks.back().clearFlag(Token::LeadingSpace); 243 244 continue; 245 } 246 247 // An argument is expanded somehow, the result is different than the 248 // input. 249 MadeChange = true; 250 251 // Otherwise, this is a use of the argument. 252 253 // In Microsoft mode, remove the comma before __VA_ARGS__ to ensure there 254 // are no trailing commas if __VA_ARGS__ is empty. 255 if (!PasteBefore && ActualArgs->isVarargsElidedUse() && 256 MaybeRemoveCommaBeforeVaArgs(ResultToks, 257 /*HasPasteOperator=*/false, 258 Macro, ArgNo, PP)) 259 continue; 260 261 // If it is not the LHS/RHS of a ## operator, we must pre-expand the 262 // argument and substitute the expanded tokens into the result. This is 263 // C99 6.10.3.1p1. 264 if (!PasteBefore && !PasteAfter) { 265 const Token *ResultArgToks; 266 267 // Only preexpand the argument if it could possibly need it. This 268 // avoids some work in common cases. 269 const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo); 270 if (ActualArgs->ArgNeedsPreexpansion(ArgTok, PP)) 271 ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, Macro, PP)[0]; 272 else 273 ResultArgToks = ArgTok; // Use non-preexpanded tokens. 274 275 // If the arg token expanded into anything, append it. 276 if (ResultArgToks->isNot(tok::eof)) { 277 unsigned FirstResult = ResultToks.size(); 278 unsigned NumToks = MacroArgs::getArgLength(ResultArgToks); 279 ResultToks.append(ResultArgToks, ResultArgToks+NumToks); 280 281 // In Microsoft-compatibility mode, we follow MSVC's preprocessing 282 // behavior by not considering single commas from nested macro 283 // expansions as argument separators. Set a flag on the token so we can 284 // test for this later when the macro expansion is processed. 285 if (PP.getLangOpts().MSVCCompat && NumToks == 1 && 286 ResultToks.back().is(tok::comma)) 287 ResultToks.back().setFlag(Token::IgnoredComma); 288 289 // If the '##' came from expanding an argument, turn it into 'unknown' 290 // to avoid pasting. 291 for (unsigned i = FirstResult, e = ResultToks.size(); i != e; ++i) { 292 Token &Tok = ResultToks[i]; 293 if (Tok.is(tok::hashhash)) 294 Tok.setKind(tok::unknown); 295 } 296 297 if(ExpandLocStart.isValid()) { 298 updateLocForMacroArgTokens(CurTok.getLocation(), 299 ResultToks.begin()+FirstResult, 300 ResultToks.end()); 301 } 302 303 // If any tokens were substituted from the argument, the whitespace 304 // before the first token should match the whitespace of the arg 305 // identifier. 306 ResultToks[FirstResult].setFlagValue(Token::LeadingSpace, 307 NextTokGetsSpace); 308 NextTokGetsSpace = false; 309 } 310 continue; 311 } 312 313 // Okay, we have a token that is either the LHS or RHS of a paste (##) 314 // argument. It gets substituted as its non-pre-expanded tokens. 315 const Token *ArgToks = ActualArgs->getUnexpArgument(ArgNo); 316 unsigned NumToks = MacroArgs::getArgLength(ArgToks); 317 if (NumToks) { // Not an empty argument? 318 // If this is the GNU ", ## __VA_ARGS__" extension, and we just learned 319 // that __VA_ARGS__ expands to multiple tokens, avoid a pasting error when 320 // the expander trys to paste ',' with the first token of the __VA_ARGS__ 321 // expansion. 322 if (NonEmptyPasteBefore && ResultToks.size() >= 2 && 323 ResultToks[ResultToks.size()-2].is(tok::comma) && 324 (unsigned)ArgNo == Macro->getNumArgs()-1 && 325 Macro->isVariadic()) { 326 // Remove the paste operator, report use of the extension. 327 PP.Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma); 328 } 329 330 ResultToks.append(ArgToks, ArgToks+NumToks); 331 332 // If the '##' came from expanding an argument, turn it into 'unknown' 333 // to avoid pasting. 334 for (unsigned i = ResultToks.size() - NumToks, e = ResultToks.size(); 335 i != e; ++i) { 336 Token &Tok = ResultToks[i]; 337 if (Tok.is(tok::hashhash)) 338 Tok.setKind(tok::unknown); 339 } 340 341 if (ExpandLocStart.isValid()) { 342 updateLocForMacroArgTokens(CurTok.getLocation(), 343 ResultToks.end()-NumToks, ResultToks.end()); 344 } 345 346 // If this token (the macro argument) was supposed to get leading 347 // whitespace, transfer this information onto the first token of the 348 // expansion. 349 // 350 // Do not do this if the paste operator occurs before the macro argument, 351 // as in "A ## MACROARG". In valid code, the first token will get 352 // smooshed onto the preceding one anyway (forming AMACROARG). In 353 // assembler-with-cpp mode, invalid pastes are allowed through: in this 354 // case, we do not want the extra whitespace to be added. For example, 355 // we want ". ## foo" -> ".foo" not ". foo". 356 if (NextTokGetsSpace) 357 ResultToks[ResultToks.size()-NumToks].setFlag(Token::LeadingSpace); 358 359 NextTokGetsSpace = false; 360 continue; 361 } 362 363 // If an empty argument is on the LHS or RHS of a paste, the standard (C99 364 // 6.10.3.3p2,3) calls for a bunch of placemarker stuff to occur. We 365 // implement this by eating ## operators when a LHS or RHS expands to 366 // empty. 367 if (PasteAfter) { 368 // Discard the argument token and skip (don't copy to the expansion 369 // buffer) the paste operator after it. 370 ++i; 371 continue; 372 } 373 374 // If this is on the RHS of a paste operator, we've already copied the 375 // paste operator to the ResultToks list, unless the LHS was empty too. 376 // Remove it. 377 assert(PasteBefore); 378 if (NonEmptyPasteBefore) { 379 assert(ResultToks.back().is(tok::hashhash)); 380 ResultToks.pop_back(); 381 } 382 383 // If this is the __VA_ARGS__ token, and if the argument wasn't provided, 384 // and if the macro had at least one real argument, and if the token before 385 // the ## was a comma, remove the comma. This is a GCC extension which is 386 // disabled when using -std=c99. 387 if (ActualArgs->isVarargsElidedUse()) 388 MaybeRemoveCommaBeforeVaArgs(ResultToks, 389 /*HasPasteOperator=*/true, 390 Macro, ArgNo, PP); 391 392 continue; 393 } 394 395 // If anything changed, install this as the new Tokens list. 396 if (MadeChange) { 397 assert(!OwnsTokens && "This would leak if we already own the token list"); 398 // This is deleted in the dtor. 399 NumTokens = ResultToks.size(); 400 // The tokens will be added to Preprocessor's cache and will be removed 401 // when this TokenLexer finishes lexing them. 402 Tokens = PP.cacheMacroExpandedTokens(this, ResultToks); 403 404 // The preprocessor cache of macro expanded tokens owns these tokens,not us. 405 OwnsTokens = false; 406 } 407} 408 409/// \brief Checks if two tokens form wide string literal. 410static bool isWideStringLiteralFromMacro(const Token &FirstTok, 411 const Token &SecondTok) { 412 return FirstTok.is(tok::identifier) && 413 FirstTok.getIdentifierInfo()->isStr("L") && SecondTok.isLiteral() && 414 SecondTok.stringifiedInMacro(); 415} 416 417/// Lex - Lex and return a token from this macro stream. 418/// 419bool TokenLexer::Lex(Token &Tok) { 420 // Lexing off the end of the macro, pop this macro off the expansion stack. 421 if (isAtEnd()) { 422 // If this is a macro (not a token stream), mark the macro enabled now 423 // that it is no longer being expanded. 424 if (Macro) Macro->EnableMacro(); 425 426 Tok.startToken(); 427 Tok.setFlagValue(Token::StartOfLine , AtStartOfLine); 428 Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace || NextTokGetsSpace); 429 if (CurToken == 0) 430 Tok.setFlag(Token::LeadingEmptyMacro); 431 return PP.HandleEndOfTokenLexer(Tok); 432 } 433 434 SourceManager &SM = PP.getSourceManager(); 435 436 // If this is the first token of the expanded result, we inherit spacing 437 // properties later. 438 bool isFirstToken = CurToken == 0; 439 440 // Get the next token to return. 441 Tok = Tokens[CurToken++]; 442 443 bool TokenIsFromPaste = false; 444 445 // If this token is followed by a token paste (##) operator, paste the tokens! 446 // Note that ## is a normal token when not expanding a macro. 447 if (!isAtEnd() && Macro && 448 (Tokens[CurToken].is(tok::hashhash) || 449 // Special processing of L#x macros in -fms-compatibility mode. 450 // Microsoft compiler is able to form a wide string literal from 451 // 'L#macro_arg' construct in a function-like macro. 452 (PP.getLangOpts().MSVCCompat && 453 isWideStringLiteralFromMacro(Tok, Tokens[CurToken])))) { 454 // When handling the microsoft /##/ extension, the final token is 455 // returned by PasteTokens, not the pasted token. 456 if (PasteTokens(Tok)) 457 return true; 458 459 TokenIsFromPaste = true; 460 } 461 462 // The token's current location indicate where the token was lexed from. We 463 // need this information to compute the spelling of the token, but any 464 // diagnostics for the expanded token should appear as if they came from 465 // ExpansionLoc. Pull this information together into a new SourceLocation 466 // that captures all of this. 467 if (ExpandLocStart.isValid() && // Don't do this for token streams. 468 // Check that the token's location was not already set properly. 469 SM.isBeforeInSLocAddrSpace(Tok.getLocation(), MacroStartSLocOffset)) { 470 SourceLocation instLoc; 471 if (Tok.is(tok::comment)) { 472 instLoc = SM.createExpansionLoc(Tok.getLocation(), 473 ExpandLocStart, 474 ExpandLocEnd, 475 Tok.getLength()); 476 } else { 477 instLoc = getExpansionLocForMacroDefLoc(Tok.getLocation()); 478 } 479 480 Tok.setLocation(instLoc); 481 } 482 483 // If this is the first token, set the lexical properties of the token to 484 // match the lexical properties of the macro identifier. 485 if (isFirstToken) { 486 Tok.setFlagValue(Token::StartOfLine , AtStartOfLine); 487 Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace); 488 } else { 489 // If this is not the first token, we may still need to pass through 490 // leading whitespace if we've expanded a macro. 491 if (AtStartOfLine) Tok.setFlag(Token::StartOfLine); 492 if (HasLeadingSpace) Tok.setFlag(Token::LeadingSpace); 493 } 494 AtStartOfLine = false; 495 HasLeadingSpace = false; 496 497 // Handle recursive expansion! 498 if (!Tok.isAnnotation() && Tok.getIdentifierInfo() != nullptr) { 499 // Change the kind of this identifier to the appropriate token kind, e.g. 500 // turning "for" into a keyword. 501 IdentifierInfo *II = Tok.getIdentifierInfo(); 502 Tok.setKind(II->getTokenID()); 503 504 // If this identifier was poisoned and from a paste, emit an error. This 505 // won't be handled by Preprocessor::HandleIdentifier because this is coming 506 // from a macro expansion. 507 if (II->isPoisoned() && TokenIsFromPaste) { 508 PP.HandlePoisonedIdentifier(Tok); 509 } 510 511 if (!DisableMacroExpansion && II->isHandleIdentifierCase()) 512 return PP.HandleIdentifier(Tok); 513 } 514 515 // Otherwise, return a normal token. 516 return true; 517} 518 519/// PasteTokens - Tok is the LHS of a ## operator, and CurToken is the ## 520/// operator. Read the ## and RHS, and paste the LHS/RHS together. If there 521/// are more ## after it, chomp them iteratively. Return the result as Tok. 522/// If this returns true, the caller should immediately return the token. 523bool TokenLexer::PasteTokens(Token &Tok) { 524 // MSVC: If previous token was pasted, this must be a recovery from an invalid 525 // paste operation. Ignore spaces before this token to mimic MSVC output. 526 // Required for generating valid UUID strings in some MS headers. 527 if (PP.getLangOpts().MicrosoftExt && (CurToken >= 2) && 528 Tokens[CurToken - 2].is(tok::hashhash)) 529 Tok.clearFlag(Token::LeadingSpace); 530 531 SmallString<128> Buffer; 532 const char *ResultTokStrPtr = nullptr; 533 SourceLocation StartLoc = Tok.getLocation(); 534 SourceLocation PasteOpLoc; 535 do { 536 // Consume the ## operator if any. 537 PasteOpLoc = Tokens[CurToken].getLocation(); 538 if (Tokens[CurToken].is(tok::hashhash)) 539 ++CurToken; 540 assert(!isAtEnd() && "No token on the RHS of a paste operator!"); 541 542 // Get the RHS token. 543 const Token &RHS = Tokens[CurToken]; 544 545 // Allocate space for the result token. This is guaranteed to be enough for 546 // the two tokens. 547 Buffer.resize(Tok.getLength() + RHS.getLength()); 548 549 // Get the spelling of the LHS token in Buffer. 550 const char *BufPtr = &Buffer[0]; 551 bool Invalid = false; 552 unsigned LHSLen = PP.getSpelling(Tok, BufPtr, &Invalid); 553 if (BufPtr != &Buffer[0]) // Really, we want the chars in Buffer! 554 memcpy(&Buffer[0], BufPtr, LHSLen); 555 if (Invalid) 556 return true; 557 558 BufPtr = Buffer.data() + LHSLen; 559 unsigned RHSLen = PP.getSpelling(RHS, BufPtr, &Invalid); 560 if (Invalid) 561 return true; 562 if (RHSLen && BufPtr != &Buffer[LHSLen]) 563 // Really, we want the chars in Buffer! 564 memcpy(&Buffer[LHSLen], BufPtr, RHSLen); 565 566 // Trim excess space. 567 Buffer.resize(LHSLen+RHSLen); 568 569 // Plop the pasted result (including the trailing newline and null) into a 570 // scratch buffer where we can lex it. 571 Token ResultTokTmp; 572 ResultTokTmp.startToken(); 573 574 // Claim that the tmp token is a string_literal so that we can get the 575 // character pointer back from CreateString in getLiteralData(). 576 ResultTokTmp.setKind(tok::string_literal); 577 PP.CreateString(Buffer, ResultTokTmp); 578 SourceLocation ResultTokLoc = ResultTokTmp.getLocation(); 579 ResultTokStrPtr = ResultTokTmp.getLiteralData(); 580 581 // Lex the resultant pasted token into Result. 582 Token Result; 583 584 if (Tok.isAnyIdentifier() && RHS.isAnyIdentifier()) { 585 // Common paste case: identifier+identifier = identifier. Avoid creating 586 // a lexer and other overhead. 587 PP.IncrementPasteCounter(true); 588 Result.startToken(); 589 Result.setKind(tok::raw_identifier); 590 Result.setRawIdentifierData(ResultTokStrPtr); 591 Result.setLocation(ResultTokLoc); 592 Result.setLength(LHSLen+RHSLen); 593 } else { 594 PP.IncrementPasteCounter(false); 595 596 assert(ResultTokLoc.isFileID() && 597 "Should be a raw location into scratch buffer"); 598 SourceManager &SourceMgr = PP.getSourceManager(); 599 FileID LocFileID = SourceMgr.getFileID(ResultTokLoc); 600 601 bool Invalid = false; 602 const char *ScratchBufStart 603 = SourceMgr.getBufferData(LocFileID, &Invalid).data(); 604 if (Invalid) 605 return false; 606 607 // Make a lexer to lex this string from. Lex just this one token. 608 // Make a lexer object so that we lex and expand the paste result. 609 Lexer TL(SourceMgr.getLocForStartOfFile(LocFileID), 610 PP.getLangOpts(), ScratchBufStart, 611 ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen); 612 613 // Lex a token in raw mode. This way it won't look up identifiers 614 // automatically, lexing off the end will return an eof token, and 615 // warnings are disabled. This returns true if the result token is the 616 // entire buffer. 617 bool isInvalid = !TL.LexFromRawLexer(Result); 618 619 // If we got an EOF token, we didn't form even ONE token. For example, we 620 // did "/ ## /" to get "//". 621 isInvalid |= Result.is(tok::eof); 622 623 // If pasting the two tokens didn't form a full new token, this is an 624 // error. This occurs with "x ## +" and other stuff. Return with Tok 625 // unmodified and with RHS as the next token to lex. 626 if (isInvalid) { 627 // Test for the Microsoft extension of /##/ turning into // here on the 628 // error path. 629 if (PP.getLangOpts().MicrosoftExt && Tok.is(tok::slash) && 630 RHS.is(tok::slash)) { 631 HandleMicrosoftCommentPaste(Tok); 632 return true; 633 } 634 635 // Do not emit the error when preprocessing assembler code. 636 if (!PP.getLangOpts().AsmPreprocessor) { 637 // Explicitly convert the token location to have proper expansion 638 // information so that the user knows where it came from. 639 SourceManager &SM = PP.getSourceManager(); 640 SourceLocation Loc = 641 SM.createExpansionLoc(PasteOpLoc, ExpandLocStart, ExpandLocEnd, 2); 642 // If we're in microsoft extensions mode, downgrade this from a hard 643 // error to an extension that defaults to an error. This allows 644 // disabling it. 645 PP.Diag(Loc, PP.getLangOpts().MicrosoftExt ? diag::ext_pp_bad_paste_ms 646 : diag::err_pp_bad_paste) 647 << Buffer; 648 } 649 650 // An error has occurred so exit loop. 651 break; 652 } 653 654 // Turn ## into 'unknown' to avoid # ## # from looking like a paste 655 // operator. 656 if (Result.is(tok::hashhash)) 657 Result.setKind(tok::unknown); 658 } 659 660 // Transfer properties of the LHS over the Result. 661 Result.setFlagValue(Token::StartOfLine , Tok.isAtStartOfLine()); 662 Result.setFlagValue(Token::LeadingSpace, Tok.hasLeadingSpace()); 663 664 // Finally, replace LHS with the result, consume the RHS, and iterate. 665 ++CurToken; 666 Tok = Result; 667 } while (!isAtEnd() && Tokens[CurToken].is(tok::hashhash)); 668 669 SourceLocation EndLoc = Tokens[CurToken - 1].getLocation(); 670 671 // The token's current location indicate where the token was lexed from. We 672 // need this information to compute the spelling of the token, but any 673 // diagnostics for the expanded token should appear as if the token was 674 // expanded from the full ## expression. Pull this information together into 675 // a new SourceLocation that captures all of this. 676 SourceManager &SM = PP.getSourceManager(); 677 if (StartLoc.isFileID()) 678 StartLoc = getExpansionLocForMacroDefLoc(StartLoc); 679 if (EndLoc.isFileID()) 680 EndLoc = getExpansionLocForMacroDefLoc(EndLoc); 681 FileID MacroFID = SM.getFileID(MacroExpansionStart); 682 while (SM.getFileID(StartLoc) != MacroFID) 683 StartLoc = SM.getImmediateExpansionRange(StartLoc).first; 684 while (SM.getFileID(EndLoc) != MacroFID) 685 EndLoc = SM.getImmediateExpansionRange(EndLoc).second; 686 687 Tok.setLocation(SM.createExpansionLoc(Tok.getLocation(), StartLoc, EndLoc, 688 Tok.getLength())); 689 690 // Now that we got the result token, it will be subject to expansion. Since 691 // token pasting re-lexes the result token in raw mode, identifier information 692 // isn't looked up. As such, if the result is an identifier, look up id info. 693 if (Tok.is(tok::raw_identifier)) { 694 // Look up the identifier info for the token. We disabled identifier lookup 695 // by saying we're skipping contents, so we need to do this manually. 696 PP.LookUpIdentifierInfo(Tok); 697 } 698 return false; 699} 700 701/// isNextTokenLParen - If the next token lexed will pop this macro off the 702/// expansion stack, return 2. If the next unexpanded token is a '(', return 703/// 1, otherwise return 0. 704unsigned TokenLexer::isNextTokenLParen() const { 705 // Out of tokens? 706 if (isAtEnd()) 707 return 2; 708 return Tokens[CurToken].is(tok::l_paren); 709} 710 711/// isParsingPreprocessorDirective - Return true if we are in the middle of a 712/// preprocessor directive. 713bool TokenLexer::isParsingPreprocessorDirective() const { 714 return Tokens[NumTokens-1].is(tok::eod) && !isAtEnd(); 715} 716 717/// HandleMicrosoftCommentPaste - In microsoft compatibility mode, /##/ pastes 718/// together to form a comment that comments out everything in the current 719/// macro, other active macros, and anything left on the current physical 720/// source line of the expanded buffer. Handle this by returning the 721/// first token on the next line. 722void TokenLexer::HandleMicrosoftCommentPaste(Token &Tok) { 723 // We 'comment out' the rest of this macro by just ignoring the rest of the 724 // tokens that have not been lexed yet, if any. 725 726 // Since this must be a macro, mark the macro enabled now that it is no longer 727 // being expanded. 728 assert(Macro && "Token streams can't paste comments"); 729 Macro->EnableMacro(); 730 731 PP.HandleMicrosoftCommentPaste(Tok); 732} 733 734/// \brief If \arg loc is a file ID and points inside the current macro 735/// definition, returns the appropriate source location pointing at the 736/// macro expansion source location entry, otherwise it returns an invalid 737/// SourceLocation. 738SourceLocation 739TokenLexer::getExpansionLocForMacroDefLoc(SourceLocation loc) const { 740 assert(ExpandLocStart.isValid() && MacroExpansionStart.isValid() && 741 "Not appropriate for token streams"); 742 assert(loc.isValid() && loc.isFileID()); 743 744 SourceManager &SM = PP.getSourceManager(); 745 assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) && 746 "Expected loc to come from the macro definition"); 747 748 unsigned relativeOffset = 0; 749 SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength, &relativeOffset); 750 return MacroExpansionStart.getLocWithOffset(relativeOffset); 751} 752 753/// \brief Finds the tokens that are consecutive (from the same FileID) 754/// creates a single SLocEntry, and assigns SourceLocations to each token that 755/// point to that SLocEntry. e.g for 756/// assert(foo == bar); 757/// There will be a single SLocEntry for the "foo == bar" chunk and locations 758/// for the 'foo', '==', 'bar' tokens will point inside that chunk. 759/// 760/// \arg begin_tokens will be updated to a position past all the found 761/// consecutive tokens. 762static void updateConsecutiveMacroArgTokens(SourceManager &SM, 763 SourceLocation InstLoc, 764 Token *&begin_tokens, 765 Token * end_tokens) { 766 assert(begin_tokens < end_tokens); 767 768 SourceLocation FirstLoc = begin_tokens->getLocation(); 769 SourceLocation CurLoc = FirstLoc; 770 771 // Compare the source location offset of tokens and group together tokens that 772 // are close, even if their locations point to different FileIDs. e.g. 773 // 774 // |bar | foo | cake | (3 tokens from 3 consecutive FileIDs) 775 // ^ ^ 776 // |bar foo cake| (one SLocEntry chunk for all tokens) 777 // 778 // we can perform this "merge" since the token's spelling location depends 779 // on the relative offset. 780 781 Token *NextTok = begin_tokens + 1; 782 for (; NextTok < end_tokens; ++NextTok) { 783 SourceLocation NextLoc = NextTok->getLocation(); 784 if (CurLoc.isFileID() != NextLoc.isFileID()) 785 break; // Token from different kind of FileID. 786 787 int RelOffs; 788 if (!SM.isInSameSLocAddrSpace(CurLoc, NextLoc, &RelOffs)) 789 break; // Token from different local/loaded location. 790 // Check that token is not before the previous token or more than 50 791 // "characters" away. 792 if (RelOffs < 0 || RelOffs > 50) 793 break; 794 CurLoc = NextLoc; 795 } 796 797 // For the consecutive tokens, find the length of the SLocEntry to contain 798 // all of them. 799 Token &LastConsecutiveTok = *(NextTok-1); 800 int LastRelOffs = 0; 801 SM.isInSameSLocAddrSpace(FirstLoc, LastConsecutiveTok.getLocation(), 802 &LastRelOffs); 803 unsigned FullLength = LastRelOffs + LastConsecutiveTok.getLength(); 804 805 // Create a macro expansion SLocEntry that will "contain" all of the tokens. 806 SourceLocation Expansion = 807 SM.createMacroArgExpansionLoc(FirstLoc, InstLoc,FullLength); 808 809 // Change the location of the tokens from the spelling location to the new 810 // expanded location. 811 for (; begin_tokens < NextTok; ++begin_tokens) { 812 Token &Tok = *begin_tokens; 813 int RelOffs = 0; 814 SM.isInSameSLocAddrSpace(FirstLoc, Tok.getLocation(), &RelOffs); 815 Tok.setLocation(Expansion.getLocWithOffset(RelOffs)); 816 } 817} 818 819/// \brief Creates SLocEntries and updates the locations of macro argument 820/// tokens to their new expanded locations. 821/// 822/// \param ArgIdDefLoc the location of the macro argument id inside the macro 823/// definition. 824/// \param Tokens the macro argument tokens to update. 825void TokenLexer::updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc, 826 Token *begin_tokens, 827 Token *end_tokens) { 828 SourceManager &SM = PP.getSourceManager(); 829 830 SourceLocation InstLoc = 831 getExpansionLocForMacroDefLoc(ArgIdSpellLoc); 832 833 while (begin_tokens < end_tokens) { 834 // If there's only one token just create a SLocEntry for it. 835 if (end_tokens - begin_tokens == 1) { 836 Token &Tok = *begin_tokens; 837 Tok.setLocation(SM.createMacroArgExpansionLoc(Tok.getLocation(), 838 InstLoc, 839 Tok.getLength())); 840 return; 841 } 842 843 updateConsecutiveMacroArgTokens(SM, InstLoc, begin_tokens, end_tokens); 844 } 845} 846 847void TokenLexer::PropagateLineStartLeadingSpaceInfo(Token &Result) { 848 AtStartOfLine = Result.isAtStartOfLine(); 849 HasLeadingSpace = Result.hasLeadingSpace(); 850} 851