Preprocessor.cpp revision cfa88f893915ceb8ae4ce2f17c46c24a4d67502f
1//===--- Preprocess.cpp - C Language Family Preprocessor Implementation ---===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the Preprocessor interface. 11// 12//===----------------------------------------------------------------------===// 13// 14// Options to support: 15// -H - Print the name of each header file used. 16// -d[DNI] - Dump various things. 17// -fworking-directory - #line's with preprocessor's working dir. 18// -fpreprocessed 19// -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD 20// -W* 21// -w 22// 23// Messages to emit: 24// "Multiple include guards may be useful for:\n" 25// 26//===----------------------------------------------------------------------===// 27 28#include "clang/Lex/Preprocessor.h" 29#include "MacroArgs.h" 30#include "clang/Basic/FileManager.h" 31#include "clang/Basic/SourceManager.h" 32#include "clang/Basic/TargetInfo.h" 33#include "clang/Lex/CodeCompletionHandler.h" 34#include "clang/Lex/ExternalPreprocessorSource.h" 35#include "clang/Lex/HeaderSearch.h" 36#include "clang/Lex/LexDiagnostic.h" 37#include "clang/Lex/LiteralSupport.h" 38#include "clang/Lex/MacroInfo.h" 39#include "clang/Lex/ModuleLoader.h" 40#include "clang/Lex/Pragma.h" 41#include "clang/Lex/PreprocessingRecord.h" 42#include "clang/Lex/PreprocessorOptions.h" 43#include "clang/Lex/ScratchBuffer.h" 44#include "llvm/ADT/APFloat.h" 45#include "llvm/ADT/SmallString.h" 46#include "llvm/Support/Capacity.h" 47#include "llvm/Support/MemoryBuffer.h" 48#include "llvm/Support/raw_ostream.h" 49using namespace clang; 50 51//===----------------------------------------------------------------------===// 52ExternalPreprocessorSource::~ExternalPreprocessorSource() { } 53 54PPMutationListener::~PPMutationListener() { } 55 56Preprocessor::Preprocessor(IntrusiveRefCntPtr<PreprocessorOptions> PPOpts, 57 DiagnosticsEngine &diags, LangOptions &opts, 58 const TargetInfo *target, SourceManager &SM, 59 HeaderSearch &Headers, ModuleLoader &TheModuleLoader, 60 IdentifierInfoLookup* IILookup, 61 bool OwnsHeaders, 62 bool DelayInitialization, 63 bool IncrProcessing) 64 : PPOpts(PPOpts), Diags(&diags), LangOpts(opts), Target(target), 65 FileMgr(Headers.getFileMgr()), 66 SourceMgr(SM), HeaderInfo(Headers), TheModuleLoader(TheModuleLoader), 67 ExternalSource(0), Identifiers(opts, IILookup), 68 IncrementalProcessing(IncrProcessing), CodeComplete(0), 69 CodeCompletionFile(0), CodeCompletionOffset(0), CodeCompletionReached(0), 70 SkipMainFilePreamble(0, true), CurPPLexer(0), 71 CurDirLookup(0), CurLexerKind(CLK_Lexer), Callbacks(0), Listener(0), 72 MacroArgCache(0), Record(0), MIChainHead(0), MICache(0) 73{ 74 OwnsHeaderSearch = OwnsHeaders; 75 76 ScratchBuf = new ScratchBuffer(SourceMgr); 77 CounterValue = 0; // __COUNTER__ starts at 0. 78 79 // Clear stats. 80 NumDirectives = NumDefined = NumUndefined = NumPragma = 0; 81 NumIf = NumElse = NumEndif = 0; 82 NumEnteredSourceFiles = 0; 83 NumMacroExpanded = NumFnMacroExpanded = NumBuiltinMacroExpanded = 0; 84 NumFastMacroExpanded = NumTokenPaste = NumFastTokenPaste = 0; 85 MaxIncludeStackDepth = 0; 86 NumSkipped = 0; 87 88 // Default to discarding comments. 89 KeepComments = false; 90 KeepMacroComments = false; 91 SuppressIncludeNotFoundError = false; 92 93 // Macro expansion is enabled. 94 DisableMacroExpansion = false; 95 MacroExpansionInDirectivesOverride = false; 96 InMacroArgs = false; 97 InMacroArgPreExpansion = false; 98 NumCachedTokenLexers = 0; 99 PragmasEnabled = true; 100 101 CachedLexPos = 0; 102 103 // We haven't read anything from the external source. 104 ReadMacrosFromExternalSource = false; 105 106 // "Poison" __VA_ARGS__, which can only appear in the expansion of a macro. 107 // This gets unpoisoned where it is allowed. 108 (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned(); 109 SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use); 110 111 // Initialize the pragma handlers. 112 PragmaHandlers = new PragmaNamespace(StringRef()); 113 RegisterBuiltinPragmas(); 114 115 // Initialize builtin macros like __LINE__ and friends. 116 RegisterBuiltinMacros(); 117 118 if(LangOpts.Borland) { 119 Ident__exception_info = getIdentifierInfo("_exception_info"); 120 Ident___exception_info = getIdentifierInfo("__exception_info"); 121 Ident_GetExceptionInfo = getIdentifierInfo("GetExceptionInformation"); 122 Ident__exception_code = getIdentifierInfo("_exception_code"); 123 Ident___exception_code = getIdentifierInfo("__exception_code"); 124 Ident_GetExceptionCode = getIdentifierInfo("GetExceptionCode"); 125 Ident__abnormal_termination = getIdentifierInfo("_abnormal_termination"); 126 Ident___abnormal_termination = getIdentifierInfo("__abnormal_termination"); 127 Ident_AbnormalTermination = getIdentifierInfo("AbnormalTermination"); 128 } else { 129 Ident__exception_info = Ident__exception_code = Ident__abnormal_termination = 0; 130 Ident___exception_info = Ident___exception_code = Ident___abnormal_termination = 0; 131 Ident_GetExceptionInfo = Ident_GetExceptionCode = Ident_AbnormalTermination = 0; 132 } 133 134 if (!DelayInitialization) { 135 assert(Target && "Must provide target information for PP initialization"); 136 Initialize(*Target); 137 } 138} 139 140Preprocessor::~Preprocessor() { 141 assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!"); 142 143 while (!IncludeMacroStack.empty()) { 144 delete IncludeMacroStack.back().TheLexer; 145 delete IncludeMacroStack.back().TheTokenLexer; 146 IncludeMacroStack.pop_back(); 147 } 148 149 // Free any macro definitions. 150 for (MacroInfoChain *I = MIChainHead ; I ; I = I->Next) 151 I->MI.Destroy(); 152 153 // Free any cached macro expanders. 154 for (unsigned i = 0, e = NumCachedTokenLexers; i != e; ++i) 155 delete TokenLexerCache[i]; 156 157 // Free any cached MacroArgs. 158 for (MacroArgs *ArgList = MacroArgCache; ArgList; ) 159 ArgList = ArgList->deallocate(); 160 161 // Release pragma information. 162 delete PragmaHandlers; 163 164 // Delete the scratch buffer info. 165 delete ScratchBuf; 166 167 // Delete the header search info, if we own it. 168 if (OwnsHeaderSearch) 169 delete &HeaderInfo; 170 171 delete Callbacks; 172} 173 174void Preprocessor::Initialize(const TargetInfo &Target) { 175 assert((!this->Target || this->Target == &Target) && 176 "Invalid override of target information"); 177 this->Target = &Target; 178 179 // Initialize information about built-ins. 180 BuiltinInfo.InitializeTarget(Target); 181 HeaderInfo.setTarget(Target); 182} 183 184void Preprocessor::setPTHManager(PTHManager* pm) { 185 PTH.reset(pm); 186 FileMgr.addStatCache(PTH->createStatCache()); 187} 188 189void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const { 190 llvm::errs() << tok::getTokenName(Tok.getKind()) << " '" 191 << getSpelling(Tok) << "'"; 192 193 if (!DumpFlags) return; 194 195 llvm::errs() << "\t"; 196 if (Tok.isAtStartOfLine()) 197 llvm::errs() << " [StartOfLine]"; 198 if (Tok.hasLeadingSpace()) 199 llvm::errs() << " [LeadingSpace]"; 200 if (Tok.isExpandDisabled()) 201 llvm::errs() << " [ExpandDisabled]"; 202 if (Tok.needsCleaning()) { 203 const char *Start = SourceMgr.getCharacterData(Tok.getLocation()); 204 llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength()) 205 << "']"; 206 } 207 208 llvm::errs() << "\tLoc=<"; 209 DumpLocation(Tok.getLocation()); 210 llvm::errs() << ">"; 211} 212 213void Preprocessor::DumpLocation(SourceLocation Loc) const { 214 Loc.dump(SourceMgr); 215} 216 217void Preprocessor::DumpMacro(const MacroInfo &MI) const { 218 llvm::errs() << "MACRO: "; 219 for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) { 220 DumpToken(MI.getReplacementToken(i)); 221 llvm::errs() << " "; 222 } 223 llvm::errs() << "\n"; 224} 225 226void Preprocessor::PrintStats() { 227 llvm::errs() << "\n*** Preprocessor Stats:\n"; 228 llvm::errs() << NumDirectives << " directives found:\n"; 229 llvm::errs() << " " << NumDefined << " #define.\n"; 230 llvm::errs() << " " << NumUndefined << " #undef.\n"; 231 llvm::errs() << " #include/#include_next/#import:\n"; 232 llvm::errs() << " " << NumEnteredSourceFiles << " source files entered.\n"; 233 llvm::errs() << " " << MaxIncludeStackDepth << " max include stack depth\n"; 234 llvm::errs() << " " << NumIf << " #if/#ifndef/#ifdef.\n"; 235 llvm::errs() << " " << NumElse << " #else/#elif.\n"; 236 llvm::errs() << " " << NumEndif << " #endif.\n"; 237 llvm::errs() << " " << NumPragma << " #pragma.\n"; 238 llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n"; 239 240 llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/" 241 << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, " 242 << NumFastMacroExpanded << " on the fast path.\n"; 243 llvm::errs() << (NumFastTokenPaste+NumTokenPaste) 244 << " token paste (##) operations performed, " 245 << NumFastTokenPaste << " on the fast path.\n"; 246 247 llvm::errs() << "\nPreprocessor Memory: " << getTotalMemory() << "B total"; 248 249 llvm::errs() << "\n BumpPtr: " << BP.getTotalMemory(); 250 llvm::errs() << "\n Macro Expanded Tokens: " 251 << llvm::capacity_in_bytes(MacroExpandedTokens); 252 llvm::errs() << "\n Predefines Buffer: " << Predefines.capacity(); 253 llvm::errs() << "\n Macros: " << llvm::capacity_in_bytes(Macros); 254 llvm::errs() << "\n #pragma push_macro Info: " 255 << llvm::capacity_in_bytes(PragmaPushMacroInfo); 256 llvm::errs() << "\n Poison Reasons: " 257 << llvm::capacity_in_bytes(PoisonReasons); 258 llvm::errs() << "\n Comment Handlers: " 259 << llvm::capacity_in_bytes(CommentHandlers) << "\n"; 260} 261 262Preprocessor::macro_iterator 263Preprocessor::macro_begin(bool IncludeExternalMacros) const { 264 if (IncludeExternalMacros && ExternalSource && 265 !ReadMacrosFromExternalSource) { 266 ReadMacrosFromExternalSource = true; 267 ExternalSource->ReadDefinedMacros(); 268 } 269 270 return Macros.begin(); 271} 272 273size_t Preprocessor::getTotalMemory() const { 274 return BP.getTotalMemory() 275 + llvm::capacity_in_bytes(MacroExpandedTokens) 276 + Predefines.capacity() /* Predefines buffer. */ 277 + llvm::capacity_in_bytes(Macros) 278 + llvm::capacity_in_bytes(PragmaPushMacroInfo) 279 + llvm::capacity_in_bytes(PoisonReasons) 280 + llvm::capacity_in_bytes(CommentHandlers); 281} 282 283Preprocessor::macro_iterator 284Preprocessor::macro_end(bool IncludeExternalMacros) const { 285 if (IncludeExternalMacros && ExternalSource && 286 !ReadMacrosFromExternalSource) { 287 ReadMacrosFromExternalSource = true; 288 ExternalSource->ReadDefinedMacros(); 289 } 290 291 return Macros.end(); 292} 293 294/// \brief Compares macro tokens with a specified token value sequence. 295static bool MacroDefinitionEquals(const MacroInfo *MI, 296 ArrayRef<TokenValue> Tokens) { 297 return Tokens.size() == MI->getNumTokens() && 298 std::equal(Tokens.begin(), Tokens.end(), MI->tokens_begin()); 299} 300 301StringRef Preprocessor::getLastMacroWithSpelling( 302 SourceLocation Loc, 303 ArrayRef<TokenValue> Tokens) const { 304 SourceLocation BestLocation; 305 StringRef BestSpelling; 306 for (Preprocessor::macro_iterator I = macro_begin(), E = macro_end(); 307 I != E; ++I) { 308 if (!I->second->isObjectLike()) 309 continue; 310 const MacroInfo *MI = I->second->findDefinitionAtLoc(Loc, SourceMgr); 311 if (!MI) 312 continue; 313 if (!MacroDefinitionEquals(MI, Tokens)) 314 continue; 315 SourceLocation Location = I->second->getDefinitionLoc(); 316 // Choose the macro defined latest. 317 if (BestLocation.isInvalid() || 318 (Location.isValid() && 319 SourceMgr.isBeforeInTranslationUnit(BestLocation, Location))) { 320 BestLocation = Location; 321 BestSpelling = I->first->getName(); 322 } 323 } 324 return BestSpelling; 325} 326 327void Preprocessor::recomputeCurLexerKind() { 328 if (CurLexer) 329 CurLexerKind = CLK_Lexer; 330 else if (CurPTHLexer) 331 CurLexerKind = CLK_PTHLexer; 332 else if (CurTokenLexer) 333 CurLexerKind = CLK_TokenLexer; 334 else 335 CurLexerKind = CLK_CachingLexer; 336} 337 338bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File, 339 unsigned CompleteLine, 340 unsigned CompleteColumn) { 341 assert(File); 342 assert(CompleteLine && CompleteColumn && "Starts from 1:1"); 343 assert(!CodeCompletionFile && "Already set"); 344 345 using llvm::MemoryBuffer; 346 347 // Load the actual file's contents. 348 bool Invalid = false; 349 const MemoryBuffer *Buffer = SourceMgr.getMemoryBufferForFile(File, &Invalid); 350 if (Invalid) 351 return true; 352 353 // Find the byte position of the truncation point. 354 const char *Position = Buffer->getBufferStart(); 355 for (unsigned Line = 1; Line < CompleteLine; ++Line) { 356 for (; *Position; ++Position) { 357 if (*Position != '\r' && *Position != '\n') 358 continue; 359 360 // Eat \r\n or \n\r as a single line. 361 if ((Position[1] == '\r' || Position[1] == '\n') && 362 Position[0] != Position[1]) 363 ++Position; 364 ++Position; 365 break; 366 } 367 } 368 369 Position += CompleteColumn - 1; 370 371 // Insert '\0' at the code-completion point. 372 if (Position < Buffer->getBufferEnd()) { 373 CodeCompletionFile = File; 374 CodeCompletionOffset = Position - Buffer->getBufferStart(); 375 376 MemoryBuffer *NewBuffer = 377 MemoryBuffer::getNewUninitMemBuffer(Buffer->getBufferSize() + 1, 378 Buffer->getBufferIdentifier()); 379 char *NewBuf = const_cast<char*>(NewBuffer->getBufferStart()); 380 char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf); 381 *NewPos = '\0'; 382 std::copy(Position, Buffer->getBufferEnd(), NewPos+1); 383 SourceMgr.overrideFileContents(File, NewBuffer); 384 } 385 386 return false; 387} 388 389void Preprocessor::CodeCompleteNaturalLanguage() { 390 if (CodeComplete) 391 CodeComplete->CodeCompleteNaturalLanguage(); 392 setCodeCompletionReached(); 393} 394 395/// getSpelling - This method is used to get the spelling of a token into a 396/// SmallVector. Note that the returned StringRef may not point to the 397/// supplied buffer if a copy can be avoided. 398StringRef Preprocessor::getSpelling(const Token &Tok, 399 SmallVectorImpl<char> &Buffer, 400 bool *Invalid) const { 401 // NOTE: this has to be checked *before* testing for an IdentifierInfo. 402 if (Tok.isNot(tok::raw_identifier)) { 403 // Try the fast path. 404 if (const IdentifierInfo *II = Tok.getIdentifierInfo()) 405 return II->getName(); 406 } 407 408 // Resize the buffer if we need to copy into it. 409 if (Tok.needsCleaning()) 410 Buffer.resize(Tok.getLength()); 411 412 const char *Ptr = Buffer.data(); 413 unsigned Len = getSpelling(Tok, Ptr, Invalid); 414 return StringRef(Ptr, Len); 415} 416 417/// CreateString - Plop the specified string into a scratch buffer and return a 418/// location for it. If specified, the source location provides a source 419/// location for the token. 420void Preprocessor::CreateString(StringRef Str, Token &Tok, 421 SourceLocation ExpansionLocStart, 422 SourceLocation ExpansionLocEnd) { 423 Tok.setLength(Str.size()); 424 425 const char *DestPtr; 426 SourceLocation Loc = ScratchBuf->getToken(Str.data(), Str.size(), DestPtr); 427 428 if (ExpansionLocStart.isValid()) 429 Loc = SourceMgr.createExpansionLoc(Loc, ExpansionLocStart, 430 ExpansionLocEnd, Str.size()); 431 Tok.setLocation(Loc); 432 433 // If this is a raw identifier or a literal token, set the pointer data. 434 if (Tok.is(tok::raw_identifier)) 435 Tok.setRawIdentifierData(DestPtr); 436 else if (Tok.isLiteral()) 437 Tok.setLiteralData(DestPtr); 438} 439 440Module *Preprocessor::getCurrentModule() { 441 if (getLangOpts().CurrentModule.empty()) 442 return 0; 443 444 return getHeaderSearchInfo().lookupModule(getLangOpts().CurrentModule); 445} 446 447//===----------------------------------------------------------------------===// 448// Preprocessor Initialization Methods 449//===----------------------------------------------------------------------===// 450 451 452/// EnterMainSourceFile - Enter the specified FileID as the main source file, 453/// which implicitly adds the builtin defines etc. 454void Preprocessor::EnterMainSourceFile() { 455 // We do not allow the preprocessor to reenter the main file. Doing so will 456 // cause FileID's to accumulate information from both runs (e.g. #line 457 // information) and predefined macros aren't guaranteed to be set properly. 458 assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!"); 459 FileID MainFileID = SourceMgr.getMainFileID(); 460 461 // If MainFileID is loaded it means we loaded an AST file, no need to enter 462 // a main file. 463 if (!SourceMgr.isLoadedFileID(MainFileID)) { 464 // Enter the main file source buffer. 465 EnterSourceFile(MainFileID, 0, SourceLocation()); 466 467 // If we've been asked to skip bytes in the main file (e.g., as part of a 468 // precompiled preamble), do so now. 469 if (SkipMainFilePreamble.first > 0) 470 CurLexer->SkipBytes(SkipMainFilePreamble.first, 471 SkipMainFilePreamble.second); 472 473 // Tell the header info that the main file was entered. If the file is later 474 // #imported, it won't be re-entered. 475 if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID)) 476 HeaderInfo.IncrementIncludeCount(FE); 477 } 478 479 // Preprocess Predefines to populate the initial preprocessor state. 480 llvm::MemoryBuffer *SB = 481 llvm::MemoryBuffer::getMemBufferCopy(Predefines, "<built-in>"); 482 assert(SB && "Cannot create predefined source buffer"); 483 FileID FID = SourceMgr.createFileIDForMemBuffer(SB); 484 assert(!FID.isInvalid() && "Could not create FileID for predefines?"); 485 486 // Start parsing the predefines. 487 EnterSourceFile(FID, 0, SourceLocation()); 488} 489 490void Preprocessor::EndSourceFile() { 491 // Notify the client that we reached the end of the source file. 492 if (Callbacks) 493 Callbacks->EndOfMainFile(); 494} 495 496//===----------------------------------------------------------------------===// 497// Lexer Event Handling. 498//===----------------------------------------------------------------------===// 499 500/// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the 501/// identifier information for the token and install it into the token, 502/// updating the token kind accordingly. 503IdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier) const { 504 assert(Identifier.getRawIdentifierData() != 0 && "No raw identifier data!"); 505 506 // Look up this token, see if it is a macro, or if it is a language keyword. 507 IdentifierInfo *II; 508 if (!Identifier.needsCleaning()) { 509 // No cleaning needed, just use the characters from the lexed buffer. 510 II = getIdentifierInfo(StringRef(Identifier.getRawIdentifierData(), 511 Identifier.getLength())); 512 } else { 513 // Cleaning needed, alloca a buffer, clean into it, then use the buffer. 514 SmallString<64> IdentifierBuffer; 515 StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer); 516 II = getIdentifierInfo(CleanedStr); 517 } 518 519 // Update the token info (identifier info and appropriate token kind). 520 Identifier.setIdentifierInfo(II); 521 Identifier.setKind(II->getTokenID()); 522 523 return II; 524} 525 526void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) { 527 PoisonReasons[II] = DiagID; 528} 529 530void Preprocessor::PoisonSEHIdentifiers(bool Poison) { 531 assert(Ident__exception_code && Ident__exception_info); 532 assert(Ident___exception_code && Ident___exception_info); 533 Ident__exception_code->setIsPoisoned(Poison); 534 Ident___exception_code->setIsPoisoned(Poison); 535 Ident_GetExceptionCode->setIsPoisoned(Poison); 536 Ident__exception_info->setIsPoisoned(Poison); 537 Ident___exception_info->setIsPoisoned(Poison); 538 Ident_GetExceptionInfo->setIsPoisoned(Poison); 539 Ident__abnormal_termination->setIsPoisoned(Poison); 540 Ident___abnormal_termination->setIsPoisoned(Poison); 541 Ident_AbnormalTermination->setIsPoisoned(Poison); 542} 543 544void Preprocessor::HandlePoisonedIdentifier(Token & Identifier) { 545 assert(Identifier.getIdentifierInfo() && 546 "Can't handle identifiers without identifier info!"); 547 llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it = 548 PoisonReasons.find(Identifier.getIdentifierInfo()); 549 if(it == PoisonReasons.end()) 550 Diag(Identifier, diag::err_pp_used_poisoned_id); 551 else 552 Diag(Identifier,it->second) << Identifier.getIdentifierInfo(); 553} 554 555/// HandleIdentifier - This callback is invoked when the lexer reads an 556/// identifier. This callback looks up the identifier in the map and/or 557/// potentially macro expands it or turns it into a named token (like 'for'). 558/// 559/// Note that callers of this method are guarded by checking the 560/// IdentifierInfo's 'isHandleIdentifierCase' bit. If this method changes, the 561/// IdentifierInfo methods that compute these properties will need to change to 562/// match. 563void Preprocessor::HandleIdentifier(Token &Identifier) { 564 assert(Identifier.getIdentifierInfo() && 565 "Can't handle identifiers without identifier info!"); 566 567 IdentifierInfo &II = *Identifier.getIdentifierInfo(); 568 569 // If the information about this identifier is out of date, update it from 570 // the external source. 571 // We have to treat __VA_ARGS__ in a special way, since it gets 572 // serialized with isPoisoned = true, but our preprocessor may have 573 // unpoisoned it if we're defining a C99 macro. 574 if (II.isOutOfDate()) { 575 bool CurrentIsPoisoned = false; 576 if (&II == Ident__VA_ARGS__) 577 CurrentIsPoisoned = Ident__VA_ARGS__->isPoisoned(); 578 579 ExternalSource->updateOutOfDateIdentifier(II); 580 Identifier.setKind(II.getTokenID()); 581 582 if (&II == Ident__VA_ARGS__) 583 II.setIsPoisoned(CurrentIsPoisoned); 584 } 585 586 // If this identifier was poisoned, and if it was not produced from a macro 587 // expansion, emit an error. 588 if (II.isPoisoned() && CurPPLexer) { 589 HandlePoisonedIdentifier(Identifier); 590 } 591 592 // If this is a macro to be expanded, do it. 593 if (MacroInfo *MI = getMacroInfo(&II)) { 594 if (!DisableMacroExpansion) { 595 if (!Identifier.isExpandDisabled() && MI->isEnabled()) { 596 if (!HandleMacroExpandedIdentifier(Identifier, MI)) 597 return; 598 } else { 599 // C99 6.10.3.4p2 says that a disabled macro may never again be 600 // expanded, even if it's in a context where it could be expanded in the 601 // future. 602 Identifier.setFlag(Token::DisableExpand); 603 if (MI->isObjectLike() || isNextPPTokenLParen()) 604 Diag(Identifier, diag::pp_disabled_macro_expansion); 605 } 606 } 607 } 608 609 // If this identifier is a keyword in C++11, produce a warning. Don't warn if 610 // we're not considering macro expansion, since this identifier might be the 611 // name of a macro. 612 // FIXME: This warning is disabled in cases where it shouldn't be, like 613 // "#define constexpr constexpr", "int constexpr;" 614 if (II.isCXX11CompatKeyword() & !DisableMacroExpansion) { 615 Diag(Identifier, diag::warn_cxx11_keyword) << II.getName(); 616 // Don't diagnose this keyword again in this translation unit. 617 II.setIsCXX11CompatKeyword(false); 618 } 619 620 // C++ 2.11p2: If this is an alternative representation of a C++ operator, 621 // then we act as if it is the actual operator and not the textual 622 // representation of it. 623 if (II.isCPlusPlusOperatorKeyword()) 624 Identifier.setIdentifierInfo(0); 625 626 // If this is an extension token, diagnose its use. 627 // We avoid diagnosing tokens that originate from macro definitions. 628 // FIXME: This warning is disabled in cases where it shouldn't be, 629 // like "#define TY typeof", "TY(1) x". 630 if (II.isExtensionToken() && !DisableMacroExpansion) 631 Diag(Identifier, diag::ext_token_used); 632 633 // If this is the 'import' contextual keyword, note 634 // that the next token indicates a module name. 635 // 636 // Note that we do not treat 'import' as a contextual 637 // keyword when we're in a caching lexer, because caching lexers only get 638 // used in contexts where import declarations are disallowed. 639 if (II.isModulesImport() && !InMacroArgs && !DisableMacroExpansion && 640 getLangOpts().Modules && CurLexerKind != CLK_CachingLexer) { 641 ModuleImportLoc = Identifier.getLocation(); 642 ModuleImportPath.clear(); 643 ModuleImportExpectsIdentifier = true; 644 CurLexerKind = CLK_LexAfterModuleImport; 645 } 646} 647 648/// \brief Lex a token following the 'import' contextual keyword. 649/// 650void Preprocessor::LexAfterModuleImport(Token &Result) { 651 // Figure out what kind of lexer we actually have. 652 recomputeCurLexerKind(); 653 654 // Lex the next token. 655 Lex(Result); 656 657 // The token sequence 658 // 659 // import identifier (. identifier)* 660 // 661 // indicates a module import directive. We already saw the 'import' 662 // contextual keyword, so now we're looking for the identifiers. 663 if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) { 664 // We expected to see an identifier here, and we did; continue handling 665 // identifiers. 666 ModuleImportPath.push_back(std::make_pair(Result.getIdentifierInfo(), 667 Result.getLocation())); 668 ModuleImportExpectsIdentifier = false; 669 CurLexerKind = CLK_LexAfterModuleImport; 670 return; 671 } 672 673 // If we're expecting a '.' or a ';', and we got a '.', then wait until we 674 // see the next identifier. 675 if (!ModuleImportExpectsIdentifier && Result.getKind() == tok::period) { 676 ModuleImportExpectsIdentifier = true; 677 CurLexerKind = CLK_LexAfterModuleImport; 678 return; 679 } 680 681 // If we have a non-empty module path, load the named module. 682 if (!ModuleImportPath.empty()) { 683 Module *Imported = TheModuleLoader.loadModule(ModuleImportLoc, 684 ModuleImportPath, 685 Module::MacrosVisible, 686 /*IsIncludeDirective=*/false); 687 if (Callbacks) 688 Callbacks->moduleImport(ModuleImportLoc, ModuleImportPath, Imported); 689 } 690} 691 692bool Preprocessor::FinishLexStringLiteral(Token &Result, std::string &String, 693 const char *DiagnosticTag, 694 bool AllowMacroExpansion) { 695 // We need at least one string literal. 696 if (Result.isNot(tok::string_literal)) { 697 Diag(Result, diag::err_expected_string_literal) 698 << /*Source='in...'*/0 << DiagnosticTag; 699 return false; 700 } 701 702 // Lex string literal tokens, optionally with macro expansion. 703 SmallVector<Token, 4> StrToks; 704 do { 705 StrToks.push_back(Result); 706 707 if (Result.hasUDSuffix()) 708 Diag(Result, diag::err_invalid_string_udl); 709 710 if (AllowMacroExpansion) 711 Lex(Result); 712 else 713 LexUnexpandedToken(Result); 714 } while (Result.is(tok::string_literal)); 715 716 // Concatenate and parse the strings. 717 StringLiteralParser Literal(&StrToks[0], StrToks.size(), *this); 718 assert(Literal.isAscii() && "Didn't allow wide strings in"); 719 720 if (Literal.hadError) 721 return false; 722 723 if (Literal.Pascal) { 724 Diag(StrToks[0].getLocation(), diag::err_expected_string_literal) 725 << /*Source='in...'*/0 << DiagnosticTag; 726 return false; 727 } 728 729 String = Literal.GetString(); 730 return true; 731} 732 733void Preprocessor::addCommentHandler(CommentHandler *Handler) { 734 assert(Handler && "NULL comment handler"); 735 assert(std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler) == 736 CommentHandlers.end() && "Comment handler already registered"); 737 CommentHandlers.push_back(Handler); 738} 739 740void Preprocessor::removeCommentHandler(CommentHandler *Handler) { 741 std::vector<CommentHandler *>::iterator Pos 742 = std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler); 743 assert(Pos != CommentHandlers.end() && "Comment handler not registered"); 744 CommentHandlers.erase(Pos); 745} 746 747bool Preprocessor::HandleComment(Token &result, SourceRange Comment) { 748 bool AnyPendingTokens = false; 749 for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(), 750 HEnd = CommentHandlers.end(); 751 H != HEnd; ++H) { 752 if ((*H)->HandleComment(*this, Comment)) 753 AnyPendingTokens = true; 754 } 755 if (!AnyPendingTokens || getCommentRetentionState()) 756 return false; 757 Lex(result); 758 return true; 759} 760 761ModuleLoader::~ModuleLoader() { } 762 763CommentHandler::~CommentHandler() { } 764 765CodeCompletionHandler::~CodeCompletionHandler() { } 766 767void Preprocessor::createPreprocessingRecord() { 768 if (Record) 769 return; 770 771 Record = new PreprocessingRecord(getSourceManager()); 772 addPPCallbacks(Record); 773} 774