Preprocessor.cpp revision 1952354bd376062c3ab3d328c0fc6c36530c9309
1//===--- Preprocess.cpp - C Language Family Preprocessor Implementation ---===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the Preprocessor interface. 11// 12//===----------------------------------------------------------------------===// 13// 14// Options to support: 15// -H - Print the name of each header file used. 16// -d[DNI] - Dump various things. 17// -fworking-directory - #line's with preprocessor's working dir. 18// -fpreprocessed 19// -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD 20// -W* 21// -w 22// 23// Messages to emit: 24// "Multiple include guards may be useful for:\n" 25// 26//===----------------------------------------------------------------------===// 27 28#include "clang/Lex/Preprocessor.h" 29#include "MacroArgs.h" 30#include "clang/Lex/ExternalPreprocessorSource.h" 31#include "clang/Lex/HeaderSearch.h" 32#include "clang/Lex/MacroInfo.h" 33#include "clang/Lex/Pragma.h" 34#include "clang/Lex/PreprocessingRecord.h" 35#include "clang/Lex/ScratchBuffer.h" 36#include "clang/Lex/LexDiagnostic.h" 37#include "clang/Lex/CodeCompletionHandler.h" 38#include "clang/Lex/ModuleLoader.h" 39#include "clang/Basic/SourceManager.h" 40#include "clang/Basic/FileManager.h" 41#include "clang/Basic/TargetInfo.h" 42#include "llvm/ADT/APFloat.h" 43#include "llvm/ADT/SmallString.h" 44#include "llvm/Support/MemoryBuffer.h" 45#include "llvm/Support/raw_ostream.h" 46#include "llvm/Support/Capacity.h" 47using namespace clang; 48 49//===----------------------------------------------------------------------===// 50ExternalPreprocessorSource::~ExternalPreprocessorSource() { } 51 52Preprocessor::Preprocessor(DiagnosticsEngine &diags, LangOptions &opts, 53 const TargetInfo *target, SourceManager &SM, 54 HeaderSearch &Headers, ModuleLoader &TheModuleLoader, 55 IdentifierInfoLookup* IILookup, 56 bool OwnsHeaders, 57 bool DelayInitialization, 58 bool IncrProcessing) 59 : Diags(&diags), LangOpts(opts), Target(target),FileMgr(Headers.getFileMgr()), 60 SourceMgr(SM), HeaderInfo(Headers), TheModuleLoader(TheModuleLoader), 61 ExternalSource(0), Identifiers(opts, IILookup), 62 IncrementalProcessing(IncrProcessing), CodeComplete(0), 63 CodeCompletionFile(0), CodeCompletionOffset(0), CodeCompletionReached(0), 64 SkipMainFilePreamble(0, true), CurPPLexer(0), 65 CurDirLookup(0), CurLexerKind(CLK_Lexer), Callbacks(0), MacroArgCache(0), 66 Record(0), MIChainHead(0), MICache(0) 67{ 68 OwnsHeaderSearch = OwnsHeaders; 69 70 ScratchBuf = new ScratchBuffer(SourceMgr); 71 CounterValue = 0; // __COUNTER__ starts at 0. 72 73 // Clear stats. 74 NumDirectives = NumDefined = NumUndefined = NumPragma = 0; 75 NumIf = NumElse = NumEndif = 0; 76 NumEnteredSourceFiles = 0; 77 NumMacroExpanded = NumFnMacroExpanded = NumBuiltinMacroExpanded = 0; 78 NumFastMacroExpanded = NumTokenPaste = NumFastTokenPaste = 0; 79 MaxIncludeStackDepth = 0; 80 NumSkipped = 0; 81 82 // Default to discarding comments. 83 KeepComments = false; 84 KeepMacroComments = false; 85 SuppressIncludeNotFoundError = false; 86 87 // Macro expansion is enabled. 88 DisableMacroExpansion = false; 89 MacroExpansionInDirectivesOverride = false; 90 InMacroArgs = false; 91 InMacroArgPreExpansion = false; 92 NumCachedTokenLexers = 0; 93 PragmasEnabled = true; 94 95 CachedLexPos = 0; 96 97 // We haven't read anything from the external source. 98 ReadMacrosFromExternalSource = false; 99 100 // "Poison" __VA_ARGS__, which can only appear in the expansion of a macro. 101 // This gets unpoisoned where it is allowed. 102 (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned(); 103 SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use); 104 105 // Initialize the pragma handlers. 106 PragmaHandlers = new PragmaNamespace(StringRef()); 107 RegisterBuiltinPragmas(); 108 109 // Initialize builtin macros like __LINE__ and friends. 110 RegisterBuiltinMacros(); 111 112 if(LangOpts.Borland) { 113 Ident__exception_info = getIdentifierInfo("_exception_info"); 114 Ident___exception_info = getIdentifierInfo("__exception_info"); 115 Ident_GetExceptionInfo = getIdentifierInfo("GetExceptionInformation"); 116 Ident__exception_code = getIdentifierInfo("_exception_code"); 117 Ident___exception_code = getIdentifierInfo("__exception_code"); 118 Ident_GetExceptionCode = getIdentifierInfo("GetExceptionCode"); 119 Ident__abnormal_termination = getIdentifierInfo("_abnormal_termination"); 120 Ident___abnormal_termination = getIdentifierInfo("__abnormal_termination"); 121 Ident_AbnormalTermination = getIdentifierInfo("AbnormalTermination"); 122 } else { 123 Ident__exception_info = Ident__exception_code = Ident__abnormal_termination = 0; 124 Ident___exception_info = Ident___exception_code = Ident___abnormal_termination = 0; 125 Ident_GetExceptionInfo = Ident_GetExceptionCode = Ident_AbnormalTermination = 0; 126 } 127 128 if (!DelayInitialization) { 129 assert(Target && "Must provide target information for PP initialization"); 130 Initialize(*Target); 131 } 132} 133 134Preprocessor::~Preprocessor() { 135 assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!"); 136 137 while (!IncludeMacroStack.empty()) { 138 delete IncludeMacroStack.back().TheLexer; 139 delete IncludeMacroStack.back().TheTokenLexer; 140 IncludeMacroStack.pop_back(); 141 } 142 143 // Free any macro definitions. 144 for (MacroInfoChain *I = MIChainHead ; I ; I = I->Next) 145 I->MI.Destroy(); 146 147 // Free any cached macro expanders. 148 for (unsigned i = 0, e = NumCachedTokenLexers; i != e; ++i) 149 delete TokenLexerCache[i]; 150 151 // Free any cached MacroArgs. 152 for (MacroArgs *ArgList = MacroArgCache; ArgList; ) 153 ArgList = ArgList->deallocate(); 154 155 // Release pragma information. 156 delete PragmaHandlers; 157 158 // Delete the scratch buffer info. 159 delete ScratchBuf; 160 161 // Delete the header search info, if we own it. 162 if (OwnsHeaderSearch) 163 delete &HeaderInfo; 164 165 delete Callbacks; 166} 167 168void Preprocessor::Initialize(const TargetInfo &Target) { 169 assert((!this->Target || this->Target == &Target) && 170 "Invalid override of target information"); 171 this->Target = &Target; 172 173 // Initialize information about built-ins. 174 BuiltinInfo.InitializeTarget(Target); 175 HeaderInfo.setTarget(Target); 176} 177 178void Preprocessor::setPTHManager(PTHManager* pm) { 179 PTH.reset(pm); 180 FileMgr.addStatCache(PTH->createStatCache()); 181} 182 183void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const { 184 llvm::errs() << tok::getTokenName(Tok.getKind()) << " '" 185 << getSpelling(Tok) << "'"; 186 187 if (!DumpFlags) return; 188 189 llvm::errs() << "\t"; 190 if (Tok.isAtStartOfLine()) 191 llvm::errs() << " [StartOfLine]"; 192 if (Tok.hasLeadingSpace()) 193 llvm::errs() << " [LeadingSpace]"; 194 if (Tok.isExpandDisabled()) 195 llvm::errs() << " [ExpandDisabled]"; 196 if (Tok.needsCleaning()) { 197 const char *Start = SourceMgr.getCharacterData(Tok.getLocation()); 198 llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength()) 199 << "']"; 200 } 201 202 llvm::errs() << "\tLoc=<"; 203 DumpLocation(Tok.getLocation()); 204 llvm::errs() << ">"; 205} 206 207void Preprocessor::DumpLocation(SourceLocation Loc) const { 208 Loc.dump(SourceMgr); 209} 210 211void Preprocessor::DumpMacro(const MacroInfo &MI) const { 212 llvm::errs() << "MACRO: "; 213 for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) { 214 DumpToken(MI.getReplacementToken(i)); 215 llvm::errs() << " "; 216 } 217 llvm::errs() << "\n"; 218} 219 220void Preprocessor::PrintStats() { 221 llvm::errs() << "\n*** Preprocessor Stats:\n"; 222 llvm::errs() << NumDirectives << " directives found:\n"; 223 llvm::errs() << " " << NumDefined << " #define.\n"; 224 llvm::errs() << " " << NumUndefined << " #undef.\n"; 225 llvm::errs() << " #include/#include_next/#import:\n"; 226 llvm::errs() << " " << NumEnteredSourceFiles << " source files entered.\n"; 227 llvm::errs() << " " << MaxIncludeStackDepth << " max include stack depth\n"; 228 llvm::errs() << " " << NumIf << " #if/#ifndef/#ifdef.\n"; 229 llvm::errs() << " " << NumElse << " #else/#elif.\n"; 230 llvm::errs() << " " << NumEndif << " #endif.\n"; 231 llvm::errs() << " " << NumPragma << " #pragma.\n"; 232 llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n"; 233 234 llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/" 235 << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, " 236 << NumFastMacroExpanded << " on the fast path.\n"; 237 llvm::errs() << (NumFastTokenPaste+NumTokenPaste) 238 << " token paste (##) operations performed, " 239 << NumFastTokenPaste << " on the fast path.\n"; 240 241 llvm::errs() << "\nPreprocessor Memory: " << getTotalMemory() << "B total"; 242 243 llvm::errs() << "\n BumpPtr: " << BP.getTotalMemory(); 244 llvm::errs() << "\n Macro Expanded Tokens: " 245 << llvm::capacity_in_bytes(MacroExpandedTokens); 246 llvm::errs() << "\n Predefines Buffer: " << Predefines.capacity(); 247 llvm::errs() << "\n Macros: " << llvm::capacity_in_bytes(Macros); 248 llvm::errs() << "\n #pragma push_macro Info: " 249 << llvm::capacity_in_bytes(PragmaPushMacroInfo); 250 llvm::errs() << "\n Poison Reasons: " 251 << llvm::capacity_in_bytes(PoisonReasons); 252 llvm::errs() << "\n Comment Handlers: " 253 << llvm::capacity_in_bytes(CommentHandlers) << "\n"; 254} 255 256Preprocessor::macro_iterator 257Preprocessor::macro_begin(bool IncludeExternalMacros) const { 258 if (IncludeExternalMacros && ExternalSource && 259 !ReadMacrosFromExternalSource) { 260 ReadMacrosFromExternalSource = true; 261 ExternalSource->ReadDefinedMacros(); 262 } 263 264 return Macros.begin(); 265} 266 267size_t Preprocessor::getTotalMemory() const { 268 return BP.getTotalMemory() 269 + llvm::capacity_in_bytes(MacroExpandedTokens) 270 + Predefines.capacity() /* Predefines buffer. */ 271 + llvm::capacity_in_bytes(Macros) 272 + llvm::capacity_in_bytes(PragmaPushMacroInfo) 273 + llvm::capacity_in_bytes(PoisonReasons) 274 + llvm::capacity_in_bytes(CommentHandlers); 275} 276 277Preprocessor::macro_iterator 278Preprocessor::macro_end(bool IncludeExternalMacros) const { 279 if (IncludeExternalMacros && ExternalSource && 280 !ReadMacrosFromExternalSource) { 281 ReadMacrosFromExternalSource = true; 282 ExternalSource->ReadDefinedMacros(); 283 } 284 285 return Macros.end(); 286} 287 288/// \brief Compares macro tokens with a specified token value sequence. 289static bool MacroDefinitionEquals(const MacroInfo *MI, 290 llvm::ArrayRef<TokenValue> Tokens) { 291 return Tokens.size() == MI->getNumTokens() && 292 std::equal(Tokens.begin(), Tokens.end(), MI->tokens_begin()); 293} 294 295StringRef Preprocessor::getLastMacroWithSpelling( 296 SourceLocation Loc, 297 ArrayRef<TokenValue> Tokens) const { 298 SourceLocation BestLocation; 299 StringRef BestSpelling; 300 for (Preprocessor::macro_iterator I = macro_begin(), E = macro_end(); 301 I != E; ++I) { 302 if (!I->second->isObjectLike()) 303 continue; 304 const MacroInfo *MI = I->second->findDefinitionAtLoc(Loc, SourceMgr); 305 if (!MI) 306 continue; 307 if (!MacroDefinitionEquals(MI, Tokens)) 308 continue; 309 SourceLocation Location = I->second->getDefinitionLoc(); 310 // Choose the macro defined latest. 311 if (BestLocation.isInvalid() || 312 (Location.isValid() && 313 SourceMgr.isBeforeInTranslationUnit(BestLocation, Location))) { 314 BestLocation = Location; 315 BestSpelling = I->first->getName(); 316 } 317 } 318 return BestSpelling; 319} 320 321void Preprocessor::recomputeCurLexerKind() { 322 if (CurLexer) 323 CurLexerKind = CLK_Lexer; 324 else if (CurPTHLexer) 325 CurLexerKind = CLK_PTHLexer; 326 else if (CurTokenLexer) 327 CurLexerKind = CLK_TokenLexer; 328 else 329 CurLexerKind = CLK_CachingLexer; 330} 331 332bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File, 333 unsigned CompleteLine, 334 unsigned CompleteColumn) { 335 assert(File); 336 assert(CompleteLine && CompleteColumn && "Starts from 1:1"); 337 assert(!CodeCompletionFile && "Already set"); 338 339 using llvm::MemoryBuffer; 340 341 // Load the actual file's contents. 342 bool Invalid = false; 343 const MemoryBuffer *Buffer = SourceMgr.getMemoryBufferForFile(File, &Invalid); 344 if (Invalid) 345 return true; 346 347 // Find the byte position of the truncation point. 348 const char *Position = Buffer->getBufferStart(); 349 for (unsigned Line = 1; Line < CompleteLine; ++Line) { 350 for (; *Position; ++Position) { 351 if (*Position != '\r' && *Position != '\n') 352 continue; 353 354 // Eat \r\n or \n\r as a single line. 355 if ((Position[1] == '\r' || Position[1] == '\n') && 356 Position[0] != Position[1]) 357 ++Position; 358 ++Position; 359 break; 360 } 361 } 362 363 Position += CompleteColumn - 1; 364 365 // Insert '\0' at the code-completion point. 366 if (Position < Buffer->getBufferEnd()) { 367 CodeCompletionFile = File; 368 CodeCompletionOffset = Position - Buffer->getBufferStart(); 369 370 MemoryBuffer *NewBuffer = 371 MemoryBuffer::getNewUninitMemBuffer(Buffer->getBufferSize() + 1, 372 Buffer->getBufferIdentifier()); 373 char *NewBuf = const_cast<char*>(NewBuffer->getBufferStart()); 374 char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf); 375 *NewPos = '\0'; 376 std::copy(Position, Buffer->getBufferEnd(), NewPos+1); 377 SourceMgr.overrideFileContents(File, NewBuffer); 378 } 379 380 return false; 381} 382 383void Preprocessor::CodeCompleteNaturalLanguage() { 384 if (CodeComplete) 385 CodeComplete->CodeCompleteNaturalLanguage(); 386 setCodeCompletionReached(); 387} 388 389/// getSpelling - This method is used to get the spelling of a token into a 390/// SmallVector. Note that the returned StringRef may not point to the 391/// supplied buffer if a copy can be avoided. 392StringRef Preprocessor::getSpelling(const Token &Tok, 393 SmallVectorImpl<char> &Buffer, 394 bool *Invalid) const { 395 // NOTE: this has to be checked *before* testing for an IdentifierInfo. 396 if (Tok.isNot(tok::raw_identifier)) { 397 // Try the fast path. 398 if (const IdentifierInfo *II = Tok.getIdentifierInfo()) 399 return II->getName(); 400 } 401 402 // Resize the buffer if we need to copy into it. 403 if (Tok.needsCleaning()) 404 Buffer.resize(Tok.getLength()); 405 406 const char *Ptr = Buffer.data(); 407 unsigned Len = getSpelling(Tok, Ptr, Invalid); 408 return StringRef(Ptr, Len); 409} 410 411/// CreateString - Plop the specified string into a scratch buffer and return a 412/// location for it. If specified, the source location provides a source 413/// location for the token. 414void Preprocessor::CreateString(StringRef Str, Token &Tok, 415 SourceLocation ExpansionLocStart, 416 SourceLocation ExpansionLocEnd) { 417 Tok.setLength(Str.size()); 418 419 const char *DestPtr; 420 SourceLocation Loc = ScratchBuf->getToken(Str.data(), Str.size(), DestPtr); 421 422 if (ExpansionLocStart.isValid()) 423 Loc = SourceMgr.createExpansionLoc(Loc, ExpansionLocStart, 424 ExpansionLocEnd, Str.size()); 425 Tok.setLocation(Loc); 426 427 // If this is a raw identifier or a literal token, set the pointer data. 428 if (Tok.is(tok::raw_identifier)) 429 Tok.setRawIdentifierData(DestPtr); 430 else if (Tok.isLiteral()) 431 Tok.setLiteralData(DestPtr); 432} 433 434Module *Preprocessor::getCurrentModule() { 435 if (getLangOpts().CurrentModule.empty()) 436 return 0; 437 438 return getHeaderSearchInfo().lookupModule(getLangOpts().CurrentModule); 439} 440 441//===----------------------------------------------------------------------===// 442// Preprocessor Initialization Methods 443//===----------------------------------------------------------------------===// 444 445 446/// EnterMainSourceFile - Enter the specified FileID as the main source file, 447/// which implicitly adds the builtin defines etc. 448void Preprocessor::EnterMainSourceFile() { 449 // We do not allow the preprocessor to reenter the main file. Doing so will 450 // cause FileID's to accumulate information from both runs (e.g. #line 451 // information) and predefined macros aren't guaranteed to be set properly. 452 assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!"); 453 FileID MainFileID = SourceMgr.getMainFileID(); 454 455 // If MainFileID is loaded it means we loaded an AST file, no need to enter 456 // a main file. 457 if (!SourceMgr.isLoadedFileID(MainFileID)) { 458 // Enter the main file source buffer. 459 EnterSourceFile(MainFileID, 0, SourceLocation()); 460 461 // If we've been asked to skip bytes in the main file (e.g., as part of a 462 // precompiled preamble), do so now. 463 if (SkipMainFilePreamble.first > 0) 464 CurLexer->SkipBytes(SkipMainFilePreamble.first, 465 SkipMainFilePreamble.second); 466 467 // Tell the header info that the main file was entered. If the file is later 468 // #imported, it won't be re-entered. 469 if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID)) 470 HeaderInfo.IncrementIncludeCount(FE); 471 } 472 473 // Preprocess Predefines to populate the initial preprocessor state. 474 llvm::MemoryBuffer *SB = 475 llvm::MemoryBuffer::getMemBufferCopy(Predefines, "<built-in>"); 476 assert(SB && "Cannot create predefined source buffer"); 477 FileID FID = SourceMgr.createFileIDForMemBuffer(SB); 478 assert(!FID.isInvalid() && "Could not create FileID for predefines?"); 479 480 // Start parsing the predefines. 481 EnterSourceFile(FID, 0, SourceLocation()); 482} 483 484void Preprocessor::EndSourceFile() { 485 // Notify the client that we reached the end of the source file. 486 if (Callbacks) 487 Callbacks->EndOfMainFile(); 488} 489 490//===----------------------------------------------------------------------===// 491// Lexer Event Handling. 492//===----------------------------------------------------------------------===// 493 494/// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the 495/// identifier information for the token and install it into the token, 496/// updating the token kind accordingly. 497IdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier) const { 498 assert(Identifier.getRawIdentifierData() != 0 && "No raw identifier data!"); 499 500 // Look up this token, see if it is a macro, or if it is a language keyword. 501 IdentifierInfo *II; 502 if (!Identifier.needsCleaning()) { 503 // No cleaning needed, just use the characters from the lexed buffer. 504 II = getIdentifierInfo(StringRef(Identifier.getRawIdentifierData(), 505 Identifier.getLength())); 506 } else { 507 // Cleaning needed, alloca a buffer, clean into it, then use the buffer. 508 SmallString<64> IdentifierBuffer; 509 StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer); 510 II = getIdentifierInfo(CleanedStr); 511 } 512 513 // Update the token info (identifier info and appropriate token kind). 514 Identifier.setIdentifierInfo(II); 515 Identifier.setKind(II->getTokenID()); 516 517 return II; 518} 519 520void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) { 521 PoisonReasons[II] = DiagID; 522} 523 524void Preprocessor::PoisonSEHIdentifiers(bool Poison) { 525 assert(Ident__exception_code && Ident__exception_info); 526 assert(Ident___exception_code && Ident___exception_info); 527 Ident__exception_code->setIsPoisoned(Poison); 528 Ident___exception_code->setIsPoisoned(Poison); 529 Ident_GetExceptionCode->setIsPoisoned(Poison); 530 Ident__exception_info->setIsPoisoned(Poison); 531 Ident___exception_info->setIsPoisoned(Poison); 532 Ident_GetExceptionInfo->setIsPoisoned(Poison); 533 Ident__abnormal_termination->setIsPoisoned(Poison); 534 Ident___abnormal_termination->setIsPoisoned(Poison); 535 Ident_AbnormalTermination->setIsPoisoned(Poison); 536} 537 538void Preprocessor::HandlePoisonedIdentifier(Token & Identifier) { 539 assert(Identifier.getIdentifierInfo() && 540 "Can't handle identifiers without identifier info!"); 541 llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it = 542 PoisonReasons.find(Identifier.getIdentifierInfo()); 543 if(it == PoisonReasons.end()) 544 Diag(Identifier, diag::err_pp_used_poisoned_id); 545 else 546 Diag(Identifier,it->second) << Identifier.getIdentifierInfo(); 547} 548 549/// HandleIdentifier - This callback is invoked when the lexer reads an 550/// identifier. This callback looks up the identifier in the map and/or 551/// potentially macro expands it or turns it into a named token (like 'for'). 552/// 553/// Note that callers of this method are guarded by checking the 554/// IdentifierInfo's 'isHandleIdentifierCase' bit. If this method changes, the 555/// IdentifierInfo methods that compute these properties will need to change to 556/// match. 557void Preprocessor::HandleIdentifier(Token &Identifier) { 558 assert(Identifier.getIdentifierInfo() && 559 "Can't handle identifiers without identifier info!"); 560 561 IdentifierInfo &II = *Identifier.getIdentifierInfo(); 562 563 // If the information about this identifier is out of date, update it from 564 // the external source. 565 // We have to treat __VA_ARGS__ in a special way, since it gets 566 // serialized with isPoisoned = true, but our preprocessor may have 567 // unpoisoned it if we're defining a C99 macro. 568 if (II.isOutOfDate()) { 569 bool CurrentIsPoisoned = false; 570 if (&II == Ident__VA_ARGS__) 571 CurrentIsPoisoned = Ident__VA_ARGS__->isPoisoned(); 572 573 ExternalSource->updateOutOfDateIdentifier(II); 574 Identifier.setKind(II.getTokenID()); 575 576 if (&II == Ident__VA_ARGS__) 577 II.setIsPoisoned(CurrentIsPoisoned); 578 } 579 580 // If this identifier was poisoned, and if it was not produced from a macro 581 // expansion, emit an error. 582 if (II.isPoisoned() && CurPPLexer) { 583 HandlePoisonedIdentifier(Identifier); 584 } 585 586 // If this is a macro to be expanded, do it. 587 if (MacroInfo *MI = getMacroInfo(&II)) { 588 if (!DisableMacroExpansion) { 589 if (Identifier.isExpandDisabled()) { 590 Diag(Identifier, diag::pp_disabled_macro_expansion); 591 } else if (MI->isEnabled()) { 592 if (!HandleMacroExpandedIdentifier(Identifier, MI)) 593 return; 594 } else { 595 // C99 6.10.3.4p2 says that a disabled macro may never again be 596 // expanded, even if it's in a context where it could be expanded in the 597 // future. 598 Identifier.setFlag(Token::DisableExpand); 599 Diag(Identifier, diag::pp_disabled_macro_expansion); 600 } 601 } 602 } 603 604 // If this identifier is a keyword in C++11, produce a warning. Don't warn if 605 // we're not considering macro expansion, since this identifier might be the 606 // name of a macro. 607 // FIXME: This warning is disabled in cases where it shouldn't be, like 608 // "#define constexpr constexpr", "int constexpr;" 609 if (II.isCXX11CompatKeyword() & !DisableMacroExpansion) { 610 Diag(Identifier, diag::warn_cxx11_keyword) << II.getName(); 611 // Don't diagnose this keyword again in this translation unit. 612 II.setIsCXX11CompatKeyword(false); 613 } 614 615 // C++ 2.11p2: If this is an alternative representation of a C++ operator, 616 // then we act as if it is the actual operator and not the textual 617 // representation of it. 618 if (II.isCPlusPlusOperatorKeyword()) 619 Identifier.setIdentifierInfo(0); 620 621 // If this is an extension token, diagnose its use. 622 // We avoid diagnosing tokens that originate from macro definitions. 623 // FIXME: This warning is disabled in cases where it shouldn't be, 624 // like "#define TY typeof", "TY(1) x". 625 if (II.isExtensionToken() && !DisableMacroExpansion) 626 Diag(Identifier, diag::ext_token_used); 627 628 // If this is the '__experimental_modules_import' contextual keyword, note 629 // that the next token indicates a module name. 630 // 631 // Note that we do not treat '__experimental_modules_import' as a contextual 632 // keyword when we're in a caching lexer, because caching lexers only get 633 // used in contexts where import declarations are disallowed. 634 if (II.isModulesImport() && !InMacroArgs && !DisableMacroExpansion && 635 getLangOpts().Modules && CurLexerKind != CLK_CachingLexer) { 636 ModuleImportLoc = Identifier.getLocation(); 637 ModuleImportPath.clear(); 638 ModuleImportExpectsIdentifier = true; 639 CurLexerKind = CLK_LexAfterModuleImport; 640 } 641} 642 643/// \brief Lex a token following the 'import' contextual keyword. 644/// 645void Preprocessor::LexAfterModuleImport(Token &Result) { 646 // Figure out what kind of lexer we actually have. 647 recomputeCurLexerKind(); 648 649 // Lex the next token. 650 Lex(Result); 651 652 // The token sequence 653 // 654 // import identifier (. identifier)* 655 // 656 // indicates a module import directive. We already saw the 'import' 657 // contextual keyword, so now we're looking for the identifiers. 658 if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) { 659 // We expected to see an identifier here, and we did; continue handling 660 // identifiers. 661 ModuleImportPath.push_back(std::make_pair(Result.getIdentifierInfo(), 662 Result.getLocation())); 663 ModuleImportExpectsIdentifier = false; 664 CurLexerKind = CLK_LexAfterModuleImport; 665 return; 666 } 667 668 // If we're expecting a '.' or a ';', and we got a '.', then wait until we 669 // see the next identifier. 670 if (!ModuleImportExpectsIdentifier && Result.getKind() == tok::period) { 671 ModuleImportExpectsIdentifier = true; 672 CurLexerKind = CLK_LexAfterModuleImport; 673 return; 674 } 675 676 // If we have a non-empty module path, load the named module. 677 if (!ModuleImportPath.empty()) { 678 Module *Imported = TheModuleLoader.loadModule(ModuleImportLoc, 679 ModuleImportPath, 680 Module::MacrosVisible, 681 /*IsIncludeDirective=*/false); 682 if (Callbacks) 683 Callbacks->moduleImport(ModuleImportLoc, ModuleImportPath, Imported); 684 } 685} 686 687void Preprocessor::addCommentHandler(CommentHandler *Handler) { 688 assert(Handler && "NULL comment handler"); 689 assert(std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler) == 690 CommentHandlers.end() && "Comment handler already registered"); 691 CommentHandlers.push_back(Handler); 692} 693 694void Preprocessor::removeCommentHandler(CommentHandler *Handler) { 695 std::vector<CommentHandler *>::iterator Pos 696 = std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler); 697 assert(Pos != CommentHandlers.end() && "Comment handler not registered"); 698 CommentHandlers.erase(Pos); 699} 700 701bool Preprocessor::HandleComment(Token &result, SourceRange Comment) { 702 bool AnyPendingTokens = false; 703 for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(), 704 HEnd = CommentHandlers.end(); 705 H != HEnd; ++H) { 706 if ((*H)->HandleComment(*this, Comment)) 707 AnyPendingTokens = true; 708 } 709 if (!AnyPendingTokens || getCommentRetentionState()) 710 return false; 711 Lex(result); 712 return true; 713} 714 715ModuleLoader::~ModuleLoader() { } 716 717CommentHandler::~CommentHandler() { } 718 719CodeCompletionHandler::~CodeCompletionHandler() { } 720 721void Preprocessor::createPreprocessingRecord(bool RecordConditionalDirectives) { 722 if (Record) 723 return; 724 725 Record = new PreprocessingRecord(getSourceManager(), 726 RecordConditionalDirectives); 727 addPPCallbacks(Record); 728} 729