Preprocessor.cpp revision cd6df6695c87ea55bc5e3e59043744e7946d5fab
1//===--- Preprocess.cpp - C Language Family Preprocessor Implementation ---===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the Preprocessor interface. 11// 12//===----------------------------------------------------------------------===// 13// 14// Options to support: 15// -H - Print the name of each header file used. 16// -d[DNI] - Dump various things. 17// -fworking-directory - #line's with preprocessor's working dir. 18// -fpreprocessed 19// -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD 20// -W* 21// -w 22// 23// Messages to emit: 24// "Multiple include guards may be useful for:\n" 25// 26//===----------------------------------------------------------------------===// 27 28#include "clang/Lex/Preprocessor.h" 29#include "MacroArgs.h" 30#include "clang/Lex/ExternalPreprocessorSource.h" 31#include "clang/Lex/HeaderSearch.h" 32#include "clang/Lex/MacroInfo.h" 33#include "clang/Lex/Pragma.h" 34#include "clang/Lex/PreprocessingRecord.h" 35#include "clang/Lex/ScratchBuffer.h" 36#include "clang/Lex/LexDiagnostic.h" 37#include "clang/Lex/CodeCompletionHandler.h" 38#include "clang/Lex/ModuleLoader.h" 39#include "clang/Basic/SourceManager.h" 40#include "clang/Basic/FileManager.h" 41#include "clang/Basic/TargetInfo.h" 42#include "llvm/ADT/APFloat.h" 43#include "llvm/ADT/SmallString.h" 44#include "llvm/Support/MemoryBuffer.h" 45#include "llvm/Support/raw_ostream.h" 46#include "llvm/Support/Capacity.h" 47using namespace clang; 48 49//===----------------------------------------------------------------------===// 50ExternalPreprocessorSource::~ExternalPreprocessorSource() { } 51 52Preprocessor::Preprocessor(DiagnosticsEngine &diags, LangOptions &opts, 53 const TargetInfo *target, SourceManager &SM, 54 HeaderSearch &Headers, ModuleLoader &TheModuleLoader, 55 IdentifierInfoLookup* IILookup, 56 bool OwnsHeaders, 57 bool DelayInitialization, 58 bool IncrProcessing) 59 : Diags(&diags), LangOpts(opts), Target(target),FileMgr(Headers.getFileMgr()), 60 SourceMgr(SM), HeaderInfo(Headers), TheModuleLoader(TheModuleLoader), 61 ExternalSource(0), Identifiers(opts, IILookup), 62 IncrementalProcessing(IncrProcessing), CodeComplete(0), 63 CodeCompletionFile(0), CodeCompletionOffset(0), CodeCompletionReached(0), 64 SkipMainFilePreamble(0, true), CurPPLexer(0), 65 CurDirLookup(0), CurLexerKind(CLK_Lexer), Callbacks(0), MacroArgCache(0), 66 Record(0), MIChainHead(0), MICache(0) 67{ 68 OwnsHeaderSearch = OwnsHeaders; 69 70 ScratchBuf = new ScratchBuffer(SourceMgr); 71 CounterValue = 0; // __COUNTER__ starts at 0. 72 73 // Clear stats. 74 NumDirectives = NumDefined = NumUndefined = NumPragma = 0; 75 NumIf = NumElse = NumEndif = 0; 76 NumEnteredSourceFiles = 0; 77 NumMacroExpanded = NumFnMacroExpanded = NumBuiltinMacroExpanded = 0; 78 NumFastMacroExpanded = NumTokenPaste = NumFastTokenPaste = 0; 79 MaxIncludeStackDepth = 0; 80 NumSkipped = 0; 81 82 // Default to discarding comments. 83 KeepComments = false; 84 KeepMacroComments = false; 85 SuppressIncludeNotFoundError = false; 86 87 // Macro expansion is enabled. 88 DisableMacroExpansion = false; 89 MacroExpansionInDirectivesOverride = false; 90 InMacroArgs = false; 91 InMacroArgPreExpansion = false; 92 NumCachedTokenLexers = 0; 93 PragmasEnabled = true; 94 95 CachedLexPos = 0; 96 97 // We haven't read anything from the external source. 98 ReadMacrosFromExternalSource = false; 99 100 // "Poison" __VA_ARGS__, which can only appear in the expansion of a macro. 101 // This gets unpoisoned where it is allowed. 102 (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned(); 103 SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use); 104 105 // Initialize the pragma handlers. 106 PragmaHandlers = new PragmaNamespace(StringRef()); 107 RegisterBuiltinPragmas(); 108 109 // Initialize builtin macros like __LINE__ and friends. 110 RegisterBuiltinMacros(); 111 112 if(LangOpts.Borland) { 113 Ident__exception_info = getIdentifierInfo("_exception_info"); 114 Ident___exception_info = getIdentifierInfo("__exception_info"); 115 Ident_GetExceptionInfo = getIdentifierInfo("GetExceptionInformation"); 116 Ident__exception_code = getIdentifierInfo("_exception_code"); 117 Ident___exception_code = getIdentifierInfo("__exception_code"); 118 Ident_GetExceptionCode = getIdentifierInfo("GetExceptionCode"); 119 Ident__abnormal_termination = getIdentifierInfo("_abnormal_termination"); 120 Ident___abnormal_termination = getIdentifierInfo("__abnormal_termination"); 121 Ident_AbnormalTermination = getIdentifierInfo("AbnormalTermination"); 122 } else { 123 Ident__exception_info = Ident__exception_code = Ident__abnormal_termination = 0; 124 Ident___exception_info = Ident___exception_code = Ident___abnormal_termination = 0; 125 Ident_GetExceptionInfo = Ident_GetExceptionCode = Ident_AbnormalTermination = 0; 126 } 127 128 if (!DelayInitialization) { 129 assert(Target && "Must provide target information for PP initialization"); 130 Initialize(*Target); 131 } 132} 133 134Preprocessor::~Preprocessor() { 135 assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!"); 136 137 while (!IncludeMacroStack.empty()) { 138 delete IncludeMacroStack.back().TheLexer; 139 delete IncludeMacroStack.back().TheTokenLexer; 140 IncludeMacroStack.pop_back(); 141 } 142 143 // Free any macro definitions. 144 for (MacroInfoChain *I = MIChainHead ; I ; I = I->Next) 145 I->MI.Destroy(); 146 147 // Free any cached macro expanders. 148 for (unsigned i = 0, e = NumCachedTokenLexers; i != e; ++i) 149 delete TokenLexerCache[i]; 150 151 // Free any cached MacroArgs. 152 for (MacroArgs *ArgList = MacroArgCache; ArgList; ) 153 ArgList = ArgList->deallocate(); 154 155 // Release pragma information. 156 delete PragmaHandlers; 157 158 // Delete the scratch buffer info. 159 delete ScratchBuf; 160 161 // Delete the header search info, if we own it. 162 if (OwnsHeaderSearch) 163 delete &HeaderInfo; 164 165 delete Callbacks; 166} 167 168void Preprocessor::Initialize(const TargetInfo &Target) { 169 assert((!this->Target || this->Target == &Target) && 170 "Invalid override of target information"); 171 this->Target = &Target; 172 173 // Initialize information about built-ins. 174 BuiltinInfo.InitializeTarget(Target); 175 HeaderInfo.setTarget(Target); 176} 177 178void Preprocessor::setPTHManager(PTHManager* pm) { 179 PTH.reset(pm); 180 FileMgr.addStatCache(PTH->createStatCache()); 181} 182 183void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const { 184 llvm::errs() << tok::getTokenName(Tok.getKind()) << " '" 185 << getSpelling(Tok) << "'"; 186 187 if (!DumpFlags) return; 188 189 llvm::errs() << "\t"; 190 if (Tok.isAtStartOfLine()) 191 llvm::errs() << " [StartOfLine]"; 192 if (Tok.hasLeadingSpace()) 193 llvm::errs() << " [LeadingSpace]"; 194 if (Tok.isExpandDisabled()) 195 llvm::errs() << " [ExpandDisabled]"; 196 if (Tok.needsCleaning()) { 197 const char *Start = SourceMgr.getCharacterData(Tok.getLocation()); 198 llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength()) 199 << "']"; 200 } 201 202 llvm::errs() << "\tLoc=<"; 203 DumpLocation(Tok.getLocation()); 204 llvm::errs() << ">"; 205} 206 207void Preprocessor::DumpLocation(SourceLocation Loc) const { 208 Loc.dump(SourceMgr); 209} 210 211void Preprocessor::DumpMacro(const MacroInfo &MI) const { 212 llvm::errs() << "MACRO: "; 213 for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) { 214 DumpToken(MI.getReplacementToken(i)); 215 llvm::errs() << " "; 216 } 217 llvm::errs() << "\n"; 218} 219 220void Preprocessor::PrintStats() { 221 llvm::errs() << "\n*** Preprocessor Stats:\n"; 222 llvm::errs() << NumDirectives << " directives found:\n"; 223 llvm::errs() << " " << NumDefined << " #define.\n"; 224 llvm::errs() << " " << NumUndefined << " #undef.\n"; 225 llvm::errs() << " #include/#include_next/#import:\n"; 226 llvm::errs() << " " << NumEnteredSourceFiles << " source files entered.\n"; 227 llvm::errs() << " " << MaxIncludeStackDepth << " max include stack depth\n"; 228 llvm::errs() << " " << NumIf << " #if/#ifndef/#ifdef.\n"; 229 llvm::errs() << " " << NumElse << " #else/#elif.\n"; 230 llvm::errs() << " " << NumEndif << " #endif.\n"; 231 llvm::errs() << " " << NumPragma << " #pragma.\n"; 232 llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n"; 233 234 llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/" 235 << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, " 236 << NumFastMacroExpanded << " on the fast path.\n"; 237 llvm::errs() << (NumFastTokenPaste+NumTokenPaste) 238 << " token paste (##) operations performed, " 239 << NumFastTokenPaste << " on the fast path.\n"; 240 241 llvm::errs() << "\nPreprocessor Memory: " << getTotalMemory() << "B total"; 242 243 llvm::errs() << "\n BumpPtr: " << BP.getTotalMemory(); 244 llvm::errs() << "\n Macro Expanded Tokens: " 245 << llvm::capacity_in_bytes(MacroExpandedTokens); 246 llvm::errs() << "\n Predefines Buffer: " << Predefines.capacity(); 247 llvm::errs() << "\n Macros: " << llvm::capacity_in_bytes(Macros); 248 llvm::errs() << "\n #pragma push_macro Info: " 249 << llvm::capacity_in_bytes(PragmaPushMacroInfo); 250 llvm::errs() << "\n Poison Reasons: " 251 << llvm::capacity_in_bytes(PoisonReasons); 252 llvm::errs() << "\n Comment Handlers: " 253 << llvm::capacity_in_bytes(CommentHandlers) << "\n"; 254} 255 256Preprocessor::macro_iterator 257Preprocessor::macro_begin(bool IncludeExternalMacros) const { 258 if (IncludeExternalMacros && ExternalSource && 259 !ReadMacrosFromExternalSource) { 260 ReadMacrosFromExternalSource = true; 261 ExternalSource->ReadDefinedMacros(); 262 } 263 264 return Macros.begin(); 265} 266 267size_t Preprocessor::getTotalMemory() const { 268 return BP.getTotalMemory() 269 + llvm::capacity_in_bytes(MacroExpandedTokens) 270 + Predefines.capacity() /* Predefines buffer. */ 271 + llvm::capacity_in_bytes(Macros) 272 + llvm::capacity_in_bytes(PragmaPushMacroInfo) 273 + llvm::capacity_in_bytes(PoisonReasons) 274 + llvm::capacity_in_bytes(CommentHandlers); 275} 276 277Preprocessor::macro_iterator 278Preprocessor::macro_end(bool IncludeExternalMacros) const { 279 if (IncludeExternalMacros && ExternalSource && 280 !ReadMacrosFromExternalSource) { 281 ReadMacrosFromExternalSource = true; 282 ExternalSource->ReadDefinedMacros(); 283 } 284 285 return Macros.end(); 286} 287 288void Preprocessor::recomputeCurLexerKind() { 289 if (CurLexer) 290 CurLexerKind = CLK_Lexer; 291 else if (CurPTHLexer) 292 CurLexerKind = CLK_PTHLexer; 293 else if (CurTokenLexer) 294 CurLexerKind = CLK_TokenLexer; 295 else 296 CurLexerKind = CLK_CachingLexer; 297} 298 299bool Preprocessor::SetCodeCompletionPoint(const FileEntry *File, 300 unsigned CompleteLine, 301 unsigned CompleteColumn) { 302 assert(File); 303 assert(CompleteLine && CompleteColumn && "Starts from 1:1"); 304 assert(!CodeCompletionFile && "Already set"); 305 306 using llvm::MemoryBuffer; 307 308 // Load the actual file's contents. 309 bool Invalid = false; 310 const MemoryBuffer *Buffer = SourceMgr.getMemoryBufferForFile(File, &Invalid); 311 if (Invalid) 312 return true; 313 314 // Find the byte position of the truncation point. 315 const char *Position = Buffer->getBufferStart(); 316 for (unsigned Line = 1; Line < CompleteLine; ++Line) { 317 for (; *Position; ++Position) { 318 if (*Position != '\r' && *Position != '\n') 319 continue; 320 321 // Eat \r\n or \n\r as a single line. 322 if ((Position[1] == '\r' || Position[1] == '\n') && 323 Position[0] != Position[1]) 324 ++Position; 325 ++Position; 326 break; 327 } 328 } 329 330 Position += CompleteColumn - 1; 331 332 // Insert '\0' at the code-completion point. 333 if (Position < Buffer->getBufferEnd()) { 334 CodeCompletionFile = File; 335 CodeCompletionOffset = Position - Buffer->getBufferStart(); 336 337 MemoryBuffer *NewBuffer = 338 MemoryBuffer::getNewUninitMemBuffer(Buffer->getBufferSize() + 1, 339 Buffer->getBufferIdentifier()); 340 char *NewBuf = const_cast<char*>(NewBuffer->getBufferStart()); 341 char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf); 342 *NewPos = '\0'; 343 std::copy(Position, Buffer->getBufferEnd(), NewPos+1); 344 SourceMgr.overrideFileContents(File, NewBuffer); 345 } 346 347 return false; 348} 349 350void Preprocessor::CodeCompleteNaturalLanguage() { 351 if (CodeComplete) 352 CodeComplete->CodeCompleteNaturalLanguage(); 353 setCodeCompletionReached(); 354} 355 356/// getSpelling - This method is used to get the spelling of a token into a 357/// SmallVector. Note that the returned StringRef may not point to the 358/// supplied buffer if a copy can be avoided. 359StringRef Preprocessor::getSpelling(const Token &Tok, 360 SmallVectorImpl<char> &Buffer, 361 bool *Invalid) const { 362 // NOTE: this has to be checked *before* testing for an IdentifierInfo. 363 if (Tok.isNot(tok::raw_identifier)) { 364 // Try the fast path. 365 if (const IdentifierInfo *II = Tok.getIdentifierInfo()) 366 return II->getName(); 367 } 368 369 // Resize the buffer if we need to copy into it. 370 if (Tok.needsCleaning()) 371 Buffer.resize(Tok.getLength()); 372 373 const char *Ptr = Buffer.data(); 374 unsigned Len = getSpelling(Tok, Ptr, Invalid); 375 return StringRef(Ptr, Len); 376} 377 378/// CreateString - Plop the specified string into a scratch buffer and return a 379/// location for it. If specified, the source location provides a source 380/// location for the token. 381void Preprocessor::CreateString(const char *Buf, unsigned Len, Token &Tok, 382 SourceLocation ExpansionLocStart, 383 SourceLocation ExpansionLocEnd) { 384 Tok.setLength(Len); 385 386 const char *DestPtr; 387 SourceLocation Loc = ScratchBuf->getToken(Buf, Len, DestPtr); 388 389 if (ExpansionLocStart.isValid()) 390 Loc = SourceMgr.createExpansionLoc(Loc, ExpansionLocStart, 391 ExpansionLocEnd, Len); 392 Tok.setLocation(Loc); 393 394 // If this is a raw identifier or a literal token, set the pointer data. 395 if (Tok.is(tok::raw_identifier)) 396 Tok.setRawIdentifierData(DestPtr); 397 else if (Tok.isLiteral()) 398 Tok.setLiteralData(DestPtr); 399} 400 401Module *Preprocessor::getCurrentModule() { 402 if (getLangOpts().CurrentModule.empty()) 403 return 0; 404 405 return getHeaderSearchInfo().lookupModule(getLangOpts().CurrentModule); 406} 407 408//===----------------------------------------------------------------------===// 409// Preprocessor Initialization Methods 410//===----------------------------------------------------------------------===// 411 412 413/// EnterMainSourceFile - Enter the specified FileID as the main source file, 414/// which implicitly adds the builtin defines etc. 415void Preprocessor::EnterMainSourceFile() { 416 // We do not allow the preprocessor to reenter the main file. Doing so will 417 // cause FileID's to accumulate information from both runs (e.g. #line 418 // information) and predefined macros aren't guaranteed to be set properly. 419 assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!"); 420 FileID MainFileID = SourceMgr.getMainFileID(); 421 422 // If MainFileID is loaded it means we loaded an AST file, no need to enter 423 // a main file. 424 if (!SourceMgr.isLoadedFileID(MainFileID)) { 425 // Enter the main file source buffer. 426 EnterSourceFile(MainFileID, 0, SourceLocation()); 427 428 // If we've been asked to skip bytes in the main file (e.g., as part of a 429 // precompiled preamble), do so now. 430 if (SkipMainFilePreamble.first > 0) 431 CurLexer->SkipBytes(SkipMainFilePreamble.first, 432 SkipMainFilePreamble.second); 433 434 // Tell the header info that the main file was entered. If the file is later 435 // #imported, it won't be re-entered. 436 if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID)) 437 HeaderInfo.IncrementIncludeCount(FE); 438 } 439 440 // Preprocess Predefines to populate the initial preprocessor state. 441 llvm::MemoryBuffer *SB = 442 llvm::MemoryBuffer::getMemBufferCopy(Predefines, "<built-in>"); 443 assert(SB && "Cannot create predefined source buffer"); 444 FileID FID = SourceMgr.createFileIDForMemBuffer(SB); 445 assert(!FID.isInvalid() && "Could not create FileID for predefines?"); 446 447 // Start parsing the predefines. 448 EnterSourceFile(FID, 0, SourceLocation()); 449} 450 451void Preprocessor::EndSourceFile() { 452 // Notify the client that we reached the end of the source file. 453 if (Callbacks) 454 Callbacks->EndOfMainFile(); 455} 456 457//===----------------------------------------------------------------------===// 458// Lexer Event Handling. 459//===----------------------------------------------------------------------===// 460 461/// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the 462/// identifier information for the token and install it into the token, 463/// updating the token kind accordingly. 464IdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier) const { 465 assert(Identifier.getRawIdentifierData() != 0 && "No raw identifier data!"); 466 467 // Look up this token, see if it is a macro, or if it is a language keyword. 468 IdentifierInfo *II; 469 if (!Identifier.needsCleaning()) { 470 // No cleaning needed, just use the characters from the lexed buffer. 471 II = getIdentifierInfo(StringRef(Identifier.getRawIdentifierData(), 472 Identifier.getLength())); 473 } else { 474 // Cleaning needed, alloca a buffer, clean into it, then use the buffer. 475 SmallString<64> IdentifierBuffer; 476 StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer); 477 II = getIdentifierInfo(CleanedStr); 478 } 479 480 // Update the token info (identifier info and appropriate token kind). 481 Identifier.setIdentifierInfo(II); 482 Identifier.setKind(II->getTokenID()); 483 484 return II; 485} 486 487void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) { 488 PoisonReasons[II] = DiagID; 489} 490 491void Preprocessor::PoisonSEHIdentifiers(bool Poison) { 492 assert(Ident__exception_code && Ident__exception_info); 493 assert(Ident___exception_code && Ident___exception_info); 494 Ident__exception_code->setIsPoisoned(Poison); 495 Ident___exception_code->setIsPoisoned(Poison); 496 Ident_GetExceptionCode->setIsPoisoned(Poison); 497 Ident__exception_info->setIsPoisoned(Poison); 498 Ident___exception_info->setIsPoisoned(Poison); 499 Ident_GetExceptionInfo->setIsPoisoned(Poison); 500 Ident__abnormal_termination->setIsPoisoned(Poison); 501 Ident___abnormal_termination->setIsPoisoned(Poison); 502 Ident_AbnormalTermination->setIsPoisoned(Poison); 503} 504 505void Preprocessor::HandlePoisonedIdentifier(Token & Identifier) { 506 assert(Identifier.getIdentifierInfo() && 507 "Can't handle identifiers without identifier info!"); 508 llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it = 509 PoisonReasons.find(Identifier.getIdentifierInfo()); 510 if(it == PoisonReasons.end()) 511 Diag(Identifier, diag::err_pp_used_poisoned_id); 512 else 513 Diag(Identifier,it->second) << Identifier.getIdentifierInfo(); 514} 515 516/// HandleIdentifier - This callback is invoked when the lexer reads an 517/// identifier. This callback looks up the identifier in the map and/or 518/// potentially macro expands it or turns it into a named token (like 'for'). 519/// 520/// Note that callers of this method are guarded by checking the 521/// IdentifierInfo's 'isHandleIdentifierCase' bit. If this method changes, the 522/// IdentifierInfo methods that compute these properties will need to change to 523/// match. 524void Preprocessor::HandleIdentifier(Token &Identifier) { 525 assert(Identifier.getIdentifierInfo() && 526 "Can't handle identifiers without identifier info!"); 527 528 IdentifierInfo &II = *Identifier.getIdentifierInfo(); 529 530 // If the information about this identifier is out of date, update it from 531 // the external source. 532 // We have to treat __VA_ARGS__ in a special way, since it gets 533 // serialized with isPoisoned = true, but our preprocessor may have 534 // unpoisoned it if we're defining a C99 macro. 535 if (II.isOutOfDate()) { 536 bool CurrentIsPoisoned = false; 537 if (&II == Ident__VA_ARGS__) 538 CurrentIsPoisoned = Ident__VA_ARGS__->isPoisoned(); 539 540 ExternalSource->updateOutOfDateIdentifier(II); 541 Identifier.setKind(II.getTokenID()); 542 543 if (&II == Ident__VA_ARGS__) 544 II.setIsPoisoned(CurrentIsPoisoned); 545 } 546 547 // If this identifier was poisoned, and if it was not produced from a macro 548 // expansion, emit an error. 549 if (II.isPoisoned() && CurPPLexer) { 550 HandlePoisonedIdentifier(Identifier); 551 } 552 553 // If this is a macro to be expanded, do it. 554 if (MacroInfo *MI = getMacroInfo(&II)) { 555 if (!DisableMacroExpansion) { 556 if (Identifier.isExpandDisabled()) { 557 Diag(Identifier, diag::pp_disabled_macro_expansion); 558 } else if (MI->isEnabled()) { 559 if (!HandleMacroExpandedIdentifier(Identifier, MI)) 560 return; 561 } else { 562 // C99 6.10.3.4p2 says that a disabled macro may never again be 563 // expanded, even if it's in a context where it could be expanded in the 564 // future. 565 Identifier.setFlag(Token::DisableExpand); 566 Diag(Identifier, diag::pp_disabled_macro_expansion); 567 } 568 } 569 } 570 571 // If this identifier is a keyword in C++11, produce a warning. Don't warn if 572 // we're not considering macro expansion, since this identifier might be the 573 // name of a macro. 574 // FIXME: This warning is disabled in cases where it shouldn't be, like 575 // "#define constexpr constexpr", "int constexpr;" 576 if (II.isCXX11CompatKeyword() & !DisableMacroExpansion) { 577 Diag(Identifier, diag::warn_cxx11_keyword) << II.getName(); 578 // Don't diagnose this keyword again in this translation unit. 579 II.setIsCXX11CompatKeyword(false); 580 } 581 582 // C++ 2.11p2: If this is an alternative representation of a C++ operator, 583 // then we act as if it is the actual operator and not the textual 584 // representation of it. 585 if (II.isCPlusPlusOperatorKeyword()) 586 Identifier.setIdentifierInfo(0); 587 588 // If this is an extension token, diagnose its use. 589 // We avoid diagnosing tokens that originate from macro definitions. 590 // FIXME: This warning is disabled in cases where it shouldn't be, 591 // like "#define TY typeof", "TY(1) x". 592 if (II.isExtensionToken() && !DisableMacroExpansion) 593 Diag(Identifier, diag::ext_token_used); 594 595 // If this is the '__experimental_modules_import' contextual keyword, note 596 // that the next token indicates a module name. 597 // 598 // Note that we do not treat '__experimental_modules_import' as a contextual 599 // keyword when we're in a caching lexer, because caching lexers only get 600 // used in contexts where import declarations are disallowed. 601 if (II.isModulesImport() && !InMacroArgs && !DisableMacroExpansion && 602 getLangOpts().Modules && CurLexerKind != CLK_CachingLexer) { 603 ModuleImportLoc = Identifier.getLocation(); 604 ModuleImportPath.clear(); 605 ModuleImportExpectsIdentifier = true; 606 CurLexerKind = CLK_LexAfterModuleImport; 607 } 608} 609 610/// \brief Lex a token following the 'import' contextual keyword. 611/// 612void Preprocessor::LexAfterModuleImport(Token &Result) { 613 // Figure out what kind of lexer we actually have. 614 recomputeCurLexerKind(); 615 616 // Lex the next token. 617 Lex(Result); 618 619 // The token sequence 620 // 621 // import identifier (. identifier)* 622 // 623 // indicates a module import directive. We already saw the 'import' 624 // contextual keyword, so now we're looking for the identifiers. 625 if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) { 626 // We expected to see an identifier here, and we did; continue handling 627 // identifiers. 628 ModuleImportPath.push_back(std::make_pair(Result.getIdentifierInfo(), 629 Result.getLocation())); 630 ModuleImportExpectsIdentifier = false; 631 CurLexerKind = CLK_LexAfterModuleImport; 632 return; 633 } 634 635 // If we're expecting a '.' or a ';', and we got a '.', then wait until we 636 // see the next identifier. 637 if (!ModuleImportExpectsIdentifier && Result.getKind() == tok::period) { 638 ModuleImportExpectsIdentifier = true; 639 CurLexerKind = CLK_LexAfterModuleImport; 640 return; 641 } 642 643 // If we have a non-empty module path, load the named module. 644 if (!ModuleImportPath.empty()) 645 (void)TheModuleLoader.loadModule(ModuleImportLoc, ModuleImportPath, 646 Module::MacrosVisible, 647 /*IsIncludeDirective=*/false); 648} 649 650void Preprocessor::addCommentHandler(CommentHandler *Handler) { 651 assert(Handler && "NULL comment handler"); 652 assert(std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler) == 653 CommentHandlers.end() && "Comment handler already registered"); 654 CommentHandlers.push_back(Handler); 655} 656 657void Preprocessor::removeCommentHandler(CommentHandler *Handler) { 658 std::vector<CommentHandler *>::iterator Pos 659 = std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler); 660 assert(Pos != CommentHandlers.end() && "Comment handler not registered"); 661 CommentHandlers.erase(Pos); 662} 663 664bool Preprocessor::HandleComment(Token &result, SourceRange Comment) { 665 bool AnyPendingTokens = false; 666 for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(), 667 HEnd = CommentHandlers.end(); 668 H != HEnd; ++H) { 669 if ((*H)->HandleComment(*this, Comment)) 670 AnyPendingTokens = true; 671 } 672 if (!AnyPendingTokens || getCommentRetentionState()) 673 return false; 674 Lex(result); 675 return true; 676} 677 678ModuleLoader::~ModuleLoader() { } 679 680CommentHandler::~CommentHandler() { } 681 682CodeCompletionHandler::~CodeCompletionHandler() { } 683 684void Preprocessor::createPreprocessingRecord(bool RecordConditionalDirectives) { 685 if (Record) 686 return; 687 688 Record = new PreprocessingRecord(getSourceManager(), 689 RecordConditionalDirectives); 690 addPPCallbacks(Record); 691} 692