Preprocessor.cpp revision 046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635
15f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===--- Preprocess.cpp - C Language Family Preprocessor Implementation ---===// 25f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 35f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// The LLVM Compiler Infrastructure 45f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 50bc735ffcfb223c0186419547abaa5c84482663eChris Lattner// This file is distributed under the University of Illinois Open Source 60bc735ffcfb223c0186419547abaa5c84482663eChris Lattner// License. See LICENSE.TXT for details. 75f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 85f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 95f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// This file implements the Preprocessor interface. 115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// Options to support: 155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// -H - Print the name of each header file used. 16f73903a1ded46748e1dfda151f5d037b7b3d31f9Chris Lattner// -d[DNI] - Dump various things. 175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// -fworking-directory - #line's with preprocessor's working dir. 185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// -fpreprocessed 195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD 205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// -W* 215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// -w 225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// Messages to emit: 245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// "Multiple include guards may be useful for:\n" 255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "clang/Lex/Preprocessor.h" 2923f77e59718385512984d4e2a021bef52b9f6ddfChris Lattner#include "MacroArgs.h" 3088a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor#include "clang/Lex/ExternalPreprocessorSource.h" 315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "clang/Lex/HeaderSearch.h" 325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "clang/Lex/MacroInfo.h" 335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "clang/Lex/Pragma.h" 345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "clang/Lex/ScratchBuffer.h" 35500d3297d2a21edeac4d46cbcbe21bc2352c2a28Chris Lattner#include "clang/Lex/LexDiagnostic.h" 365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "clang/Basic/SourceManager.h" 37337edcdbec05316b407d0d64865c88ff8597d910Ted Kremenek#include "clang/Basic/FileManager.h" 385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "clang/Basic/TargetInfo.h" 392db78dd977fbba8fdf24cf8a4593436c031cfbcbChris Lattner#include "llvm/ADT/APFloat.h" 405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "llvm/ADT/SmallVector.h" 4197ba77cf09bf7b83b679165ce67ad7d49ffd568cChris Lattner#include "llvm/Support/MemoryBuffer.h" 426cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer#include "llvm/Support/raw_ostream.h" 433daed52a57d03765223021f5f921bdc280c8f3ccChris Lattner#include <cstdio> 445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerusing namespace clang; 455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 4788a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas GregorExternalPreprocessorSource::~ExternalPreprocessorSource() { } 485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid SpencerPreprocessor::Preprocessor(Diagnostic &diags, const LangOptions &opts, 50444be7366d0a1e172c0290a1ea54c1cb16b5947cDaniel Dunbar const TargetInfo &target, SourceManager &SM, 5172b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek HeaderSearch &Headers, 525814e657c9ad9ef6049a2a4af0d2aad248a8a15cDaniel Dunbar IdentifierInfoLookup* IILookup, 535814e657c9ad9ef6049a2a4af0d2aad248a8a15cDaniel Dunbar bool OwnsHeaders) 54836040f9eafe862fb1607df5c30cd3df0c22c832Chris Lattner : Diags(&diags), Features(opts), Target(target),FileMgr(Headers.getFileMgr()), 5588a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor SourceMgr(SM), HeaderInfo(Headers), ExternalSource(0), 5688a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor Identifiers(opts, IILookup), BuiltinInfo(Target), CodeCompletionFile(0), 5788a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor CurPPLexer(0), CurDirLookup(0), Callbacks(0), MacroArgCache(0) { 585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ScratchBuf = new ScratchBuffer(SourceMgr); 59c1f9d828c733ec1eba06d01070735d1f36fda733Chris Lattner CounterValue = 0; // __COUNTER__ starts at 0. 605814e657c9ad9ef6049a2a4af0d2aad248a8a15cDaniel Dunbar OwnsHeaderSearch = OwnsHeaders; 611eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Clear stats. 635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NumDirectives = NumDefined = NumUndefined = NumPragma = 0; 645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NumIf = NumElse = NumEndif = 0; 655f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NumEnteredSourceFiles = 0; 665f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NumMacroExpanded = NumFnMacroExpanded = NumBuiltinMacroExpanded = 0; 675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NumFastMacroExpanded = NumTokenPaste = NumFastTokenPaste = 0; 681eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump MaxIncludeStackDepth = 0; 695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NumSkipped = 0; 705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Default to discarding comments. 725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer KeepComments = false; 735f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer KeepMacroComments = false; 741eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Macro expansion is enabled. 765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer DisableMacroExpansion = false; 775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer InMacroArgs = false; 786cfe7594a46b5d270142cfcb688a9c1a3a487a48Chris Lattner NumCachedTokenLexers = 0; 795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 8003db1b31dd926409b7defc1c90b66549464652c0Argyrios Kyrtzidis CachedLexPos = 0; 8103db1b31dd926409b7defc1c90b66549464652c0Argyrios Kyrtzidis 8288a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor // We haven't read anything from the external source. 8388a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor ReadMacrosFromExternalSource = false; 8488a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor 855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // "Poison" __VA_ARGS__, which can only appear in the expansion of a macro. 865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // This gets unpoisoned where it is allowed. 875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned(); 881eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Initialize the pragma handlers. 905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer PragmaHandlers = new PragmaNamespace(0); 915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer RegisterBuiltinPragmas(); 921eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Initialize builtin macros like __LINE__ and friends. 945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer RegisterBuiltinMacros(); 955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid SpencerPreprocessor::~Preprocessor() { 982174a4ff834b72c0a0e3f24e4b62123f30f4cce6Argyrios Kyrtzidis assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!"); 992174a4ff834b72c0a0e3f24e4b62123f30f4cce6Argyrios Kyrtzidis 1005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (!IncludeMacroStack.empty()) { 1015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer delete IncludeMacroStack.back().TheLexer; 1026cfe7594a46b5d270142cfcb688a9c1a3a487a48Chris Lattner delete IncludeMacroStack.back().TheTokenLexer; 1035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer IncludeMacroStack.pop_back(); 1045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 105cc1a875f94630e58d24a55577ffbf0e89b7da8c7Chris Lattner 106cc1a875f94630e58d24a55577ffbf0e89b7da8c7Chris Lattner // Free any macro definitions. 107cc1a875f94630e58d24a55577ffbf0e89b7da8c7Chris Lattner for (llvm::DenseMap<IdentifierInfo*, MacroInfo*>::iterator I = 108cc1a875f94630e58d24a55577ffbf0e89b7da8c7Chris Lattner Macros.begin(), E = Macros.end(); I != E; ++I) { 1090ea76727ae91bca918a8414ed85b530eddcfedebTed Kremenek // We don't need to free the MacroInfo objects directly. These 1100ea76727ae91bca918a8414ed85b530eddcfedebTed Kremenek // will be released when the BumpPtrAllocator 'BP' object gets 1112b9d6377f1b749ab8ada8fc2cff05954cd4731aeChris Lattner // destroyed. We still need to run the dtor, however, to free 1129ee7d456b427e002f7fe1844b64bec8055c8db52Ted Kremenek // memory alocated by MacroInfo. 113685befeb5f6472585bae473a6389e47cab9eac67Chris Lattner I->second->Destroy(BP); 114cc1a875f94630e58d24a55577ffbf0e89b7da8c7Chris Lattner I->first->setHasMacroDefinition(false); 115cc1a875f94630e58d24a55577ffbf0e89b7da8c7Chris Lattner } 1161eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1179594acf32de2939b15eafa8fe818607bfc56bf66Chris Lattner // Free any cached macro expanders. 1186cfe7594a46b5d270142cfcb688a9c1a3a487a48Chris Lattner for (unsigned i = 0, e = NumCachedTokenLexers; i != e; ++i) 1196cfe7594a46b5d270142cfcb688a9c1a3a487a48Chris Lattner delete TokenLexerCache[i]; 12023f77e59718385512984d4e2a021bef52b9f6ddfChris Lattner 12123f77e59718385512984d4e2a021bef52b9f6ddfChris Lattner // Free any cached MacroArgs. 12223f77e59718385512984d4e2a021bef52b9f6ddfChris Lattner for (MacroArgs *ArgList = MacroArgCache; ArgList; ) 12323f77e59718385512984d4e2a021bef52b9f6ddfChris Lattner ArgList = ArgList->deallocate(); 1241eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Release pragma information. 1265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer delete PragmaHandlers; 1275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Delete the scratch buffer info. 1295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer delete ScratchBuf; 130eb50ed88c2aa040fac08bf2a50bde4dd3da6eb19Chris Lattner 1315814e657c9ad9ef6049a2a4af0d2aad248a8a15cDaniel Dunbar // Delete the header search info, if we own it. 1325814e657c9ad9ef6049a2a4af0d2aad248a8a15cDaniel Dunbar if (OwnsHeaderSearch) 1335814e657c9ad9ef6049a2a4af0d2aad248a8a15cDaniel Dunbar delete &HeaderInfo; 1345814e657c9ad9ef6049a2a4af0d2aad248a8a15cDaniel Dunbar 135eb50ed88c2aa040fac08bf2a50bde4dd3da6eb19Chris Lattner delete Callbacks; 1365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 1375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 138337edcdbec05316b407d0d64865c88ff8597d910Ted Kremenekvoid Preprocessor::setPTHManager(PTHManager* pm) { 139337edcdbec05316b407d0d64865c88ff8597d910Ted Kremenek PTH.reset(pm); 14052e7108f51a4a9f4d6e84f33fb594d06e1d79560Douglas Gregor FileMgr.addStatCache(PTH->createStatCache()); 141337edcdbec05316b407d0d64865c88ff8597d910Ted Kremenek} 142337edcdbec05316b407d0d64865c88ff8597d910Ted Kremenek 143d217773f106856a11879ec79dc468efefaf2ee75Chris Lattnervoid Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const { 1446cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer llvm::errs() << tok::getTokenName(Tok.getKind()) << " '" 1456cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer << getSpelling(Tok) << "'"; 1461eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (!DumpFlags) return; 1481eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1496cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer llvm::errs() << "\t"; 1505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Tok.isAtStartOfLine()) 1516cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer llvm::errs() << " [StartOfLine]"; 1525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Tok.hasLeadingSpace()) 1536cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer llvm::errs() << " [LeadingSpace]"; 1545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Tok.isExpandDisabled()) 1556cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer llvm::errs() << " [ExpandDisabled]"; 1565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Tok.needsCleaning()) { 1575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *Start = SourceMgr.getCharacterData(Tok.getLocation()); 1586cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer llvm::errs() << " [UnClean='" << std::string(Start, Start+Tok.getLength()) 1596cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer << "']"; 1605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 1611eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1626cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer llvm::errs() << "\tLoc=<"; 163c3d8d57b010e2ed15a2a7685d5761db14f5d2252Chris Lattner DumpLocation(Tok.getLocation()); 1646cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer llvm::errs() << ">"; 165c3d8d57b010e2ed15a2a7685d5761db14f5d2252Chris Lattner} 166c3d8d57b010e2ed15a2a7685d5761db14f5d2252Chris Lattner 167c3d8d57b010e2ed15a2a7685d5761db14f5d2252Chris Lattnervoid Preprocessor::DumpLocation(SourceLocation Loc) const { 168b9c3f966b103f7cfe8e5e60007c4c8b38f7298ebChris Lattner Loc.dump(SourceMgr); 1695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 1705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencervoid Preprocessor::DumpMacro(const MacroInfo &MI) const { 1726cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer llvm::errs() << "MACRO: "; 1735f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) { 1745f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer DumpToken(MI.getReplacementToken(i)); 1756cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer llvm::errs() << " "; 1765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 1776cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer llvm::errs() << "\n"; 1785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 1795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1805f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencervoid Preprocessor::PrintStats() { 1816cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer llvm::errs() << "\n*** Preprocessor Stats:\n"; 1826cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer llvm::errs() << NumDirectives << " directives found:\n"; 1836cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer llvm::errs() << " " << NumDefined << " #define.\n"; 1846cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer llvm::errs() << " " << NumUndefined << " #undef.\n"; 1856cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer llvm::errs() << " #include/#include_next/#import:\n"; 1866cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer llvm::errs() << " " << NumEnteredSourceFiles << " source files entered.\n"; 1876cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer llvm::errs() << " " << MaxIncludeStackDepth << " max include stack depth\n"; 1886cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer llvm::errs() << " " << NumIf << " #if/#ifndef/#ifdef.\n"; 1896cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer llvm::errs() << " " << NumElse << " #else/#elif.\n"; 1906cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer llvm::errs() << " " << NumEndif << " #endif.\n"; 1916cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer llvm::errs() << " " << NumPragma << " #pragma.\n"; 1926cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n"; 1936cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer 1946cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/" 195bdd30c26bd93a89ae0fe125e9aae8b7c05e55324Ted Kremenek << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, " 196bdd30c26bd93a89ae0fe125e9aae8b7c05e55324Ted Kremenek << NumFastMacroExpanded << " on the fast path.\n"; 1976cb7c1a43b0c8f739d1f54b7fdae5ede86033496Benjamin Kramer llvm::errs() << (NumFastTokenPaste+NumTokenPaste) 198bdd30c26bd93a89ae0fe125e9aae8b7c05e55324Ted Kremenek << " token paste (##) operations performed, " 199bdd30c26bd93a89ae0fe125e9aae8b7c05e55324Ted Kremenek << NumFastTokenPaste << " on the fast path.\n"; 2005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 2015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 20288a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas GregorPreprocessor::macro_iterator 20388a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas GregorPreprocessor::macro_begin(bool IncludeExternalMacros) const { 20488a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor if (IncludeExternalMacros && ExternalSource && 20588a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor !ReadMacrosFromExternalSource) { 20688a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor ReadMacrosFromExternalSource = true; 20788a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor ExternalSource->ReadDefinedMacros(); 20888a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor } 20988a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor 21088a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor return Macros.begin(); 21188a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor} 21288a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor 21388a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas GregorPreprocessor::macro_iterator 21488a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas GregorPreprocessor::macro_end(bool IncludeExternalMacros) const { 21588a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor if (IncludeExternalMacros && ExternalSource && 21688a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor !ReadMacrosFromExternalSource) { 21788a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor ReadMacrosFromExternalSource = true; 21888a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor ExternalSource->ReadDefinedMacros(); 21988a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor } 22088a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor 22188a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor return Macros.end(); 22288a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor} 22388a35862fbe473f2a4f0c19f24dbe536937e1dc6Douglas Gregor 2242968442603b029949246467253eeac8139a5b6d8Douglas Gregorbool Preprocessor::SetCodeCompletionPoint(const FileEntry *File, 2252968442603b029949246467253eeac8139a5b6d8Douglas Gregor unsigned TruncateAtLine, 2262968442603b029949246467253eeac8139a5b6d8Douglas Gregor unsigned TruncateAtColumn) { 2272968442603b029949246467253eeac8139a5b6d8Douglas Gregor using llvm::MemoryBuffer; 2282968442603b029949246467253eeac8139a5b6d8Douglas Gregor 2292968442603b029949246467253eeac8139a5b6d8Douglas Gregor CodeCompletionFile = File; 2302968442603b029949246467253eeac8139a5b6d8Douglas Gregor 2312968442603b029949246467253eeac8139a5b6d8Douglas Gregor // Okay to clear out the code-completion point by passing NULL. 2322968442603b029949246467253eeac8139a5b6d8Douglas Gregor if (!CodeCompletionFile) 2332968442603b029949246467253eeac8139a5b6d8Douglas Gregor return false; 2342968442603b029949246467253eeac8139a5b6d8Douglas Gregor 2352968442603b029949246467253eeac8139a5b6d8Douglas Gregor // Load the actual file's contents. 2362968442603b029949246467253eeac8139a5b6d8Douglas Gregor const MemoryBuffer *Buffer = SourceMgr.getMemoryBufferForFile(File); 2372968442603b029949246467253eeac8139a5b6d8Douglas Gregor if (!Buffer) 2382968442603b029949246467253eeac8139a5b6d8Douglas Gregor return true; 2392968442603b029949246467253eeac8139a5b6d8Douglas Gregor 2402968442603b029949246467253eeac8139a5b6d8Douglas Gregor // Find the byte position of the truncation point. 2412968442603b029949246467253eeac8139a5b6d8Douglas Gregor const char *Position = Buffer->getBufferStart(); 2422968442603b029949246467253eeac8139a5b6d8Douglas Gregor for (unsigned Line = 1; Line < TruncateAtLine; ++Line) { 2432968442603b029949246467253eeac8139a5b6d8Douglas Gregor for (; *Position; ++Position) { 2442968442603b029949246467253eeac8139a5b6d8Douglas Gregor if (*Position != '\r' && *Position != '\n') 2452968442603b029949246467253eeac8139a5b6d8Douglas Gregor continue; 2462968442603b029949246467253eeac8139a5b6d8Douglas Gregor 2472968442603b029949246467253eeac8139a5b6d8Douglas Gregor // Eat \r\n or \n\r as a single line. 2482968442603b029949246467253eeac8139a5b6d8Douglas Gregor if ((Position[1] == '\r' || Position[1] == '\n') && 2492968442603b029949246467253eeac8139a5b6d8Douglas Gregor Position[0] != Position[1]) 2502968442603b029949246467253eeac8139a5b6d8Douglas Gregor ++Position; 2512968442603b029949246467253eeac8139a5b6d8Douglas Gregor ++Position; 2522968442603b029949246467253eeac8139a5b6d8Douglas Gregor break; 2532968442603b029949246467253eeac8139a5b6d8Douglas Gregor } 2542968442603b029949246467253eeac8139a5b6d8Douglas Gregor } 2552968442603b029949246467253eeac8139a5b6d8Douglas Gregor 256b760fe8fafde1d9d9f1f01db57a88cd9ce636342Douglas Gregor Position += TruncateAtColumn - 1; 2572968442603b029949246467253eeac8139a5b6d8Douglas Gregor 2582968442603b029949246467253eeac8139a5b6d8Douglas Gregor // Truncate the buffer. 259b760fe8fafde1d9d9f1f01db57a88cd9ce636342Douglas Gregor if (Position < Buffer->getBufferEnd()) { 2602968442603b029949246467253eeac8139a5b6d8Douglas Gregor MemoryBuffer *TruncatedBuffer 2612968442603b029949246467253eeac8139a5b6d8Douglas Gregor = MemoryBuffer::getMemBufferCopy(Buffer->getBufferStart(), Position, 2622968442603b029949246467253eeac8139a5b6d8Douglas Gregor Buffer->getBufferIdentifier()); 2632968442603b029949246467253eeac8139a5b6d8Douglas Gregor SourceMgr.overrideFileContents(File, TruncatedBuffer); 2642968442603b029949246467253eeac8139a5b6d8Douglas Gregor } 2652968442603b029949246467253eeac8139a5b6d8Douglas Gregor 2662968442603b029949246467253eeac8139a5b6d8Douglas Gregor return false; 2672968442603b029949246467253eeac8139a5b6d8Douglas Gregor} 2682968442603b029949246467253eeac8139a5b6d8Douglas Gregor 269109ae73ec71a1d27358510ce049b59637b4a6b40Douglas Gregorbool Preprocessor::isCodeCompletionFile(SourceLocation FileLoc) const { 2702968442603b029949246467253eeac8139a5b6d8Douglas Gregor return CodeCompletionFile && FileLoc.isFileID() && 2712968442603b029949246467253eeac8139a5b6d8Douglas Gregor SourceMgr.getFileEntryForID(SourceMgr.getFileID(FileLoc)) 2722968442603b029949246467253eeac8139a5b6d8Douglas Gregor == CodeCompletionFile; 2732968442603b029949246467253eeac8139a5b6d8Douglas Gregor} 2742968442603b029949246467253eeac8139a5b6d8Douglas Gregor 2755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 2765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// Token Spelling 2775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 2785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 2795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// getSpelling() - Return the 'spelling' of this token. The spelling of a 2805f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// token are the characters used to represent the token in the source file 2815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// after trigraph expansion and escaped-newline folding. In particular, this 2825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// wants to get the true, uncanonicalized, spelling of things like digraphs 2835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// UCNs, etc. 2840ff1042ddaad1419264be0de6da17f3b378482a4Daniel Dunbarstd::string Preprocessor::getSpelling(const Token &Tok, 2850ff1042ddaad1419264be0de6da17f3b378482a4Daniel Dunbar const SourceManager &SourceMgr, 2860ff1042ddaad1419264be0de6da17f3b378482a4Daniel Dunbar const LangOptions &Features) { 2875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer assert((int)Tok.getLength() >= 0 && "Token character range is bogus!"); 288277faca30c9f8f72b79f55695cbe3395ec246e7cTed Kremenek 2895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this token contains nothing interesting, return it directly. 290277faca30c9f8f72b79f55695cbe3395ec246e7cTed Kremenek const char* TokStart = SourceMgr.getCharacterData(Tok.getLocation()); 2915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (!Tok.needsCleaning()) 2925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return std::string(TokStart, TokStart+Tok.getLength()); 2931eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer std::string Result; 2955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Result.reserve(Tok.getLength()); 2961eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, hard case, relex the characters into the string. 2985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer for (const char *Ptr = TokStart, *End = TokStart+Tok.getLength(); 2995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Ptr != End; ) { 3005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned CharSize; 3015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Result.push_back(Lexer::getCharAndSizeNoWarn(Ptr, CharSize, Features)); 3025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Ptr += CharSize; 3035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 3045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer assert(Result.size() != unsigned(Tok.getLength()) && 3055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer "NeedsCleaning flag set on something that didn't need cleaning!"); 3065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return Result; 3075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 3085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3090ff1042ddaad1419264be0de6da17f3b378482a4Daniel Dunbar/// getSpelling() - Return the 'spelling' of this token. The spelling of a 3100ff1042ddaad1419264be0de6da17f3b378482a4Daniel Dunbar/// token are the characters used to represent the token in the source file 3110ff1042ddaad1419264be0de6da17f3b378482a4Daniel Dunbar/// after trigraph expansion and escaped-newline folding. In particular, this 3120ff1042ddaad1419264be0de6da17f3b378482a4Daniel Dunbar/// wants to get the true, uncanonicalized, spelling of things like digraphs 3130ff1042ddaad1419264be0de6da17f3b378482a4Daniel Dunbar/// UCNs, etc. 3140ff1042ddaad1419264be0de6da17f3b378482a4Daniel Dunbarstd::string Preprocessor::getSpelling(const Token &Tok) const { 3150ff1042ddaad1419264be0de6da17f3b378482a4Daniel Dunbar return getSpelling(Tok, SourceMgr, Features); 3160ff1042ddaad1419264be0de6da17f3b378482a4Daniel Dunbar} 3170ff1042ddaad1419264be0de6da17f3b378482a4Daniel Dunbar 3185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// getSpelling - This method is used to get the spelling of a token into a 3195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// preallocated buffer, instead of as an std::string. The caller is required 3205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// to allocate enough space for the token, which is guaranteed to be at least 3215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// Tok.getLength() bytes long. The actual length of the token is returned. 3225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// 3235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// Note that this method may do two possible things: it may either fill in 3245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// the buffer specified with characters, or it may *change the input pointer* 3255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// to point to a constant buffer with the data already in it (avoiding a 3265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// copy). The caller is not allowed to modify the returned buffer pointer 3275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// if an internal buffer is returned. 328d217773f106856a11879ec79dc468efefaf2ee75Chris Lattnerunsigned Preprocessor::getSpelling(const Token &Tok, 3295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *&Buffer) const { 3305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer assert((int)Tok.getLength() >= 0 && "Token character range is bogus!"); 3311eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 3325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this token is an identifier, just return the string from the identifier 3335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // table, which is very quick. 3345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (const IdentifierInfo *II = Tok.getIdentifierInfo()) { 335e013d685c6689ac7ae103ee88acf573422d1ed6aDaniel Dunbar Buffer = II->getNameStart(); 336e1dccaefe2d44a9f83f8c3f087438a3297254cacChris Lattner return II->getLength(); 3375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 338b70e3dafb9618f34017061400dc19ac5e3539a6dTed Kremenek 3395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, compute the start of the token in the input lexer buffer. 34047246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner const char *TokStart = 0; 3411eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 34247246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner if (Tok.isLiteral()) 34347246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner TokStart = Tok.getLiteralData(); 3441eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 34547246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner if (TokStart == 0) 34647246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner TokStart = SourceMgr.getCharacterData(Tok.getLocation()); 3475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this token contains nothing interesting, return it directly. 3495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (!Tok.needsCleaning()) { 3505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Buffer = TokStart; 3515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return Tok.getLength(); 3525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 3531eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 3545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, hard case, relex the characters into the string. 3555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char *OutBuf = const_cast<char*>(Buffer); 3565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer for (const char *Ptr = TokStart, *End = TokStart+Tok.getLength(); 3575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Ptr != End; ) { 3585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned CharSize; 3595f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer *OutBuf++ = Lexer::getCharAndSizeNoWarn(Ptr, CharSize, Features); 3605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Ptr += CharSize; 3615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 3625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer assert(unsigned(OutBuf-Buffer) != Tok.getLength() && 3635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer "NeedsCleaning flag set on something that didn't need cleaning!"); 3641eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 3655f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return OutBuf-Buffer; 3665f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 3675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// CreateString - Plop the specified string into a scratch buffer and return a 3695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// location for it. If specified, the source location provides a source 3705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// location for the token. 37147246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattnervoid Preprocessor::CreateString(const char *Buf, unsigned Len, Token &Tok, 37247246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner SourceLocation InstantiationLoc) { 37347246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner Tok.setLength(Len); 3741eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 37547246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner const char *DestPtr; 37647246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner SourceLocation Loc = ScratchBuf->getToken(Buf, Len, DestPtr); 3771eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 37847246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner if (InstantiationLoc.isValid()) 379e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner Loc = SourceMgr.createInstantiationLoc(Loc, InstantiationLoc, 380e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner InstantiationLoc, Len); 38147246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner Tok.setLocation(Loc); 3821eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 38347246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner // If this is a literal token, set the pointer data. 38447246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner if (Tok.isLiteral()) 38547246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner Tok.setLiteralData(DestPtr); 3865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 3875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 3885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 38997ba77cf09bf7b83b679165ce67ad7d49ffd568cChris Lattner/// AdvanceToTokenCharacter - Given a location that specifies the start of a 39097ba77cf09bf7b83b679165ce67ad7d49ffd568cChris Lattner/// token, return a new location that specifies a character within the token. 3911eb4433ac451dc16f4133a88af2d002ac26c58efMike StumpSourceLocation Preprocessor::AdvanceToTokenCharacter(SourceLocation TokStart, 39297ba77cf09bf7b83b679165ce67ad7d49ffd568cChris Lattner unsigned CharNo) { 393f7cf85b330bedd2877e1371fb0a83e99751ae162Chris Lattner // Figure out how many physical characters away the specified instantiation 39497ba77cf09bf7b83b679165ce67ad7d49ffd568cChris Lattner // character is. This needs to take into consideration newlines and 39597ba77cf09bf7b83b679165ce67ad7d49ffd568cChris Lattner // trigraphs. 3969dc1f530c086d2c16f8cba758b0f59a5bf41323aChris Lattner const char *TokPtr = SourceMgr.getCharacterData(TokStart); 3971eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 39888e2524b8f3393975872a23bbe6e8b3cf50d6773Chris Lattner // If they request the first char of the token, we're trivially done. 39988e2524b8f3393975872a23bbe6e8b3cf50d6773Chris Lattner if (CharNo == 0 && Lexer::isObviouslySimpleCharacter(*TokPtr)) 40088e2524b8f3393975872a23bbe6e8b3cf50d6773Chris Lattner return TokStart; 4011eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 4029dc1f530c086d2c16f8cba758b0f59a5bf41323aChris Lattner unsigned PhysOffset = 0; 4031eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 40497ba77cf09bf7b83b679165ce67ad7d49ffd568cChris Lattner // The usual case is that tokens don't contain anything interesting. Skip 40597ba77cf09bf7b83b679165ce67ad7d49ffd568cChris Lattner // over the uninteresting characters. If a token only consists of simple 40697ba77cf09bf7b83b679165ce67ad7d49ffd568cChris Lattner // chars, this method is extremely fast. 40788e2524b8f3393975872a23bbe6e8b3cf50d6773Chris Lattner while (Lexer::isObviouslySimpleCharacter(*TokPtr)) { 40888e2524b8f3393975872a23bbe6e8b3cf50d6773Chris Lattner if (CharNo == 0) 40988e2524b8f3393975872a23bbe6e8b3cf50d6773Chris Lattner return TokStart.getFileLocWithOffset(PhysOffset); 4109dc1f530c086d2c16f8cba758b0f59a5bf41323aChris Lattner ++TokPtr, --CharNo, ++PhysOffset; 41188e2524b8f3393975872a23bbe6e8b3cf50d6773Chris Lattner } 4121eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 41328c90ad7ef7c609a0b10c2f68b519ef69e7084d1Chris Lattner // If we have a character that may be a trigraph or escaped newline, use a 41497ba77cf09bf7b83b679165ce67ad7d49ffd568cChris Lattner // lexer to parse it correctly. 41588e2524b8f3393975872a23bbe6e8b3cf50d6773Chris Lattner for (; CharNo; --CharNo) { 41688e2524b8f3393975872a23bbe6e8b3cf50d6773Chris Lattner unsigned Size; 41788e2524b8f3393975872a23bbe6e8b3cf50d6773Chris Lattner Lexer::getCharAndSizeNoWarn(TokPtr, Size, Features); 41888e2524b8f3393975872a23bbe6e8b3cf50d6773Chris Lattner TokPtr += Size; 41988e2524b8f3393975872a23bbe6e8b3cf50d6773Chris Lattner PhysOffset += Size; 42097ba77cf09bf7b83b679165ce67ad7d49ffd568cChris Lattner } 4211eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 42288e2524b8f3393975872a23bbe6e8b3cf50d6773Chris Lattner // Final detail: if we end up on an escaped newline, we want to return the 42388e2524b8f3393975872a23bbe6e8b3cf50d6773Chris Lattner // location of the actual byte of the token. For example foo\<newline>bar 42488e2524b8f3393975872a23bbe6e8b3cf50d6773Chris Lattner // advanced by 3 should return the location of b, not of \\. One compounding 42588e2524b8f3393975872a23bbe6e8b3cf50d6773Chris Lattner // detail of this is that the escape may be made by a trigraph. 42688e2524b8f3393975872a23bbe6e8b3cf50d6773Chris Lattner if (!Lexer::isObviouslySimpleCharacter(*TokPtr)) 42788e2524b8f3393975872a23bbe6e8b3cf50d6773Chris Lattner PhysOffset = Lexer::SkipEscapedNewLines(TokPtr)-TokPtr; 4281eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 4299dc1f530c086d2c16f8cba758b0f59a5bf41323aChris Lattner return TokStart.getFileLocWithOffset(PhysOffset); 43097ba77cf09bf7b83b679165ce67ad7d49ffd568cChris Lattner} 43197ba77cf09bf7b83b679165ce67ad7d49ffd568cChris Lattner 432b2fb6de9070fea9abc56c8e8d5469066e964cefeDouglas Gregor/// \brief Computes the source location just past the end of the 433b2fb6de9070fea9abc56c8e8d5469066e964cefeDouglas Gregor/// token at this source location. 434b2fb6de9070fea9abc56c8e8d5469066e964cefeDouglas Gregor/// 435b2fb6de9070fea9abc56c8e8d5469066e964cefeDouglas Gregor/// This routine can be used to produce a source location that 436b2fb6de9070fea9abc56c8e8d5469066e964cefeDouglas Gregor/// points just past the end of the token referenced by \p Loc, and 437b2fb6de9070fea9abc56c8e8d5469066e964cefeDouglas Gregor/// is generally used when a diagnostic needs to point just after a 438b2fb6de9070fea9abc56c8e8d5469066e964cefeDouglas Gregor/// token where it expected something different that it received. If 439b2fb6de9070fea9abc56c8e8d5469066e964cefeDouglas Gregor/// the returned source location would not be meaningful (e.g., if 440b2fb6de9070fea9abc56c8e8d5469066e964cefeDouglas Gregor/// it points into a macro), this routine returns an invalid 441b2fb6de9070fea9abc56c8e8d5469066e964cefeDouglas Gregor/// source location. 442b2fb6de9070fea9abc56c8e8d5469066e964cefeDouglas GregorSourceLocation Preprocessor::getLocForEndOfToken(SourceLocation Loc) { 443b2fb6de9070fea9abc56c8e8d5469066e964cefeDouglas Gregor if (Loc.isInvalid() || !Loc.isFileID()) 444b2fb6de9070fea9abc56c8e8d5469066e964cefeDouglas Gregor return SourceLocation(); 445b2fb6de9070fea9abc56c8e8d5469066e964cefeDouglas Gregor 4462c78b873f4f3823ae859c15674cb3d76c8554113Chris Lattner unsigned Len = Lexer::MeasureTokenLength(Loc, getSourceManager(), Features); 447b2fb6de9070fea9abc56c8e8d5469066e964cefeDouglas Gregor return AdvanceToTokenCharacter(Loc, Len); 448b2fb6de9070fea9abc56c8e8d5469066e964cefeDouglas Gregor} 449b2fb6de9070fea9abc56c8e8d5469066e964cefeDouglas Gregor 450b2fb6de9070fea9abc56c8e8d5469066e964cefeDouglas Gregor 45197ba77cf09bf7b83b679165ce67ad7d49ffd568cChris Lattner 45253b0dabbe52219a8057659b90539837394ef0fa1Chris Lattner//===----------------------------------------------------------------------===// 45353b0dabbe52219a8057659b90539837394ef0fa1Chris Lattner// Preprocessor Initialization Methods 45453b0dabbe52219a8057659b90539837394ef0fa1Chris Lattner//===----------------------------------------------------------------------===// 45553b0dabbe52219a8057659b90539837394ef0fa1Chris Lattner 45653b0dabbe52219a8057659b90539837394ef0fa1Chris Lattner 45753b0dabbe52219a8057659b90539837394ef0fa1Chris Lattner/// EnterMainSourceFile - Enter the specified FileID as the main source file, 4586b61602bc7695d40b3c4d09032be1ca1dd0b3c31Nate Begeman/// which implicitly adds the builtin defines etc. 45995041a2029a069386ee67439f6d0fb524a9d184fTed Kremenekvoid Preprocessor::EnterMainSourceFile() { 46005db4278ecd193edcff63fb8c54818226fceaad2Chris Lattner // We do not allow the preprocessor to reenter the main file. Doing so will 46105db4278ecd193edcff63fb8c54818226fceaad2Chris Lattner // cause FileID's to accumulate information from both runs (e.g. #line 46205db4278ecd193edcff63fb8c54818226fceaad2Chris Lattner // information) and predefined macros aren't guaranteed to be set properly. 46305db4278ecd193edcff63fb8c54818226fceaad2Chris Lattner assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!"); 4642b2453a7d8fe732561795431f39ceb2b2a832d84Chris Lattner FileID MainFileID = SourceMgr.getMainFileID(); 4651eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 46653b0dabbe52219a8057659b90539837394ef0fa1Chris Lattner // Enter the main file source buffer. 46763ceaa32a2371e38d1f912080fe471285e6b6e56Daniel Dunbar std::string ErrorStr; 46863ceaa32a2371e38d1f912080fe471285e6b6e56Daniel Dunbar bool Res = EnterSourceFile(MainFileID, 0, ErrorStr); 46963ceaa32a2371e38d1f912080fe471285e6b6e56Daniel Dunbar assert(!Res && "Entering main file should not fail!"); 4701eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 471b283298f03d2b512204d14ca687f9de263eeff0fChris Lattner // Tell the header info that the main file was entered. If the file is later 472b283298f03d2b512204d14ca687f9de263eeff0fChris Lattner // #imported, it won't be re-entered. 4732b2453a7d8fe732561795431f39ceb2b2a832d84Chris Lattner if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID)) 474b283298f03d2b512204d14ca687f9de263eeff0fChris Lattner HeaderInfo.IncrementIncludeCount(FE); 4751eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 476ffd6e39006a7177c4c537a6ce701bcd852fb3e7cBenjamin Kramer // Preprocess Predefines to populate the initial preprocessor state. 4771eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump llvm::MemoryBuffer *SB = 478ffd6e39006a7177c4c537a6ce701bcd852fb3e7cBenjamin Kramer llvm::MemoryBuffer::getMemBufferCopy(Predefines.data(), 479ffd6e39006a7177c4c537a6ce701bcd852fb3e7cBenjamin Kramer Predefines.data() + Predefines.size(), 480f2e880a99c8b08976069331283392cc22b07701dChris Lattner "<built-in>"); 48153b0dabbe52219a8057659b90539837394ef0fa1Chris Lattner assert(SB && "Cannot fail to create predefined source buffer"); 4822b2453a7d8fe732561795431f39ceb2b2a832d84Chris Lattner FileID FID = SourceMgr.createFileIDForMemBuffer(SB); 4832b2453a7d8fe732561795431f39ceb2b2a832d84Chris Lattner assert(!FID.isInvalid() && "Could not create FileID for predefines?"); 4841eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 48553b0dabbe52219a8057659b90539837394ef0fa1Chris Lattner // Start parsing the predefines. 48663ceaa32a2371e38d1f912080fe471285e6b6e56Daniel Dunbar Res = EnterSourceFile(FID, 0, ErrorStr); 48763ceaa32a2371e38d1f912080fe471285e6b6e56Daniel Dunbar assert(!Res && "Entering predefines should not fail!"); 48853b0dabbe52219a8057659b90539837394ef0fa1Chris Lattner} 48997ba77cf09bf7b83b679165ce67ad7d49ffd568cChris Lattner 4905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 4915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 4925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// Lexer Event Handling. 4935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 4945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 4955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// LookUpIdentifierInfo - Given a tok::identifier token, look up the 4965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// identifier information for the token and install it into the token. 497d217773f106856a11879ec79dc468efefaf2ee75Chris LattnerIdentifierInfo *Preprocessor::LookUpIdentifierInfo(Token &Identifier, 498c3222091e1ffa35d0264ca6b680a88c9dc84ede2Daniel Dunbar const char *BufPtr) const { 49922f6bbcafa8871f4f20c4402d9cbc5c024fee99aChris Lattner assert(Identifier.is(tok::identifier) && "Not an identifier!"); 5005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer assert(Identifier.getIdentifierInfo() == 0 && "Identinfo already exists!"); 5011eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 5025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Look up this token, see if it is a macro, or if it is a language keyword. 5035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer IdentifierInfo *II; 5045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (BufPtr && !Identifier.needsCleaning()) { 5055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // No cleaning needed, just use the characters from the lexed buffer. 5063da736c1143126be19b253804b3b135ebcd3d6ffDaniel Dunbar II = getIdentifierInfo(llvm::StringRef(BufPtr, Identifier.getLength())); 5075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 5085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Cleaning needed, alloca a buffer, clean into it, then use the buffer. 509c35717a1b34269fc5d78226b1624f53e1c073ed6Chris Lattner llvm::SmallVector<char, 64> IdentifierBuffer; 510c35717a1b34269fc5d78226b1624f53e1c073ed6Chris Lattner IdentifierBuffer.resize(Identifier.getLength()); 511c35717a1b34269fc5d78226b1624f53e1c073ed6Chris Lattner const char *TmpBuf = &IdentifierBuffer[0]; 5125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned Size = getSpelling(Identifier, TmpBuf); 5133da736c1143126be19b253804b3b135ebcd3d6ffDaniel Dunbar II = getIdentifierInfo(llvm::StringRef(TmpBuf, Size)); 5145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 5155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Identifier.setIdentifierInfo(II); 5165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return II; 5175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 5185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 5195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 5205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// HandleIdentifier - This callback is invoked when the lexer reads an 5215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// identifier. This callback looks up the identifier in the map and/or 5225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// potentially macro expands it or turns it into a named token (like 'for'). 5236a170eb3ea6d6319277becabef68eb1a26bf8766Chris Lattner/// 5246a170eb3ea6d6319277becabef68eb1a26bf8766Chris Lattner/// Note that callers of this method are guarded by checking the 5256a170eb3ea6d6319277becabef68eb1a26bf8766Chris Lattner/// IdentifierInfo's 'isHandleIdentifierCase' bit. If this method changes, the 5266a170eb3ea6d6319277becabef68eb1a26bf8766Chris Lattner/// IdentifierInfo methods that compute these properties will need to change to 5276a170eb3ea6d6319277becabef68eb1a26bf8766Chris Lattner/// match. 528d217773f106856a11879ec79dc468efefaf2ee75Chris Lattnervoid Preprocessor::HandleIdentifier(Token &Identifier) { 5295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer assert(Identifier.getIdentifierInfo() && 5305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer "Can't handle identifiers without identifier info!"); 5311eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 5325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer IdentifierInfo &II = *Identifier.getIdentifierInfo(); 5335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 5345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this identifier was poisoned, and if it was not produced from a macro 5355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // expansion, emit an error. 5361a531570d294ffc300cce1bc5e73de847a370f4fTed Kremenek if (II.isPoisoned() && CurPPLexer) { 5375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (&II != Ident__VA_ARGS__) // We warn about __VA_ARGS__ with poisoning. 5385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Diag(Identifier, diag::err_pp_used_poisoned_id); 5395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer else 5405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Diag(Identifier, diag::ext_pp_bad_vaargs_use); 5415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 5421eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 5435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is a macro to be expanded, do it. 544cc1a875f94630e58d24a55577ffbf0e89b7da8c7Chris Lattner if (MacroInfo *MI = getMacroInfo(&II)) { 5455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (!DisableMacroExpansion && !Identifier.isExpandDisabled()) { 5465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (MI->isEnabled()) { 5475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (!HandleMacroExpandedIdentifier(Identifier, MI)) 5485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return; 5495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 5505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // C99 6.10.3.4p2 says that a disabled macro may never again be 5515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // expanded, even if it's in a context where it could be expanded in the 5525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // future. 553d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Identifier.setFlag(Token::DisableExpand); 5545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 5555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 5565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 5575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 5585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // C++ 2.11p2: If this is an alternative representation of a C++ operator, 5595f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // then we act as if it is the actual operator and not the textual 5605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // representation of it. 5615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (II.isCPlusPlusOperatorKeyword()) 5625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Identifier.setIdentifierInfo(0); 5635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 5645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is an extension token, diagnose its use. 565b4eaf9cf5f79c86a3f7564f4dfdea57f165ca45cSteve Naroff // We avoid diagnosing tokens that originate from macro definitions. 5662962f4d71d26817780e7441b23e0e91214fceb5eEli Friedman // FIXME: This warning is disabled in cases where it shouldn't be, 5672962f4d71d26817780e7441b23e0e91214fceb5eEli Friedman // like "#define TY typeof", "TY(1) x". 5682962f4d71d26817780e7441b23e0e91214fceb5eEli Friedman if (II.isExtensionToken() && !DisableMacroExpansion) 5695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Diag(Identifier, diag::ext_token_used); 5705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 5712e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregor 5722e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregorvoid Preprocessor::AddCommentHandler(CommentHandler *Handler) { 5732e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregor assert(Handler && "NULL comment handler"); 5742e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregor assert(std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler) == 5752e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregor CommentHandlers.end() && "Comment handler already registered"); 5762e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregor CommentHandlers.push_back(Handler); 5772e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregor} 5782e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregor 5792e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregorvoid Preprocessor::RemoveCommentHandler(CommentHandler *Handler) { 5802e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregor std::vector<CommentHandler *>::iterator Pos 5812e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregor = std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler); 5822e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregor assert(Pos != CommentHandlers.end() && "Comment handler not registered"); 5832e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregor CommentHandlers.erase(Pos); 5842e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregor} 5852e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregor 586046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattnerbool Preprocessor::HandleComment(Token &result, SourceRange Comment) { 587046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner bool AnyPendingTokens = false; 5882e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregor for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(), 5892e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregor HEnd = CommentHandlers.end(); 590046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner H != HEnd; ++H) { 591046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner if ((*H)->HandleComment(*this, Comment)) 592046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner AnyPendingTokens = true; 593046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner } 594046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner if (!AnyPendingTokens || getCommentRetentionState()) 595046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner return false; 596046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner Lex(result); 597046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner return true; 5982e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregor} 5992e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregor 6002e22253e03e175144aeb9d13350a12fd83f858beDouglas GregorCommentHandler::~CommentHandler() { } 601