Lexer.cpp revision 20afc2977cd0a6bacbe6218a633cd59a24463e2f
15f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===--- Lexer.cpp - C Language Family Lexer ------------------------------===// 25f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 35f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// The LLVM Compiler Infrastructure 45f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 50bc735ffcfb223c0186419547abaa5c84482663eChris Lattner// This file is distributed under the University of Illinois Open Source 60bc735ffcfb223c0186419547abaa5c84482663eChris Lattner// License. See LICENSE.TXT for details. 75f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 85f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 95f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 10d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner// This file implements the Lexer and Token interfaces. 115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// TODO: GCC Diagnostics emitted by the lexer: 155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// PEDWARN: (form feed|vertical tab) in preprocessing directive 165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// Universal characters, unicode, char mapping: 185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// WARNING: `%.*s' is not in NFKC 195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// WARNING: `%.*s' is not in NFC 205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// Other: 225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// TODO: Options to support: 235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// -fexec-charset,-fwide-exec-charset 245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// 255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "clang/Lex/Lexer.h" 289dc1f530c086d2c16f8cba758b0f59a5bf41323aChris Lattner#include "clang/Basic/SourceManager.h" 2955fc873017f10f6f566b182b70f6fc22aefa3464Chandler Carruth#include "clang/Lex/CodeCompletionHandler.h" 3055fc873017f10f6f566b182b70f6fc22aefa3464Chandler Carruth#include "clang/Lex/LexDiagnostic.h" 3155fc873017f10f6f566b182b70f6fc22aefa3464Chandler Carruth#include "clang/Lex/Preprocessor.h" 32d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis#include "llvm/ADT/STLExtras.h" 33c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose#include "llvm/ADT/StringExtras.h" 3455fc873017f10f6f566b182b70f6fc22aefa3464Chandler Carruth#include "llvm/ADT/StringSwitch.h" 35409a03671224d4e5bdab1594c43baf070148f830Chris Lattner#include "llvm/Support/Compiler.h" 36cb5620c9b213f4bd323912159fdddda35e258a14Dmitri Gribenko#include "llvm/Support/ConvertUTF.h" 375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include "llvm/Support/MemoryBuffer.h" 382fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper#include <cstring> 395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerusing namespace clang; 405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 41a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattnerstatic void InitCharacterInfo(); 425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 43dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner//===----------------------------------------------------------------------===// 44dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner// Token Class Implementation 45dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner//===----------------------------------------------------------------------===// 46dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner 471eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump/// isObjCAtKeyword - Return true if we have an ObjC keyword identifier. 48dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattnerbool Token::isObjCAtKeyword(tok::ObjCKeywordKind objcKey) const { 49bec1c9d6f3feeec112cc8eeae90c1be29c6aaf13Douglas Gregor if (IdentifierInfo *II = getIdentifierInfo()) 50bec1c9d6f3feeec112cc8eeae90c1be29c6aaf13Douglas Gregor return II->getObjCKeywordID() == objcKey; 51bec1c9d6f3feeec112cc8eeae90c1be29c6aaf13Douglas Gregor return false; 52dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner} 53dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner 54dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner/// getObjCKeywordID - Return the ObjC keyword kind. 55dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattnertok::ObjCKeywordKind Token::getObjCKeywordID() const { 56dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner IdentifierInfo *specId = getIdentifierInfo(); 57dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner return specId ? specId->getObjCKeywordID() : tok::objc_not_keyword; 58dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner} 59dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner 6053702cd401b8fdca985aede7732c2f6a82ad9b1cChris Lattner 61dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner//===----------------------------------------------------------------------===// 62dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner// Lexer Class Implementation 63dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner//===----------------------------------------------------------------------===// 64dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner 6599ba9e3bd70671f3441fb974895f226a83ce0e66David Blaikievoid Lexer::anchor() { } 6699ba9e3bd70671f3441fb974895f226a83ce0e66David Blaikie 671eb4433ac451dc16f4133a88af2d002ac26c58efMike Stumpvoid Lexer::InitLexer(const char *BufStart, const char *BufPtr, 6822d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner const char *BufEnd) { 69a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner InitCharacterInfo(); 701eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 7122d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner BufferStart = BufStart; 7222d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner BufferPtr = BufPtr; 7322d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner BufferEnd = BufEnd; 741eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 7522d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner assert(BufEnd[0] == 0 && 7622d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner "We assume that the input buffer has a null character at the end" 7722d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner " to simplify lexing!"); 781eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 79156119df1d076b63609618976281961283f871dbEric Christopher // Check whether we have a BOM in the beginning of the buffer. If yes - act 80156119df1d076b63609618976281961283f871dbEric Christopher // accordingly. Right now we support only UTF-8 with and without BOM, so, just 81156119df1d076b63609618976281961283f871dbEric Christopher // skip the UTF-8 BOM if it's present. 82156119df1d076b63609618976281961283f871dbEric Christopher if (BufferStart == BufferPtr) { 83156119df1d076b63609618976281961283f871dbEric Christopher // Determine the size of the BOM. 845f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner StringRef Buf(BufferStart, BufferEnd - BufferStart); 85969f9d47338fc36ebb6d24ad3a51e45eda07fd58Eli Friedman size_t BOMLength = llvm::StringSwitch<size_t>(Buf) 86156119df1d076b63609618976281961283f871dbEric Christopher .StartsWith("\xEF\xBB\xBF", 3) // UTF-8 BOM 87156119df1d076b63609618976281961283f871dbEric Christopher .Default(0); 88156119df1d076b63609618976281961283f871dbEric Christopher 89156119df1d076b63609618976281961283f871dbEric Christopher // Skip the BOM. 90156119df1d076b63609618976281961283f871dbEric Christopher BufferPtr += BOMLength; 91156119df1d076b63609618976281961283f871dbEric Christopher } 92156119df1d076b63609618976281961283f871dbEric Christopher 9322d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner Is_PragmaLexer = false; 94d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith CurrentConflictMarkerState = CMK_None; 95156119df1d076b63609618976281961283f871dbEric Christopher 9622d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner // Start of the file is a start of line. 9722d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner IsAtStartOfLine = true; 981eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 9922d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner // We are not after parsing a #. 10022d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner ParsingPreprocessorDirective = false; 1011eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 10222d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner // We are not after parsing #include. 10322d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner ParsingFilename = false; 1041eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 10522d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner // We are not in raw mode. Raw mode disables diagnostics and interpretation 10622d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner // of tokens (e.g. identifiers, thus disabling macro expansion). It is used 10722d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner // to quickly lex the tokens of the buffer, e.g. when handling a "#if 0" block 10822d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner // or otherwise skipping over tokens. 10922d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner LexingRawMode = false; 1101eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 11122d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner // Default to not keeping comments. 11222d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner ExtendedTokenMode = 0; 11322d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner} 11422d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner 1150770dabb1ae81a2a9c2e7199262067103062a0b3Chris Lattner/// Lexer constructor - Create a new lexer object for the specified buffer 1160770dabb1ae81a2a9c2e7199262067103062a0b3Chris Lattner/// with the specified preprocessor managing the lexing process. This lexer 1170770dabb1ae81a2a9c2e7199262067103062a0b3Chris Lattner/// assumes that the associated file buffer and Preprocessor objects will 1180770dabb1ae81a2a9c2e7199262067103062a0b3Chris Lattner/// outlive it, so it doesn't take ownership of either of them. 1196e2901407bff59aeb4cc301cc58b034723d0eb49Chris LattnerLexer::Lexer(FileID FID, const llvm::MemoryBuffer *InputFile, Preprocessor &PP) 12088d3ac1341aa016cabd966c5b113a95ac05ea43fChris Lattner : PreprocessorLexer(&PP, FID), 12188d3ac1341aa016cabd966c5b113a95ac05ea43fChris Lattner FileLoc(PP.getSourceManager().getLocForStartOfFile(FID)), 1224e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie LangOpts(PP.getLangOpts()) { 1231eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1240770dabb1ae81a2a9c2e7199262067103062a0b3Chris Lattner InitLexer(InputFile->getBufferStart(), InputFile->getBufferStart(), 1250770dabb1ae81a2a9c2e7199262067103062a0b3Chris Lattner InputFile->getBufferEnd()); 1261eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1270770dabb1ae81a2a9c2e7199262067103062a0b3Chris Lattner // Default to keeping comments if the preprocessor wants them. 1280770dabb1ae81a2a9c2e7199262067103062a0b3Chris Lattner SetCommentRetentionState(PP.getCommentRetentionState()); 1290770dabb1ae81a2a9c2e7199262067103062a0b3Chris Lattner} 130dbf388b54bb1688ee4470c61235494cbc34b8f3cChris Lattner 131168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner/// Lexer constructor - Create a new raw lexer object. This object is only 132092bf67e5ca560d2fc6aa70be1f172b8b3a5ff96Dmitri Gribenko/// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text 133590f0cc643274267d4d41125b62557e1d87886c3Chris Lattner/// range will outlive it, so it doesn't take ownership of it. 1344e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid BlaikieLexer::Lexer(SourceLocation fileloc, const LangOptions &langOpts, 135de96c0f29c4cacabe6ea577c61db87c2a85aea6cChris Lattner const char *BufStart, const char *BufPtr, const char *BufEnd) 1364e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie : FileLoc(fileloc), LangOpts(langOpts) { 13722d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner 13822d91ca8d7c134eac5cc6a4869e6a84c461ad624Chris Lattner InitLexer(BufStart, BufPtr, BufEnd); 1391eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 140168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner // We *are* in raw mode. 141168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner LexingRawMode = true; 1425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 1435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 144025c3a66402fb713c2d9bf5dc174ff264765379aChris Lattner/// Lexer constructor - Create a new raw lexer object. This object is only 145092bf67e5ca560d2fc6aa70be1f172b8b3a5ff96Dmitri Gribenko/// suitable for calls to 'LexFromRawLexer'. This lexer assumes that the text 146025c3a66402fb713c2d9bf5dc174ff264765379aChris Lattner/// range will outlive it, so it doesn't take ownership of it. 1476e2901407bff59aeb4cc301cc58b034723d0eb49Chris LattnerLexer::Lexer(FileID FID, const llvm::MemoryBuffer *FromFile, 1484e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie const SourceManager &SM, const LangOptions &langOpts) 1494e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie : FileLoc(SM.getLocForStartOfFile(FID)), LangOpts(langOpts) { 150025c3a66402fb713c2d9bf5dc174ff264765379aChris Lattner 1511eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump InitLexer(FromFile->getBufferStart(), FromFile->getBufferStart(), 152025c3a66402fb713c2d9bf5dc174ff264765379aChris Lattner FromFile->getBufferEnd()); 1531eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 154025c3a66402fb713c2d9bf5dc174ff264765379aChris Lattner // We *are* in raw mode. 155025c3a66402fb713c2d9bf5dc174ff264765379aChris Lattner LexingRawMode = true; 156025c3a66402fb713c2d9bf5dc174ff264765379aChris Lattner} 157025c3a66402fb713c2d9bf5dc174ff264765379aChris Lattner 15842e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// Create_PragmaLexer: Lexer constructor - Create a new lexer object for 15942e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// _Pragma expansion. This has a variety of magic semantics that this method 16042e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// sets up. It returns a new'd Lexer that must be delete'd when done. 16142e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// 16242e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// On entrance to this routine, TokStartLoc is a macro location which has a 16342e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// spelling loc that indicates the bytes to be lexed for the token and an 164433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth/// expansion location that indicates where all lexed tokens should be 16542e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// "expanded from". 16642e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// 16742e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// FIXME: It would really be nice to make _Pragma just be a wrapper around a 16842e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// normal lexer that remaps tokens as they fly by. This would require making 16942e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// Preprocessor::Lex virtual. Given that, we could just dump in a magic lexer 17042e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// interface that could handle this stuff. This would pull GetMappedTokenLoc 17142e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// out of the critical path of the lexer! 17242e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner/// 1731eb4433ac451dc16f4133a88af2d002ac26c58efMike StumpLexer *Lexer::Create_PragmaLexer(SourceLocation SpellingLoc, 174433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth SourceLocation ExpansionLocStart, 175433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth SourceLocation ExpansionLocEnd, 176bcc2a67e5180612417727cbdd8afd0f79fdf726dChris Lattner unsigned TokLen, Preprocessor &PP) { 17742e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner SourceManager &SM = PP.getSourceManager(); 17842e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner 17942e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner // Create the lexer as if we were going to lex the file normally. 180a11d61793341fea195c29a0dab3fbd74f2b39a8cChris Lattner FileID SpellingFID = SM.getFileID(SpellingLoc); 1816e2901407bff59aeb4cc301cc58b034723d0eb49Chris Lattner const llvm::MemoryBuffer *InputFile = SM.getBuffer(SpellingFID); 1826e2901407bff59aeb4cc301cc58b034723d0eb49Chris Lattner Lexer *L = new Lexer(SpellingFID, InputFile, PP); 1831eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 18442e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner // Now that the lexer is created, change the start/end locations so that we 18542e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner // just lex the subsection of the file that we want. This is lexing from a 18642e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner // scratch buffer. 18742e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner const char *StrData = SM.getCharacterData(SpellingLoc); 1881eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 18942e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner L->BufferPtr = StrData; 19042e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner L->BufferEnd = StrData+TokLen; 1911fa495304c81e03f07f278a47b5efe9317104aabChris Lattner assert(L->BufferEnd[0] == 0 && "Buffer is not nul terminated!"); 19242e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner 19342e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner // Set the SourceLocation with the remapping information. This ensures that 19442e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner // GetMappedTokenLoc will remap the tokens as they are lexed. 195bf340e452339e374ea6eef78c1f0a2abdd16c5a3Chandler Carruth L->FileLoc = SM.createExpansionLoc(SM.getLocForStartOfFile(SpellingFID), 196bf340e452339e374ea6eef78c1f0a2abdd16c5a3Chandler Carruth ExpansionLocStart, 197bf340e452339e374ea6eef78c1f0a2abdd16c5a3Chandler Carruth ExpansionLocEnd, TokLen); 1981eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 19942e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner // Ensure that the lexer thinks it is inside a directive, so that end \n will 20084021556baceb76eedf7d44be8ba71d9b8cfaccePeter Collingbourne // return an EOD token. 20142e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner L->ParsingPreprocessorDirective = true; 2021eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 20342e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner // This lexer really is for _Pragma. 20442e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner L->Is_PragmaLexer = true; 20542e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner return L; 20642e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner} 20742e00d19b0dac64732eb5449d52a076282fcbf77Chris Lattner 208168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner 2095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// Stringify - Convert the specified string into a C string, with surrounding 2105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// ""'s, and with escaped \ and " characters. 2115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerstd::string Lexer::Stringify(const std::string &Str, bool Charify) { 2125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer std::string Result = Str; 2135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char Quote = Charify ? '\'' : '"'; 2145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer for (unsigned i = 0, e = Result.size(); i != e; ++i) { 2155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Result[i] == '\\' || Result[i] == Quote) { 2165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Result.insert(Result.begin()+i, '\\'); 2175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++i; ++e; 2185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 2195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 2205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return Result; 2215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 2225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 223d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner/// Stringify - Convert the specified string into a C string by escaping '\' 224d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner/// and " characters. This does not add surrounding ""'s to the string. 2255f9e272e632e951b1efe824cd16acb4d96077930Chris Lattnervoid Lexer::Stringify(SmallVectorImpl<char> &Str) { 226d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner for (unsigned i = 0, e = Str.size(); i != e; ++i) { 227d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner if (Str[i] == '\\' || Str[i] == '"') { 228d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner Str.insert(Str.begin()+i, '\\'); 229d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner ++i; ++e; 230d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner } 231d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner } 232d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner} 233d8e3083840fef752d11ca183f42786470ed061e3Chris Lattner 234b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner//===----------------------------------------------------------------------===// 235b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner// Token Spelling 236b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner//===----------------------------------------------------------------------===// 237b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner 23830cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith/// \brief Slow case of getSpelling. Extract the characters comprising the 23930cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith/// spelling of this token from the provided input buffer. 24030cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smithstatic size_t getSpellingSlow(const Token &Tok, const char *BufPtr, 24130cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith const LangOptions &LangOpts, char *Spelling) { 24230cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith assert(Tok.needsCleaning() && "getSpellingSlow called on simple token"); 24330cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith 24430cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith size_t Length = 0; 24530cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith const char *BufEnd = BufPtr + Tok.getLength(); 24630cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith 24730cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith if (Tok.is(tok::string_literal)) { 24830cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith // Munch the encoding-prefix and opening double-quote. 24930cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith while (BufPtr < BufEnd) { 25030cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith unsigned Size; 25130cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts); 25230cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith BufPtr += Size; 25330cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith 25430cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith if (Spelling[Length - 1] == '"') 25530cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith break; 25630cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith } 25730cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith 25830cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith // Raw string literals need special handling; trigraph expansion and line 25930cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith // splicing do not occur within their d-char-sequence nor within their 26030cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith // r-char-sequence. 26130cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith if (Length >= 2 && 26230cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith Spelling[Length - 2] == 'R' && Spelling[Length - 1] == '"') { 26330cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith // Search backwards from the end of the token to find the matching closing 26430cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith // quote. 26530cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith const char *RawEnd = BufEnd; 26630cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith do --RawEnd; while (*RawEnd != '"'); 26730cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith size_t RawLength = RawEnd - BufPtr + 1; 26830cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith 26930cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith // Everything between the quotes is included verbatim in the spelling. 27030cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith memcpy(Spelling + Length, BufPtr, RawLength); 27130cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith Length += RawLength; 27230cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith BufPtr += RawLength; 27330cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith 27430cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith // The rest of the token is lexed normally. 27530cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith } 27630cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith } 27730cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith 27830cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith while (BufPtr < BufEnd) { 27930cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith unsigned Size; 28030cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith Spelling[Length++] = Lexer::getCharAndSizeNoWarn(BufPtr, Size, LangOpts); 28130cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith BufPtr += Size; 28230cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith } 28330cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith 28430cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith assert(Length < Tok.getLength() && 28530cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith "NeedsCleaning flag set on token that didn't need cleaning!"); 28630cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith return Length; 28730cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith} 28830cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith 289b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// getSpelling() - Return the 'spelling' of this token. The spelling of a 290b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// token are the characters used to represent the token in the source file 291b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// after trigraph expansion and escaped-newline folding. In particular, this 292b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// wants to get the true, uncanonicalized, spelling of things like digraphs 293b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// UCNs, etc. 2945f9e272e632e951b1efe824cd16acb4d96077930Chris LattnerStringRef Lexer::getSpelling(SourceLocation loc, 29530cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith SmallVectorImpl<char> &buffer, 29630cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith const SourceManager &SM, 29730cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith const LangOptions &options, 29830cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith bool *invalid) { 299834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall // Break down the source location. 300834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall std::pair<FileID, unsigned> locInfo = SM.getDecomposedLoc(loc); 301834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall 302834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall // Try to the load the file buffer. 303834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall bool invalidTemp = false; 3045f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner StringRef file = SM.getBufferData(locInfo.first, &invalidTemp); 305834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall if (invalidTemp) { 306834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall if (invalid) *invalid = true; 3075f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner return StringRef(); 308834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall } 309834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall 310834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall const char *tokenBegin = file.data() + locInfo.second; 311834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall 312834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall // Lex from the start of the given location. 313834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall Lexer lexer(SM.getLocForStartOfFile(locInfo.first), options, 314834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall file.begin(), tokenBegin, file.end()); 315834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall Token token; 316834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall lexer.LexFromRawLexer(token); 317834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall 318834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall unsigned length = token.getLength(); 319834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall 320834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall // Common case: no need for cleaning. 321834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall if (!token.needsCleaning()) 3225f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner return StringRef(tokenBegin, length); 323834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall 32430cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith // Hard case, we need to relex the characters into the string. 32530cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith buffer.resize(length); 32630cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith buffer.resize(getSpellingSlow(token, tokenBegin, options, buffer.data())); 3275f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner return StringRef(buffer.data(), buffer.size()); 328834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall} 329834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall 330834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall/// getSpelling() - Return the 'spelling' of this token. The spelling of a 331834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall/// token are the characters used to represent the token in the source file 332834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall/// after trigraph expansion and escaped-newline folding. In particular, this 333834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall/// wants to get the true, uncanonicalized, spelling of things like digraphs 334834e3f6c77d9ac03997a3f0c56934edcf406a355John McCall/// UCNs, etc. 335b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattnerstd::string Lexer::getSpelling(const Token &Tok, const SourceManager &SourceMgr, 3364e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie const LangOptions &LangOpts, bool *Invalid) { 337b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner assert((int)Tok.getLength() >= 0 && "Token character range is bogus!"); 33830cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith 339b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner bool CharDataInvalid = false; 34030cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith const char *TokStart = SourceMgr.getCharacterData(Tok.getLocation(), 341b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner &CharDataInvalid); 342b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner if (Invalid) 343b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner *Invalid = CharDataInvalid; 344b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner if (CharDataInvalid) 345b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner return std::string(); 34630cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith 34730cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith // If this token contains nothing interesting, return it directly. 348b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner if (!Tok.needsCleaning()) 34930cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith return std::string(TokStart, TokStart + Tok.getLength()); 35030cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith 351b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner std::string Result; 35230cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith Result.resize(Tok.getLength()); 35330cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith Result.resize(getSpellingSlow(Tok, TokStart, LangOpts, &*Result.begin())); 354b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner return Result; 355b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner} 356b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner 357b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// getSpelling - This method is used to get the spelling of a token into a 358b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// preallocated buffer, instead of as an std::string. The caller is required 359b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// to allocate enough space for the token, which is guaranteed to be at least 360b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// Tok.getLength() bytes long. The actual length of the token is returned. 361b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// 362b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// Note that this method may do two possible things: it may either fill in 363b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// the buffer specified with characters, or it may *change the input pointer* 364b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// to point to a constant buffer with the data already in it (avoiding a 365b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// copy). The caller is not allowed to modify the returned buffer pointer 366b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner/// if an internal buffer is returned. 367b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattnerunsigned Lexer::getSpelling(const Token &Tok, const char *&Buffer, 368b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner const SourceManager &SourceMgr, 3694e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie const LangOptions &LangOpts, bool *Invalid) { 370b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner assert((int)Tok.getLength() >= 0 && "Token character range is bogus!"); 371c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara 372c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara const char *TokStart = 0; 373c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara // NOTE: this has to be checked *before* testing for an IdentifierInfo. 374c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara if (Tok.is(tok::raw_identifier)) 375c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara TokStart = Tok.getRawIdentifierData(); 376c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose else if (!Tok.hasUCN()) { 377c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose if (const IdentifierInfo *II = Tok.getIdentifierInfo()) { 378c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // Just return the string from the identifier table, which is very quick. 379c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose Buffer = II->getNameStart(); 380c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose return II->getLength(); 381c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose } 382b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner } 383c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara 384c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara // NOTE: this can be checked even after testing for an IdentifierInfo. 385b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner if (Tok.isLiteral()) 386b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner TokStart = Tok.getLiteralData(); 387c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara 388b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner if (TokStart == 0) { 389c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara // Compute the start of the token in the input lexer buffer. 390b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner bool CharDataInvalid = false; 391b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner TokStart = SourceMgr.getCharacterData(Tok.getLocation(), &CharDataInvalid); 392b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner if (Invalid) 393b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner *Invalid = CharDataInvalid; 394b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner if (CharDataInvalid) { 395b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner Buffer = ""; 396b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner return 0; 397b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner } 398b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner } 399c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara 400b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner // If this token contains nothing interesting, return it directly. 401b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner if (!Tok.needsCleaning()) { 402b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner Buffer = TokStart; 403b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner return Tok.getLength(); 404b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner } 405c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara 406b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner // Otherwise, hard case, relex the characters into the string. 40730cddaec99fa6c3207613efdaedbb51dd8d70c77Richard Smith return getSpellingSlow(Tok, TokStart, LangOpts, const_cast<char*>(Buffer)); 408b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner} 409b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner 410b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner 411b0607279cb98bbf2bbfe0db170aed39ef91e86a2Chris Lattner 41233e9abd21083a0191a7676a04b497006d2da184dDouglas Gregorstatic bool isWhitespace(unsigned char c); 4135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 4149a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner/// MeasureTokenLength - Relex the token at the specified location and return 4159a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner/// its length in bytes in the input file. If the token needs cleaning (e.g. 4169a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner/// includes a trigraph or an escaped newline) then this count includes bytes 4179a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner/// that are part of that. 4189a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattnerunsigned Lexer::MeasureTokenLength(SourceLocation Loc, 4192c78b873f4f3823ae859c15674cb3d76c8554113Chris Lattner const SourceManager &SM, 4202c78b873f4f3823ae859c15674cb3d76c8554113Chris Lattner const LangOptions &LangOpts) { 421d93335c43fd462145fee3ea8f4d84d430577c821Argyrios Kyrtzidis Token TheTok; 422d93335c43fd462145fee3ea8f4d84d430577c821Argyrios Kyrtzidis if (getRawToken(Loc, TheTok, SM, LangOpts)) 423d93335c43fd462145fee3ea8f4d84d430577c821Argyrios Kyrtzidis return 0; 424d93335c43fd462145fee3ea8f4d84d430577c821Argyrios Kyrtzidis return TheTok.getLength(); 425d93335c43fd462145fee3ea8f4d84d430577c821Argyrios Kyrtzidis} 426d93335c43fd462145fee3ea8f4d84d430577c821Argyrios Kyrtzidis 427d93335c43fd462145fee3ea8f4d84d430577c821Argyrios Kyrtzidis/// \brief Relex the token at the specified location. 428d93335c43fd462145fee3ea8f4d84d430577c821Argyrios Kyrtzidis/// \returns true if there was a failure, false on success. 429d93335c43fd462145fee3ea8f4d84d430577c821Argyrios Kyrtzidisbool Lexer::getRawToken(SourceLocation Loc, Token &Result, 430d93335c43fd462145fee3ea8f4d84d430577c821Argyrios Kyrtzidis const SourceManager &SM, 431d93335c43fd462145fee3ea8f4d84d430577c821Argyrios Kyrtzidis const LangOptions &LangOpts) { 4329a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner // TODO: this could be special cased for common tokens like identifiers, ')', 4339a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner // etc to make this faster, if it mattered. Just look at StrData[0] to handle 4341eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump // all obviously single-char tokens. This could use 4359a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner // Lexer::isObviouslySimpleCharacter for example to handle identifiers or 4369a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner // something. 437de7aeefc5573d669ed476d7bda7a8940d3bcadb7Chris Lattner 438de7aeefc5573d669ed476d7bda7a8940d3bcadb7Chris Lattner // If this comes from a macro expansion, we really do want the macro name, not 439de7aeefc5573d669ed476d7bda7a8940d3bcadb7Chris Lattner // the token this macro expanded to. 440402785357ab053dd53f4fdd858b9630a5e0f8badChandler Carruth Loc = SM.getExpansionLoc(Loc); 441363fdc29656cc03c1817268888f95e6343470aa8Chris Lattner std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 442f715ca12bfc9fddfde75f98a197424434428b821Douglas Gregor bool Invalid = false; 4435f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); 444f715ca12bfc9fddfde75f98a197424434428b821Douglas Gregor if (Invalid) 445d93335c43fd462145fee3ea8f4d84d430577c821Argyrios Kyrtzidis return true; 446f6ac97b101c8840efa92bf29166077ce4049e293Benjamin Kramer 447f6ac97b101c8840efa92bf29166077ce4049e293Benjamin Kramer const char *StrData = Buffer.data()+LocInfo.second; 4488350394c65b81bba3986dfe44ae17423873741deChris Lattner 44933e9abd21083a0191a7676a04b497006d2da184dDouglas Gregor if (isWhitespace(StrData[0])) 450d93335c43fd462145fee3ea8f4d84d430577c821Argyrios Kyrtzidis return true; 45133e9abd21083a0191a7676a04b497006d2da184dDouglas Gregor 4529a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner // Create a lexer starting at the beginning of this token. 453c3526d89ef9c31639ec8b25180cfb22354344241Sebastian Redl Lexer TheLexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, 454c3526d89ef9c31639ec8b25180cfb22354344241Sebastian Redl Buffer.begin(), StrData, Buffer.end()); 45539de7409bffb6b725a8aa64f0ba77ab51e8c9eb3Chris Lattner TheLexer.SetCommentRetentionState(true); 456d93335c43fd462145fee3ea8f4d84d430577c821Argyrios Kyrtzidis TheLexer.LexFromRawLexer(Result); 457d93335c43fd462145fee3ea8f4d84d430577c821Argyrios Kyrtzidis return false; 4589a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner} 4599a6119437672f42be5f50c3fe89fe843b1bfa5b5Chris Lattner 4600e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidisstatic SourceLocation getBeginningOfFileToken(SourceLocation Loc, 4610e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis const SourceManager &SM, 4620e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis const LangOptions &LangOpts) { 4630e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis assert(Loc.isFileID()); 464a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 4653de84241d90f3dd280126fdf2c4651667151c967Douglas Gregor if (LocInfo.first.isInvalid()) 4663de84241d90f3dd280126fdf2c4651667151c967Douglas Gregor return Loc; 4673de84241d90f3dd280126fdf2c4651667151c967Douglas Gregor 468a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor bool Invalid = false; 4695f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner StringRef Buffer = SM.getBufferData(LocInfo.first, &Invalid); 470a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor if (Invalid) 471a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor return Loc; 472a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor 473a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // Back up from the current location until we hit the beginning of a line 474a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // (or the buffer). We'll relex from that point. 475a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor const char *BufStart = Buffer.data(); 4763de84241d90f3dd280126fdf2c4651667151c967Douglas Gregor if (LocInfo.second >= Buffer.size()) 4773de84241d90f3dd280126fdf2c4651667151c967Douglas Gregor return Loc; 4783de84241d90f3dd280126fdf2c4651667151c967Douglas Gregor 479a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor const char *StrData = BufStart+LocInfo.second; 480a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor if (StrData[0] == '\n' || StrData[0] == '\r') 481a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor return Loc; 482a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor 483a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor const char *LexStart = StrData; 484a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor while (LexStart != BufStart) { 485a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor if (LexStart[0] == '\n' || LexStart[0] == '\r') { 486a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor ++LexStart; 487a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor break; 488a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor } 489a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor 490a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor --LexStart; 491a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor } 492a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor 493a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // Create a lexer starting at the beginning of this token. 494a64ccefdf0ea4e03ec88805d71b0af74950c7472Argyrios Kyrtzidis SourceLocation LexerStartLoc = Loc.getLocWithOffset(-LocInfo.second); 495a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor Lexer TheLexer(LexerStartLoc, LangOpts, BufStart, LexStart, Buffer.end()); 496a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor TheLexer.SetCommentRetentionState(true); 497a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor 498a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // Lex tokens until we find the token that contains the source location. 499a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor Token TheTok; 500a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor do { 501a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor TheLexer.LexFromRawLexer(TheTok); 502a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor 503a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor if (TheLexer.getBufferLocation() > StrData) { 504a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // Lexing this token has taken the lexer past the source location we're 505a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // looking for. If the current token encompasses our source location, 506a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // return the beginning of that token. 507a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor if (TheLexer.getBufferLocation() - TheTok.getLength() <= StrData) 508a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor return TheTok.getLocation(); 509a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor 510a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // We ended up skipping over the source location entirely, which means 511a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // that it points into whitespace. We're done here. 512a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor break; 513a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor } 514a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor } while (TheTok.getKind() != tok::eof); 515a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor 516a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor // We've passed our source location; just return the original source location. 517a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor return Loc; 518a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor} 519a8e5c5bdbe387b2552c1c23b828f54abcf085a40Douglas Gregor 5200e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios KyrtzidisSourceLocation Lexer::GetBeginningOfToken(SourceLocation Loc, 5210e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis const SourceManager &SM, 5220e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis const LangOptions &LangOpts) { 5230e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis if (Loc.isFileID()) 5240e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis return getBeginningOfFileToken(Loc, SM, LangOpts); 5250e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis 5260e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis if (!SM.isMacroArgExpansion(Loc)) 5270e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis return Loc; 5280e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis 5290e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis SourceLocation FileLoc = SM.getSpellingLoc(Loc); 5300e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis SourceLocation BeginFileLoc = getBeginningOfFileToken(FileLoc, SM, LangOpts); 5310e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis std::pair<FileID, unsigned> FileLocInfo = SM.getDecomposedLoc(FileLoc); 532ae9f85b2c062ac20512a484cfa6e66239dd3d0d9Chandler Carruth std::pair<FileID, unsigned> BeginFileLocInfo 533ae9f85b2c062ac20512a484cfa6e66239dd3d0d9Chandler Carruth = SM.getDecomposedLoc(BeginFileLoc); 5340e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis assert(FileLocInfo.first == BeginFileLocInfo.first && 5350e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis FileLocInfo.second >= BeginFileLocInfo.second); 536ae9f85b2c062ac20512a484cfa6e66239dd3d0d9Chandler Carruth return Loc.getLocWithOffset(BeginFileLocInfo.second - FileLocInfo.second); 5370e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis} 5380e870622e4d4b2ecb7bc6ffd2c97f74fd14220b6Argyrios Kyrtzidis 539f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregornamespace { 540f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor enum PreambleDirectiveKind { 541f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor PDK_Skipped, 542f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor PDK_StartIf, 543f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor PDK_EndIf, 544f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor PDK_Unknown 545f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor }; 546f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor} 547f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 548f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregorstd::pair<unsigned, bool> 54903c107a42fae79e89d0016999a1a04c07d65591aArgyrios KyrtzidisLexer::ComputePreamble(const llvm::MemoryBuffer *Buffer, 5504e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie const LangOptions &LangOpts, unsigned MaxLines) { 551f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // Create a lexer starting at the beginning of the file. Note that we use a 552f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // "fake" file source location at offset 1 so that the lexer will track our 553f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // position within the file. 554f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor const unsigned StartOffset = 1; 5551cb7142b66ac1844b91511314cce2b309a9a180dArgyrios Kyrtzidis SourceLocation FileLoc = SourceLocation::getFromRawEncoding(StartOffset); 5561cb7142b66ac1844b91511314cce2b309a9a180dArgyrios Kyrtzidis Lexer TheLexer(FileLoc, LangOpts, Buffer->getBufferStart(), 557f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor Buffer->getBufferStart(), Buffer->getBufferEnd()); 5581cb7142b66ac1844b91511314cce2b309a9a180dArgyrios Kyrtzidis 5591cb7142b66ac1844b91511314cce2b309a9a180dArgyrios Kyrtzidis // StartLoc will differ from FileLoc if there is a BOM that was skipped. 5601cb7142b66ac1844b91511314cce2b309a9a180dArgyrios Kyrtzidis SourceLocation StartLoc = TheLexer.getSourceLocation(); 5611cb7142b66ac1844b91511314cce2b309a9a180dArgyrios Kyrtzidis 562f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor bool InPreprocessorDirective = false; 563f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor Token TheTok; 564f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor Token IfStartTok; 565f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor unsigned IfCount = 0; 566c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis 567c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis unsigned MaxLineOffset = 0; 568c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis if (MaxLines) { 569c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis const char *CurPtr = Buffer->getBufferStart(); 570c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis unsigned CurLine = 0; 571c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis while (CurPtr != Buffer->getBufferEnd()) { 572c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis char ch = *CurPtr++; 573c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis if (ch == '\n') { 574c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis ++CurLine; 575c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis if (CurLine == MaxLines) 576c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis break; 577c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis } 578c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis } 579c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis if (CurPtr != Buffer->getBufferEnd()) 580c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis MaxLineOffset = CurPtr - Buffer->getBufferStart(); 581c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis } 582df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor 583f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor do { 584f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor TheLexer.LexFromRawLexer(TheTok); 585f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 586f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor if (InPreprocessorDirective) { 587f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // If we've hit the end of the file, we're done. 588f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor if (TheTok.getKind() == tok::eof) { 589f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor break; 590f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor } 591f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 592f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // If we haven't hit the end of the preprocessor directive, skip this 593f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // token. 594f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor if (!TheTok.isAtStartOfLine()) 595f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor continue; 596f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 597f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // We've passed the end of the preprocessor directive, and will look 598f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // at this token again below. 599f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor InPreprocessorDirective = false; 600f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor } 601f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 602df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor // Keep track of the # of lines in the preamble. 603df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor if (TheTok.isAtStartOfLine()) { 604c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis unsigned TokOffset = TheTok.getLocation().getRawEncoding() - StartOffset; 605df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor 606df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor // If we were asked to limit the number of lines in the preamble, 607df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor // and we're about to exceed that limit, we're done. 608c8c97a03eb0fdeb4f5fc9c4dea308ebbf46c2c93Argyrios Kyrtzidis if (MaxLineOffset && TokOffset >= MaxLineOffset) 609df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor break; 610df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor } 611df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor 612f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // Comments are okay; skip over them. 613f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor if (TheTok.getKind() == tok::comment) 614f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor continue; 615f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 616f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor if (TheTok.isAtStartOfLine() && TheTok.getKind() == tok::hash) { 617f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // This is the start of a preprocessor directive. 618f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor Token HashTok = TheTok; 619f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor InPreprocessorDirective = true; 620f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 62119207f1e5f51261a33492602501fb7ada50ea546Joerg Sonnenberger // Figure out which directive this is. Since we're lexing raw tokens, 622f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // we don't have an identifier table available. Instead, just look at 623f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // the raw identifier to recognize and categorize preprocessor directives. 624f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor TheLexer.LexFromRawLexer(TheTok); 625c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara if (TheTok.getKind() == tok::raw_identifier && !TheTok.needsCleaning()) { 6265f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner StringRef Keyword(TheTok.getRawIdentifierData(), 627c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara TheTok.getLength()); 628f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor PreambleDirectiveKind PDK 629f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor = llvm::StringSwitch<PreambleDirectiveKind>(Keyword) 630f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("include", PDK_Skipped) 631f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("__include_macros", PDK_Skipped) 632f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("define", PDK_Skipped) 633f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("undef", PDK_Skipped) 634f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("line", PDK_Skipped) 635f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("error", PDK_Skipped) 636f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("pragma", PDK_Skipped) 637f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("import", PDK_Skipped) 638f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("include_next", PDK_Skipped) 639f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("warning", PDK_Skipped) 640f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("ident", PDK_Skipped) 641f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("sccs", PDK_Skipped) 642f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("assert", PDK_Skipped) 643f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("unassert", PDK_Skipped) 644f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("if", PDK_StartIf) 645f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("ifdef", PDK_StartIf) 646f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("ifndef", PDK_StartIf) 647f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("elif", PDK_Skipped) 648f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("else", PDK_Skipped) 649f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Case("endif", PDK_EndIf) 650f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor .Default(PDK_Unknown); 651f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 652f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor switch (PDK) { 653f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor case PDK_Skipped: 654f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor continue; 655f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 656f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor case PDK_StartIf: 657f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor if (IfCount == 0) 658f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor IfStartTok = HashTok; 659f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 660f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor ++IfCount; 661f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor continue; 662f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 663f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor case PDK_EndIf: 664f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // Mismatched #endif. The preamble ends here. 665f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor if (IfCount == 0) 666f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor break; 667f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 668f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor --IfCount; 669f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor continue; 670f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 671f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor case PDK_Unknown: 672f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // We don't know what this directive is; stop at the '#'. 673f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor break; 674f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor } 675f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor } 676f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 677f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // We only end up here if we didn't recognize the preprocessor 678f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // directive or it was one that can't occur in the preamble at this 679f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor // point. Roll back the current token to the location of the '#'. 680f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor InPreprocessorDirective = false; 681f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor TheTok = HashTok; 682f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor } 683f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 684df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor // We hit a token that we don't recognize as being in the 685df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor // "preprocessing only" part of the file, so we're no longer in 686df95a13ec73d2cdaea79555cb412d767f4963120Douglas Gregor // the preamble. 687f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor break; 688f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor } while (true); 689f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 690f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor SourceLocation End = IfCount? IfStartTok.getLocation() : TheTok.getLocation(); 691f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor return std::make_pair(End.getRawEncoding() - StartLoc.getRawEncoding(), 692f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor IfCount? IfStartTok.isAtStartOfLine() 693f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor : TheTok.isAtStartOfLine()); 694f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor} 695f033f1da4a34f8df6e95e9929dc04ff54bb8fb01Douglas Gregor 6967ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 6977ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// AdvanceToTokenCharacter - Given a location that specifies the start of a 6987ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// token, return a new location that specifies a character within the token. 6997ef5c27eb6e8ebe58b52013246c06753c3613263Chris LattnerSourceLocation Lexer::AdvanceToTokenCharacter(SourceLocation TokStart, 7007ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner unsigned CharNo, 7017ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner const SourceManager &SM, 7024e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie const LangOptions &LangOpts) { 703433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth // Figure out how many physical characters away the specified expansion 7047ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // character is. This needs to take into consideration newlines and 7057ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // trigraphs. 7067ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner bool Invalid = false; 7077ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner const char *TokPtr = SM.getCharacterData(TokStart, &Invalid); 7087ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 7097ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // If they request the first char of the token, we're trivially done. 7107ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner if (Invalid || (CharNo == 0 && Lexer::isObviouslySimpleCharacter(*TokPtr))) 7117ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner return TokStart; 7127ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 7137ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner unsigned PhysOffset = 0; 7147ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 7157ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // The usual case is that tokens don't contain anything interesting. Skip 7167ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // over the uninteresting characters. If a token only consists of simple 7177ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // chars, this method is extremely fast. 7187ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner while (Lexer::isObviouslySimpleCharacter(*TokPtr)) { 7197ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner if (CharNo == 0) 720a64ccefdf0ea4e03ec88805d71b0af74950c7472Argyrios Kyrtzidis return TokStart.getLocWithOffset(PhysOffset); 7217ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner ++TokPtr, --CharNo, ++PhysOffset; 7227ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner } 7237ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 7247ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // If we have a character that may be a trigraph or escaped newline, use a 7257ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // lexer to parse it correctly. 7267ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner for (; CharNo; --CharNo) { 7277ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner unsigned Size; 7284e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie Lexer::getCharAndSizeNoWarn(TokPtr, Size, LangOpts); 7297ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner TokPtr += Size; 7307ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner PhysOffset += Size; 7317ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner } 7327ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 7337ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // Final detail: if we end up on an escaped newline, we want to return the 7347ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // location of the actual byte of the token. For example foo\<newline>bar 7357ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // advanced by 3 should return the location of b, not of \\. One compounding 7367ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner // detail of this is that the escape may be made by a trigraph. 7377ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner if (!Lexer::isObviouslySimpleCharacter(*TokPtr)) 7387ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner PhysOffset += Lexer::SkipEscapedNewLines(TokPtr)-TokPtr; 7397ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 740a64ccefdf0ea4e03ec88805d71b0af74950c7472Argyrios Kyrtzidis return TokStart.getLocWithOffset(PhysOffset); 7417ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner} 7427ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 7437ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// \brief Computes the source location just past the end of the 7447ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// token at this source location. 7457ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// 7467ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// This routine can be used to produce a source location that 7477ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// points just past the end of the token referenced by \p Loc, and 7487ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// is generally used when a diagnostic needs to point just after a 7497ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// token where it expected something different that it received. If 7507ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// the returned source location would not be meaningful (e.g., if 7517ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// it points into a macro), this routine returns an invalid 7527ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// source location. 7537ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// 7547ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// \param Offset an offset from the end of the token, where the source 7557ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// location should refer to. The default offset (0) produces a source 7567ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// location pointing just past the end of the token; an offset of 1 produces 7577ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner/// a source location pointing to the last character in the token, etc. 7587ef5c27eb6e8ebe58b52013246c06753c3613263Chris LattnerSourceLocation Lexer::getLocForEndOfToken(SourceLocation Loc, unsigned Offset, 7597ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner const SourceManager &SM, 7604e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie const LangOptions &LangOpts) { 7617ddf6b2d77ac4cb27f78d817d7884e6ce17afd0cArgyrios Kyrtzidis if (Loc.isInvalid()) 7627ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner return SourceLocation(); 7637ddf6b2d77ac4cb27f78d817d7884e6ce17afd0cArgyrios Kyrtzidis 7647ddf6b2d77ac4cb27f78d817d7884e6ce17afd0cArgyrios Kyrtzidis if (Loc.isMacroID()) { 7654e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie if (Offset > 0 || !isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc)) 766433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth return SourceLocation(); // Points inside the macro expansion. 7677ddf6b2d77ac4cb27f78d817d7884e6ce17afd0cArgyrios Kyrtzidis } 7687ddf6b2d77ac4cb27f78d817d7884e6ce17afd0cArgyrios Kyrtzidis 7694e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie unsigned Len = Lexer::MeasureTokenLength(Loc, SM, LangOpts); 7707ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner if (Len > Offset) 7717ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner Len = Len - Offset; 7727ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner else 7737ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner return Loc; 7747ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 775a64ccefdf0ea4e03ec88805d71b0af74950c7472Argyrios Kyrtzidis return Loc.getLocWithOffset(Len); 7767ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner} 7777ef5c27eb6e8ebe58b52013246c06753c3613263Chris Lattner 7787a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis/// \brief Returns true if the given MacroID location points at the first 779433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth/// token of the macro expansion. 780433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruthbool Lexer::isAtStartOfMacroExpansion(SourceLocation loc, 781f62d43d2afe1960755a1b5813cae1e5983bcac1bDouglas Gregor const SourceManager &SM, 78269bda4c027671df7163619f215209529eb236620Argyrios Kyrtzidis const LangOptions &LangOpts, 78369bda4c027671df7163619f215209529eb236620Argyrios Kyrtzidis SourceLocation *MacroBegin) { 7847a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc"); 7857a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis 7867a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis std::pair<FileID, unsigned> infoLoc = SM.getDecomposedLoc(loc); 7877a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis // FIXME: If the token comes from the macro token paste operator ('##') 7887a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis // this function will always return false; 7897a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis if (infoLoc.second > 0) 7907a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis return false; // Does not point at the start of token. 7917a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis 792433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth SourceLocation expansionLoc = 7931728762d5a8cfaf8d64385f47b311e84de1ae7a2Chandler Carruth SM.getSLocEntry(infoLoc.first).getExpansion().getExpansionLocStart(); 79469bda4c027671df7163619f215209529eb236620Argyrios Kyrtzidis if (expansionLoc.isFileID()) { 79569bda4c027671df7163619f215209529eb236620Argyrios Kyrtzidis // No other macro expansions, this is the first. 79669bda4c027671df7163619f215209529eb236620Argyrios Kyrtzidis if (MacroBegin) 79769bda4c027671df7163619f215209529eb236620Argyrios Kyrtzidis *MacroBegin = expansionLoc; 79869bda4c027671df7163619f215209529eb236620Argyrios Kyrtzidis return true; 79969bda4c027671df7163619f215209529eb236620Argyrios Kyrtzidis } 8007a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis 80169bda4c027671df7163619f215209529eb236620Argyrios Kyrtzidis return isAtStartOfMacroExpansion(expansionLoc, SM, LangOpts, MacroBegin); 8027a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis} 8037a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis 8047a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis/// \brief Returns true if the given MacroID location points at the last 805433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth/// token of the macro expansion. 806433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruthbool Lexer::isAtEndOfMacroExpansion(SourceLocation loc, 80769bda4c027671df7163619f215209529eb236620Argyrios Kyrtzidis const SourceManager &SM, 80869bda4c027671df7163619f215209529eb236620Argyrios Kyrtzidis const LangOptions &LangOpts, 80969bda4c027671df7163619f215209529eb236620Argyrios Kyrtzidis SourceLocation *MacroEnd) { 8107a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis assert(loc.isValid() && loc.isMacroID() && "Expected a valid macro loc"); 8117a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis 8127a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis SourceLocation spellLoc = SM.getSpellingLoc(loc); 8137a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis unsigned tokLen = MeasureTokenLength(spellLoc, SM, LangOpts); 8147a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis if (tokLen == 0) 8157a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis return false; 8167a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis 8177a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis FileID FID = SM.getFileID(loc); 818a64ccefdf0ea4e03ec88805d71b0af74950c7472Argyrios Kyrtzidis SourceLocation afterLoc = loc.getLocWithOffset(tokLen+1); 819f8c50652f7b224e66b0b6098d1fba07e036019b4Argyrios Kyrtzidis if (SM.isInFileID(afterLoc, FID)) 820f8c50652f7b224e66b0b6098d1fba07e036019b4Argyrios Kyrtzidis return false; // Still in the same FileID, does not point to the last token. 8217a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis 8227a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis // FIXME: If the token comes from the macro token paste operator ('##') 8237a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis // or the stringify operator ('#') this function will always return false; 824f8c50652f7b224e66b0b6098d1fba07e036019b4Argyrios Kyrtzidis 825433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth SourceLocation expansionLoc = 8261728762d5a8cfaf8d64385f47b311e84de1ae7a2Chandler Carruth SM.getSLocEntry(FID).getExpansion().getExpansionLocEnd(); 82769bda4c027671df7163619f215209529eb236620Argyrios Kyrtzidis if (expansionLoc.isFileID()) { 82869bda4c027671df7163619f215209529eb236620Argyrios Kyrtzidis // No other macro expansions. 82969bda4c027671df7163619f215209529eb236620Argyrios Kyrtzidis if (MacroEnd) 83069bda4c027671df7163619f215209529eb236620Argyrios Kyrtzidis *MacroEnd = expansionLoc; 83169bda4c027671df7163619f215209529eb236620Argyrios Kyrtzidis return true; 83269bda4c027671df7163619f215209529eb236620Argyrios Kyrtzidis } 8337a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis 83469bda4c027671df7163619f215209529eb236620Argyrios Kyrtzidis return isAtEndOfMacroExpansion(expansionLoc, SM, LangOpts, MacroEnd); 8357a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis} 8367a759606d93975866051f67104ae58446e55f404Argyrios Kyrtzidis 837a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidisstatic CharSourceRange makeRangeFromFileLocs(CharSourceRange Range, 838d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis const SourceManager &SM, 839d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis const LangOptions &LangOpts) { 840a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis SourceLocation Begin = Range.getBegin(); 841a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis SourceLocation End = Range.getEnd(); 842d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis assert(Begin.isFileID() && End.isFileID()); 843a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis if (Range.isTokenRange()) { 844a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis End = Lexer::getLocForEndOfToken(End, 0, SM,LangOpts); 845a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis if (End.isInvalid()) 846a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis return CharSourceRange(); 847a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis } 848d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis 849d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis // Break down the source locations. 850d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis FileID FID; 851d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis unsigned BeginOffs; 852d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis llvm::tie(FID, BeginOffs) = SM.getDecomposedLoc(Begin); 853d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis if (FID.isInvalid()) 854d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis return CharSourceRange(); 855d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis 856d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis unsigned EndOffs; 857d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis if (!SM.isInFileID(End, FID, &EndOffs) || 858d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis BeginOffs > EndOffs) 859d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis return CharSourceRange(); 860d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis 861d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis return CharSourceRange::getCharRange(Begin, End); 862d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis} 863d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis 864a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios KyrtzidisCharSourceRange Lexer::makeFileCharRange(CharSourceRange Range, 86511b652d41d0d97380ab321a1dba48ecb044f9de8Argyrios Kyrtzidis const SourceManager &SM, 86611b652d41d0d97380ab321a1dba48ecb044f9de8Argyrios Kyrtzidis const LangOptions &LangOpts) { 867a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis SourceLocation Begin = Range.getBegin(); 868a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis SourceLocation End = Range.getEnd(); 869d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis if (Begin.isInvalid() || End.isInvalid()) 87011b652d41d0d97380ab321a1dba48ecb044f9de8Argyrios Kyrtzidis return CharSourceRange(); 87111b652d41d0d97380ab321a1dba48ecb044f9de8Argyrios Kyrtzidis 872d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis if (Begin.isFileID() && End.isFileID()) 873a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis return makeRangeFromFileLocs(Range, SM, LangOpts); 874d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis 875d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis if (Begin.isMacroID() && End.isFileID()) { 87611b652d41d0d97380ab321a1dba48ecb044f9de8Argyrios Kyrtzidis if (!isAtStartOfMacroExpansion(Begin, SM, LangOpts, &Begin)) 87711b652d41d0d97380ab321a1dba48ecb044f9de8Argyrios Kyrtzidis return CharSourceRange(); 878a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis Range.setBegin(Begin); 879a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis return makeRangeFromFileLocs(Range, SM, LangOpts); 880d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis } 88111b652d41d0d97380ab321a1dba48ecb044f9de8Argyrios Kyrtzidis 882d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis if (Begin.isFileID() && End.isMacroID()) { 883a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis if ((Range.isTokenRange() && !isAtEndOfMacroExpansion(End, SM, LangOpts, 884a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis &End)) || 885a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis (Range.isCharRange() && !isAtStartOfMacroExpansion(End, SM, LangOpts, 886a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis &End))) 887d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis return CharSourceRange(); 888a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis Range.setEnd(End); 889a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis return makeRangeFromFileLocs(Range, SM, LangOpts); 890d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis } 89111b652d41d0d97380ab321a1dba48ecb044f9de8Argyrios Kyrtzidis 892d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis assert(Begin.isMacroID() && End.isMacroID()); 893d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis SourceLocation MacroBegin, MacroEnd; 894d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis if (isAtStartOfMacroExpansion(Begin, SM, LangOpts, &MacroBegin) && 895a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis ((Range.isTokenRange() && isAtEndOfMacroExpansion(End, SM, LangOpts, 896a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis &MacroEnd)) || 897a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis (Range.isCharRange() && isAtStartOfMacroExpansion(End, SM, LangOpts, 898a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis &MacroEnd)))) { 899a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis Range.setBegin(MacroBegin); 900a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis Range.setEnd(MacroEnd); 901a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis return makeRangeFromFileLocs(Range, SM, LangOpts); 902a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis } 903d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis 904d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis FileID FID; 905d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis unsigned BeginOffs; 906d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis llvm::tie(FID, BeginOffs) = SM.getDecomposedLoc(Begin); 907d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis if (FID.isInvalid()) 908e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis return CharSourceRange(); 909e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis 91011b652d41d0d97380ab321a1dba48ecb044f9de8Argyrios Kyrtzidis unsigned EndOffs; 911d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis if (!SM.isInFileID(End, FID, &EndOffs) || 912d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis BeginOffs > EndOffs) 91311b652d41d0d97380ab321a1dba48ecb044f9de8Argyrios Kyrtzidis return CharSourceRange(); 91411b652d41d0d97380ab321a1dba48ecb044f9de8Argyrios Kyrtzidis 915d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis const SrcMgr::SLocEntry *E = &SM.getSLocEntry(FID); 916d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis const SrcMgr::ExpansionInfo &Expansion = E->getExpansion(); 917d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis if (Expansion.isMacroArgExpansion() && 918d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis Expansion.getSpellingLoc().isFileID()) { 919d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis SourceLocation SpellLoc = Expansion.getSpellingLoc(); 920a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis Range.setBegin(SpellLoc.getLocWithOffset(BeginOffs)); 921a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis Range.setEnd(SpellLoc.getLocWithOffset(EndOffs)); 922a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis return makeRangeFromFileLocs(Range, SM, LangOpts); 923d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis } 924d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis 925d9806c912ae3e870a733acfd83c26e8a1f6a5ffcArgyrios Kyrtzidis return CharSourceRange(); 92611b652d41d0d97380ab321a1dba48ecb044f9de8Argyrios Kyrtzidis} 92711b652d41d0d97380ab321a1dba48ecb044f9de8Argyrios Kyrtzidis 928e64d9037658a1b95c79ea275af6167a110b3c563Argyrios KyrtzidisStringRef Lexer::getSourceText(CharSourceRange Range, 929e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis const SourceManager &SM, 930e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis const LangOptions &LangOpts, 931e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis bool *Invalid) { 932a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis Range = makeFileCharRange(Range, SM, LangOpts); 933a83f4d2315dbeb3914868f1ccb8e74fb2ccdbb0cArgyrios Kyrtzidis if (Range.isInvalid()) { 934e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis if (Invalid) *Invalid = true; 935e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis return StringRef(); 936e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis } 937e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis 938e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis // Break down the source location. 939e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis std::pair<FileID, unsigned> beginInfo = SM.getDecomposedLoc(Range.getBegin()); 940e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis if (beginInfo.first.isInvalid()) { 941e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis if (Invalid) *Invalid = true; 942e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis return StringRef(); 943e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis } 944e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis 945e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis unsigned EndOffs; 946e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis if (!SM.isInFileID(Range.getEnd(), beginInfo.first, &EndOffs) || 947e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis beginInfo.second > EndOffs) { 948e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis if (Invalid) *Invalid = true; 949e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis return StringRef(); 950e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis } 951e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis 952e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis // Try to the load the file buffer. 953e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis bool invalidTemp = false; 954e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis StringRef file = SM.getBufferData(beginInfo.first, &invalidTemp); 955e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis if (invalidTemp) { 956e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis if (Invalid) *Invalid = true; 957e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis return StringRef(); 958e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis } 959e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis 960e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis if (Invalid) *Invalid = false; 961e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis return file.substr(beginInfo.second, EndOffs - beginInfo.second); 962e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis} 963e64d9037658a1b95c79ea275af6167a110b3c563Argyrios Kyrtzidis 964c2a8d6cee01fc4845f5409bf5c021a64616ac8c3Anna ZaksStringRef Lexer::getImmediateMacroName(SourceLocation Loc, 965c2a8d6cee01fc4845f5409bf5c021a64616ac8c3Anna Zaks const SourceManager &SM, 966c2a8d6cee01fc4845f5409bf5c021a64616ac8c3Anna Zaks const LangOptions &LangOpts) { 967c2a8d6cee01fc4845f5409bf5c021a64616ac8c3Anna Zaks assert(Loc.isMacroID() && "Only reasonble to call this on macros"); 9687f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis 9697f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis // Find the location of the immediate macro expansion. 9707f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis while (1) { 9717f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis FileID FID = SM.getFileID(Loc); 9727f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis const SrcMgr::SLocEntry *E = &SM.getSLocEntry(FID); 9737f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis const SrcMgr::ExpansionInfo &Expansion = E->getExpansion(); 9747f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis Loc = Expansion.getExpansionLocStart(); 9757f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis if (!Expansion.isMacroArgExpansion()) 9767f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis break; 9777f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis 9787f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis // For macro arguments we need to check that the argument did not come 9797f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis // from an inner macro, e.g: "MAC1( MAC2(foo) )" 9807f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis 9817f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis // Loc points to the argument id of the macro definition, move to the 9827f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis // macro expansion. 983c2a8d6cee01fc4845f5409bf5c021a64616ac8c3Anna Zaks Loc = SM.getImmediateExpansionRange(Loc).first; 9847f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis SourceLocation SpellLoc = Expansion.getSpellingLoc(); 9857f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis if (SpellLoc.isFileID()) 9867f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis break; // No inner macro. 9877f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis 9887f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis // If spelling location resides in the same FileID as macro expansion 9897f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis // location, it means there is no inner macro. 9907f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis FileID MacroFID = SM.getFileID(Loc); 9917f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis if (SM.isInFileID(SpellLoc, MacroFID)) 9927f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis break; 9937f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis 9947f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis // Argument came from inner macro. 9957f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis Loc = SpellLoc; 9967f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis } 997c2a8d6cee01fc4845f5409bf5c021a64616ac8c3Anna Zaks 998c2a8d6cee01fc4845f5409bf5c021a64616ac8c3Anna Zaks // Find the spelling location of the start of the non-argument expansion 999c2a8d6cee01fc4845f5409bf5c021a64616ac8c3Anna Zaks // range. This is where the macro name was spelled in order to begin 1000c2a8d6cee01fc4845f5409bf5c021a64616ac8c3Anna Zaks // expanding this macro. 10017f6cf9764b33381e03fcf7c44f7985a333212b06Argyrios Kyrtzidis Loc = SM.getSpellingLoc(Loc); 1002c2a8d6cee01fc4845f5409bf5c021a64616ac8c3Anna Zaks 1003c2a8d6cee01fc4845f5409bf5c021a64616ac8c3Anna Zaks // Dig out the buffer where the macro name was spelled and the extents of the 1004c2a8d6cee01fc4845f5409bf5c021a64616ac8c3Anna Zaks // name so that we can render it into the expansion note. 1005c2a8d6cee01fc4845f5409bf5c021a64616ac8c3Anna Zaks std::pair<FileID, unsigned> ExpansionInfo = SM.getDecomposedLoc(Loc); 1006c2a8d6cee01fc4845f5409bf5c021a64616ac8c3Anna Zaks unsigned MacroTokenLength = Lexer::MeasureTokenLength(Loc, SM, LangOpts); 1007c2a8d6cee01fc4845f5409bf5c021a64616ac8c3Anna Zaks StringRef ExpansionBuffer = SM.getBufferData(ExpansionInfo.first); 1008c2a8d6cee01fc4845f5409bf5c021a64616ac8c3Anna Zaks return ExpansionBuffer.substr(ExpansionInfo.second, MacroTokenLength); 1009c2a8d6cee01fc4845f5409bf5c021a64616ac8c3Anna Zaks} 1010c2a8d6cee01fc4845f5409bf5c021a64616ac8c3Anna Zaks 10115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 10125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// Character information. 10135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 10145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 10155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerenum { 10165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CHAR_HORZ_WS = 0x01, // ' ', '\t', '\f', '\v'. Note, no '\0' 10175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CHAR_VERT_WS = 0x02, // '\r', '\n' 10185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CHAR_LETTER = 0x04, // a-z,A-Z 10195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CHAR_NUMBER = 0x08, // 0-9 10205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CHAR_UNDER = 0x10, // _ 10212fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CHAR_PERIOD = 0x20, // . 10222fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CHAR_RAWDEL = 0x40 // {}[]#<>%:;?*+-/^&|~!=,"' 10235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer}; 10245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 102503b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner// Statically initialize CharInfo table based on ASCII character set 102603b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner// Reference: FreeBSD 7.2 /usr/share/misc/ascii 1027a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattnerstatic const unsigned char CharInfo[256] = 102803b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner{ 102903b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner// 0 NUL 1 SOH 2 STX 3 ETX 103003b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner// 4 EOT 5 ENQ 6 ACK 7 BEL 103103b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 0 , 0 , 0 , 0 , 103203b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 0 , 0 , 0 , 0 , 103303b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner// 8 BS 9 HT 10 NL 11 VT 103403b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//12 NP 13 CR 14 SO 15 SI 103503b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 0 , CHAR_HORZ_WS, CHAR_VERT_WS, CHAR_HORZ_WS, 103603b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_HORZ_WS, CHAR_VERT_WS, 0 , 0 , 103703b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//16 DLE 17 DC1 18 DC2 19 DC3 103803b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//20 DC4 21 NAK 22 SYN 23 ETB 103903b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 0 , 0 , 0 , 0 , 104003b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 0 , 0 , 0 , 0 , 104103b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//24 CAN 25 EM 26 SUB 27 ESC 104203b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//28 FS 29 GS 30 RS 31 US 104303b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 0 , 0 , 0 , 0 , 104403b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 0 , 0 , 0 , 0 , 104503b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//32 SP 33 ! 34 " 35 # 104603b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//36 $ 37 % 38 & 39 ' 10472fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CHAR_HORZ_WS, CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL , 10482fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 0 , CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL , 104903b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//40 ( 41 ) 42 * 43 + 105003b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//44 , 45 - 46 . 47 / 10512fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 0 , 0 , CHAR_RAWDEL , CHAR_RAWDEL , 10522fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CHAR_RAWDEL , CHAR_RAWDEL , CHAR_PERIOD , CHAR_RAWDEL , 105303b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//48 0 49 1 50 2 51 3 105403b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//52 4 53 5 54 6 55 7 105503b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_NUMBER , CHAR_NUMBER , CHAR_NUMBER , CHAR_NUMBER , 105603b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_NUMBER , CHAR_NUMBER , CHAR_NUMBER , CHAR_NUMBER , 105703b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//56 8 57 9 58 : 59 ; 105803b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//60 < 61 = 62 > 63 ? 10592fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CHAR_NUMBER , CHAR_NUMBER , CHAR_RAWDEL , CHAR_RAWDEL , 10602fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL , 106103b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//64 @ 65 A 66 B 67 C 106203b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//68 D 69 E 70 F 71 G 106303b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 0 , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 106403b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 106503b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//72 H 73 I 74 J 75 K 106603b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//76 L 77 M 78 N 79 O 106703b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 106803b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 106903b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//80 P 81 Q 82 R 83 S 107003b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//84 T 85 U 86 V 87 W 107103b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 107203b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 107303b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//88 X 89 Y 90 Z 91 [ 107403b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//92 \ 93 ] 94 ^ 95 _ 10752fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_RAWDEL , 10762fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 0 , CHAR_RAWDEL , CHAR_RAWDEL , CHAR_UNDER , 107703b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//96 ` 97 a 98 b 99 c 107803b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//100 d 101 e 102 f 103 g 107903b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 0 , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 108003b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 108103b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//104 h 105 i 106 j 107 k 108203b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//108 l 109 m 110 n 111 o 108303b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 108403b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 108503b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//112 p 113 q 114 r 115 s 108603b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//116 t 117 u 118 v 119 w 108703b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 108803b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , 108903b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner//120 x 121 y 122 z 123 { 10902fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper//124 | 125 } 126 ~ 127 DEL 10912fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CHAR_LETTER , CHAR_LETTER , CHAR_LETTER , CHAR_RAWDEL , 10922fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CHAR_RAWDEL , CHAR_RAWDEL , CHAR_RAWDEL , 0 109303b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner}; 109403b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 1095a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattnerstatic void InitCharacterInfo() { 10965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer static bool isInited = false; 10975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (isInited) return; 109803b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner // check the statically-initialized CharInfo table 109903b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_HORZ_WS == CharInfo[(int)' ']); 110003b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_HORZ_WS == CharInfo[(int)'\t']); 110103b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_HORZ_WS == CharInfo[(int)'\f']); 110203b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_HORZ_WS == CharInfo[(int)'\v']); 110303b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_VERT_WS == CharInfo[(int)'\n']); 110403b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_VERT_WS == CharInfo[(int)'\r']); 110503b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_UNDER == CharInfo[(int)'_']); 110603b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_PERIOD == CharInfo[(int)'.']); 110703b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner for (unsigned i = 'a'; i <= 'z'; ++i) { 110803b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_LETTER == CharInfo[i]); 110903b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_LETTER == CharInfo[i+'A'-'a']); 111003b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner } 11115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer for (unsigned i = '0'; i <= '9'; ++i) 111203b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner assert(CHAR_NUMBER == CharInfo[i]); 11137b682656310b0d7e94a98d982444bc7ddedc653fSteve Naroff 111403b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner isInited = true; 11155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 11165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 111703b986660cc9ae7a08588f99913f2d91e359a65fChris Lattner 11185cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smith/// isIdentifierHead - Return true if this is the first character of an 11195cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smith/// identifier, which is [a-zA-Z_]. 11205cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smithstatic inline bool isIdentifierHead(unsigned char c) { 11215cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smith return (CharInfo[c] & (CHAR_LETTER|CHAR_UNDER)) ? true : false; 11225cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smith} 11235cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smith 11245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// isIdentifierBody - Return true if this is the body character of an 11255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// identifier, which is [a-zA-Z0-9_]. 11265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerstatic inline bool isIdentifierBody(unsigned char c) { 112795c062b94e2af5380617cf5dc115497ae7a5f817Hartmut Kaiser return (CharInfo[c] & (CHAR_LETTER|CHAR_NUMBER|CHAR_UNDER)) ? true : false; 11285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 11295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 11305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// isHorizontalWhitespace - Return true if this character is horizontal 1131a05369fbb7ec2840d6e4d53cbc48b70a8e9e37acJames Dennett/// whitespace: ' ', '\\t', '\\f', '\\v'. Note that this returns false for 1132a05369fbb7ec2840d6e4d53cbc48b70a8e9e37acJames Dennett/// '\\0'. 11335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerstatic inline bool isHorizontalWhitespace(unsigned char c) { 113495c062b94e2af5380617cf5dc115497ae7a5f817Hartmut Kaiser return (CharInfo[c] & CHAR_HORZ_WS) ? true : false; 11355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 11365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1137aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks/// isVerticalWhitespace - Return true if this character is vertical 1138a05369fbb7ec2840d6e4d53cbc48b70a8e9e37acJames Dennett/// whitespace: '\\n', '\\r'. Note that this returns false for '\\0'. 1139aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaksstatic inline bool isVerticalWhitespace(unsigned char c) { 1140aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks return (CharInfo[c] & CHAR_VERT_WS) ? true : false; 1141aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks} 1142aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks 11435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// isWhitespace - Return true if this character is horizontal or vertical 1144a05369fbb7ec2840d6e4d53cbc48b70a8e9e37acJames Dennett/// whitespace: ' ', '\\t', '\\f', '\\v', '\\n', '\\r'. Note that this returns 1145a05369fbb7ec2840d6e4d53cbc48b70a8e9e37acJames Dennett/// false for '\\0'. 11465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerstatic inline bool isWhitespace(unsigned char c) { 114795c062b94e2af5380617cf5dc115497ae7a5f817Hartmut Kaiser return (CharInfo[c] & (CHAR_HORZ_WS|CHAR_VERT_WS)) ? true : false; 11485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 11495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 11505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// isNumberBody - Return true if this is the body character of an 11515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// preprocessing number, which is [a-zA-Z0-9_.]. 11525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerstatic inline bool isNumberBody(unsigned char c) { 11531eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump return (CharInfo[c] & (CHAR_LETTER|CHAR_NUMBER|CHAR_UNDER|CHAR_PERIOD)) ? 115495c062b94e2af5380617cf5dc115497ae7a5f817Hartmut Kaiser true : false; 11555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 11565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 11572fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper/// isRawStringDelimBody - Return true if this is the body character of a 11582fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper/// raw string delimiter. 11592fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topperstatic inline bool isRawStringDelimBody(unsigned char c) { 11602fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return (CharInfo[c] & 11612fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper (CHAR_LETTER|CHAR_NUMBER|CHAR_UNDER|CHAR_PERIOD|CHAR_RAWDEL)) ? 11622fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper true : false; 11632fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper} 11642fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 1165d880b3aa6d594d1a7f2d307c29378c6f59b216ffJordan Rose// Allow external clients to make use of CharInfo. 1166d880b3aa6d594d1a7f2d307c29378c6f59b216ffJordan Rosebool Lexer::isIdentifierBodyChar(char c, const LangOptions &LangOpts) { 1167d880b3aa6d594d1a7f2d307c29378c6f59b216ffJordan Rose return isIdentifierBody(c) || (c == '$' && LangOpts.DollarIdents); 1168d880b3aa6d594d1a7f2d307c29378c6f59b216ffJordan Rose} 1169d880b3aa6d594d1a7f2d307c29378c6f59b216ffJordan Rose 11705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 11715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 11725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// Diagnostics forwarding code. 11735f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 11745f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1175409a03671224d4e5bdab1594c43baf070148f830Chris Lattner/// GetMappedTokenLoc - If lexing out of a 'mapped buffer', where we pretend the 1176433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth/// lexer buffer was all expanded at a single point, perform the mapping. 1177409a03671224d4e5bdab1594c43baf070148f830Chris Lattner/// This is currently only used for _Pragma implementation, so it is the slow 1178409a03671224d4e5bdab1594c43baf070148f830Chris Lattner/// path of the hot getSourceLocation method. Do not allow it to be inlined. 117914bd96571ef6f0e97dc79ec4d01b547d60e8fa68Chandler Carruthstatic LLVM_ATTRIBUTE_NOINLINE SourceLocation GetMappedTokenLoc( 118014bd96571ef6f0e97dc79ec4d01b547d60e8fa68Chandler Carruth Preprocessor &PP, SourceLocation FileLoc, unsigned CharNo, unsigned TokLen); 1181409a03671224d4e5bdab1594c43baf070148f830Chris Lattnerstatic SourceLocation GetMappedTokenLoc(Preprocessor &PP, 1182409a03671224d4e5bdab1594c43baf070148f830Chris Lattner SourceLocation FileLoc, 1183de7aeefc5573d669ed476d7bda7a8940d3bcadb7Chris Lattner unsigned CharNo, unsigned TokLen) { 1184433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth assert(FileLoc.isMacroID() && "Must be a macro expansion"); 11851eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1186409a03671224d4e5bdab1594c43baf070148f830Chris Lattner // Otherwise, we're lexing "mapped tokens". This is used for things like 1187433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth // _Pragma handling. Combine the expansion location of FileLoc with the 1188df7c17a8d02fe09a3466786bae3e40fc3252687aChris Lattner // spelling location. 1189e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner SourceManager &SM = PP.getSourceManager(); 11901eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1191433db06b614f26dc6829e86d6ff469e2cca7d4f9Chandler Carruth // Create a new SLoc which is expanded from Expansion(FileLoc) but whose 1192df7c17a8d02fe09a3466786bae3e40fc3252687aChris Lattner // characters come from spelling(FileLoc)+Offset. 1193e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner SourceLocation SpellingLoc = SM.getSpellingLoc(FileLoc); 1194a64ccefdf0ea4e03ec88805d71b0af74950c7472Argyrios Kyrtzidis SpellingLoc = SpellingLoc.getLocWithOffset(CharNo); 11951eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1196e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner // Figure out the expansion loc range, which is the range covered by the 1197e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner // original _Pragma(...) sequence. 1198e7fb48466afcbf2c4ccdfa658824282fdc3c512cChris Lattner std::pair<SourceLocation,SourceLocation> II = 1199999f739404edf2078cf9f9c28b4dc45c19765842Chandler Carruth SM.getImmediateExpansionRange(FileLoc); 12001eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1201bf340e452339e374ea6eef78c1f0a2abdd16c5a3Chandler Carruth return SM.createExpansionLoc(SpellingLoc, II.first, II.second, TokLen); 1202409a03671224d4e5bdab1594c43baf070148f830Chris Lattner} 1203409a03671224d4e5bdab1594c43baf070148f830Chris Lattner 12045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// getSourceLocation - Return a source location identifier for the specified 12055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// offset in the current file. 1206de7aeefc5573d669ed476d7bda7a8940d3bcadb7Chris LattnerSourceLocation Lexer::getSourceLocation(const char *Loc, 1207de7aeefc5573d669ed476d7bda7a8940d3bcadb7Chris Lattner unsigned TokLen) const { 1208448cec4c1c3705f6f49ffdefb58a7329942a2dd8Chris Lattner assert(Loc >= BufferStart && Loc <= BufferEnd && 12095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer "Location out of range for this buffer!"); 12109dc1f530c086d2c16f8cba758b0f59a5bf41323aChris Lattner 12119dc1f530c086d2c16f8cba758b0f59a5bf41323aChris Lattner // In the normal case, we're just lexing from a simple file buffer, return 12129dc1f530c086d2c16f8cba758b0f59a5bf41323aChris Lattner // the file id from FileLoc with the offset specified. 1213448cec4c1c3705f6f49ffdefb58a7329942a2dd8Chris Lattner unsigned CharNo = Loc-BufferStart; 12149dc1f530c086d2c16f8cba758b0f59a5bf41323aChris Lattner if (FileLoc.isFileID()) 1215a64ccefdf0ea4e03ec88805d71b0af74950c7472Argyrios Kyrtzidis return FileLoc.getLocWithOffset(CharNo); 12161eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 12172b2453a7d8fe732561795431f39ceb2b2a832d84Chris Lattner // Otherwise, this is the _Pragma lexer case, which pretends that all of the 12182b2453a7d8fe732561795431f39ceb2b2a832d84Chris Lattner // tokens are lexed from where the _Pragma was defined. 1219168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner assert(PP && "This doesn't work on raw lexers"); 1220de7aeefc5573d669ed476d7bda7a8940d3bcadb7Chris Lattner return GetMappedTokenLoc(*PP, FileLoc, CharNo, TokLen); 12215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 12225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 12235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// Diag - Forwarding function for diagnostics. This translate a source 12245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// position in the current buffer into a SourceLocation object for rendering. 12253cbfe2c4159e0a219ae660d50625c013aa4afbd0Chris LattnerDiagnosticBuilder Lexer::Diag(const char *Loc, unsigned DiagID) const { 12263692b09faa9fe346f39bc922db6dce48cdcc3f63Chris Lattner return PP->Diag(getSourceLocation(Loc), DiagID); 12275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 12285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 12295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 12305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// Trigraph and Escaped Newline Handling Code. 12315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 12325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 12335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// GetTrigraphCharForLetter - Given a character that occurs after a ?? pair, 12345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// return the decoded trigraph letter it corresponds to, or '\0' if nothing. 12355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerstatic char GetTrigraphCharForLetter(char Letter) { 12365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer switch (Letter) { 12375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer default: return 0; 12385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '=': return '#'; 12395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case ')': return ']'; 12405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '(': return '['; 12415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '!': return '|'; 12425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\'': return '^'; 12435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '>': return '}'; 12445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '/': return '\\'; 12455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '<': return '{'; 12465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '-': return '~'; 12475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 12485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 12495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 12505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// DecodeTrigraphChar - If the specified character is a legal trigraph when 12515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// prefixed with ??, emit a trigraph warning. If trigraphs are enabled, 12525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// return the result character. Finally, emit a warning about trigraph use 12535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// whether trigraphs are enabled or not. 12545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerstatic char DecodeTrigraphChar(const char *CP, Lexer *L) { 12555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char Res = GetTrigraphCharForLetter(*CP); 12563692b09faa9fe346f39bc922db6dce48cdcc3f63Chris Lattner if (!Res || !L) return Res; 12571eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 12584e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie if (!L->getLangOpts().Trigraphs) { 125974d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!L->isLexingRawMode()) 126074d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner L->Diag(CP-2, diag::trigraph_ignored); 12613692b09faa9fe346f39bc922db6dce48cdcc3f63Chris Lattner return 0; 12625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 12631eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 126474d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!L->isLexingRawMode()) 12655f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner L->Diag(CP-2, diag::trigraph_converted) << StringRef(&Res, 1); 12665f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return Res; 12675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 12685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 126924f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner/// getEscapedNewLineSize - Return the size of the specified escaped newline, 127024f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner/// or 0 if it is not an escaped newline. P[-1] is known to be a "\" or a 12711eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump/// trigraph equivalent on entry to this function. 127224f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattnerunsigned Lexer::getEscapedNewLineSize(const char *Ptr) { 127324f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner unsigned Size = 0; 127424f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner while (isWhitespace(Ptr[Size])) { 127524f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner ++Size; 12761eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 127724f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner if (Ptr[Size-1] != '\n' && Ptr[Size-1] != '\r') 127824f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner continue; 127924f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner 128024f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner // If this is a \r\n or \n\r, skip the other half. 128124f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner if ((Ptr[Size] == '\r' || Ptr[Size] == '\n') && 128224f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner Ptr[Size-1] != Ptr[Size]) 128324f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner ++Size; 12841eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 128524f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner return Size; 12861eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } 12871eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 128824f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner // Not an escaped newline, must be a \t or something else. 128924f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner return 0; 129024f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner} 129124f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner 1292033749571f8d4c804eeb357c70b06424aa24503bChris Lattner/// SkipEscapedNewLines - If P points to an escaped newline (or a series of 1293033749571f8d4c804eeb357c70b06424aa24503bChris Lattner/// them), skip over them and return the first non-escaped-newline found, 1294033749571f8d4c804eeb357c70b06424aa24503bChris Lattner/// otherwise return P. 1295033749571f8d4c804eeb357c70b06424aa24503bChris Lattnerconst char *Lexer::SkipEscapedNewLines(const char *P) { 1296033749571f8d4c804eeb357c70b06424aa24503bChris Lattner while (1) { 1297033749571f8d4c804eeb357c70b06424aa24503bChris Lattner const char *AfterEscape; 1298033749571f8d4c804eeb357c70b06424aa24503bChris Lattner if (*P == '\\') { 1299033749571f8d4c804eeb357c70b06424aa24503bChris Lattner AfterEscape = P+1; 1300033749571f8d4c804eeb357c70b06424aa24503bChris Lattner } else if (*P == '?') { 1301033749571f8d4c804eeb357c70b06424aa24503bChris Lattner // If not a trigraph for escape, bail out. 1302033749571f8d4c804eeb357c70b06424aa24503bChris Lattner if (P[1] != '?' || P[2] != '/') 1303033749571f8d4c804eeb357c70b06424aa24503bChris Lattner return P; 1304033749571f8d4c804eeb357c70b06424aa24503bChris Lattner AfterEscape = P+3; 1305033749571f8d4c804eeb357c70b06424aa24503bChris Lattner } else { 1306033749571f8d4c804eeb357c70b06424aa24503bChris Lattner return P; 1307033749571f8d4c804eeb357c70b06424aa24503bChris Lattner } 13081eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1309033749571f8d4c804eeb357c70b06424aa24503bChris Lattner unsigned NewLineSize = Lexer::getEscapedNewLineSize(AfterEscape); 1310033749571f8d4c804eeb357c70b06424aa24503bChris Lattner if (NewLineSize == 0) return P; 1311033749571f8d4c804eeb357c70b06424aa24503bChris Lattner P = AfterEscape+NewLineSize; 1312033749571f8d4c804eeb357c70b06424aa24503bChris Lattner } 1313033749571f8d4c804eeb357c70b06424aa24503bChris Lattner} 1314033749571f8d4c804eeb357c70b06424aa24503bChris Lattner 1315aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks/// \brief Checks that the given token is the first token that occurs after the 1316aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks/// given location (this excludes comments and whitespace). Returns the location 1317aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks/// immediately after the specified token. If the token is not found or the 1318aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks/// location is inside a macro, the returned source location will be invalid. 1319aca25bccefe56121b686706afc84c8cb5d46e65bAnna ZaksSourceLocation Lexer::findLocationAfterToken(SourceLocation Loc, 1320aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks tok::TokenKind TKind, 1321aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks const SourceManager &SM, 1322aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks const LangOptions &LangOpts, 1323aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks bool SkipTrailingWhitespaceAndNewLine) { 1324aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks if (Loc.isMacroID()) { 132569bda4c027671df7163619f215209529eb236620Argyrios Kyrtzidis if (!Lexer::isAtEndOfMacroExpansion(Loc, SM, LangOpts, &Loc)) 1326aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks return SourceLocation(); 1327aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks } 1328aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks Loc = Lexer::getLocForEndOfToken(Loc, 0, SM, LangOpts); 1329aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks 1330aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks // Break down the source location. 1331aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(Loc); 1332aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks 1333aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks // Try to load the file buffer. 1334aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks bool InvalidTemp = false; 1335cfa88f893915ceb8ae4ce2f17c46c24a4d67502fDmitri Gribenko StringRef File = SM.getBufferData(LocInfo.first, &InvalidTemp); 1336aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks if (InvalidTemp) 1337aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks return SourceLocation(); 1338aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks 1339aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks const char *TokenBegin = File.data() + LocInfo.second; 1340aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks 1341aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks // Lex from the start of the given location. 1342aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks Lexer lexer(SM.getLocForStartOfFile(LocInfo.first), LangOpts, File.begin(), 1343aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks TokenBegin, File.end()); 1344aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks // Find the token. 1345aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks Token Tok; 1346aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks lexer.LexFromRawLexer(Tok); 1347aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks if (Tok.isNot(TKind)) 1348aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks return SourceLocation(); 1349aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks SourceLocation TokenLoc = Tok.getLocation(); 1350aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks 1351aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks // Calculate how much whitespace needs to be skipped if any. 1352aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks unsigned NumWhitespaceChars = 0; 1353aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks if (SkipTrailingWhitespaceAndNewLine) { 1354aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks const char *TokenEnd = SM.getCharacterData(TokenLoc) + 1355aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks Tok.getLength(); 1356aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks unsigned char C = *TokenEnd; 1357aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks while (isHorizontalWhitespace(C)) { 1358aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks C = *(++TokenEnd); 1359aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks NumWhitespaceChars++; 1360aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks } 136135a2b798efd61fec425553f387d76be9c522f184Eli Friedman 136235a2b798efd61fec425553f387d76be9c522f184Eli Friedman // Skip \r, \n, \r\n, or \n\r 136335a2b798efd61fec425553f387d76be9c522f184Eli Friedman if (C == '\n' || C == '\r') { 136435a2b798efd61fec425553f387d76be9c522f184Eli Friedman char PrevC = C; 136535a2b798efd61fec425553f387d76be9c522f184Eli Friedman C = *(++TokenEnd); 1366aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks NumWhitespaceChars++; 136735a2b798efd61fec425553f387d76be9c522f184Eli Friedman if ((C == '\n' || C == '\r') && C != PrevC) 136835a2b798efd61fec425553f387d76be9c522f184Eli Friedman NumWhitespaceChars++; 136935a2b798efd61fec425553f387d76be9c522f184Eli Friedman } 1370aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks } 1371aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks 1372a64ccefdf0ea4e03ec88805d71b0af74950c7472Argyrios Kyrtzidis return TokenLoc.getLocWithOffset(Tok.getLength() + NumWhitespaceChars); 1373aca25bccefe56121b686706afc84c8cb5d46e65bAnna Zaks} 137424f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner 13755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// getCharAndSizeSlow - Peek a single 'character' from the specified buffer, 13765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// get its size, and return it. This is tricky in several cases: 13775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// 1. If currently at the start of a trigraph, we warn about the trigraph, 13785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// then either return the trigraph (skipping 3 chars) or the '?', 13795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// depending on whether trigraphs are enabled or not. 13805f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// 2. If this is an escaped newline (potentially with whitespace between 13815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// the backslash and newline), implicitly skip the newline and return 13825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// the char after it. 13835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// 13845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// This handles the slow/uncommon case of the getCharAndSize method. Here we 13855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// know that we can accumulate into Size, and that we have already incremented 13865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// Ptr by Size bytes. 13875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// 13885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// NOTE: When this method is updated, getCharAndSizeSlowNoWarn (below) should 13895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// be updated to match. 13905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// 13915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerchar Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size, 1392d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Token *Tok) { 13935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we have a slash, look for an escaped newline. 13945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Ptr[0] == '\\') { 13955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++Size; 13965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++Ptr; 13975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid SpencerSlash: 13985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Common case, backslash-char where the char is not whitespace. 13995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (!isWhitespace(Ptr[0])) return '\\'; 14001eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 14015636a3b6ece2c1f413464b72545e08eb0b7f06e4Chris Lattner // See if we have optional whitespace characters between the slash and 14025636a3b6ece2c1f413464b72545e08eb0b7f06e4Chris Lattner // newline. 140324f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) { 140424f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner // Remember that this token needs to be cleaned. 140524f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner if (Tok) Tok->setFlag(Token::NeedsCleaning); 140624f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner 140724f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner // Warn if there was whitespace between the backslash and newline. 14085636a3b6ece2c1f413464b72545e08eb0b7f06e4Chris Lattner if (Ptr[0] != '\n' && Ptr[0] != '\r' && Tok && !isLexingRawMode()) 140924f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner Diag(Ptr, diag::backslash_newline_space); 14101eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 141124f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner // Found backslash<whitespace><newline>. Parse the char after it. 141224f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner Size += EscapedNewLineSize; 141324f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner Ptr += EscapedNewLineSize; 1414f132dcaae82ebfc44c4fe0e84bf0b1f95e9d1251Argyrios Kyrtzidis 141504a94bcc56438b17e88db592708324041f75d48cArgyrios Kyrtzidis // If the char that we finally got was a \n, then we must have had 141604a94bcc56438b17e88db592708324041f75d48cArgyrios Kyrtzidis // something like \<newline><newline>. We don't want to consume the 141704a94bcc56438b17e88db592708324041f75d48cArgyrios Kyrtzidis // second newline. 141804a94bcc56438b17e88db592708324041f75d48cArgyrios Kyrtzidis if (*Ptr == '\n' || *Ptr == '\r' || *Ptr == '\0') 141904a94bcc56438b17e88db592708324041f75d48cArgyrios Kyrtzidis return ' '; 1420f132dcaae82ebfc44c4fe0e84bf0b1f95e9d1251Argyrios Kyrtzidis 142124f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner // Use slow version to accumulate a correct size field. 142224f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner return getCharAndSizeSlow(Ptr, Size, Tok); 142324f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner } 14241eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 14255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, this is not an escaped newline, just return the slash. 14265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return '\\'; 14275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 14281eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 14295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is a trigraph, process it. 14305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Ptr[0] == '?' && Ptr[1] == '?') { 14315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is actually a legal trigraph (not something like "??x"), emit 14325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // a trigraph warning. If so, and if trigraphs are enabled, return it. 14335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (char C = DecodeTrigraphChar(Ptr+2, Tok ? this : 0)) { 14345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Remember that this token needs to be cleaned. 1435d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner if (Tok) Tok->setFlag(Token::NeedsCleaning); 14365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 14375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Ptr += 3; 14385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Size += 3; 14395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (C == '\\') goto Slash; 14405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return C; 14415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 14425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 14431eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 14445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is neither, return a single character. 14455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++Size; 14465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return *Ptr; 14475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 14485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 14495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 14505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// getCharAndSizeSlowNoWarn - Handle the slow/uncommon case of the 14515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// getCharAndSizeNoWarn method. Here we know that we can accumulate into Size, 14525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// and that we have already incremented Ptr by Size bytes. 14535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// 14545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// NOTE: When this method is updated, getCharAndSizeSlow (above) should 14555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// be updated to match. 14565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerchar Lexer::getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size, 14574e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie const LangOptions &LangOpts) { 14585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we have a slash, look for an escaped newline. 14595f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Ptr[0] == '\\') { 14605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++Size; 14615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++Ptr; 14625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid SpencerSlash: 14635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Common case, backslash-char where the char is not whitespace. 14645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (!isWhitespace(Ptr[0])) return '\\'; 14651eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 14665f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // See if we have optional whitespace characters followed by a newline. 146724f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner if (unsigned EscapedNewLineSize = getEscapedNewLineSize(Ptr)) { 146824f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner // Found backslash<whitespace><newline>. Parse the char after it. 146924f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner Size += EscapedNewLineSize; 147024f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner Ptr += EscapedNewLineSize; 14711eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 147204a94bcc56438b17e88db592708324041f75d48cArgyrios Kyrtzidis // If the char that we finally got was a \n, then we must have had 147304a94bcc56438b17e88db592708324041f75d48cArgyrios Kyrtzidis // something like \<newline><newline>. We don't want to consume the 147404a94bcc56438b17e88db592708324041f75d48cArgyrios Kyrtzidis // second newline. 147504a94bcc56438b17e88db592708324041f75d48cArgyrios Kyrtzidis if (*Ptr == '\n' || *Ptr == '\r' || *Ptr == '\0') 147604a94bcc56438b17e88db592708324041f75d48cArgyrios Kyrtzidis return ' '; 1477f132dcaae82ebfc44c4fe0e84bf0b1f95e9d1251Argyrios Kyrtzidis 147824f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner // Use slow version to accumulate a correct size field. 14794e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie return getCharAndSizeSlowNoWarn(Ptr, Size, LangOpts); 148024f0e48c0aa62f2268e061aad70f9b19a59e7b52Chris Lattner } 14811eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 14825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, this is not an escaped newline, just return the slash. 14835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return '\\'; 14845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 14851eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 14865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is a trigraph, process it. 14874e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie if (LangOpts.Trigraphs && Ptr[0] == '?' && Ptr[1] == '?') { 14885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is actually a legal trigraph (not something like "??x"), return 14895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // it. 14905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (char C = GetTrigraphCharForLetter(Ptr[2])) { 14915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Ptr += 3; 14925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Size += 3; 14935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (C == '\\') goto Slash; 14945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return C; 14955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 14965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 14971eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 14985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is neither, return a single character. 14995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++Size; 15005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return *Ptr; 15015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 15025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 15035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 15045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// Helper methods for lexing. 15055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 15065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1507f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor/// \brief Routine that indiscriminately skips bytes in the source file. 1508f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregorvoid Lexer::SkipBytes(unsigned Bytes, bool StartOfLine) { 1509f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor BufferPtr += Bytes; 1510f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor if (BufferPtr > BufferEnd) 1511f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor BufferPtr = BufferEnd; 1512f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor IsAtStartOfLine = StartOfLine; 1513f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor} 1514f4f6c9db68465b886ec2e596feaa6ecc782395a4Douglas Gregor 1515c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rosenamespace { 1516c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose struct UCNCharRange { 1517c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose uint32_t Lower; 1518c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose uint32_t Upper; 1519c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose }; 1520c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 1521c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // C11 D.1, C++11 [charname.allowed] 1522c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // FIXME: C99 and C++03 each have a different set of allowed UCNs. 1523c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose const UCNCharRange UCNAllowedCharRanges[] = { 1524c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // 1 1525c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose { 0x00A8, 0x00A8 }, { 0x00AA, 0x00AA }, { 0x00AD, 0x00AD }, 1526c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose { 0x00AF, 0x00AF }, { 0x00B2, 0x00B5 }, { 0x00B7, 0x00BA }, 1527c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose { 0x00BC, 0x00BE }, { 0x00C0, 0x00D6 }, { 0x00D8, 0x00F6 }, 1528c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose { 0x00F8, 0x00FF }, 1529c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // 2 1530c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose { 0x0100, 0x167F }, { 0x1681, 0x180D }, { 0x180F, 0x1FFF }, 1531c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // 3 1532c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose { 0x200B, 0x200D }, { 0x202A, 0x202E }, { 0x203F, 0x2040 }, 1533c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose { 0x2054, 0x2054 }, { 0x2060, 0x206F }, 1534c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // 4 1535c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose { 0x2070, 0x218F }, { 0x2460, 0x24FF }, { 0x2776, 0x2793 }, 1536c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose { 0x2C00, 0x2DFF }, { 0x2E80, 0x2FFF }, 1537c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // 5 1538c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose { 0x3004, 0x3007 }, { 0x3021, 0x302F }, { 0x3031, 0x303F }, 1539c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // 6 1540c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose { 0x3040, 0xD7FF }, 1541c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // 7 1542c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose { 0xF900, 0xFD3D }, { 0xFD40, 0xFDCF }, { 0xFDF0, 0xFE44 }, 1543c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose { 0xFE47, 0xFFFD }, 1544c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // 8 1545c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose { 0x10000, 0x1FFFD }, { 0x20000, 0x2FFFD }, { 0x30000, 0x3FFFD }, 1546c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose { 0x40000, 0x4FFFD }, { 0x50000, 0x5FFFD }, { 0x60000, 0x6FFFD }, 1547c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose { 0x70000, 0x7FFFD }, { 0x80000, 0x8FFFD }, { 0x90000, 0x9FFFD }, 1548c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose { 0xA0000, 0xAFFFD }, { 0xB0000, 0xBFFFD }, { 0xC0000, 0xCFFFD }, 1549c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose { 0xD0000, 0xDFFFD }, { 0xE0000, 0xEFFFD } 1550c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose }; 1551c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose} 1552c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 1553c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rosestatic bool isAllowedIDChar(uint32_t c) { 1554c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose unsigned LowPoint = 0; 1555c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose unsigned HighPoint = llvm::array_lengthof(UCNAllowedCharRanges); 1556c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 1557c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // Binary search the UCNAllowedCharRanges set. 1558c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose while (HighPoint != LowPoint) { 1559c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose unsigned MidPoint = (HighPoint + LowPoint) / 2; 1560c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose if (c < UCNAllowedCharRanges[MidPoint].Lower) 1561c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose HighPoint = MidPoint; 1562c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose else if (c > UCNAllowedCharRanges[MidPoint].Upper) 1563c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose LowPoint = MidPoint + 1; 1564c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose else 1565c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose return true; 1566c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose } 1567c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 1568c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose return false; 1569c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose} 1570c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 1571c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rosestatic bool isAllowedInitiallyIDChar(uint32_t c) { 1572c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // C11 D.2, C++11 [charname.disallowed] 1573c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // FIXME: C99 only forbids "digits", presumably as described in C99 Annex D. 1574c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // FIXME: C++03 does not forbid any initial characters. 1575c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose return !(0x0300 <= c && c <= 0x036F) && 1576c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose !(0x1DC0 <= c && c <= 0x1DFF) && 1577c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose !(0x20D0 <= c && c <= 0x20FF) && 1578c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose !(0xFE20 <= c && c <= 0xFE2F); 1579c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose} 1580c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 1581c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rosestatic inline bool isASCII(char C) { 1582c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose return static_cast<signed char>(C) >= 0; 1583c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose} 1584c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 1585c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 1586d217773f106856a11879ec79dc468efefaf2ee75Chris Lattnervoid Lexer::LexIdentifier(Token &Result, const char *CurPtr) { 15875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Match [_A-Za-z0-9]*, we have already matched [_A-Za-z$] 15885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned Size; 15895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned char C = *CurPtr++; 1590cd991dbb12f24368753cef086c7ad3ec203c9ea6Chris Lattner while (isIdentifierBody(C)) 15915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = *CurPtr++; 1592cd991dbb12f24368753cef086c7ad3ec203c9ea6Chris Lattner 15935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer --CurPtr; // Back up over the skipped character. 15945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 15955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Fast path, no $,\,? in identifier found. '\' might be an escaped newline 15965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // or UCN, and ? might be a trigraph for '\', an escaped newline or UCN. 1597cd991dbb12f24368753cef086c7ad3ec203c9ea6Chris Lattner // 1598cd991dbb12f24368753cef086c7ad3ec203c9ea6Chris Lattner // TODO: Could merge these checks into a CharInfo flag to make the comparison 1599cd991dbb12f24368753cef086c7ad3ec203c9ea6Chris Lattner // cheaper 1600c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose if (isASCII(C) && C != '\\' && C != '?' && 1601c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose (C != '$' || !LangOpts.DollarIdents)) { 16025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid SpencerFinishIdentifier: 16035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *IdStart = BufferPtr; 1604c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara FormTokenWithChars(Result, CurPtr, tok::raw_identifier); 1605c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara Result.setRawIdentifierData(IdStart); 16061eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 16075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we are in raw mode, return this identifier raw. There is no need to 16085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // look up identifier information or attempt to macro expand it. 1609c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara if (LexingRawMode) 1610c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara return; 16111eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1612c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara // Fill in Result.IdentifierInfo and update the token kind, 1613c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara // looking up the identifier in the identifier table. 1614c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara IdentifierInfo *II = PP->LookUpIdentifierInfo(Result); 16151eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 16165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Finally, now that we know we have an identifier, pass this off to the 16175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // preprocessor, which may macro expand it or something. 1618d1186fa38166a581b51975f0382a45fc3a0733d0Chris Lattner if (II->isHandleIdentifierCase()) 16196a170eb3ea6d6319277becabef68eb1a26bf8766Chris Lattner PP->HandleIdentifier(Result); 16206aa52ec6b969faabf3764baf79d89810b8249a7eDouglas Gregor 16216a170eb3ea6d6319277becabef68eb1a26bf8766Chris Lattner return; 16225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 16231eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 16245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, $,\,? in identifier found. Enter slower path. 16251eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 16265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getCharAndSize(CurPtr, Size); 16275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (1) { 16285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (C == '$') { 16295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we hit a $ and they are not supported in identifiers, we are done. 16304e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie if (!LangOpts.DollarIdents) goto FinishIdentifier; 16311eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 16325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, emit a diagnostic and continue. 163374d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!isLexingRawMode()) 163474d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(CurPtr, diag::ext_dollar_in_identifier); 16355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, Size, Result); 16365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getCharAndSize(CurPtr, Size); 16375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer continue; 1638c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 1639c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose } else if (C == '\\') { 1640c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose const char *UCNPtr = CurPtr + Size; 1641c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose uint32_t CodePoint = tryReadUCN(UCNPtr, CurPtr, /*Token=*/0); 1642c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose if (CodePoint == 0 || !isAllowedIDChar(CodePoint)) 1643c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose goto FinishIdentifier; 1644c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 1645c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose Result.setFlag(Token::HasUCN); 1646c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose if ((UCNPtr - CurPtr == 6 && CurPtr[1] == 'u') || 1647c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose (UCNPtr - CurPtr == 10 && CurPtr[1] == 'U')) 1648c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose CurPtr = UCNPtr; 1649c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose else 1650c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose while (CurPtr != UCNPtr) 1651c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose (void)getAndAdvanceChar(CurPtr, Result); 1652c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 1653c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose C = getCharAndSize(CurPtr, Size); 1654c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose continue; 1655c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose } else if (!isASCII(C)) { 1656c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose const char *UnicodePtr = CurPtr; 1657c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose UTF32 CodePoint; 1658cb5620c9b213f4bd323912159fdddda35e258a14Dmitri Gribenko ConversionResult Result = 1659cb5620c9b213f4bd323912159fdddda35e258a14Dmitri Gribenko llvm::convertUTF8Sequence((const UTF8 **)&UnicodePtr, 1660cb5620c9b213f4bd323912159fdddda35e258a14Dmitri Gribenko (const UTF8 *)BufferEnd, 1661cb5620c9b213f4bd323912159fdddda35e258a14Dmitri Gribenko &CodePoint, 1662cb5620c9b213f4bd323912159fdddda35e258a14Dmitri Gribenko strictConversion); 1663c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose if (Result != conversionOK || 1664c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose !isAllowedIDChar(static_cast<uint32_t>(CodePoint))) 1665c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose goto FinishIdentifier; 1666c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 1667c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose CurPtr = UnicodePtr; 1668c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose C = getCharAndSize(CurPtr, Size); 1669c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose continue; 1670c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose } else if (!isIdentifierBody(C)) { 16715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer goto FinishIdentifier; 16725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 16735f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 16745f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, this character is good, consume it. 16755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, Size, Result); 16765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 16775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getCharAndSize(CurPtr, Size); 1678c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose while (isIdentifierBody(C)) { 16795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, Size, Result); 16805f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getCharAndSize(CurPtr, Size); 16815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 16825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 16835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 16845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1685a75ec43d625753b4439b0d6f70bd988444c74617Douglas Gregor/// isHexaLiteral - Return true if Start points to a hex constant. 16864a551000bee716ac8b1bbe16134a53f0ad221a5aChris Lattner/// in microsoft mode (where this is supposed to be several different tokens). 1687e506f8a41063410c75945ebb804758bd0202947fEli Friedmanbool Lexer::isHexaLiteral(const char *Start, const LangOptions &LangOpts) { 16886ab55ebab20086f725c4017b48ef8d7691ef870fChris Lattner unsigned Size; 16894e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie char C1 = Lexer::getCharAndSizeNoWarn(Start, Size, LangOpts); 16906ab55ebab20086f725c4017b48ef8d7691ef870fChris Lattner if (C1 != '0') 16916ab55ebab20086f725c4017b48ef8d7691ef870fChris Lattner return false; 16924e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie char C2 = Lexer::getCharAndSizeNoWarn(Start + Size, Size, LangOpts); 16936ab55ebab20086f725c4017b48ef8d7691ef870fChris Lattner return (C2 == 'x' || C2 == 'X'); 1694a75ec43d625753b4439b0d6f70bd988444c74617Douglas Gregor} 16955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 16965253c7ff266ae79308050c9f43d60dd1a67c5fb9Nate Begeman/// LexNumericConstant - Lex the remainder of a integer or floating point 16975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// constant. From[-1] is the first character lexed. Return the end of the 16985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// constant. 1699d217773f106856a11879ec79dc468efefaf2ee75Chris Lattnervoid Lexer::LexNumericConstant(Token &Result, const char *CurPtr) { 17005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned Size; 17015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char C = getCharAndSize(CurPtr, Size); 17025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char PrevCh = 0; 1703dd81731d82a33a242e7b088e249e56873ef52807Nico Weber while (isNumberBody(C)) { // FIXME: UCNs in ud-suffix. 17045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, Size, Result); 17055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer PrevCh = C; 17065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getCharAndSize(CurPtr, Size); 17075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 17081eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 17095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we fell out, check for a sign, due to 1e+12. If we have one, continue. 1710b2f4a20ddc281b194caa00e850ad74759e0d50baChris Lattner if ((C == '-' || C == '+') && (PrevCh == 'E' || PrevCh == 'e')) { 1711b2f4a20ddc281b194caa00e850ad74759e0d50baChris Lattner // If we are in Microsoft mode, don't continue if the constant is hex. 1712b2f4a20ddc281b194caa00e850ad74759e0d50baChris Lattner // For example, MSVC will accept the following as 3 tokens: 0x1234567e+1 17134e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie if (!LangOpts.MicrosoftExt || !isHexaLiteral(BufferPtr, LangOpts)) 1714b2f4a20ddc281b194caa00e850ad74759e0d50baChris Lattner return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result)); 1715b2f4a20ddc281b194caa00e850ad74759e0d50baChris Lattner } 17165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 17175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we have a hex FP constant, continue. 1718d2e95d1538ff91fe902464f02f83429f96117af5Richard Smith if ((C == '-' || C == '+') && (PrevCh == 'P' || PrevCh == 'p')) { 1719d2e95d1538ff91fe902464f02f83429f96117af5Richard Smith // Outside C99, we accept hexadecimal floating point numbers as a 1720d2e95d1538ff91fe902464f02f83429f96117af5Richard Smith // not-quite-conforming extension. Only do so if this looks like it's 1721d2e95d1538ff91fe902464f02f83429f96117af5Richard Smith // actually meant to be a hexfloat, and not if it has a ud-suffix. 1722d2e95d1538ff91fe902464f02f83429f96117af5Richard Smith bool IsHexFloat = true; 1723d2e95d1538ff91fe902464f02f83429f96117af5Richard Smith if (!LangOpts.C99) { 1724d2e95d1538ff91fe902464f02f83429f96117af5Richard Smith if (!isHexaLiteral(BufferPtr, LangOpts)) 1725d2e95d1538ff91fe902464f02f83429f96117af5Richard Smith IsHexFloat = false; 1726d2e95d1538ff91fe902464f02f83429f96117af5Richard Smith else if (std::find(BufferPtr, CurPtr, '_') != CurPtr) 1727d2e95d1538ff91fe902464f02f83429f96117af5Richard Smith IsHexFloat = false; 1728d2e95d1538ff91fe902464f02f83429f96117af5Richard Smith } 1729d2e95d1538ff91fe902464f02f83429f96117af5Richard Smith if (IsHexFloat) 1730d2e95d1538ff91fe902464f02f83429f96117af5Richard Smith return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result)); 1731d2e95d1538ff91fe902464f02f83429f96117af5Richard Smith } 17321eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 17335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Update the location of token as well as BufferPtr. 173447246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner const char *TokStart = BufferPtr; 17359e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, tok::numeric_constant); 173647246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner Result.setLiteralData(TokStart); 17375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 17385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 17395cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smith/// LexUDSuffix - Lex the ud-suffix production for user-defined literal suffixes 1740e816c717d4de1de6c67f1fd5ef4a927fe2bf2ea7Richard Smith/// in C++11, or warn on a ud-suffix in C++98. 17415cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smithconst char *Lexer::LexUDSuffix(Token &Result, const char *CurPtr) { 17424e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie assert(getLangOpts().CPlusPlus); 17435cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smith 17445cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smith // Maximally munch an identifier. FIXME: UCNs. 17455cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smith unsigned Size; 17465cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smith char C = getCharAndSize(CurPtr, Size); 17475cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smith if (isIdentifierHead(C)) { 174880ad52f327b532bded5c5b0ee38779d841c6cd35Richard Smith if (!getLangOpts().CPlusPlus11) { 1749e816c717d4de1de6c67f1fd5ef4a927fe2bf2ea7Richard Smith if (!isLexingRawMode()) 17502fb4ae36825ca3a0cbe7e845c5747062870066beRichard Smith Diag(CurPtr, 17512fb4ae36825ca3a0cbe7e845c5747062870066beRichard Smith C == '_' ? diag::warn_cxx11_compat_user_defined_literal 17522fb4ae36825ca3a0cbe7e845c5747062870066beRichard Smith : diag::warn_cxx11_compat_reserved_user_defined_literal) 17532fb4ae36825ca3a0cbe7e845c5747062870066beRichard Smith << FixItHint::CreateInsertion(getSourceLocation(CurPtr), " "); 17542fb4ae36825ca3a0cbe7e845c5747062870066beRichard Smith return CurPtr; 17552fb4ae36825ca3a0cbe7e845c5747062870066beRichard Smith } 17562fb4ae36825ca3a0cbe7e845c5747062870066beRichard Smith 17572fb4ae36825ca3a0cbe7e845c5747062870066beRichard Smith // C++11 [lex.ext]p10, [usrlit.suffix]p1: A program containing a ud-suffix 17582fb4ae36825ca3a0cbe7e845c5747062870066beRichard Smith // that does not start with an underscore is ill-formed. As a conforming 17592fb4ae36825ca3a0cbe7e845c5747062870066beRichard Smith // extension, we treat all such suffixes as if they had whitespace before 17602fb4ae36825ca3a0cbe7e845c5747062870066beRichard Smith // them. 17612fb4ae36825ca3a0cbe7e845c5747062870066beRichard Smith if (C != '_') { 17622fb4ae36825ca3a0cbe7e845c5747062870066beRichard Smith if (!isLexingRawMode()) 1763b0afd5df3c427c329f6c5e00fe264c5bada3bf36Francois Pichet Diag(CurPtr, getLangOpts().MicrosoftMode ? 1764b0afd5df3c427c329f6c5e00fe264c5bada3bf36Francois Pichet diag::ext_ms_reserved_user_defined_literal : 1765b0afd5df3c427c329f6c5e00fe264c5bada3bf36Francois Pichet diag::ext_reserved_user_defined_literal) 1766e816c717d4de1de6c67f1fd5ef4a927fe2bf2ea7Richard Smith << FixItHint::CreateInsertion(getSourceLocation(CurPtr), " "); 1767e816c717d4de1de6c67f1fd5ef4a927fe2bf2ea7Richard Smith return CurPtr; 1768e816c717d4de1de6c67f1fd5ef4a927fe2bf2ea7Richard Smith } 1769e816c717d4de1de6c67f1fd5ef4a927fe2bf2ea7Richard Smith 177099831e4677a7e2e051af636221694d60ba31fcdbRichard Smith Result.setFlag(Token::HasUDSuffix); 17715cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smith do { 17725cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smith CurPtr = ConsumeChar(CurPtr, Size, Result); 17735cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smith C = getCharAndSize(CurPtr, Size); 17745cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smith } while (isIdentifierBody(C)); 17755cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smith } 17765cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smith return CurPtr; 17775cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smith} 17785cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smith 17795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// LexStringLiteral - Lex the remainder of a string literal, after having lexed 17805cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor/// either " or L" or u8" or u" or U". 17815cee1195584fa8672253139c86e922daeda69b9eDouglas Gregorvoid Lexer::LexStringLiteral(Token &Result, const char *CurPtr, 17825cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor tok::TokenKind Kind) { 17835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *NulCharacter = 0; // Does this string contain the \0 character? 17841eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1785661a99690bc133bbaa029da925481d4a860dec90Richard Smith if (!isLexingRawMode() && 1786661a99690bc133bbaa029da925481d4a860dec90Richard Smith (Kind == tok::utf8_string_literal || 1787661a99690bc133bbaa029da925481d4a860dec90Richard Smith Kind == tok::utf16_string_literal || 1788661a99690bc133bbaa029da925481d4a860dec90Richard Smith Kind == tok::utf32_string_literal)) 1789661a99690bc133bbaa029da925481d4a860dec90Richard Smith Diag(BufferPtr, diag::warn_cxx98_compat_unicode_literal); 1790661a99690bc133bbaa029da925481d4a860dec90Richard Smith 17915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char C = getAndAdvanceChar(CurPtr, Result); 17925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (C != '"') { 1793571339c215dc3fa44acccb89c7c6b1d5b962b7f2Chris Lattner // Skip escaped characters. Escaped newlines will already be processed by 1794571339c215dc3fa44acccb89c7c6b1d5b962b7f2Chris Lattner // getAndAdvanceChar. 1795571339c215dc3fa44acccb89c7c6b1d5b962b7f2Chris Lattner if (C == '\\') 17965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getAndAdvanceChar(CurPtr, Result); 179733611e0d5ab1372608a7649b1877cd4300621c71Douglas Gregor 1798571339c215dc3fa44acccb89c7c6b1d5b962b7f2Chris Lattner if (C == '\n' || C == '\r' || // Newline. 179933611e0d5ab1372608a7649b1877cd4300621c71Douglas Gregor (C == 0 && CurPtr-1 == BufferEnd)) { // End of file. 18004e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) 1801b6ebd4490235c9ea6016530d623c46d0b9ce565bRichard Smith Diag(BufferPtr, diag::ext_unterminated_string); 18029e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr-1, tok::unknown); 18035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return; 18045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 1805571339c215dc3fa44acccb89c7c6b1d5b962b7f2Chris Lattner 18067d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (C == 0) { 18077d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (isCodeCompletionPoint(CurPtr-1)) { 18087d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis PP->CodeCompleteNaturalLanguage(); 18097d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis FormTokenWithChars(Result, CurPtr-1, tok::unknown); 18107d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis return cutOffLexing(); 18117d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis } 18127d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis 1813571339c215dc3fa44acccb89c7c6b1d5b962b7f2Chris Lattner NulCharacter = CurPtr-1; 18147d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis } 18155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getAndAdvanceChar(CurPtr, Result); 18165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 18171eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 18185cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smith // If we are in C++11, lex the optional ud-suffix. 18194e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie if (getLangOpts().CPlusPlus) 18205cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smith CurPtr = LexUDSuffix(Result, CurPtr); 18215cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smith 18225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If a nul character existed in the string, warn about it. 182374d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (NulCharacter && !isLexingRawMode()) 182474d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(NulCharacter, diag::null_in_string); 18255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 18265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Update the location of the token as well as the BufferPtr instance var. 182747246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner const char *TokStart = BufferPtr; 18285cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor FormTokenWithChars(Result, CurPtr, Kind); 182947246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner Result.setLiteralData(TokStart); 18305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 18315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 18322fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper/// LexRawStringLiteral - Lex the remainder of a raw string literal, after 18332fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper/// having lexed R", LR", u8R", uR", or UR". 18342fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Toppervoid Lexer::LexRawStringLiteral(Token &Result, const char *CurPtr, 18352fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper tok::TokenKind Kind) { 18362fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // This function doesn't use getAndAdvanceChar because C++0x [lex.pptoken]p3: 18372fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // Between the initial and final double quote characters of the raw string, 18382fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // any transformations performed in phases 1 and 2 (trigraphs, 18392fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // universal-character-names, and line splicing) are reverted. 18402fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 1841661a99690bc133bbaa029da925481d4a860dec90Richard Smith if (!isLexingRawMode()) 1842661a99690bc133bbaa029da925481d4a860dec90Richard Smith Diag(BufferPtr, diag::warn_cxx98_compat_raw_string_literal); 1843661a99690bc133bbaa029da925481d4a860dec90Richard Smith 18442fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper unsigned PrefixLen = 0; 18452fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 18462fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper while (PrefixLen != 16 && isRawStringDelimBody(CurPtr[PrefixLen])) 18472fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper ++PrefixLen; 18482fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 18492fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // If the last character was not a '(', then we didn't lex a valid delimiter. 18502fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (CurPtr[PrefixLen] != '(') { 18512fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (!isLexingRawMode()) { 18522fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper const char *PrefixEnd = &CurPtr[PrefixLen]; 18532fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (PrefixLen == 16) { 18542fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper Diag(PrefixEnd, diag::err_raw_delim_too_long); 18552fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } else { 18562fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper Diag(PrefixEnd, diag::err_invalid_char_raw_delim) 18572fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper << StringRef(PrefixEnd, 1); 18582fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 18592fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 18602fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 18612fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // Search for the next '"' in hopes of salvaging the lexer. Unfortunately, 18622fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // it's possible the '"' was intended to be part of the raw string, but 18632fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // there's not much we can do about that. 18642fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper while (1) { 18652fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper char C = *CurPtr++; 18662fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 18672fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (C == '"') 18682fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper break; 18692fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (C == 0 && CurPtr-1 == BufferEnd) { 18702fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper --CurPtr; 18712fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper break; 18722fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 18732fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 18742fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 18752fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper FormTokenWithChars(Result, CurPtr, tok::unknown); 18762fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return; 18772fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 18782fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 18792fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // Save prefix and move CurPtr past it 18802fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper const char *Prefix = CurPtr; 18812fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CurPtr += PrefixLen + 1; // skip over prefix and '(' 18822fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 18832fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper while (1) { 18842fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper char C = *CurPtr++; 18852fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 18862fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (C == ')') { 18872fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // Check for prefix match and closing quote. 18882fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (strncmp(CurPtr, Prefix, PrefixLen) == 0 && CurPtr[PrefixLen] == '"') { 18892fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper CurPtr += PrefixLen + 1; // skip over prefix and '"' 18902fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper break; 18912fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 18922fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } else if (C == 0 && CurPtr-1 == BufferEnd) { // End of file. 18932fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (!isLexingRawMode()) 18942fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper Diag(BufferPtr, diag::err_unterminated_raw_string) 18952fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper << StringRef(Prefix, PrefixLen); 18962fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper FormTokenWithChars(Result, CurPtr-1, tok::unknown); 18972fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return; 18982fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 18992fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 19002fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 19015cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smith // If we are in C++11, lex the optional ud-suffix. 19024e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie if (getLangOpts().CPlusPlus) 19035cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smith CurPtr = LexUDSuffix(Result, CurPtr); 19045cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smith 19052fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // Update the location of token as well as BufferPtr. 19062fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper const char *TokStart = BufferPtr; 19072fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper FormTokenWithChars(Result, CurPtr, Kind); 19082fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper Result.setLiteralData(TokStart); 19092fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper} 19102fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 19115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// LexAngledStringLiteral - Lex the remainder of an angled string literal, 19125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// after having lexed the '<' character. This is used for #include filenames. 1913d217773f106856a11879ec79dc468efefaf2ee75Chris Lattnervoid Lexer::LexAngledStringLiteral(Token &Result, const char *CurPtr) { 19145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *NulCharacter = 0; // Does this string contain the \0 character? 19159cb51ce03caf0755759f7e67aea7d4be684ff738Chris Lattner const char *AfterLessPos = CurPtr; 19165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char C = getAndAdvanceChar(CurPtr, Result); 19175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (C != '>') { 19185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Skip escaped characters. 19195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (C == '\\') { 19205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Skip the escaped character. 192160b202c5eb6fb2d608bfef05523f40fdf5091d48Dmitri Gribenko getAndAdvanceChar(CurPtr, Result); 19225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (C == '\n' || C == '\r' || // Newline. 19237d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis (C == 0 && (CurPtr-1 == BufferEnd || // End of file. 19247d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis isCodeCompletionPoint(CurPtr-1)))) { 19259cb51ce03caf0755759f7e67aea7d4be684ff738Chris Lattner // If the filename is unterminated, then it must just be a lone < 19269cb51ce03caf0755759f7e67aea7d4be684ff738Chris Lattner // character. Return this as such. 19279cb51ce03caf0755759f7e67aea7d4be684ff738Chris Lattner FormTokenWithChars(Result, AfterLessPos, tok::less); 19285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return; 19295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (C == 0) { 19305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer NulCharacter = CurPtr-1; 19315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 19325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getAndAdvanceChar(CurPtr, Result); 19335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 19341eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 19355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If a nul character existed in the string, warn about it. 193674d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (NulCharacter && !isLexingRawMode()) 193774d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(NulCharacter, diag::null_in_string); 19381eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 19395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Update the location of token as well as BufferPtr. 194047246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner const char *TokStart = BufferPtr; 19419e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, tok::angle_string_literal); 194247246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner Result.setLiteralData(TokStart); 19435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 19445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 19455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 19465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// LexCharConstant - Lex the remainder of a character constant, after having 19475cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor/// lexed either ' or L' or u' or U'. 19485cee1195584fa8672253139c86e922daeda69b9eDouglas Gregorvoid Lexer::LexCharConstant(Token &Result, const char *CurPtr, 19495cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor tok::TokenKind Kind) { 19505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *NulCharacter = 0; // Does this character contain the \0 character? 19515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 1952661a99690bc133bbaa029da925481d4a860dec90Richard Smith if (!isLexingRawMode() && 1953661a99690bc133bbaa029da925481d4a860dec90Richard Smith (Kind == tok::utf16_char_constant || Kind == tok::utf32_char_constant)) 1954661a99690bc133bbaa029da925481d4a860dec90Richard Smith Diag(BufferPtr, diag::warn_cxx98_compat_unicode_literal); 1955661a99690bc133bbaa029da925481d4a860dec90Richard Smith 19565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char C = getAndAdvanceChar(CurPtr, Result); 19575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (C == '\'') { 19584e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) 1959b6ebd4490235c9ea6016530d623c46d0b9ce565bRichard Smith Diag(BufferPtr, diag::ext_empty_character); 19609e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, tok::unknown); 19615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return; 19625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 19631eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 1964d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner while (C != '\'') { 1965d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner // Skip escaped characters. 19666d926ae667cca926e77ddce1734514911706ed0aNico Weber if (C == '\\') 19676d926ae667cca926e77ddce1734514911706ed0aNico Weber C = getAndAdvanceChar(CurPtr, Result); 19686d926ae667cca926e77ddce1734514911706ed0aNico Weber 19696d926ae667cca926e77ddce1734514911706ed0aNico Weber if (C == '\n' || C == '\r' || // Newline. 19706d926ae667cca926e77ddce1734514911706ed0aNico Weber (C == 0 && CurPtr-1 == BufferEnd)) { // End of file. 19714e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie if (!isLexingRawMode() && !LangOpts.AsmPreprocessor) 1972b6ebd4490235c9ea6016530d623c46d0b9ce565bRichard Smith Diag(BufferPtr, diag::ext_unterminated_char); 1973d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner FormTokenWithChars(Result, CurPtr-1, tok::unknown); 1974d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner return; 19756d926ae667cca926e77ddce1734514911706ed0aNico Weber } 19766d926ae667cca926e77ddce1734514911706ed0aNico Weber 19776d926ae667cca926e77ddce1734514911706ed0aNico Weber if (C == 0) { 19787d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (isCodeCompletionPoint(CurPtr-1)) { 19797d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis PP->CodeCompleteNaturalLanguage(); 19807d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis FormTokenWithChars(Result, CurPtr-1, tok::unknown); 19817d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis return cutOffLexing(); 19827d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis } 19837d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis 1984d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner NulCharacter = CurPtr-1; 1985d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner } 1986d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner C = getAndAdvanceChar(CurPtr, Result); 19875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 19881eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 19895cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smith // If we are in C++11, lex the optional ud-suffix. 19904e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie if (getLangOpts().CPlusPlus) 19915cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smith CurPtr = LexUDSuffix(Result, CurPtr); 19925cc2c6eb67b6e5361bbe96f79b519fd62ec666d6Richard Smith 1993d80f786689d608e5c22d6e1045884de7aff76c40Chris Lattner // If a nul character existed in the character, warn about it. 199474d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (NulCharacter && !isLexingRawMode()) 199574d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(NulCharacter, diag::null_in_char); 19965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 19975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Update the location of token as well as BufferPtr. 199847246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner const char *TokStart = BufferPtr; 19995cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor FormTokenWithChars(Result, CurPtr, Kind); 200047246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner Result.setLiteralData(TokStart); 20015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 20025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 20035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// SkipWhitespace - Efficiently skip over a series of whitespace characters. 20045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// Update BufferPtr to point to the next non-whitespace character and return. 2005d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner/// 2006d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner/// This method forms a token and returns true if KeepWhitespaceMode is enabled. 2007d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner/// 2008d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattnerbool Lexer::SkipWhitespace(Token &Result, const char *CurPtr) { 20095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Whitespace - Skip it, then return the token after the whitespace. 20105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned char Char = *CurPtr; // Skip consequtive spaces efficiently. 20115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (1) { 20125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Skip horizontal whitespace very aggressively. 20135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (isHorizontalWhitespace(Char)) 20145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = *++CurPtr; 20151eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2016ddd3e8b90a92c39f620bda7a0945320c0b9e60dbDaniel Dunbar // Otherwise if we have something other than whitespace, we're done. 20175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char != '\n' && Char != '\r') 20185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 20191eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 20205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (ParsingPreprocessorDirective) { 20215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // End of preprocessor directive line, let LexTokenInternal handle this. 20225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufferPtr = CurPtr; 2023d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner return false; 20245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 20251eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 20265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // ok, but handle newline. 20275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // The returned token is at the start of the line. 2028d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::StartOfLine); 20295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // No leading whitespace seen so far. 2030d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.clearFlag(Token::LeadingSpace); 20315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = *++CurPtr; 20325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 20335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 20345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this isn't immediately after a newline, there is leading space. 20355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char PrevChar = CurPtr[-1]; 20365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (PrevChar != '\n' && PrevChar != '\r') 2037d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::LeadingSpace); 20385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 2039d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner // If the client wants us to return whitespace, return it now. 2040d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner if (isKeepWhitespaceMode()) { 20419e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, tok::unknown); 2042d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner return true; 2043d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner } 20441eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 20455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufferPtr = CurPtr; 2046d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner return false; 20475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 20485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 2049bb23628148f555a4cf71f98c27096a7a804c085cNico Weber/// We have just read the // characters from input. Skip until we find the 2050bb23628148f555a4cf71f98c27096a7a804c085cNico Weber/// newline character thats terminate the comment. Then update BufferPtr and 2051bb23628148f555a4cf71f98c27096a7a804c085cNico Weber/// return. 2052046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner/// 2053046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner/// If we're in KeepCommentMode or any CommentHandler has inserted 2054046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner/// some tokens, this will store the first token and return true. 2055bb23628148f555a4cf71f98c27096a7a804c085cNico Weberbool Lexer::SkipLineComment(Token &Result, const char *CurPtr) { 2056bb23628148f555a4cf71f98c27096a7a804c085cNico Weber // If Line comments aren't explicitly enabled for this language, emit an 20575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // extension warning. 2058bb23628148f555a4cf71f98c27096a7a804c085cNico Weber if (!LangOpts.LineComment && !isLexingRawMode()) { 2059bb23628148f555a4cf71f98c27096a7a804c085cNico Weber Diag(BufferPtr, diag::ext_line_comment); 20601eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 20615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Mark them enabled so we only emit one warning for this translation 20625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // unit. 2063bb23628148f555a4cf71f98c27096a7a804c085cNico Weber LangOpts.LineComment = true; 20645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 20651eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 20665f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Scan over the body of the comment. The common case, when scanning, is that 20675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // the comment contains normal ascii characters with nothing interesting in 20685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // them. As such, optimize for this case with the inner loop. 20695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char C; 20705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer do { 20715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = *CurPtr; 20725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Skip over characters in the fast loop. 20735f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (C != 0 && // Potentially EOF. 20745f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C != '\n' && C != '\r') // Newline or DOS-style newline. 20755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = *++CurPtr; 20765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 20771daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer const char *NextLine = CurPtr; 20781daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer if (C != 0) { 20791daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer // We found a newline, see if it's escaped. 20801daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer const char *EscapePtr = CurPtr-1; 20811daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer while (isHorizontalWhitespace(*EscapePtr)) // Skip whitespace. 20821daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer --EscapePtr; 20831daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer 20841daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer if (*EscapePtr == '\\') // Escaped newline. 20851daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer CurPtr = EscapePtr; 20861daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer else if (EscapePtr[0] == '/' && EscapePtr[-1] == '?' && 20871daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer EscapePtr[-2] == '?') // Trigraph-escaped newline. 20881daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer CurPtr = EscapePtr-2; 20891daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer else 20901daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer break; // This is a newline, we're done. 20911daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer } 20921eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 20935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, this is a hard case. Fall back on getAndAdvanceChar to 2094bc3e9842212b78bcf49682641307e4f3419ecc5eChris Lattner // properly decode the character. Read it in raw mode to avoid emitting 2095bc3e9842212b78bcf49682641307e4f3419ecc5eChris Lattner // diagnostics about things like trigraphs. If we see an escaped newline, 2096bc3e9842212b78bcf49682641307e4f3419ecc5eChris Lattner // we'll handle it below. 20975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *OldPtr = CurPtr; 2098bc3e9842212b78bcf49682641307e4f3419ecc5eChris Lattner bool OldRawMode = isLexingRawMode(); 2099bc3e9842212b78bcf49682641307e4f3419ecc5eChris Lattner LexingRawMode = true; 21005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = getAndAdvanceChar(CurPtr, Result); 2101bc3e9842212b78bcf49682641307e4f3419ecc5eChris Lattner LexingRawMode = OldRawMode; 2102ead616c5d8681a41b11273327813e61bda01907aChris Lattner 21031daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer // If we only read only one character, then no special handling is needed. 21041daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer // We're done and can skip forward to the newline. 21051daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer if (C != 0 && CurPtr == OldPtr+1) { 21061daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer CurPtr = NextLine; 21071daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer break; 21081daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer } 21091daa58ea8973854b62e79f0939fa37cb6fed00e6Benjamin Kramer 21105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we read multiple characters, and one of those characters was a \r or 21115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // \n, then we had an escaped newline within the comment. Emit diagnostic 21125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // unless the next line is also a // comment. 21135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurPtr != OldPtr+1 && C != '/' && CurPtr[0] != '/') { 21145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer for (; OldPtr != CurPtr; ++OldPtr) 21155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (OldPtr[0] == '\n' || OldPtr[0] == '\r') { 21165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Okay, we found a // comment that ends in a newline, if the next 21175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // line is also a // comment, but has spaces, don't emit a diagnostic. 21185d6ae288bc661572e52ea71fc14b249eb5e2d9e9Benjamin Kramer if (isWhitespace(C)) { 21195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *ForwardPtr = CurPtr; 21205d6ae288bc661572e52ea71fc14b249eb5e2d9e9Benjamin Kramer while (isWhitespace(*ForwardPtr)) // Skip whitespace. 21215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++ForwardPtr; 21225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (ForwardPtr[0] == '/' && ForwardPtr[1] == '/') 21235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 21245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 21251eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 212674d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!isLexingRawMode()) 2127bb23628148f555a4cf71f98c27096a7a804c085cNico Weber Diag(OldPtr-1, diag::ext_multi_line_line_comment); 21285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 21295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 21305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 21311eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 213255817afdf9d453a443262a733f6caf6692dca118Douglas Gregor if (CurPtr == BufferEnd+1) { 213355817afdf9d453a443262a733f6caf6692dca118Douglas Gregor --CurPtr; 213455817afdf9d453a443262a733f6caf6692dca118Douglas Gregor break; 213555817afdf9d453a443262a733f6caf6692dca118Douglas Gregor } 21367d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis 21377d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) { 21387d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis PP->CodeCompleteNaturalLanguage(); 21397d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis cutOffLexing(); 21407d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis return false; 21417d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis } 21427d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis 21435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } while (C != '\n' && C != '\r'); 21445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 21453d0ad58b28e0d50fca7f21c6a078b05370510288Chris Lattner // Found but did not consume the newline. Notify comment handlers about the 21463d0ad58b28e0d50fca7f21c6a078b05370510288Chris Lattner // comment unless we're in a #if 0 block. 21473d0ad58b28e0d50fca7f21c6a078b05370510288Chris Lattner if (PP && !isLexingRawMode() && 21483d0ad58b28e0d50fca7f21c6a078b05370510288Chris Lattner PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr), 21493d0ad58b28e0d50fca7f21c6a078b05370510288Chris Lattner getSourceLocation(CurPtr)))) { 2150046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner BufferPtr = CurPtr; 2151046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner return true; // A token has to be returned. 2152046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner } 21531eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 21545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we are returning comments as tokens, return this comment as a token. 2155fa95a019da00b926d64ff83358ba73bbc6ae1e37Chris Lattner if (inKeepCommentMode()) 2156bb23628148f555a4cf71f98c27096a7a804c085cNico Weber return SaveLineComment(Result, CurPtr); 21575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 21585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we are inside a preprocessor directive and we see the end of line, 215984021556baceb76eedf7d44be8ba71d9b8cfaccePeter Collingbourne // return immediately, so that the lexer can return this as an EOD token. 21605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (ParsingPreprocessorDirective || CurPtr == BufferEnd) { 21615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufferPtr = CurPtr; 21622d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner return false; 21635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 21641eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 21655f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, eat the \n character. We don't care if this is a \n\r or 21667a4f004a77421af876152281b92c4f8ea24afb2dChris Lattner // \r\n sequence. This is an efficiency hack (because we know the \n can't 2167d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner // contribute to another token), it isn't needed for correctness. Note that 2168d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner // this is ok even in KeepWhitespaceMode, because we would have returned the 2169d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner /// comment above in that mode. 21705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++CurPtr; 21711eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 21725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // The next returned token is at the start of the line. 2173d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::StartOfLine); 21745f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // No leading whitespace seen so far. 2175d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.clearFlag(Token::LeadingSpace); 21765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufferPtr = CurPtr; 21772d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner return false; 21785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 21795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 2180bb23628148f555a4cf71f98c27096a7a804c085cNico Weber/// If in save-comment mode, package up this Line comment in an appropriate 2181bb23628148f555a4cf71f98c27096a7a804c085cNico Weber/// way and return it. 2182bb23628148f555a4cf71f98c27096a7a804c085cNico Weberbool Lexer::SaveLineComment(Token &Result, const char *CurPtr) { 21839e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner // If we're not in a preprocessor directive, just return the // comment 21849e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner // directly. 21859e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, tok::comment); 21861eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 21878c0b3787e7ccc7978b42dfbb84da2b802c743a5dDavid Blaikie if (!ParsingPreprocessorDirective || LexingRawMode) 21889e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner return true; 21891eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2190bb23628148f555a4cf71f98c27096a7a804c085cNico Weber // If this Line-style comment is in a macro definition, transmogrify it into 21919e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner // a C-style block comment. 2192453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor bool Invalid = false; 2193453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor std::string Spelling = PP->getSpelling(Result, &Invalid); 2194453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor if (Invalid) 2195453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor return true; 2196453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor 2197bb23628148f555a4cf71f98c27096a7a804c085cNico Weber assert(Spelling[0] == '/' && Spelling[1] == '/' && "Not line comment?"); 21989e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Spelling[1] = '*'; // Change prefix to "/*". 21999e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Spelling += "*/"; // add suffix. 22001eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 22019e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Result.setKind(tok::comment); 2202374b3837d676133fcc1eb70a25c8baf8ec4a5c4aDmitri Gribenko PP->CreateString(Spelling, Result, 2203a08529cc3f00e0b47a3c028823634129ac46847bAbramo Bagnara Result.getLocation(), Result.getLocation()); 22042d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner return true; 22055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 22065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 22075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// isBlockCommentEndOfEscapedNewLine - Return true if the specified newline 220880d7c52653800d3338cca2c4388348010d2b1921David Blaikie/// character (either \\n or \\r) is part of an escaped newline sequence. Issue 220980d7c52653800d3338cca2c4388348010d2b1921David Blaikie/// a diagnostic if so. We know that the newline is inside of a block comment. 22101eb4433ac451dc16f4133a88af2d002ac26c58efMike Stumpstatic bool isEndOfBlockCommentWithEscapedNewLine(const char *CurPtr, 22115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Lexer *L) { 22125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer assert(CurPtr[0] == '\n' || CurPtr[0] == '\r'); 22131eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 22145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Back up off the newline. 22155f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer --CurPtr; 22161eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 22175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is a two-character newline sequence, skip the other character. 22185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurPtr[0] == '\n' || CurPtr[0] == '\r') { 22195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // \n\n or \r\r -> not escaped newline. 22205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurPtr[0] == CurPtr[1]) 22215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return false; 22225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // \n\r or \r\n -> skip the newline. 22235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer --CurPtr; 22245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 22251eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 22265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we have horizontal whitespace, skip over it. We allow whitespace 22275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // between the slash and newline. 22285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer bool HasSpace = false; 22295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (isHorizontalWhitespace(*CurPtr) || *CurPtr == 0) { 22305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer --CurPtr; 22315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer HasSpace = true; 22325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 22331eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 22345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we have a slash, we know this is an escaped newline. 22355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (*CurPtr == '\\') { 22365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurPtr[-1] != '*') return false; 22375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 22385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // It isn't a slash, is it the ?? / trigraph? 22395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurPtr[0] != '/' || CurPtr[-1] != '?' || CurPtr[-2] != '?' || 22405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr[-3] != '*') 22415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return false; 22421eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 22435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // This is the trigraph ending the comment. Emit a stern warning! 22445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr -= 2; 22455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 22465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If no trigraphs are enabled, warn that we ignored this trigraph and 22475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // ignore this * character. 22484e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie if (!L->getLangOpts().Trigraphs) { 224974d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!L->isLexingRawMode()) 225074d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner L->Diag(CurPtr, diag::trigraph_ignored_block_comment); 22515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return false; 22525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 225374d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!L->isLexingRawMode()) 225474d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner L->Diag(CurPtr, diag::trigraph_ends_block_comment); 22555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 22561eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 22575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Warn about having an escaped newline between the */ characters. 225874d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!L->isLexingRawMode()) 225974d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner L->Diag(CurPtr, diag::escaped_newline_block_comment_end); 22601eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 22615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If there was space between the backslash and newline, warn about it. 226274d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (HasSpace && !L->isLexingRawMode()) 226374d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner L->Diag(CurPtr, diag::backslash_newline_space); 22641eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 22655f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return true; 22665f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 22675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 22685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#ifdef __SSE2__ 22695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include <emmintrin.h> 22705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#elif __ALTIVEC__ 22715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#include <altivec.h> 22725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#undef bool 22735f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#endif 22745f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 2275ec7699398bad018e8ea8134ef760a259598fefc1James Dennett/// We have just read from input the / and * characters that started a comment. 2276ec7699398bad018e8ea8134ef760a259598fefc1James Dennett/// Read until we find the * and / characters that terminate the comment. 2277ec7699398bad018e8ea8134ef760a259598fefc1James Dennett/// Note that we don't bother decoding trigraphs or escaped newlines in block 2278ec7699398bad018e8ea8134ef760a259598fefc1James Dennett/// comments, because they cannot cause the comment to end. The only thing 2279ec7699398bad018e8ea8134ef760a259598fefc1James Dennett/// that can happen is the comment could end with an escaped newline between 2280ec7699398bad018e8ea8134ef760a259598fefc1James Dennett/// the terminating * and /. 22812d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner/// 2282046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner/// If we're in KeepCommentMode or any CommentHandler has inserted 2283046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner/// some tokens, this will store the first token and return true. 2284d217773f106856a11879ec79dc468efefaf2ee75Chris Lattnerbool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) { 22855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Scan one character past where we should, looking for a '/' character. Once 2286fc8f0e14ad142ed811e90fbd9a30e419e301c717Chris Lattner // we find it, check to see if it was preceded by a *. This common 22875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // optimization helps people who like to put a lot of * characters in their 22885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // comments. 22898146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner 22908146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner // The first character we get with newlines and trigraphs skipped to handle 22918146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner // the degenerate /*/ case below correctly if the * has an escaped newline 22928146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner // after it. 22938146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner unsigned CharSize; 22948146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner unsigned char C = getCharAndSize(CurPtr, CharSize); 22958146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner CurPtr += CharSize; 22965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (C == 0 && CurPtr == BufferEnd+1) { 22977d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (!isLexingRawMode()) 22980af574270d3be2b0e73a3379dfaa633746f8fc6fChris Lattner Diag(BufferPtr, diag::err_unterminated_block_comment); 229931f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner --CurPtr; 23001eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 230131f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner // KeepWhitespaceMode should return this broken comment as a token. Since 230231f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner // it isn't a well formed comment, just return it as an 'unknown' token. 230331f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner if (isKeepWhitespaceMode()) { 23049e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, tok::unknown); 230531f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner return true; 230631f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner } 23071eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 230831f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner BufferPtr = CurPtr; 23092d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner return false; 23105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 23111eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 23128146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner // Check to see if the first character after the '/*' is another /. If so, 23138146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner // then this slash does not end the block comment, it is part of it. 23148146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner if (C == '/') 23158146b6851e873eab71341f1b1d3198894bc4c0acChris Lattner C = *CurPtr++; 23161eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 23175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (1) { 23185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Skip over all non-interesting characters until we find end of buffer or a 23195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // (probably ending) '/' character. 23207d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (CurPtr + 24 < BufferEnd && 23217d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis // If there is a code-completion point avoid the fast scan because it 23227d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis // doesn't check for '\0'. 23237d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis !(PP && PP->getCodeCompletionFileLoc() == FileLoc)) { 23245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // While not aligned to a 16-byte boundary. 23255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (C != '/' && ((intptr_t)CurPtr & 0x0F) != 0) 23265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = *CurPtr++; 23271eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 23285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (C == '/') goto FoundSlash; 23295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 23305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#ifdef __SSE2__ 23313f6f4e65e7806fd49137cf3cf59a1432ed776b2aBenjamin Kramer __m128i Slashes = _mm_set1_epi8('/'); 23323f6f4e65e7806fd49137cf3cf59a1432ed776b2aBenjamin Kramer while (CurPtr+16 <= BufferEnd) { 233331ba6135375433b617a8587ea6cc836a014ebd86Roman Divacky int cmp = _mm_movemask_epi8(_mm_cmpeq_epi8(*(const __m128i*)CurPtr, 233431ba6135375433b617a8587ea6cc836a014ebd86Roman Divacky Slashes)); 23353f6f4e65e7806fd49137cf3cf59a1432ed776b2aBenjamin Kramer if (cmp != 0) { 23366300f5b4382d5128c9346a4d4f05e7bac2e3d771Benjamin Kramer // Adjust the pointer to point directly after the first slash. It's 23376300f5b4382d5128c9346a4d4f05e7bac2e3d771Benjamin Kramer // not necessary to set C here, it will be overwritten at the end of 23386300f5b4382d5128c9346a4d4f05e7bac2e3d771Benjamin Kramer // the outer loop. 23396300f5b4382d5128c9346a4d4f05e7bac2e3d771Benjamin Kramer CurPtr += llvm::CountTrailingZeros_32(cmp) + 1; 23403f6f4e65e7806fd49137cf3cf59a1432ed776b2aBenjamin Kramer goto FoundSlash; 23413f6f4e65e7806fd49137cf3cf59a1432ed776b2aBenjamin Kramer } 23425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr += 16; 23433f6f4e65e7806fd49137cf3cf59a1432ed776b2aBenjamin Kramer } 23445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#elif __ALTIVEC__ 23455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer __vector unsigned char Slashes = { 23461eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump '/', '/', '/', '/', '/', '/', '/', '/', 23475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer '/', '/', '/', '/', '/', '/', '/', '/' 23485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer }; 23495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (CurPtr+16 <= BufferEnd && 23505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer !vec_any_eq(*(vector unsigned char*)CurPtr, Slashes)) 23515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr += 16; 23521eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump#else 23535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Scan for '/' quickly. Many block comments are very large. 23545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (CurPtr[0] != '/' && 23555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr[1] != '/' && 23565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr[2] != '/' && 23575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr[3] != '/' && 23585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr+4 < BufferEnd) { 23595f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr += 4; 23605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 23615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer#endif 23621eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 23635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // It has to be one of the bytes scanned, increment to it and read one. 23645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = *CurPtr++; 23655f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 23661eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 23675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Loop to scan the remainder. 23685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (C != '/' && C != '\0') 23695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = *CurPtr++; 23701eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 23715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (C == '/') { 23723f6f4e65e7806fd49137cf3cf59a1432ed776b2aBenjamin Kramer FoundSlash: 23735f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurPtr[-2] == '*') // We found the final */. We're done! 23745f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 23751eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 23765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if ((CurPtr[-2] == '\n' || CurPtr[-2] == '\r')) { 23775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (isEndOfBlockCommentWithEscapedNewLine(CurPtr-2, this)) { 23785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // We found the final */, though it had an escaped newline between the 23795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // * and /. We're done! 23805f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 23815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 23825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 23835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurPtr[0] == '*' && CurPtr[1] != '/') { 23845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If this is a /* inside of the comment, emit a warning. Don't do this 23855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // if this is a /*/, which will end the comment. This misses cases with 23865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // embedded escaped newlines, but oh well. 238774d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!isLexingRawMode()) 238874d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(CurPtr-1, diag::warn_nested_block_comment); 23895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 23905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (C == 0 && CurPtr == BufferEnd+1) { 23917d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (!isLexingRawMode()) 239274d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(BufferPtr, diag::err_unterminated_block_comment); 23935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Note: the user probably forgot a */. We could continue immediately 23945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // after the /*, but this would involve lexing a lot of what really is the 23955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // comment, which surely would confuse the parser. 239631f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner --CurPtr; 23971eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 239831f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner // KeepWhitespaceMode should return this broken comment as a token. Since 239931f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner // it isn't a well formed comment, just return it as an 'unknown' token. 240031f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner if (isKeepWhitespaceMode()) { 24019e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, tok::unknown); 240231f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner return true; 240331f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner } 24041eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 240531f0ecae4b2d4d268d04c562a43d9fbfee96f189Chris Lattner BufferPtr = CurPtr; 24062d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner return false; 24077d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis } else if (C == '\0' && isCodeCompletionPoint(CurPtr-1)) { 24087d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis PP->CodeCompleteNaturalLanguage(); 24097d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis cutOffLexing(); 24107d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis return false; 24115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 24127d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis 24135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer C = *CurPtr++; 24145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 24151eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 24163d0ad58b28e0d50fca7f21c6a078b05370510288Chris Lattner // Notify comment handlers about the comment unless we're in a #if 0 block. 24173d0ad58b28e0d50fca7f21c6a078b05370510288Chris Lattner if (PP && !isLexingRawMode() && 24183d0ad58b28e0d50fca7f21c6a078b05370510288Chris Lattner PP->HandleComment(Result, SourceRange(getSourceLocation(BufferPtr), 24193d0ad58b28e0d50fca7f21c6a078b05370510288Chris Lattner getSourceLocation(CurPtr)))) { 2420046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner BufferPtr = CurPtr; 2421046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner return true; // A token has to be returned. 2422046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner } 24232e22253e03e175144aeb9d13350a12fd83f858beDouglas Gregor 24245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we are returning comments as tokens, return this comment as a token. 2425fa95a019da00b926d64ff83358ba73bbc6ae1e37Chris Lattner if (inKeepCommentMode()) { 24269e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, tok::comment); 24272d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner return true; 24285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 24295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 24305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // It is common for the tokens immediately after a /**/ comment to be 24315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // whitespace. Instead of going through the big switch, handle it 2432d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner // efficiently now. This is safe even in KeepWhitespaceMode because we would 2433d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner // have already returned above with the comment as a token. 24345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (isHorizontalWhitespace(*CurPtr)) { 2435d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::LeadingSpace); 24365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer SkipWhitespace(Result, CurPtr+1); 24372d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner return false; 24385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 24395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 24405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Otherwise, just return so that the next character will be lexed as a token. 24415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufferPtr = CurPtr; 2442d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::LeadingSpace); 24432d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner return false; 24445f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 24455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 24465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 24475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer// Primary Lexing Entry Points 24485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer//===----------------------------------------------------------------------===// 24495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 24505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// ReadToEndOfLine - Read the rest of the current preprocessor line as an 24515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// uninterpreted string. This switches the lexer out of directive mode. 24523093b20d824a953d8bc7a786dd952414898f8d6dBenjamin Kramervoid Lexer::ReadToEndOfLine(SmallVectorImpl<char> *Result) { 24535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer assert(ParsingPreprocessorDirective && ParsingFilename == false && 24545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer "Must be in a preprocessing directive!"); 2455d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Token Tmp; 24565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 24575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // CurPtr - Cache BufferPtr in an automatic variable. 24585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *CurPtr = BufferPtr; 24595f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (1) { 24605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char Char = getAndAdvanceChar(CurPtr, Tmp); 24615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer switch (Char) { 24625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer default: 24633093b20d824a953d8bc7a786dd952414898f8d6dBenjamin Kramer if (Result) 24643093b20d824a953d8bc7a786dd952414898f8d6dBenjamin Kramer Result->push_back(Char); 24655f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 24665f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case 0: // Null. 24675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Found end of file? 24685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurPtr-1 != BufferEnd) { 24697d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (isCodeCompletionPoint(CurPtr-1)) { 24707d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis PP->CodeCompleteNaturalLanguage(); 24717d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis cutOffLexing(); 24723093b20d824a953d8bc7a786dd952414898f8d6dBenjamin Kramer return; 24737d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis } 24747d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis 24755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Nope, normal character, continue. 24763093b20d824a953d8bc7a786dd952414898f8d6dBenjamin Kramer if (Result) 24773093b20d824a953d8bc7a786dd952414898f8d6dBenjamin Kramer Result->push_back(Char); 24785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 24795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 24805f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // FALL THROUGH. 24815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\r': 24825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\n': 24835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Okay, we found the end of the line. First, back up past the \0, \r, \n. 24845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer assert(CurPtr[-1] == Char && "Trigraphs for newline?"); 24855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufferPtr = CurPtr-1; 24861eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 248784021556baceb76eedf7d44be8ba71d9b8cfaccePeter Collingbourne // Next, lex the character, which should handle the EOD transition. 24885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Lex(Tmp); 248955817afdf9d453a443262a733f6caf6692dca118Douglas Gregor if (Tmp.is(tok::code_completion)) { 24907d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (PP) 24917d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis PP->CodeCompleteNaturalLanguage(); 249255817afdf9d453a443262a733f6caf6692dca118Douglas Gregor Lex(Tmp); 249355817afdf9d453a443262a733f6caf6692dca118Douglas Gregor } 249484021556baceb76eedf7d44be8ba71d9b8cfaccePeter Collingbourne assert(Tmp.is(tok::eod) && "Unexpected token!"); 24951eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 24963093b20d824a953d8bc7a786dd952414898f8d6dBenjamin Kramer // Finally, we're done; 24973093b20d824a953d8bc7a786dd952414898f8d6dBenjamin Kramer return; 24985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 24995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 25005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 25015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 25025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// LexEndOfFile - CurPtr points to the end of this file. Handle this 25035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// condition, reporting diagnostics and handling other edge cases as required. 25045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// This returns true if Result contains a token, false if PP.Lex should be 25055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// called again. 2506d217773f106856a11879ec79dc468efefaf2ee75Chris Lattnerbool Lexer::LexEndOfFile(Token &Result, const char *CurPtr) { 25075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we hit the end of the file while parsing a preprocessor directive, 25085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // end the preprocessor directive first. The next token returned will 25095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // then be the end of file. 25105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (ParsingPreprocessorDirective) { 25115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Done parsing the "line". 25125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ParsingPreprocessorDirective = false; 25135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Update the location of token as well as BufferPtr. 251484021556baceb76eedf7d44be8ba71d9b8cfaccePeter Collingbourne FormTokenWithChars(Result, CurPtr, tok::eod); 25151eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 25165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Restore comment saving mode, in case it was disabled for directive. 2517f744d133e8d36eec65828d046ee695bda238125cChris Lattner SetCommentRetentionState(PP->getCommentRetentionState()); 25185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return true; // Have a token. 25191eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } 252086d9a52c24d390631a888d4ff812e1b15445e0a0Douglas Gregor 252186d9a52c24d390631a888d4ff812e1b15445e0a0Douglas Gregor // If we are in raw mode, return this event as an EOF token. Let the caller 252286d9a52c24d390631a888d4ff812e1b15445e0a0Douglas Gregor // that put us in raw mode handle the event. 252386d9a52c24d390631a888d4ff812e1b15445e0a0Douglas Gregor if (isLexingRawMode()) { 252486d9a52c24d390631a888d4ff812e1b15445e0a0Douglas Gregor Result.startToken(); 252586d9a52c24d390631a888d4ff812e1b15445e0a0Douglas Gregor BufferPtr = BufferEnd; 252686d9a52c24d390631a888d4ff812e1b15445e0a0Douglas Gregor FormTokenWithChars(Result, BufferEnd, tok::eof); 252786d9a52c24d390631a888d4ff812e1b15445e0a0Douglas Gregor return true; 252886d9a52c24d390631a888d4ff812e1b15445e0a0Douglas Gregor } 252981b747b7fcc91c2fba9a3183d8fac80adbfc1d3eDouglas Gregor 2530f44e854ed1e3aa86d2ed6d615ccd109d50ddcff9Douglas Gregor // Issue diagnostics for unterminated #if and missing newline. 2531f44e854ed1e3aa86d2ed6d615ccd109d50ddcff9Douglas Gregor 25325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we are in a #if directive, emit an error. 25335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while (!ConditionalStack.empty()) { 25347d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (PP->getCodeCompletionFileLoc() != FileLoc) 25352d474ba9e8ae43a1a5a9f72718c0d79092b9453fDouglas Gregor PP->Diag(ConditionalStack.back().IfLoc, 25362d474ba9e8ae43a1a5a9f72718c0d79092b9453fDouglas Gregor diag::err_pp_unterminated_conditional); 25375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ConditionalStack.pop_back(); 25385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 25391eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2540b25e5d79d9d1967df058a242e96a62d0d0ace074Chris Lattner // C99 5.1.1.2p2: If the file is non-empty and didn't end in a newline, issue 2541b25e5d79d9d1967df058a242e96a62d0d0ace074Chris Lattner // a pedwarn. 25425e6c3f0397254e2a1a9ef85cf13232f89e653640Seth Cantrell if (CurPtr != BufferStart && (CurPtr[-1] != '\n' && CurPtr[-1] != '\r')) 254380ad52f327b532bded5c5b0ee38779d841c6cd35Richard Smith Diag(BufferEnd, LangOpts.CPlusPlus11 ? // C++11 [lex.phases] 2.2 p2 25445e6c3f0397254e2a1a9ef85cf13232f89e653640Seth Cantrell diag::warn_cxx98_compat_no_newline_eof : diag::ext_no_newline_eof) 25455e6c3f0397254e2a1a9ef85cf13232f89e653640Seth Cantrell << FixItHint::CreateInsertion(getSourceLocation(BufferEnd), "\n"); 25461eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 25475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufferPtr = CurPtr; 25485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 25495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Finally, let the preprocessor handle this. 25500cdd1fe3ec29b5cbff9a728966ace5c5b5d614f7Jordan Rose return PP->HandleEndOfFile(Result, isPragmaLexer()); 25515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 25525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 25535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// isNextPPTokenLParen - Return 1 if the next unexpanded token lexed from 25545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// the specified lexer will return a tok::l_paren token, 0 if it is something 25555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// else and 2 if there are no more tokens in the buffer controlled by the 25565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// lexer. 25575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencerunsigned Lexer::isNextPPTokenLParen() { 25585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer assert(!LexingRawMode && "How can we expand a macro from a skipping buffer?"); 25591eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 25605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Switch to 'skipping' mode. This will ensure that we can lex a token 25615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // without emitting diagnostics, disables macro expansion, and will cause EOF 25625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // to return an EOF token instead of popping the include stack. 25635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer LexingRawMode = true; 25641eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 25655f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Save state that can be changed while lexing so that we can restore it. 25665f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *TmpBufferPtr = BufferPtr; 2567a864cf7c1d774a0f790bfc46befc87d9dbf1f65cChris Lattner bool inPPDirectiveMode = ParsingPreprocessorDirective; 25681eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2569d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Token Tok; 25705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Tok.startToken(); 25715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer LexTokenInternal(Tok); 25721eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 25735f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Restore state that may have changed. 25745f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufferPtr = TmpBufferPtr; 2575a864cf7c1d774a0f790bfc46befc87d9dbf1f65cChris Lattner ParsingPreprocessorDirective = inPPDirectiveMode; 25761eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 25775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Restore the lexer back to non-skipping mode. 25785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer LexingRawMode = false; 25791eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 258022f6bbcafa8871f4f20c4402d9cbc5c024fee99aChris Lattner if (Tok.is(tok::eof)) 25815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return 2; 258222f6bbcafa8871f4f20c4402d9cbc5c024fee99aChris Lattner return Tok.is(tok::l_paren); 25835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 25845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 2585ec7699398bad018e8ea8134ef760a259598fefc1James Dennett/// \brief Find the end of a version control conflict marker. 2586d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smithstatic const char *FindConflictEnd(const char *CurPtr, const char *BufferEnd, 2587d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith ConflictMarkerKind CMK) { 2588d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith const char *Terminator = CMK == CMK_Perforce ? "<<<<\n" : ">>>>>>>"; 2589d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith size_t TermLen = CMK == CMK_Perforce ? 5 : 7; 2590d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith StringRef RestOfBuffer(CurPtr+TermLen, BufferEnd-CurPtr-TermLen); 2591d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith size_t Pos = RestOfBuffer.find(Terminator); 25925f9e272e632e951b1efe824cd16acb4d96077930Chris Lattner while (Pos != StringRef::npos) { 259334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // Must occur at start of line. 259434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (RestOfBuffer[Pos-1] != '\r' && 259534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner RestOfBuffer[Pos-1] != '\n') { 2596d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith RestOfBuffer = RestOfBuffer.substr(Pos+TermLen); 2597d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith Pos = RestOfBuffer.find(Terminator); 259834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner continue; 259934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } 260034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return RestOfBuffer.data()+Pos; 260134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } 260234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return 0; 260334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner} 260434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 260534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner/// IsStartOfConflictMarker - If the specified pointer is the start of a version 260634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner/// control conflict marker like '<<<<<<<', recognize it as such, emit an error 260734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner/// and recover nicely. This returns true if it is a conflict marker and false 260834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner/// if not. 260934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattnerbool Lexer::IsStartOfConflictMarker(const char *CurPtr) { 261034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // Only a conflict marker if it starts at the beginning of a line. 261134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (CurPtr != BufferStart && 261234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner CurPtr[-1] != '\n' && CurPtr[-1] != '\r') 261334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return false; 261434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 2615d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith // Check to see if we have <<<<<<< or >>>>. 2616d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith if ((BufferEnd-CurPtr < 8 || StringRef(CurPtr, 7) != "<<<<<<<") && 2617d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith (BufferEnd-CurPtr < 6 || StringRef(CurPtr, 5) != ">>>> ")) 261834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return false; 261934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 262034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // If we have a situation where we don't care about conflict markers, ignore 262134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // it. 2622d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith if (CurrentConflictMarkerState || isLexingRawMode()) 262334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return false; 262434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 2625d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith ConflictMarkerKind Kind = *CurPtr == '<' ? CMK_Normal : CMK_Perforce; 2626d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith 2627d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith // Check to see if there is an ending marker somewhere in the buffer at the 2628d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith // start of a line to terminate this conflict marker. 2629d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith if (FindConflictEnd(CurPtr, BufferEnd, Kind)) { 263034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // We found a match. We are really in a conflict marker. 263134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // Diagnose this, and ignore to the end of line. 263234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner Diag(CurPtr, diag::err_conflict_marker); 2633d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith CurrentConflictMarkerState = Kind; 263434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 263534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // Skip ahead to the end of line. We know this exists because the 263634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // end-of-conflict marker starts with \r or \n. 263734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner while (*CurPtr != '\r' && *CurPtr != '\n') { 263834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner assert(CurPtr != BufferEnd && "Didn't find end of line"); 263934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner ++CurPtr; 264034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } 264134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner BufferPtr = CurPtr; 264234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return true; 264334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } 264434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 264534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // No end of conflict marker found. 264634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return false; 264734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner} 264834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 264934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 2650d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith/// HandleEndOfConflictMarker - If this is a '====' or '||||' or '>>>>', or if 2651d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith/// it is '<<<<' and the conflict marker started with a '>>>>' marker, then it 2652d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith/// is the end of a conflict marker. Handle it by ignoring up until the end of 2653d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith/// the line. This returns true if it is a conflict marker and false if not. 265434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattnerbool Lexer::HandleEndOfConflictMarker(const char *CurPtr) { 265534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // Only a conflict marker if it starts at the beginning of a line. 265634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (CurPtr != BufferStart && 265734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner CurPtr[-1] != '\n' && CurPtr[-1] != '\r') 265834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return false; 265934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 266034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // If we have a situation where we don't care about conflict markers, ignore 266134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // it. 2662d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith if (!CurrentConflictMarkerState || isLexingRawMode()) 266334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return false; 266434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 2665d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith // Check to see if we have the marker (4 characters in a row). 2666d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith for (unsigned i = 1; i != 4; ++i) 266734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (CurPtr[i] != CurPtr[0]) 266834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return false; 266934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 267034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // If we do have it, search for the end of the conflict marker. This could 267134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // fail if it got skipped with a '#if 0' or something. Note that CurPtr might 267234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // be the end of conflict marker. 2673d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith if (const char *End = FindConflictEnd(CurPtr, BufferEnd, 2674d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith CurrentConflictMarkerState)) { 267534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner CurPtr = End; 267634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 267734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // Skip ahead to the end of line. 267834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner while (CurPtr != BufferEnd && *CurPtr != '\r' && *CurPtr != '\n') 267934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner ++CurPtr; 268034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 268134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner BufferPtr = CurPtr; 268234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 268334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // No longer in the conflict marker. 2684d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith CurrentConflictMarkerState = CMK_None; 268534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return true; 268634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } 268734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 268834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner return false; 268934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner} 269034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 26917d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidisbool Lexer::isCodeCompletionPoint(const char *CurPtr) const { 26927d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (PP && PP->isCodeCompletionEnabled()) { 2693a64ccefdf0ea4e03ec88805d71b0af74950c7472Argyrios Kyrtzidis SourceLocation Loc = FileLoc.getLocWithOffset(CurPtr-BufferStart); 26947d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis return Loc == PP->getCodeCompletionLoc(); 26957d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis } 26967d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis 26977d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis return false; 26987d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis} 26997d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis 2700c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Roseuint32_t Lexer::tryReadUCN(const char *&StartPtr, const char *SlashLoc, 2701c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose Token *Result) { 2702c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose unsigned CharSize; 2703c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose char Kind = getCharAndSize(StartPtr, CharSize); 2704c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 2705c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose unsigned NumHexDigits; 2706c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose if (Kind == 'u') 2707c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose NumHexDigits = 4; 2708c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose else if (Kind == 'U') 2709c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose NumHexDigits = 8; 2710c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose else 2711c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose return 0; 2712c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 2713bfec916e5fc40f22ac11267e78a024cd8dcf3bbfJordan Rose if (!LangOpts.CPlusPlus && !LangOpts.C99) { 27148094bac4e987caf90e8fd719c24545add8dafcb6Jordan Rose if (Result && !isLexingRawMode()) 27158094bac4e987caf90e8fd719c24545add8dafcb6Jordan Rose Diag(SlashLoc, diag::warn_ucn_not_valid_in_c89); 2716bfec916e5fc40f22ac11267e78a024cd8dcf3bbfJordan Rose return 0; 2717bfec916e5fc40f22ac11267e78a024cd8dcf3bbfJordan Rose } 2718bfec916e5fc40f22ac11267e78a024cd8dcf3bbfJordan Rose 2719c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose const char *CurPtr = StartPtr + CharSize; 2720c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose const char *KindLoc = &CurPtr[-1]; 2721c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 2722c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose uint32_t CodePoint = 0; 2723c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose for (unsigned i = 0; i < NumHexDigits; ++i) { 2724c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose char C = getCharAndSize(CurPtr, CharSize); 2725c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 2726c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose unsigned Value = llvm::hexDigitValue(C); 2727c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose if (Value == -1U) { 2728c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose if (Result && !isLexingRawMode()) { 2729c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose if (i == 0) { 2730c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose Diag(BufferPtr, diag::warn_ucn_escape_no_digits) 2731c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose << StringRef(KindLoc, 1); 2732c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose } else { 2733c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose Diag(BufferPtr, diag::warn_ucn_escape_incomplete); 2734b87672b124ab4fbe6f2cabc2ad71655fc71230eaJordan Rose 2735b87672b124ab4fbe6f2cabc2ad71655fc71230eaJordan Rose // If the user wrote \U1234, suggest a fixit to \u. 2736b87672b124ab4fbe6f2cabc2ad71655fc71230eaJordan Rose if (i == 4 && NumHexDigits == 8) { 2737b87672b124ab4fbe6f2cabc2ad71655fc71230eaJordan Rose CharSourceRange URange = 2738b87672b124ab4fbe6f2cabc2ad71655fc71230eaJordan Rose CharSourceRange::getCharRange(getSourceLocation(KindLoc), 2739b87672b124ab4fbe6f2cabc2ad71655fc71230eaJordan Rose getSourceLocation(KindLoc + 1)); 2740b87672b124ab4fbe6f2cabc2ad71655fc71230eaJordan Rose Diag(KindLoc, diag::note_ucn_four_not_eight) 2741b87672b124ab4fbe6f2cabc2ad71655fc71230eaJordan Rose << FixItHint::CreateReplacement(URange, "u"); 2742b87672b124ab4fbe6f2cabc2ad71655fc71230eaJordan Rose } 2743c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose } 2744c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose } 2745bfec916e5fc40f22ac11267e78a024cd8dcf3bbfJordan Rose 2746c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose return 0; 2747c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose } 2748c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 2749c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose CodePoint <<= 4; 2750c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose CodePoint += Value; 2751c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 2752c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose CurPtr += CharSize; 2753c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose } 2754c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 2755c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose if (Result) { 2756c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose Result->setFlag(Token::HasUCN); 2757b6c08a64145485a7c233761220b8d82b74aa7546NAKAMURA Takumi if (CurPtr - StartPtr == (ptrdiff_t)NumHexDigits + 2) 2758c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose StartPtr = CurPtr; 2759c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose else 2760c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose while (StartPtr != CurPtr) 2761c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose (void)getAndAdvanceChar(StartPtr, *Result); 2762c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose } else { 2763c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose StartPtr = CurPtr; 2764c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose } 2765c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 2766c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // C99 6.4.3p2: A universal character name shall not specify a character whose 2767c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // short identifier is less than 00A0 other than 0024 ($), 0040 (@), or 2768c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // 0060 (`), nor one in the range D800 through DFFF inclusive.) 2769c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // C++11 [lex.charset]p2: If the hexadecimal value for a 2770c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // universal-character-name corresponds to a surrogate code point (in the 2771c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // range 0xD800-0xDFFF, inclusive), the program is ill-formed. Additionally, 2772c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // if the hexadecimal value for a universal-character-name outside the 2773c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // c-char-sequence, s-char-sequence, or r-char-sequence of a character or 2774c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // string literal corresponds to a control character (in either of the 2775c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // ranges 0x00-0x1F or 0x7F-0x9F, both inclusive) or to a character in the 2776c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // basic source character set, the program is ill-formed. 2777c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose if (CodePoint < 0xA0) { 2778c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose if (CodePoint == 0x24 || CodePoint == 0x40 || CodePoint == 0x60) 2779c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose return CodePoint; 2780c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 2781c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // We don't use isLexingRawMode() here because we need to warn about bad 2782c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // UCNs even when skipping preprocessing tokens in a #if block. 2783c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose if (Result && PP) { 2784c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose if (CodePoint < 0x20 || CodePoint >= 0x7F) 2785c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose Diag(BufferPtr, diag::err_ucn_control_character); 2786c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose else { 2787c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose char C = static_cast<char>(CodePoint); 2788c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose Diag(BufferPtr, diag::err_ucn_escape_basic_scs) << StringRef(&C, 1); 2789c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose } 2790c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose } 2791c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 2792c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose return 0; 2793c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 2794c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose } else if ((!LangOpts.CPlusPlus || LangOpts.CPlusPlus11) && 2795c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose (CodePoint >= 0xD800 && CodePoint <= 0xDFFF)) { 2796c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // C++03 allows UCNs representing surrogate characters. C99 and C++11 don't. 2797c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // We don't use isLexingRawMode() here because we need to warn about bad 2798c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // UCNs even when skipping preprocessing tokens in a #if block. 2799c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose if (Result && PP) 2800c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose Diag(BufferPtr, diag::err_ucn_escape_invalid); 2801c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose return 0; 2802c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose } 2803c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 2804c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose return CodePoint; 2805c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose} 2806c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 2807fc12060ed595fd23d731b8a86adb21ddbb8c7bfbJordan Rosestatic bool isUnicodeWhitespace(uint32_t C) { 2808fc12060ed595fd23d731b8a86adb21ddbb8c7bfbJordan Rose return (C == 0x0085 || C == 0x00A0 || C == 0x1680 || 2809fc12060ed595fd23d731b8a86adb21ddbb8c7bfbJordan Rose C == 0x180E || (C >= 0x2000 && C <= 0x200A) || 2810fc12060ed595fd23d731b8a86adb21ddbb8c7bfbJordan Rose C == 0x2028 || C == 0x2029 || C == 0x202F || 2811fc12060ed595fd23d731b8a86adb21ddbb8c7bfbJordan Rose C == 0x205F || C == 0x3000); 2812fc12060ed595fd23d731b8a86adb21ddbb8c7bfbJordan Rose} 2813fc12060ed595fd23d731b8a86adb21ddbb8c7bfbJordan Rose 2814c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rosevoid Lexer::LexUnicode(Token &Result, uint32_t C, const char *CurPtr) { 281574c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose if (!isLexingRawMode() && !PP->isPreprocessedOutput() && 281674c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose isUnicodeWhitespace(C)) { 281774c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose CharSourceRange CharRange = 281874c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose CharSourceRange::getCharRange(getSourceLocation(), 281974c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose getSourceLocation(CurPtr)); 282074c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose Diag(BufferPtr, diag::ext_unicode_whitespace) 282174c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose << CharRange; 2822fc12060ed595fd23d731b8a86adb21ddbb8c7bfbJordan Rose 2823fc12060ed595fd23d731b8a86adb21ddbb8c7bfbJordan Rose Result.setFlag(Token::LeadingSpace); 2824fc12060ed595fd23d731b8a86adb21ddbb8c7bfbJordan Rose if (SkipWhitespace(Result, CurPtr)) 2825fc12060ed595fd23d731b8a86adb21ddbb8c7bfbJordan Rose return; // KeepWhitespaceMode 2826fc12060ed595fd23d731b8a86adb21ddbb8c7bfbJordan Rose 2827fc12060ed595fd23d731b8a86adb21ddbb8c7bfbJordan Rose return LexTokenInternal(Result); 2828fc12060ed595fd23d731b8a86adb21ddbb8c7bfbJordan Rose } 2829fc12060ed595fd23d731b8a86adb21ddbb8c7bfbJordan Rose 2830c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose if (isAllowedIDChar(C) && isAllowedInitiallyIDChar(C)) { 2831c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose MIOpt.ReadToken(); 2832c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose return LexIdentifier(Result, CurPtr); 2833c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose } 2834c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 283574c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose if (!isLexingRawMode() && !PP->isPreprocessedOutput() && 283674c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose !isASCII(*BufferPtr) && !isAllowedIDChar(C)) { 2837c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // Non-ASCII characters tend to creep into source code unintentionally. 2838c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // Instead of letting the parser complain about the unknown token, 2839c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // just drop the character. 2840c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // Note that we can /only/ do this when the non-ASCII character is actually 2841c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // spelled as Unicode, not written as a UCN. The standard requires that 2842c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // we not throw away any possible preprocessor tokens, but there's a 2843c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // loophole in the mapping of Unicode characters to basic character set 2844c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // characters that allows us to map these particular characters to, say, 2845c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // whitespace. 284674c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose CharSourceRange CharRange = 284774c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose CharSourceRange::getCharRange(getSourceLocation(), 284874c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose getSourceLocation(CurPtr)); 284974c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose Diag(BufferPtr, diag::err_non_ascii) 285074c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose << FixItHint::CreateRemoval(CharRange); 2851c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 2852c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose BufferPtr = CurPtr; 2853c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose return LexTokenInternal(Result); 2854c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose } 2855c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 2856c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // Otherwise, we have an explicit UCN or a character that's unlikely to show 2857c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // up by accident. 2858c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose MIOpt.ReadToken(); 2859c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose FormTokenWithChars(Result, CurPtr, tok::unknown); 2860c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose} 2861c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 28625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 28635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// LexTokenInternal - This implements a simple C family lexer. It is an 28645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer/// extremely performance critical piece of code. This assumes that the buffer 2865efb173ddd95325c7fd3c15070392b27c07a49a85Chris Lattner/// has a null character at the end of the file. This returns a preprocessing 2866efb173ddd95325c7fd3c15070392b27c07a49a85Chris Lattner/// token, not a normal token, as such, it is an internal interface. It assumes 2867efb173ddd95325c7fd3c15070392b27c07a49a85Chris Lattner/// that the Flags of result have been cleared before calling this. 2868d217773f106856a11879ec79dc468efefaf2ee75Chris Lattnervoid Lexer::LexTokenInternal(Token &Result) { 28695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid SpencerLexNextToken: 28705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // New token, can't need cleaning yet. 2871d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.clearFlag(Token::NeedsCleaning); 28725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Result.setIdentifierInfo(0); 28731eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 28745f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // CurPtr - Cache BufferPtr in an automatic variable. 28755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer const char *CurPtr = BufferPtr; 28765f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 28775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Small amounts of horizontal whitespace is very common between tokens. 28785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if ((*CurPtr == ' ') || (*CurPtr == '\t')) { 28795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++CurPtr; 28805f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer while ((*CurPtr == ' ') || (*CurPtr == '\t')) 28815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ++CurPtr; 28821eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2883d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner // If we are keeping whitespace and other tokens, just return what we just 2884d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner // skipped. The next lexer invocation will return the token after the 2885d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner // whitespace. 2886d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner if (isKeepWhitespaceMode()) { 28879e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, tok::unknown); 2888d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner return; 2889d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner } 28901eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 28915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer BufferPtr = CurPtr; 2892d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::LeadingSpace); 28935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 28941eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 28955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer unsigned SizeTmp, SizeTmp2; // Temporaries for use in cases below. 28961eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 28975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Read a character, advancing over it. 28985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer char Char = getAndAdvanceChar(CurPtr, Result); 28999e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner tok::TokenKind Kind; 29001eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 29015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer switch (Char) { 29025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case 0: // Null. 29035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Found end of file? 29045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (CurPtr-1 == BufferEnd) { 29055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Read the PP instance variable into an automatic variable, because 29065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // LexEndOfFile will often delete 'this'. 2907168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner Preprocessor *PPCache = PP; 29085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (LexEndOfFile(Result, CurPtr-1)) // Retreat back into the file. 29095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return; // Got a token to return. 2910168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner assert(PPCache && "Raw buffer::LexEndOfFile should return a token"); 2911168ae2d44a443da75ea85db5f3b5081eb0bce113Chris Lattner return PPCache->Lex(Result); 29125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 29131eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 29147d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis // Check if we are performing code completion. 29157d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis if (isCodeCompletionPoint(CurPtr-1)) { 29167d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis // Return the code-completion token. 29177d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis Result.startToken(); 29187d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis FormTokenWithChars(Result, CurPtr, tok::code_completion); 29197d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis return; 29207d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis } 29217d100872341f233c81e1d7b72b40457e62c36862Argyrios Kyrtzidis 292274d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!isLexingRawMode()) 292374d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(CurPtr-1, diag::null_in_file); 2924d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::LeadingSpace); 2925d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner if (SkipWhitespace(Result, CurPtr)) 2926d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner return; // KeepWhitespaceMode 29271eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 29285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer goto LexNextToken; // GCC isn't tail call eliminating. 2929a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner 2930a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner case 26: // DOS & CP/M EOF: "^Z". 2931a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner // If we're in Microsoft extensions mode, treat this as end of file. 29324e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie if (LangOpts.MicrosoftExt) { 2933a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner // Read the PP instance variable into an automatic variable, because 2934a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner // LexEndOfFile will often delete 'this'. 2935a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner Preprocessor *PPCache = PP; 2936a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner if (LexEndOfFile(Result, CurPtr-1)) // Retreat back into the file. 2937a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner return; // Got a token to return. 2938a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner assert(PPCache && "Raw buffer::LexEndOfFile should return a token"); 2939a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner return PPCache->Lex(Result); 2940a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner } 2941a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner // If Microsoft extensions are disabled, this is just random garbage. 2942a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner Kind = tok::unknown; 2943a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner break; 2944a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner 29455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\n': 29465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\r': 29475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // If we are inside a preprocessor directive and we see the end of line, 294884021556baceb76eedf7d44be8ba71d9b8cfaccePeter Collingbourne // we know we are done with the directive, so return an EOD token. 29495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (ParsingPreprocessorDirective) { 29505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Done parsing the "line". 29515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer ParsingPreprocessorDirective = false; 29521eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 29535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Restore comment saving mode, in case it was disabled for directive. 29541a8354659a6007bbae3b5d9161a56ecc8f61a219David Blaikie if (PP) 29558c0b3787e7ccc7978b42dfbb84da2b802c743a5dDavid Blaikie SetCommentRetentionState(PP->getCommentRetentionState()); 29561eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 29575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Since we consumed a newline, we are back at the start of a line. 29585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer IsAtStartOfLine = true; 29591eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 296084021556baceb76eedf7d44be8ba71d9b8cfaccePeter Collingbourne Kind = tok::eod; 29615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 29625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 29635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // The returned token is at the start of the line. 2964d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::StartOfLine); 29655f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // No leading whitespace seen so far. 2966d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.clearFlag(Token::LeadingSpace); 29671eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 2968d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner if (SkipWhitespace(Result, CurPtr)) 2969d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner return; // KeepWhitespaceMode 29705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer goto LexNextToken; // GCC isn't tail call eliminating. 29715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case ' ': 29725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\t': 29735f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\f': 29745f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\v': 29758133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner SkipHorizontalWhitespace: 2976d217773f106856a11879ec79dc468efefaf2ee75Chris Lattner Result.setFlag(Token::LeadingSpace); 2977d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner if (SkipWhitespace(Result, CurPtr)) 2978d88dc48e33d71732708960170e57a3d1bdc8f847Chris Lattner return; // KeepWhitespaceMode 29798133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner 29808133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner SkipIgnoredUnits: 29818133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner CurPtr = BufferPtr; 29821eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 29838133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner // If the next token is obviously a // or /* */ comment, skip it efficiently 29848133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner // too (without going through the big switch stmt). 29858402c73dd880e8af46c826d873681820aebe32ebChris Lattner if (CurPtr[0] == '/' && CurPtr[1] == '/' && !inKeepCommentMode() && 2986bb23628148f555a4cf71f98c27096a7a804c085cNico Weber LangOpts.LineComment && !LangOpts.TraditionalCPP) { 2987bb23628148f555a4cf71f98c27096a7a804c085cNico Weber if (SkipLineComment(Result, CurPtr+2)) 2988046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner return; // There is a token to return. 29898133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner goto SkipIgnoredUnits; 2990fa95a019da00b926d64ff83358ba73bbc6ae1e37Chris Lattner } else if (CurPtr[0] == '/' && CurPtr[1] == '*' && !inKeepCommentMode()) { 2991046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner if (SkipBlockComment(Result, CurPtr+2)) 2992046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner return; // There is a token to return. 29938133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner goto SkipIgnoredUnits; 29948133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner } else if (isHorizontalWhitespace(*CurPtr)) { 29958133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner goto SkipHorizontalWhitespace; 29968133cfcc0b672cc86d1ef73ce0c61eccecdb3690Chris Lattner } 29975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer goto LexNextToken; // GCC isn't tail call eliminating. 2998a2bf105739cef7aea8e4d219629c627fcd11742eChris Lattner 29993a5707766850f9ee9daa35299794328b5caf96dcChris Lattner // C99 6.4.4.1: Integer Constants. 30003a5707766850f9ee9daa35299794328b5caf96dcChris Lattner // C99 6.4.4.2: Floating Constants. 30013a5707766850f9ee9daa35299794328b5caf96dcChris Lattner case '0': case '1': case '2': case '3': case '4': 30023a5707766850f9ee9daa35299794328b5caf96dcChris Lattner case '5': case '6': case '7': case '8': case '9': 30033a5707766850f9ee9daa35299794328b5caf96dcChris Lattner // Notify MIOpt that we read a non-whitespace/non-comment token. 30043a5707766850f9ee9daa35299794328b5caf96dcChris Lattner MIOpt.ReadToken(); 30053a5707766850f9ee9daa35299794328b5caf96dcChris Lattner return LexNumericConstant(Result, CurPtr); 30061eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 30075cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor case 'u': // Identifier (uber) or C++0x UTF-8 or UTF-16 string literal 30085cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor // Notify MIOpt that we read a non-whitespace/non-comment token. 30095cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor MIOpt.ReadToken(); 30105cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 301180ad52f327b532bded5c5b0ee38779d841c6cd35Richard Smith if (LangOpts.CPlusPlus11) { 30125cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor Char = getCharAndSize(CurPtr, SizeTmp); 30135cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 30145cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor // UTF-16 string literal 30155cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor if (Char == '"') 30165cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 30175cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor tok::utf16_string_literal); 30185cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 30195cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor // UTF-16 character constant 30205cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor if (Char == '\'') 30215cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 30225cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor tok::utf16_char_constant); 30235cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 30242fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // UTF-16 raw string literal 30252fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (Char == 'R' && getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 30262fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return LexRawStringLiteral(Result, 30272fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 30282fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper SizeTmp2, Result), 30292fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper tok::utf16_string_literal); 30302fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 30312fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (Char == '8') { 30322fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper char Char2 = getCharAndSize(CurPtr + SizeTmp, SizeTmp2); 30332fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 30342fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // UTF-8 string literal 30352fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (Char2 == '"') 30362fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return LexStringLiteral(Result, 30372fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 30382fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper SizeTmp2, Result), 30392fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper tok::utf8_string_literal); 30402fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 30412fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (Char2 == 'R') { 30422fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper unsigned SizeTmp3; 30432fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper char Char3 = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3); 30442fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // UTF-8 raw string literal 30452fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (Char3 == '"') { 30462fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return LexRawStringLiteral(Result, 30472fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper ConsumeChar(ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 30482fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper SizeTmp2, Result), 30492fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper SizeTmp3, Result), 30502fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper tok::utf8_string_literal); 30512fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 30522fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 30532fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 30545cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor } 30555cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 30565cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor // treat u like the start of an identifier. 30575cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor return LexIdentifier(Result, CurPtr); 30585cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 30595cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor case 'U': // Identifier (Uber) or C++0x UTF-32 string literal 30605cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor // Notify MIOpt that we read a non-whitespace/non-comment token. 30615cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor MIOpt.ReadToken(); 30625cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 306380ad52f327b532bded5c5b0ee38779d841c6cd35Richard Smith if (LangOpts.CPlusPlus11) { 30645cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor Char = getCharAndSize(CurPtr, SizeTmp); 30655cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 30665cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor // UTF-32 string literal 30675cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor if (Char == '"') 30685cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 30695cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor tok::utf32_string_literal); 30705cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 30715cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor // UTF-32 character constant 30725cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor if (Char == '\'') 30735cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 30745cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor tok::utf32_char_constant); 30752fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 30762fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // UTF-32 raw string literal 30772fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (Char == 'R' && getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 30782fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return LexRawStringLiteral(Result, 30792fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 30802fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper SizeTmp2, Result), 30812fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper tok::utf32_string_literal); 30825cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor } 30835cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 30845cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor // treat U like the start of an identifier. 30855cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor return LexIdentifier(Result, CurPtr); 30865cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor 30872fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper case 'R': // Identifier or C++0x raw string literal 30882fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // Notify MIOpt that we read a non-whitespace/non-comment token. 30892fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper MIOpt.ReadToken(); 30902fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 309180ad52f327b532bded5c5b0ee38779d841c6cd35Richard Smith if (LangOpts.CPlusPlus11) { 30922fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper Char = getCharAndSize(CurPtr, SizeTmp); 30932fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 30942fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper if (Char == '"') 30952fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return LexRawStringLiteral(Result, 30962fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper ConsumeChar(CurPtr, SizeTmp, Result), 30972fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper tok::string_literal); 30982fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper } 30992fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 31002fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // treat R like the start of an identifier. 31012fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return LexIdentifier(Result, CurPtr); 31022fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 31033a5707766850f9ee9daa35299794328b5caf96dcChris Lattner case 'L': // Identifier (Loony) or wide literal (L'x' or L"xyz"). 31045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Notify MIOpt that we read a non-whitespace/non-comment token. 31055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer MIOpt.ReadToken(); 31065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 31075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 31085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Wide string literal. 31095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '"') 31105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result), 31115cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor tok::wide_string_literal); 31125f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 31132fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper // Wide raw string literal. 311480ad52f327b532bded5c5b0ee38779d841c6cd35Richard Smith if (LangOpts.CPlusPlus11 && Char == 'R' && 31152fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == '"') 31162fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper return LexRawStringLiteral(Result, 31172fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 31182fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper SizeTmp2, Result), 31192fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper tok::wide_string_literal); 31202fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper 31215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Wide character constant. 31225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '\'') 31235cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result), 31245cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor tok::wide_char_constant); 31255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // FALL THROUGH, treating L like the start of an identifier. 31261eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 31275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // C99 6.4.2: Identifiers. 31285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': 31295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case 'H': case 'I': case 'J': case 'K': /*'L'*/case 'M': case 'N': 31302fa4e86b4fdada3b9ecbbbd99965b83ed879f69bCraig Topper case 'O': case 'P': case 'Q': /*'R'*/case 'S': case 'T': /*'U'*/ 31315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case 'V': case 'W': case 'X': case 'Y': case 'Z': 31325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': 31335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': 31345cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor case 'o': case 'p': case 'q': case 'r': case 's': case 't': /*'u'*/ 31355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case 'v': case 'w': case 'x': case 'y': case 'z': 31365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '_': 31375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Notify MIOpt that we read a non-whitespace/non-comment token. 31385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer MIOpt.ReadToken(); 31395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return LexIdentifier(Result, CurPtr); 31403a5707766850f9ee9daa35299794328b5caf96dcChris Lattner 31413a5707766850f9ee9daa35299794328b5caf96dcChris Lattner case '$': // $ in identifiers. 31424e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie if (LangOpts.DollarIdents) { 314374d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!isLexingRawMode()) 314474d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner Diag(CurPtr-1, diag::ext_dollar_in_identifier); 31453a5707766850f9ee9daa35299794328b5caf96dcChris Lattner // Notify MIOpt that we read a non-whitespace/non-comment token. 31463a5707766850f9ee9daa35299794328b5caf96dcChris Lattner MIOpt.ReadToken(); 31473a5707766850f9ee9daa35299794328b5caf96dcChris Lattner return LexIdentifier(Result, CurPtr); 31483a5707766850f9ee9daa35299794328b5caf96dcChris Lattner } 31491eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 31509e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::unknown; 31513a5707766850f9ee9daa35299794328b5caf96dcChris Lattner break; 31521eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 31535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // C99 6.4.4: Character Constants. 31545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\'': 31555f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Notify MIOpt that we read a non-whitespace/non-comment token. 31565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer MIOpt.ReadToken(); 31575cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor return LexCharConstant(Result, CurPtr, tok::char_constant); 31585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 31595f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // C99 6.4.5: String Literals. 31605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '"': 31615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Notify MIOpt that we read a non-whitespace/non-comment token. 31625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer MIOpt.ReadToken(); 31635cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor return LexStringLiteral(Result, CurPtr, tok::string_literal); 31645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 31655f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // C99 6.4.6: Punctuators. 31665f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '?': 31679e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::question; 31685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 31695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '[': 31709e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::l_square; 31715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 31725f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case ']': 31739e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::r_square; 31745f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 31755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '(': 31769e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::l_paren; 31775f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 31785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case ')': 31799e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::r_paren; 31805f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 31815f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '{': 31829e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::l_brace; 31835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 31845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '}': 31859e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::r_brace; 31865f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 31875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '.': 31885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 31895f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char >= '0' && Char <= '9') { 31905f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Notify MIOpt that we read a non-whitespace/non-comment token. 31915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer MIOpt.ReadToken(); 31925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 31935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer return LexNumericConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result)); 31944e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie } else if (LangOpts.CPlusPlus && Char == '*') { 31959e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::periodstar; 31965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr += SizeTmp; 31975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Char == '.' && 31985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '.') { 31999e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::ellipsis; 32005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 32015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer SizeTmp2, Result); 32025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 32039e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::period; 32045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 32055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 32065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '&': 32075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 32085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '&') { 32099e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::ampamp; 32105f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 32115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Char == '=') { 32129e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::ampequal; 32135f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 32145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 32159e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::amp; 32165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 32175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 32181eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump case '*': 32195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (getCharAndSize(CurPtr, SizeTmp) == '=') { 32209e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::starequal; 32215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 32225f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 32239e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::star; 32245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 32255f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 32265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '+': 32275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 32285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '+') { 32295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 32309e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::plusplus; 32315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Char == '=') { 32325f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 32339e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::plusequal; 32345f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 32359e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::plus; 32365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 32375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 32385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '-': 32395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 32409e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner if (Char == '-') { // -- 32415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 32429e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::minusminus; 32434e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie } else if (Char == '>' && LangOpts.CPlusPlus && 32449e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '*') { // C++ ->* 32455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 32465f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer SizeTmp2, Result); 32479e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::arrowstar; 32489e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner } else if (Char == '>') { // -> 32495f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 32509e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::arrow; 32519e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner } else if (Char == '=') { // -= 32525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 32539e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::minusequal; 32545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 32559e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::minus; 32565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 32575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 32585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '~': 32599e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::tilde; 32605f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 32615f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '!': 32625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (getCharAndSize(CurPtr, SizeTmp) == '=') { 32639e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::exclaimequal; 32645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 32655f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 32669e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::exclaim; 32675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 32685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 32695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '/': 32705f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // 6.4.9: Comments 32715f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 3272bb23628148f555a4cf71f98c27096a7a804c085cNico Weber if (Char == '/') { // Line comment. 3273bb23628148f555a4cf71f98c27096a7a804c085cNico Weber // Even if Line comments are disabled (e.g. in C89 mode), we generally 32748402c73dd880e8af46c826d873681820aebe32ebChris Lattner // want to lex this as a comment. There is one problem with this though, 32758402c73dd880e8af46c826d873681820aebe32ebChris Lattner // that in one particular corner case, this can change the behavior of the 32768402c73dd880e8af46c826d873681820aebe32ebChris Lattner // resultant program. For example, In "foo //**/ bar", C89 would lex 3277bb23628148f555a4cf71f98c27096a7a804c085cNico Weber // this as "foo / bar" and langauges with Line comments would lex it as 32788402c73dd880e8af46c826d873681820aebe32ebChris Lattner // "foo". Check to see if the character after the second slash is a '*'. 32798402c73dd880e8af46c826d873681820aebe32ebChris Lattner // If so, we will lex that as a "/" instead of the start of a comment. 32802ed42287b9b88a0bb08ab10e95e306da3f294ba9Daniel Dunbar // However, we never do this in -traditional-cpp mode. 3281bb23628148f555a4cf71f98c27096a7a804c085cNico Weber if ((LangOpts.LineComment || 32822ed42287b9b88a0bb08ab10e95e306da3f294ba9Daniel Dunbar getCharAndSize(CurPtr+SizeTmp, SizeTmp2) != '*') && 32834e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie !LangOpts.TraditionalCPP) { 3284bb23628148f555a4cf71f98c27096a7a804c085cNico Weber if (SkipLineComment(Result, ConsumeChar(CurPtr, SizeTmp, Result))) 3285046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner return; // There is a token to return. 32861eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 32878402c73dd880e8af46c826d873681820aebe32ebChris Lattner // It is common for the tokens immediately after a // comment to be 32888402c73dd880e8af46c826d873681820aebe32ebChris Lattner // whitespace (indentation for the next line). Instead of going through 32898402c73dd880e8af46c826d873681820aebe32ebChris Lattner // the big switch, handle it efficiently now. 32908402c73dd880e8af46c826d873681820aebe32ebChris Lattner goto SkipIgnoredUnits; 32918402c73dd880e8af46c826d873681820aebe32ebChris Lattner } 32928402c73dd880e8af46c826d873681820aebe32ebChris Lattner } 32931eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 32948402c73dd880e8af46c826d873681820aebe32ebChris Lattner if (Char == '*') { // /**/ comment. 32955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (SkipBlockComment(Result, ConsumeChar(CurPtr, SizeTmp, Result))) 3296046c2277dcbcc8eb89dbb5b1b8c5226b7cb81635Chris Lattner return; // There is a token to return. 32972d3818959df7b71bfcf8c2f2b573856bfc4c3a21Chris Lattner goto LexNextToken; // GCC isn't tail call eliminating. 32988402c73dd880e8af46c826d873681820aebe32ebChris Lattner } 32991eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 33008402c73dd880e8af46c826d873681820aebe32ebChris Lattner if (Char == '=') { 33015f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 33029e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::slashequal; 33035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 33049e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::slash; 33055f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 33065f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 33075f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '%': 33085f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 33095f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '=') { 33109e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::percentequal; 33115f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 33124e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie } else if (LangOpts.Digraphs && Char == '>') { 33139e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::r_brace; // '%>' -> '}' 33145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 33154e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie } else if (LangOpts.Digraphs && Char == ':') { 33165f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 33175f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 33185f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '%' && getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == ':') { 33199e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::hashhash; // '%:%:' -> '##' 33205f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 33215f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer SizeTmp2, Result); 33224e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie } else if (Char == '@' && LangOpts.MicrosoftExt) {// %:@ -> #@ -> Charize 33235f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 332474d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!isLexingRawMode()) 332566d5ce11b9426f6a59f61a03cbd8dbf047cc9350Ted Kremenek Diag(BufferPtr, diag::ext_charize_microsoft); 33269e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::hashat; 3327e91e93225db2e66906878513c6ef4dd6a7ee2b6aChris Lattner } else { // '%:' -> '#' 33285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // We parsed a # character. If this occurs at the start of the line, 33295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // it's actually the start of a preprocessing directive. Callback to 33305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // the preprocessor to handle it. 33315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // FIXME: -fpreprocessed mode?? 33323185d4ac30378995ef70421e2848f77524c2b5d5Argyrios Kyrtzidis if (Result.isAtStartOfLine() && !LexingRawMode && !Is_PragmaLexer) 33333185d4ac30378995ef70421e2848f77524c2b5d5Argyrios Kyrtzidis goto HandleDirective; 33341eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 3335e91e93225db2e66906878513c6ef4dd6a7ee2b6aChris Lattner Kind = tok::hash; 33365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 33375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 33389e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::percent; 33395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 33405f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 33415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '<': 33425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 33435f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (ParsingFilename) { 33449cb51ce03caf0755759f7e67aea7d4be684ff738Chris Lattner return LexAngledStringLiteral(Result, CurPtr); 33455f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Char == '<') { 334634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); 334734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (After == '=') { 334834f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner Kind = tok::lesslessequal; 334934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 335034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner SizeTmp2, Result); 335134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } else if (After == '<' && IsStartOfConflictMarker(CurPtr-1)) { 335234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // If this is actually a '<<<<<<<' version control conflict marker, 335334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // recognize it as such and recover nicely. 335434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner goto LexNextToken; 3355d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith } else if (After == '<' && HandleEndOfConflictMarker(CurPtr-1)) { 3356d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith // If this is '<<<<' and we're in a Perforce-style conflict marker, 3357d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith // ignore it. 3358d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith goto LexNextToken; 33594e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie } else if (LangOpts.CUDA && After == '<') { 33601b791d6465d42a9763927be1dd8af229efcbbf5ePeter Collingbourne Kind = tok::lesslessless; 33611b791d6465d42a9763927be1dd8af229efcbbf5ePeter Collingbourne CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 33621b791d6465d42a9763927be1dd8af229efcbbf5ePeter Collingbourne SizeTmp2, Result); 336334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } else { 336434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 336534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner Kind = tok::lessless; 336634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } 33675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Char == '=') { 33685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 33699e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::lessequal; 33704e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie } else if (LangOpts.Digraphs && Char == ':') { // '<:' -> '[' 337180ad52f327b532bded5c5b0ee38779d841c6cd35Richard Smith if (LangOpts.CPlusPlus11 && 337287a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith getCharAndSize(CurPtr + SizeTmp, SizeTmp2) == ':') { 337387a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith // C++0x [lex.pptoken]p3: 337487a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith // Otherwise, if the next three characters are <:: and the subsequent 337587a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith // character is neither : nor >, the < is treated as a preprocessor 337687a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith // token by itself and not as the first character of the alternative 337787a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith // token <:. 337887a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith unsigned SizeTmp3; 337987a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith char After = getCharAndSize(CurPtr + SizeTmp + SizeTmp2, SizeTmp3); 338087a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith if (After != ':' && After != '>') { 338187a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith Kind = tok::less; 3382661a99690bc133bbaa029da925481d4a860dec90Richard Smith if (!isLexingRawMode()) 3383661a99690bc133bbaa029da925481d4a860dec90Richard Smith Diag(BufferPtr, diag::warn_cxx98_compat_less_colon_colon); 338487a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith break; 338587a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith } 338687a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith } 338787a1e19735a27d84edc2ac1331c040e2fb4c3b1aRichard Smith 33885f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 33899e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::l_square; 33904e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie } else if (LangOpts.Digraphs && Char == '%') { // '<%' -> '{' 33915f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 33929e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::l_brace; 33935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 33949e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::less; 33955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 33965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 33975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '>': 33985f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 33995f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '=') { 34005f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 34019e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::greaterequal; 34025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Char == '>') { 340334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner char After = getCharAndSize(CurPtr+SizeTmp, SizeTmp2); 340434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (After == '=') { 340534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 340634f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner SizeTmp2, Result); 340734f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner Kind = tok::greatergreaterequal; 3408d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith } else if (After == '>' && IsStartOfConflictMarker(CurPtr-1)) { 3409d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith // If this is actually a '>>>>' conflict marker, recognize it as such 3410d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith // and recover nicely. 3411d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith goto LexNextToken; 341234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } else if (After == '>' && HandleEndOfConflictMarker(CurPtr-1)) { 341334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // If this is '>>>>>>>' and we're in a conflict marker, ignore it. 341434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner goto LexNextToken; 34154e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie } else if (LangOpts.CUDA && After == '>') { 34161b791d6465d42a9763927be1dd8af229efcbbf5ePeter Collingbourne Kind = tok::greatergreatergreater; 34171b791d6465d42a9763927be1dd8af229efcbbf5ePeter Collingbourne CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), 34181b791d6465d42a9763927be1dd8af229efcbbf5ePeter Collingbourne SizeTmp2, Result); 341934f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } else { 342034f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 342134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner Kind = tok::greatergreater; 342234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner } 342334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 34245f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 34259e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::greater; 34265f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 34275f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 34285f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '^': 34295f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 34305f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '=') { 34315f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 34329e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::caretequal; 34335f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 34349e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::caret; 34355f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 34365f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 34375f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '|': 34385f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 34395f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '=') { 34409e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::pipeequal; 34415f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 34425f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else if (Char == '|') { 344334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner // If this is '|||||||' and we're in a conflict marker, ignore it. 344434f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (CurPtr[1] == '|' && HandleEndOfConflictMarker(CurPtr-1)) 344534f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner goto LexNextToken; 34469e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::pipepipe; 34475f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 34485f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 34499e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::pipe; 34505f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 34515f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 34525f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case ':': 34535f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 34544e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie if (LangOpts.Digraphs && Char == '>') { 34559e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::r_square; // ':>' -> ']' 34565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 34574e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie } else if (LangOpts.CPlusPlus && Char == ':') { 34589e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::coloncolon; 34595f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 34601eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } else { 34619e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::colon; 34625f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 34635f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 34645f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case ';': 34659e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::semi; 34665f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 34675f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '=': 34685f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 34695f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '=') { 3470d5e1d606f8c22ebda17c6fbf952f8c1696428758Richard Smith // If this is '====' and we're in a conflict marker, ignore it. 347134f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner if (CurPtr[1] == '=' && HandleEndOfConflictMarker(CurPtr-1)) 347234f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner goto LexNextToken; 347334f349da38a7bd99103e12d8ea6c73bc8d025193Chris Lattner 34749e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::equalequal; 34755f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 34761eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump } else { 34779e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::equal; 34785f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 34795f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 34805f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case ',': 34819e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::comma; 34825f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 34835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '#': 34845f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer Char = getCharAndSize(CurPtr, SizeTmp); 34855f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer if (Char == '#') { 34869e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::hashhash; 34875f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 34884e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie } else if (Char == '@' && LangOpts.MicrosoftExt) { // #@ -> Charize 34899e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::hashat; 349074d15dfd183b2082e8a5d4dfbf66bd861b220901Chris Lattner if (!isLexingRawMode()) 349166d5ce11b9426f6a59f61a03cbd8dbf047cc9350Ted Kremenek Diag(BufferPtr, diag::ext_charize_microsoft); 34925f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); 34935f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } else { 34945f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // We parsed a # character. If this occurs at the start of the line, 34955f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // it's actually the start of a preprocessing directive. Callback to 34965f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // the preprocessor to handle it. 34975f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // FIXME: -fpreprocessed mode?? 34983185d4ac30378995ef70421e2848f77524c2b5d5Argyrios Kyrtzidis if (Result.isAtStartOfLine() && !LexingRawMode && !Is_PragmaLexer) 34993185d4ac30378995ef70421e2848f77524c2b5d5Argyrios Kyrtzidis goto HandleDirective; 35001eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 3501e91e93225db2e66906878513c6ef4dd6a7ee2b6aChris Lattner Kind = tok::hash; 35025f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 35035f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 35045f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 35053a5707766850f9ee9daa35299794328b5caf96dcChris Lattner case '@': 35063a5707766850f9ee9daa35299794328b5caf96dcChris Lattner // Objective C support. 35074e4d08403ca5cfd4d558fa2936215d3a4e5a528dDavid Blaikie if (CurPtr[-1] == '@' && LangOpts.ObjC1) 35089e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::at; 35093a5707766850f9ee9daa35299794328b5caf96dcChris Lattner else 35109e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::unknown; 35113a5707766850f9ee9daa35299794328b5caf96dcChris Lattner break; 35121eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 3513c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // UCNs (C99 6.4.3, C++11 [lex.charset]p2) 35145f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer case '\\': 3515c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose if (uint32_t CodePoint = tryReadUCN(CurPtr, BufferPtr, &Result)) 3516c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose return LexUnicode(Result, CodePoint, CurPtr); 3517c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 35189e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner Kind = tok::unknown; 35195f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer break; 3520c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 3521c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose default: { 3522c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose if (isASCII(Char)) { 3523c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose Kind = tok::unknown; 3524c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose break; 3525c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose } 3526c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 3527c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose UTF32 CodePoint; 3528c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 3529c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // We can't just reset CurPtr to BufferPtr because BufferPtr may point to 3530c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // an escaped newline. 3531c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose --CurPtr; 3532cb5620c9b213f4bd323912159fdddda35e258a14Dmitri Gribenko ConversionResult Status = 3533cb5620c9b213f4bd323912159fdddda35e258a14Dmitri Gribenko llvm::convertUTF8Sequence((const UTF8 **)&CurPtr, 3534cb5620c9b213f4bd323912159fdddda35e258a14Dmitri Gribenko (const UTF8 *)BufferEnd, 3535cb5620c9b213f4bd323912159fdddda35e258a14Dmitri Gribenko &CodePoint, 3536cb5620c9b213f4bd323912159fdddda35e258a14Dmitri Gribenko strictConversion); 3537c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose if (Status == conversionOK) 3538c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose return LexUnicode(Result, CodePoint, CurPtr); 3539c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 354074c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose if (isLexingRawMode() || PP->isPreprocessedOutput()) { 354120afc2977cd0a6bacbe6218a633cd59a24463e2fJordan Rose ++CurPtr; 354274c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose Kind = tok::unknown; 354374c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose break; 354474c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose } 354574c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose 3546c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // Non-ASCII characters tend to creep into source code unintentionally. 3547c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose // Instead of letting the parser complain about the unknown token, 3548ae82c2b7b62d742f56638fa3cfb0f550ddcaf315Jordan Rose // just diagnose the invalid UTF-8, then drop the character. 354974c2498bb9e339345ee32bdd095e76157cec3b86Jordan Rose Diag(CurPtr, diag::err_invalid_utf8); 3550c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose 3551c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose BufferPtr = CurPtr+1; 3552c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose goto LexNextToken; 3553c7629d941557f7179eb8fa8a2e2a74d749cbaf7cJordan Rose } 35545f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer } 35551eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 35565f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Notify MIOpt that we read a non-whitespace/non-comment token. 35575f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer MIOpt.ReadToken(); 35585f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer 35595f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer // Update the location of token as well as BufferPtr. 35609e6293d4dfd688429f77ee3b6edba9dfd7ada3a2Chris Lattner FormTokenWithChars(Result, CurPtr, Kind); 35613185d4ac30378995ef70421e2848f77524c2b5d5Argyrios Kyrtzidis return; 35623185d4ac30378995ef70421e2848f77524c2b5d5Argyrios Kyrtzidis 35633185d4ac30378995ef70421e2848f77524c2b5d5Argyrios KyrtzidisHandleDirective: 35643185d4ac30378995ef70421e2848f77524c2b5d5Argyrios Kyrtzidis // We parsed a # character and it's the start of a preprocessing directive. 35653185d4ac30378995ef70421e2848f77524c2b5d5Argyrios Kyrtzidis 35663185d4ac30378995ef70421e2848f77524c2b5d5Argyrios Kyrtzidis FormTokenWithChars(Result, CurPtr, tok::hash); 35673185d4ac30378995ef70421e2848f77524c2b5d5Argyrios Kyrtzidis PP->HandleDirective(Result); 35683185d4ac30378995ef70421e2848f77524c2b5d5Argyrios Kyrtzidis 35693185d4ac30378995ef70421e2848f77524c2b5d5Argyrios Kyrtzidis // As an optimization, if the preprocessor didn't switch lexers, tail 35703185d4ac30378995ef70421e2848f77524c2b5d5Argyrios Kyrtzidis // recurse. 35713185d4ac30378995ef70421e2848f77524c2b5d5Argyrios Kyrtzidis if (PP->isCurrentLexer(this)) { 35723185d4ac30378995ef70421e2848f77524c2b5d5Argyrios Kyrtzidis // Start a new token. If this is a #include or something, the PP may 35733185d4ac30378995ef70421e2848f77524c2b5d5Argyrios Kyrtzidis // want us starting at the beginning of the line again. If so, set 35743185d4ac30378995ef70421e2848f77524c2b5d5Argyrios Kyrtzidis // the StartOfLine flag and clear LeadingSpace. 35753185d4ac30378995ef70421e2848f77524c2b5d5Argyrios Kyrtzidis if (IsAtStartOfLine) { 35763185d4ac30378995ef70421e2848f77524c2b5d5Argyrios Kyrtzidis Result.setFlag(Token::StartOfLine); 35773185d4ac30378995ef70421e2848f77524c2b5d5Argyrios Kyrtzidis Result.clearFlag(Token::LeadingSpace); 35783185d4ac30378995ef70421e2848f77524c2b5d5Argyrios Kyrtzidis IsAtStartOfLine = false; 35793185d4ac30378995ef70421e2848f77524c2b5d5Argyrios Kyrtzidis } 35803185d4ac30378995ef70421e2848f77524c2b5d5Argyrios Kyrtzidis goto LexNextToken; // GCC isn't tail call eliminating. 35813185d4ac30378995ef70421e2848f77524c2b5d5Argyrios Kyrtzidis } 35823185d4ac30378995ef70421e2848f77524c2b5d5Argyrios Kyrtzidis return PP->Lex(Result); 35835f016e2cb5d11daeb237544de1c5d59f20fe1a6eReid Spencer} 3584