1e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner//===--- TokenLexer.cpp - Lex from a token stream -------------------------===// 2e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner// 3e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner// The LLVM Compiler Infrastructure 4e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner// 5e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner// This file is distributed under the University of Illinois Open Source 6e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner// License. See LICENSE.TXT for details. 7e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner// 8e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner//===----------------------------------------------------------------------===// 9e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner// 10e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner// This file implements the TokenLexer interface. 11e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner// 12e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner//===----------------------------------------------------------------------===// 13e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner 14e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner#include "MacroArgs.h" 15e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner#include "clang/Lex/MacroInfo.h" 16e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner#include "clang/Lex/Preprocessor.h" 17500d3297d2a21edeac4d46cbcbe21bc2352c2a28Chris Lattner#include "clang/Lex/LexDiagnostic.h" 18d7bb6a014afec8c8e2962d8282db412e664dc2f9David Blaikie 19d7bb6a014afec8c8e2962d8282db412e664dc2f9David Blaikie#include <algorithm> 20d7bb6a014afec8c8e2962d8282db412e664dc2f9David Blaikie 21e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattnerusing namespace clang; 22e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner 23e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner/// MacroArgs ctor function - This destroys the vector passed in. 24e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris LattnerMacroArgs *MacroArgs::create(const MacroInfo *MI, 25d7bb6a014afec8c8e2962d8282db412e664dc2f9David Blaikie llvm::ArrayRef<Token> UnexpArgTokens, 26d7bb6a014afec8c8e2962d8282db412e664dc2f9David Blaikie bool VarargsElided, Preprocessor &PP) { 27e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner assert(MI->isFunctionLike() && 28e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner "Can't have args for an object-like macro!"); 294fe739f293057516e70b15a3e852f36614e07e73Chris Lattner MacroArgs **ResultEnt = 0; 304fe739f293057516e70b15a3e852f36614e07e73Chris Lattner unsigned ClosestMatch = ~0U; 314608459d918261275af147b4c22351469f7ebea1Chris Lattner 324608459d918261275af147b4c22351469f7ebea1Chris Lattner // See if we have an entry with a big enough argument list to reuse on the 334608459d918261275af147b4c22351469f7ebea1Chris Lattner // free list. If so, reuse it. 344608459d918261275af147b4c22351469f7ebea1Chris Lattner for (MacroArgs **Entry = &PP.MacroArgCache; *Entry; 354608459d918261275af147b4c22351469f7ebea1Chris Lattner Entry = &(*Entry)->ArgCache) 36d7bb6a014afec8c8e2962d8282db412e664dc2f9David Blaikie if ((*Entry)->NumUnexpArgTokens >= UnexpArgTokens.size() && 374fe739f293057516e70b15a3e852f36614e07e73Chris Lattner (*Entry)->NumUnexpArgTokens < ClosestMatch) { 384fe739f293057516e70b15a3e852f36614e07e73Chris Lattner ResultEnt = Entry; 394fe739f293057516e70b15a3e852f36614e07e73Chris Lattner 404fe739f293057516e70b15a3e852f36614e07e73Chris Lattner // If we have an exact match, use it. 41d7bb6a014afec8c8e2962d8282db412e664dc2f9David Blaikie if ((*Entry)->NumUnexpArgTokens == UnexpArgTokens.size()) 424fe739f293057516e70b15a3e852f36614e07e73Chris Lattner break; 434fe739f293057516e70b15a3e852f36614e07e73Chris Lattner // Otherwise, use the best fit. 444fe739f293057516e70b15a3e852f36614e07e73Chris Lattner ClosestMatch = (*Entry)->NumUnexpArgTokens; 454608459d918261275af147b4c22351469f7ebea1Chris Lattner } 464608459d918261275af147b4c22351469f7ebea1Chris Lattner 474fe739f293057516e70b15a3e852f36614e07e73Chris Lattner MacroArgs *Result; 484fe739f293057516e70b15a3e852f36614e07e73Chris Lattner if (ResultEnt == 0) { 494608459d918261275af147b4c22351469f7ebea1Chris Lattner // Allocate memory for a MacroArgs object with the lexer tokens at the end. 50d7bb6a014afec8c8e2962d8282db412e664dc2f9David Blaikie Result = (MacroArgs*)malloc(sizeof(MacroArgs) + 51d7bb6a014afec8c8e2962d8282db412e664dc2f9David Blaikie UnexpArgTokens.size() * sizeof(Token)); 524608459d918261275af147b4c22351469f7ebea1Chris Lattner // Construct the MacroArgs object. 53d7bb6a014afec8c8e2962d8282db412e664dc2f9David Blaikie new (Result) MacroArgs(UnexpArgTokens.size(), VarargsElided); 544608459d918261275af147b4c22351469f7ebea1Chris Lattner } else { 554fe739f293057516e70b15a3e852f36614e07e73Chris Lattner Result = *ResultEnt; 564fe739f293057516e70b15a3e852f36614e07e73Chris Lattner // Unlink this node from the preprocessors singly linked list. 574fe739f293057516e70b15a3e852f36614e07e73Chris Lattner *ResultEnt = Result->ArgCache; 58d7bb6a014afec8c8e2962d8282db412e664dc2f9David Blaikie Result->NumUnexpArgTokens = UnexpArgTokens.size(); 594608459d918261275af147b4c22351469f7ebea1Chris Lattner Result->VarargsElided = VarargsElided; 604608459d918261275af147b4c22351469f7ebea1Chris Lattner } 611eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 62e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // Copy the actual unexpanded tokens to immediately after the result ptr. 63d7bb6a014afec8c8e2962d8282db412e664dc2f9David Blaikie if (!UnexpArgTokens.empty()) 64d7bb6a014afec8c8e2962d8282db412e664dc2f9David Blaikie std::copy(UnexpArgTokens.begin(), UnexpArgTokens.end(), 65d7bb6a014afec8c8e2962d8282db412e664dc2f9David Blaikie const_cast<Token*>(Result->getUnexpArgument(0))); 661eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 67e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner return Result; 68e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner} 69e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner 70e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner/// destroy - Destroy and deallocate the memory for this object. 71e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner/// 72561395bead16b9dc9b0d5bfb6b257a9ed4545db6Chris Lattnervoid MacroArgs::destroy(Preprocessor &PP) { 734608459d918261275af147b4c22351469f7ebea1Chris Lattner StringifiedArgs.clear(); 744608459d918261275af147b4c22351469f7ebea1Chris Lattner 754608459d918261275af147b4c22351469f7ebea1Chris Lattner // Don't clear PreExpArgTokens, just clear the entries. Clearing the entries 764608459d918261275af147b4c22351469f7ebea1Chris Lattner // would deallocate the element vectors. 774608459d918261275af147b4c22351469f7ebea1Chris Lattner for (unsigned i = 0, e = PreExpArgTokens.size(); i != e; ++i) 784608459d918261275af147b4c22351469f7ebea1Chris Lattner PreExpArgTokens[i].clear(); 794608459d918261275af147b4c22351469f7ebea1Chris Lattner 804608459d918261275af147b4c22351469f7ebea1Chris Lattner // Add this to the preprocessor's free list. 814608459d918261275af147b4c22351469f7ebea1Chris Lattner ArgCache = PP.MacroArgCache; 824608459d918261275af147b4c22351469f7ebea1Chris Lattner PP.MacroArgCache = this; 83e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner} 84e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner 8523f77e59718385512984d4e2a021bef52b9f6ddfChris Lattner/// deallocate - This should only be called by the Preprocessor when managing 8623f77e59718385512984d4e2a021bef52b9f6ddfChris Lattner/// its freelist. 8723f77e59718385512984d4e2a021bef52b9f6ddfChris LattnerMacroArgs *MacroArgs::deallocate() { 8823f77e59718385512984d4e2a021bef52b9f6ddfChris Lattner MacroArgs *Next = ArgCache; 8923f77e59718385512984d4e2a021bef52b9f6ddfChris Lattner 9023f77e59718385512984d4e2a021bef52b9f6ddfChris Lattner // Run the dtor to deallocate the vectors. 9123f77e59718385512984d4e2a021bef52b9f6ddfChris Lattner this->~MacroArgs(); 9223f77e59718385512984d4e2a021bef52b9f6ddfChris Lattner // Release the memory for the object. 9323f77e59718385512984d4e2a021bef52b9f6ddfChris Lattner free(this); 9423f77e59718385512984d4e2a021bef52b9f6ddfChris Lattner 9523f77e59718385512984d4e2a021bef52b9f6ddfChris Lattner return Next; 9623f77e59718385512984d4e2a021bef52b9f6ddfChris Lattner} 9723f77e59718385512984d4e2a021bef52b9f6ddfChris Lattner 98e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner 99e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner/// getArgLength - Given a pointer to an expanded or unexpanded argument, 100e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner/// return the number of tokens, not counting the EOF, that make up the 101e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner/// argument. 102e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattnerunsigned MacroArgs::getArgLength(const Token *ArgPtr) { 103e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner unsigned NumArgTokens = 0; 104e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner for (; ArgPtr->isNot(tok::eof); ++ArgPtr) 105e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner ++NumArgTokens; 106e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner return NumArgTokens; 107e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner} 108e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner 109e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner 110e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner/// getUnexpArgument - Return the unexpanded tokens for the specified formal. 111e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner/// 112e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattnerconst Token *MacroArgs::getUnexpArgument(unsigned Arg) const { 113e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // The unexpanded argument tokens start immediately after the MacroArgs object 114e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // in memory. 115e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner const Token *Start = (const Token *)(this+1); 116e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner const Token *Result = Start; 117e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // Scan to find Arg. 118e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner for (; Arg; ++Result) { 119e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner assert(Result < Start+NumUnexpArgTokens && "Invalid arg #"); 120e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner if (Result->is(tok::eof)) 121e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner --Arg; 122e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner } 1239fc9e77c2fd15a88bd19ff3fdc9eb9716742f720Chris Lattner assert(Result < Start+NumUnexpArgTokens && "Invalid arg #"); 124e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner return Result; 125e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner} 126e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner 127e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner 128e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner/// ArgNeedsPreexpansion - If we can prove that the argument won't be affected 129e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner/// by pre-expansion, return false. Otherwise, conservatively return true. 130e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattnerbool MacroArgs::ArgNeedsPreexpansion(const Token *ArgTok, 131e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner Preprocessor &PP) const { 132e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // If there are no identifiers in the argument list, or if the identifiers are 133e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // known to not be macros, pre-expansion won't modify it. 134e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner for (; ArgTok->isNot(tok::eof); ++ArgTok) 135e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner if (IdentifierInfo *II = ArgTok->getIdentifierInfo()) { 136e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner if (II->hasMacroDefinition() && PP.getMacroInfo(II)->isEnabled()) 137e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // Return true even though the macro could be a function-like macro 138e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // without a following '(' token. 139e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner return true; 140e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner } 141e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner return false; 142e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner} 143e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner 144e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner/// getPreExpArgument - Return the pre-expanded form of the specified 145e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner/// argument. 146e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattnerconst std::vector<Token> & 147f5809a7b76fd5a86ae737d9b525a1eddb9339ee7Chris LattnerMacroArgs::getPreExpArgument(unsigned Arg, const MacroInfo *MI, 148f5809a7b76fd5a86ae737d9b525a1eddb9339ee7Chris Lattner Preprocessor &PP) { 149f5809a7b76fd5a86ae737d9b525a1eddb9339ee7Chris Lattner assert(Arg < MI->getNumArgs() && "Invalid argument number!"); 1501eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 151e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // If we have already computed this, return it. 152f5809a7b76fd5a86ae737d9b525a1eddb9339ee7Chris Lattner if (PreExpArgTokens.size() < MI->getNumArgs()) 153f5809a7b76fd5a86ae737d9b525a1eddb9339ee7Chris Lattner PreExpArgTokens.resize(MI->getNumArgs()); 154f5809a7b76fd5a86ae737d9b525a1eddb9339ee7Chris Lattner 155e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner std::vector<Token> &Result = PreExpArgTokens[Arg]; 156e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner if (!Result.empty()) return Result; 157e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner 158e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner const Token *AT = getUnexpArgument(Arg); 159e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner unsigned NumToks = getArgLength(AT)+1; // Include the EOF. 1601eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 161e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // Otherwise, we have to pre-expand this argument, populating Result. To do 162e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // this, we set up a fake TokenLexer to lex from the unexpanded argument 163e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // list. With this installed, we lex expanded tokens until we hit the EOF 164e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // token at the end of the unexp list. 1651eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump PP.EnterTokenStream(AT, NumToks, false /*disable expand*/, 1666b884508c3bc97cc9df9516adb92fbf88dd0a2e4Chris Lattner false /*owns tokens*/); 167e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner 168e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // Lex all of the macro-expanded tokens into Result. 169e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner do { 170e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner Result.push_back(Token()); 1717c35122df2998d9f615c3ebd4b71a91a2c6287d6Chris Lattner Token &Tok = Result.back(); 1727c35122df2998d9f615c3ebd4b71a91a2c6287d6Chris Lattner PP.Lex(Tok); 173e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner } while (Result.back().isNot(tok::eof)); 1741eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 175e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // Pop the token stream off the top of the stack. We know that the internal 176e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // pointer inside of it is to the "end" of the token stream, but the stack 177e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // will not otherwise be popped until the next token is lexed. The problem is 178e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // that the token may be lexed sometime after the vector of tokens itself is 179e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // destroyed, which would be badness. 180e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner PP.RemoveTopOfLexerStack(); 181e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner return Result; 182e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner} 183e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner 184e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner 185e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner/// StringifyArgument - Implement C99 6.10.3.2p2, converting a sequence of 186e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner/// tokens into the literal string token that should be produced by the C # 187e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner/// preprocessor operator. If Charify is true, then it should be turned into 188e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner/// a character literal for the Microsoft charize (#@) extension. 189e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner/// 190e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris LattnerToken MacroArgs::StringifyArgument(const Token *ArgToks, 191b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis Preprocessor &PP, bool Charify, 192a08529cc3f00e0b47a3c028823634129ac46847bAbramo Bagnara SourceLocation ExpansionLocStart, 193a08529cc3f00e0b47a3c028823634129ac46847bAbramo Bagnara SourceLocation ExpansionLocEnd) { 194e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner Token Tok; 195e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner Tok.startToken(); 1966633522aaea663ebb18044a997b963bf92cd1a74Chris Lattner Tok.setKind(Charify ? tok::char_constant : tok::string_literal); 197e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner 198e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner const Token *ArgTokStart = ArgToks; 1991eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 200e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // Stringify all the tokens. 201c19e8a2b92f5ab50d1d0da064ae078760cab369fChris Lattner llvm::SmallString<128> Result; 202c19e8a2b92f5ab50d1d0da064ae078760cab369fChris Lattner Result += "\""; 2031eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 204e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner bool isFirst = true; 205e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner for (; ArgToks->isNot(tok::eof); ++ArgToks) { 206e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner const Token &Tok = *ArgToks; 207e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner if (!isFirst && (Tok.hasLeadingSpace() || Tok.isAtStartOfLine())) 208e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner Result += ' '; 209e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner isFirst = false; 2101eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 211e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // If this is a string or character constant, escape the token as specified 212e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // by 6.10.3.2p2. 213e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner if (Tok.is(tok::string_literal) || // "foo" 214e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner Tok.is(tok::wide_string_literal) || // L"foo" 2155cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor Tok.is(tok::utf8_string_literal) || // u8"foo" 2165cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor Tok.is(tok::utf16_string_literal) || // u"foo" 2175cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor Tok.is(tok::utf32_string_literal) || // U"foo" 2185cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor Tok.is(tok::char_constant) || // 'x' 2195cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor Tok.is(tok::wide_char_constant) || // L'x'. 2205cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor Tok.is(tok::utf16_char_constant) || // u'x'. 2215cee1195584fa8672253139c86e922daeda69b9eDouglas Gregor Tok.is(tok::utf32_char_constant)) { // U'x'. 222453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor bool Invalid = false; 223453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor std::string TokStr = PP.getSpelling(Tok, &Invalid); 224453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor if (!Invalid) { 225453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor std::string Str = Lexer::Stringify(TokStr); 226453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor Result.append(Str.begin(), Str.end()); 227453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor } 2288e85e85b8c3dbbc7e9863fe88f0992d825fd0e32Argyrios Kyrtzidis } else if (Tok.is(tok::code_completion)) { 2298e85e85b8c3dbbc7e9863fe88f0992d825fd0e32Argyrios Kyrtzidis PP.CodeCompleteNaturalLanguage(); 230e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner } else { 231c19e8a2b92f5ab50d1d0da064ae078760cab369fChris Lattner // Otherwise, just append the token. Do some gymnastics to get the token 232c19e8a2b92f5ab50d1d0da064ae078760cab369fChris Lattner // in place and avoid copies where possible. 233c19e8a2b92f5ab50d1d0da064ae078760cab369fChris Lattner unsigned CurStrLen = Result.size(); 234c19e8a2b92f5ab50d1d0da064ae078760cab369fChris Lattner Result.resize(CurStrLen+Tok.getLength()); 235c19e8a2b92f5ab50d1d0da064ae078760cab369fChris Lattner const char *BufPtr = &Result[CurStrLen]; 236453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor bool Invalid = false; 237453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor unsigned ActualTokLen = PP.getSpelling(Tok, BufPtr, &Invalid); 238453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor 239453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor if (!Invalid) { 240453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor // If getSpelling returned a pointer to an already uniqued version of 241453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor // the string instead of filling in BufPtr, memcpy it onto our string. 242453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor if (BufPtr != &Result[CurStrLen]) 243453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor memcpy(&Result[CurStrLen], BufPtr, ActualTokLen); 244453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor 245453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor // If the token was dirty, the spelling may be shorter than the token. 246453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor if (ActualTokLen != Tok.getLength()) 247453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor Result.resize(CurStrLen+ActualTokLen); 248453091cc2082e207ea2c2dda645a9bc01b37fb0cDouglas Gregor } 249e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner } 250e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner } 2511eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 252e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // If the last character of the string is a \, and if it isn't escaped, this 253e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // is an invalid string literal, diagnose it as specified in C99. 254c19e8a2b92f5ab50d1d0da064ae078760cab369fChris Lattner if (Result.back() == '\\') { 255e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // Count the number of consequtive \ characters. If even, then they are 256e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // just escaped backslashes, otherwise it's an error. 257e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner unsigned FirstNonSlash = Result.size()-2; 258e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // Guaranteed to find the starting " if nothing else. 259e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner while (Result[FirstNonSlash] == '\\') 260e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner --FirstNonSlash; 261e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner if ((Result.size()-1-FirstNonSlash) & 1) { 262e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // Diagnose errors for things like: #define F(X) #X / F(\) 263e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner PP.Diag(ArgToks[-1], diag::pp_invalid_string_literal); 264c19e8a2b92f5ab50d1d0da064ae078760cab369fChris Lattner Result.pop_back(); // remove one of the \'s. 265e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner } 266e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner } 267e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner Result += '"'; 2681eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 269e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // If this is the charify operation and the result is not a legal character 270e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // constant, diagnose it. 271e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner if (Charify) { 272e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // First step, turn double quotes into single quotes: 273e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner Result[0] = '\''; 274e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner Result[Result.size()-1] = '\''; 2751eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 276e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner // Check for bogus character. 277e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner bool isBad = false; 278c19e8a2b92f5ab50d1d0da064ae078760cab369fChris Lattner if (Result.size() == 3) 279e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner isBad = Result[1] == '\''; // ''' is not legal. '\' already fixed above. 280c19e8a2b92f5ab50d1d0da064ae078760cab369fChris Lattner else 281e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner isBad = (Result.size() != 4 || Result[1] != '\\'); // Not '\x' 2821eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 283e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner if (isBad) { 284e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner PP.Diag(ArgTokStart[0], diag::err_invalid_character_to_charify); 285e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner Result = "' '"; // Use something arbitrary, but legal. 286e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner } 287e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner } 2881eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 289a08529cc3f00e0b47a3c028823634129ac46847bAbramo Bagnara PP.CreateString(&Result[0], Result.size(), Tok, 290a08529cc3f00e0b47a3c028823634129ac46847bAbramo Bagnara ExpansionLocStart, ExpansionLocEnd); 291e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner return Tok; 292e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner} 293e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner 294e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner/// getStringifiedArgument - Compute, cache, and return the specified argument 295e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner/// that has been 'stringified' as required by the # operator. 296e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattnerconst Token &MacroArgs::getStringifiedArgument(unsigned ArgNo, 297b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis Preprocessor &PP, 298a08529cc3f00e0b47a3c028823634129ac46847bAbramo Bagnara SourceLocation ExpansionLocStart, 299a08529cc3f00e0b47a3c028823634129ac46847bAbramo Bagnara SourceLocation ExpansionLocEnd) { 300e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner assert(ArgNo < NumUnexpArgTokens && "Invalid argument number!"); 301e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner if (StringifiedArgs.empty()) { 302e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner StringifiedArgs.resize(getNumArguments()); 30375c4064932d481ac710a80aa88b3370ad8a6af1dChandler Carruth memset((void*)&StringifiedArgs[0], 0, 304e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner sizeof(StringifiedArgs[0])*getNumArguments()); 305e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner } 306e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner if (StringifiedArgs[ArgNo].isNot(tok::string_literal)) 307b73377eeb3eff76be134203aebb6068244b177f3Argyrios Kyrtzidis StringifiedArgs[ArgNo] = StringifyArgument(getUnexpArgument(ArgNo), PP, 308a08529cc3f00e0b47a3c028823634129ac46847bAbramo Bagnara /*Charify=*/false, 309a08529cc3f00e0b47a3c028823634129ac46847bAbramo Bagnara ExpansionLocStart, 310a08529cc3f00e0b47a3c028823634129ac46847bAbramo Bagnara ExpansionLocEnd); 311e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner return StringifiedArgs[ArgNo]; 312e5c8ffe09f8fec836b90f23c2a3b996bfc5b09c4Chris Lattner} 313