1cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner//===--- TokenRewriter.cpp - Token-based code rewriting interface ---------===// 2cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner// 3cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner// The LLVM Compiler Infrastructure 4cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner// 5cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner// This file is distributed under the University of Illinois Open Source 6cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner// License. See LICENSE.TXT for details. 7cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner// 8cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner//===----------------------------------------------------------------------===// 9cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner// 10cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner// This file implements the TokenRewriter class, which is used for code 11cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner// transformations. 12cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner// 13cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner//===----------------------------------------------------------------------===// 14cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner 15305c613af6cfc40e519c75d9d2c84c6fa9a841c0Ted Kremenek#include "clang/Rewrite/Core/TokenRewriter.h" 1655fc873017f10f6f566b182b70f6fc22aefa3464Chandler Carruth#include "clang/Basic/SourceManager.h" 17cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner#include "clang/Lex/Lexer.h" 1899bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner#include "clang/Lex/ScratchBuffer.h" 19cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattnerusing namespace clang; 20cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner 212b2453a7d8fe732561795431f39ceb2b2a832d84Chris LattnerTokenRewriter::TokenRewriter(FileID FID, SourceManager &SM, 22cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner const LangOptions &LangOpts) { 2399bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner ScratchBuf.reset(new ScratchBuffer(SM)); 241eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 25cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner // Create a lexer to lex all the tokens of the main file in raw mode. 266e2901407bff59aeb4cc301cc58b034723d0eb49Chris Lattner const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID); 276e2901407bff59aeb4cc301cc58b034723d0eb49Chris Lattner Lexer RawLex(FID, FromFile, SM, LangOpts); 281eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 29cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner // Return all comments and whitespace as tokens. 30cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner RawLex.SetKeepWhitespaceMode(true); 31cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner 32cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner // Lex the file, populating our datastructures. 33cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner Token RawTok; 34cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner RawLex.LexFromRawLexer(RawTok); 35cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner while (RawTok.isNot(tok::eof)) { 3699bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner#if 0 37c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara if (Tok.is(tok::raw_identifier)) { 3899bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner // Look up the identifier info for the token. This should use 3999bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner // IdentifierTable directly instead of PP. 40c4bf2b9afb7d47445a9dc6bc848657098a4e3851Abramo Bagnara PP.LookUpIdentifierInfo(Tok); 4199bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner } 4299bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner#endif 431eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 44cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner AddToken(RawTok, TokenList.end()); 45cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner RawLex.LexFromRawLexer(RawTok); 46cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner } 4799bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner} 4899bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner 4999bd46c018ece10d6541a4b6bf0dbe97ad162477Chris LattnerTokenRewriter::~TokenRewriter() { 5099bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner} 5199bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner 5299bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner 5399bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner/// RemapIterator - Convert from token_iterator (a const iterator) to 5499bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner/// TokenRefTy (a non-const iterator). 5599bd46c018ece10d6541a4b6bf0dbe97ad162477Chris LattnerTokenRewriter::TokenRefTy TokenRewriter::RemapIterator(token_iterator I) { 5699bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner if (I == token_end()) return TokenList.end(); 571eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 5899bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner // FIXME: This is horrible, we should use our own list or something to avoid 5999bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner // this. 601eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump std::map<SourceLocation, TokenRefTy>::iterator MapIt = 6199bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner TokenAtLoc.find(I->getLocation()); 6299bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner assert(MapIt != TokenAtLoc.end() && "iterator not in rewriter?"); 6399bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner return MapIt->second; 64cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner} 65cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner 6699bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner 67cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner/// AddToken - Add the specified token into the Rewriter before the other 68cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner/// position. 691eb4433ac451dc16f4133a88af2d002ac26c58efMike StumpTokenRewriter::TokenRefTy 7099bd46c018ece10d6541a4b6bf0dbe97ad162477Chris LattnerTokenRewriter::AddToken(const Token &T, TokenRefTy Where) { 71cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner Where = TokenList.insert(Where, T); 721eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 73cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner bool InsertSuccess = TokenAtLoc.insert(std::make_pair(T.getLocation(), 74cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner Where)).second; 75cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner assert(InsertSuccess && "Token location already in rewriter!"); 76b7ec67a43d0b5408d6a97e92240207b5a2374456Jakob Stoklund Olesen (void)InsertSuccess; 7799bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner return Where; 78cff9cc95de367a3aea885a7f8fee304fe2707b92Chris Lattner} 791eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 8099bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner 8199bd46c018ece10d6541a4b6bf0dbe97ad162477Chris LattnerTokenRewriter::token_iterator 8247246be8ac5b0ddde6c402b8fc6946b6135487b5Chris LattnerTokenRewriter::AddTokenBefore(token_iterator I, const char *Val) { 8399bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner unsigned Len = strlen(Val); 841eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 8599bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner // Plop the string into the scratch buffer, then create a token for this 8699bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner // string. 8799bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner Token Tok; 8899bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner Tok.startToken(); 8947246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner const char *Spelling; 9047246be8ac5b0ddde6c402b8fc6946b6135487b5Chris Lattner Tok.setLocation(ScratchBuf->getToken(Val, Len, Spelling)); 9199bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner Tok.setLength(Len); 921eb4433ac451dc16f4133a88af2d002ac26c58efMike Stump 9399bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner // TODO: Form a whole lexer around this and relex the token! For now, just 9499bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner // set kind to tok::unknown. 9599bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner Tok.setKind(tok::unknown); 9699bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner 9799bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner return AddToken(Tok, RemapIterator(I)); 9899bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner} 9999bd46c018ece10d6541a4b6bf0dbe97ad162477Chris Lattner 100