LexerTest.cpp revision c50c6ff49aa3648ae031349de6f09439f52425f0 
1//===- unittests/Lex/LexerTest.cpp ------ Lexer tests ---------------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9
10#include "clang/Lex/Lexer.h"
11#include "clang/Basic/Diagnostic.h"
12#include "clang/Basic/DiagnosticOptions.h"
13#include "clang/Basic/FileManager.h"
14#include "clang/Basic/LangOptions.h"
15#include "clang/Basic/SourceManager.h"
16#include "clang/Basic/TargetInfo.h"
17#include "clang/Basic/TargetOptions.h"
18#include "clang/Lex/HeaderSearch.h"
19#include "clang/Lex/HeaderSearchOptions.h"
20#include "clang/Lex/ModuleLoader.h"
21#include "clang/Lex/Preprocessor.h"
22#include "clang/Lex/PreprocessorOptions.h"
23#include "llvm/Config/config.h"
24#include "gtest/gtest.h"
25
26using namespace llvm;
27using namespace clang;
28
29namespace {
30
31class VoidModuleLoader : public ModuleLoader {
32  virtual ModuleLoadResult loadModule(SourceLocation ImportLoc,
33                                      ModuleIdPath Path,
34                                      Module::NameVisibilityKind Visibility,
35                                      bool IsInclusionDirective) {
36    return ModuleLoadResult();
37  }
38
39  virtual void makeModuleVisible(Module *Mod,
40                                 Module::NameVisibilityKind Visibility,
41                                 SourceLocation ImportLoc,
42                                 bool Complain) { }
43};
44
45// The test fixture.
46class LexerTest : public ::testing::Test {
47protected:
48  LexerTest()
49    : FileMgr(FileMgrOpts),
50      DiagID(new DiagnosticIDs()),
51      Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()),
52      SourceMgr(Diags, FileMgr),
53      TargetOpts(new TargetOptions)
54  {
55    TargetOpts->Triple = "x86_64-apple-darwin11.1.0";
56    Target = TargetInfo::CreateTargetInfo(Diags, &*TargetOpts);
57  }
58
59  std::vector<Token> CheckLex(StringRef Source,
60                              ArrayRef<tok::TokenKind> ExpectedTokens) {
61    MemoryBuffer *buf = MemoryBuffer::getMemBuffer(Source);
62    (void) SourceMgr.createMainFileIDForMemBuffer(buf);
63
64    VoidModuleLoader ModLoader;
65    HeaderSearch HeaderInfo(new HeaderSearchOptions, FileMgr, Diags, LangOpts,
66                            Target.getPtr());
67    Preprocessor PP(new PreprocessorOptions(), Diags, LangOpts, Target.getPtr(),
68                    SourceMgr, HeaderInfo, ModLoader, /*IILookup =*/ 0,
69                    /*OwnsHeaderSearch =*/ false,
70                    /*DelayInitialization =*/ false);
71    PP.EnterMainSourceFile();
72
73    std::vector<Token> toks;
74    while (1) {
75      Token tok;
76      PP.Lex(tok);
77      if (tok.is(tok::eof))
78        break;
79      toks.push_back(tok);
80    }
81
82    EXPECT_EQ(ExpectedTokens.size(), toks.size());
83    for (unsigned i = 0, e = ExpectedTokens.size(); i != e; ++i) {
84      EXPECT_EQ(ExpectedTokens[i], toks[i].getKind());
85    }
86
87    return toks;
88  }
89
90  std::string getSourceText(Token Begin, Token End) {
91    bool Invalid;
92    StringRef Str =
93        Lexer::getSourceText(CharSourceRange::getTokenRange(SourceRange(
94                                    Begin.getLocation(), End.getLocation())),
95                             SourceMgr, LangOpts, &Invalid);
96    if (Invalid)
97      return "<INVALID>";
98    return Str;
99  }
100
101  FileSystemOptions FileMgrOpts;
102  FileManager FileMgr;
103  IntrusiveRefCntPtr<DiagnosticIDs> DiagID;
104  DiagnosticsEngine Diags;
105  SourceManager SourceMgr;
106  LangOptions LangOpts;
107  IntrusiveRefCntPtr<TargetOptions> TargetOpts;
108  IntrusiveRefCntPtr<TargetInfo> Target;
109};
110
111TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgument) {
112  std::vector<tok::TokenKind> ExpectedTokens;
113  ExpectedTokens.push_back(tok::identifier);
114  ExpectedTokens.push_back(tok::l_paren);
115  ExpectedTokens.push_back(tok::identifier);
116  ExpectedTokens.push_back(tok::r_paren);
117
118  std::vector<Token> toks = CheckLex("#define M(x) x\n"
119                                     "M(f(M(i)))",
120                                     ExpectedTokens);
121
122  EXPECT_EQ("M(i)", getSourceText(toks[2], toks[2]));
123}
124
125TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgumentForEndOfMacro) {
126  std::vector<tok::TokenKind> ExpectedTokens;
127  ExpectedTokens.push_back(tok::identifier);
128  ExpectedTokens.push_back(tok::identifier);
129
130  std::vector<Token> toks = CheckLex("#define M(x) x\n"
131                                     "M(M(i) c)",
132                                     ExpectedTokens);
133
134  EXPECT_EQ("M(i)", getSourceText(toks[0], toks[0]));
135}
136
137TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForBeginOfMacro) {
138  std::vector<tok::TokenKind> ExpectedTokens;
139  ExpectedTokens.push_back(tok::identifier);
140  ExpectedTokens.push_back(tok::identifier);
141  ExpectedTokens.push_back(tok::identifier);
142
143  std::vector<Token> toks = CheckLex("#define M(x) x\n"
144                                     "M(c c M(i))",
145                                     ExpectedTokens);
146
147  EXPECT_EQ("c M(i)", getSourceText(toks[1], toks[2]));
148}
149
150TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForEndOfMacro) {
151  std::vector<tok::TokenKind> ExpectedTokens;
152  ExpectedTokens.push_back(tok::identifier);
153  ExpectedTokens.push_back(tok::identifier);
154  ExpectedTokens.push_back(tok::identifier);
155
156  std::vector<Token> toks = CheckLex("#define M(x) x\n"
157                                     "M(M(i) c c)",
158                                     ExpectedTokens);
159
160  EXPECT_EQ("M(i) c", getSourceText(toks[0], toks[1]));
161}
162
163TEST_F(LexerTest, GetSourceTextInSeparateFnMacros) {
164  std::vector<tok::TokenKind> ExpectedTokens;
165  ExpectedTokens.push_back(tok::identifier);
166  ExpectedTokens.push_back(tok::identifier);
167  ExpectedTokens.push_back(tok::identifier);
168  ExpectedTokens.push_back(tok::identifier);
169
170  std::vector<Token> toks = CheckLex("#define M(x) x\n"
171                                     "M(c M(i)) M(M(i) c)",
172                                     ExpectedTokens);
173
174  EXPECT_EQ("<INVALID>", getSourceText(toks[1], toks[2]));
175}
176
177TEST_F(LexerTest, GetSourceTextWorksAcrossTokenPastes) {
178  std::vector<tok::TokenKind> ExpectedTokens;
179  ExpectedTokens.push_back(tok::identifier);
180  ExpectedTokens.push_back(tok::l_paren);
181  ExpectedTokens.push_back(tok::identifier);
182  ExpectedTokens.push_back(tok::r_paren);
183
184  std::vector<Token> toks = CheckLex("#define M(x) x\n"
185                                     "#define C(x) M(x##c)\n"
186                                     "M(f(C(i)))",
187                                     ExpectedTokens);
188
189  EXPECT_EQ("C(i)", getSourceText(toks[2], toks[2]));
190}
191
192TEST_F(LexerTest, GetSourceTextExpandsAcrossMultipleMacroCalls) {
193  std::vector<tok::TokenKind> ExpectedTokens;
194  ExpectedTokens.push_back(tok::identifier);
195  ExpectedTokens.push_back(tok::l_paren);
196  ExpectedTokens.push_back(tok::identifier);
197  ExpectedTokens.push_back(tok::r_paren);
198
199  std::vector<Token> toks = CheckLex("#define M(x) x\n"
200                                     "f(M(M(i)))",
201                                     ExpectedTokens);
202  EXPECT_EQ("M(M(i))", getSourceText(toks[2], toks[2]));
203}
204
205TEST_F(LexerTest, GetSourceTextInMiddleOfMacroArgument) {
206  std::vector<tok::TokenKind> ExpectedTokens;
207  ExpectedTokens.push_back(tok::identifier);
208  ExpectedTokens.push_back(tok::l_paren);
209  ExpectedTokens.push_back(tok::identifier);
210  ExpectedTokens.push_back(tok::r_paren);
211
212  std::vector<Token> toks = CheckLex("#define M(x) x\n"
213                                     "M(f(i))",
214                                     ExpectedTokens);
215  EXPECT_EQ("i", getSourceText(toks[2], toks[2]));
216}
217
218TEST_F(LexerTest, GetSourceTextExpandsAroundDifferentMacroCalls) {
219  std::vector<tok::TokenKind> ExpectedTokens;
220  ExpectedTokens.push_back(tok::identifier);
221  ExpectedTokens.push_back(tok::l_paren);
222  ExpectedTokens.push_back(tok::identifier);
223  ExpectedTokens.push_back(tok::r_paren);
224
225  std::vector<Token> toks = CheckLex("#define M(x) x\n"
226                                     "#define C(x) x\n"
227                                     "f(C(M(i)))",
228                                     ExpectedTokens);
229  EXPECT_EQ("C(M(i))", getSourceText(toks[2], toks[2]));
230}
231
232TEST_F(LexerTest, GetSourceTextOnlyExpandsIfFirstTokenInMacro) {
233  std::vector<tok::TokenKind> ExpectedTokens;
234  ExpectedTokens.push_back(tok::identifier);
235  ExpectedTokens.push_back(tok::l_paren);
236  ExpectedTokens.push_back(tok::identifier);
237  ExpectedTokens.push_back(tok::identifier);
238  ExpectedTokens.push_back(tok::r_paren);
239
240  std::vector<Token> toks = CheckLex("#define M(x) x\n"
241                                     "#define C(x) c x\n"
242                                     "f(C(M(i)))",
243                                     ExpectedTokens);
244  EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3]));
245}
246
247TEST_F(LexerTest, GetSourceTextExpandsRecursively) {
248  std::vector<tok::TokenKind> ExpectedTokens;
249  ExpectedTokens.push_back(tok::identifier);
250  ExpectedTokens.push_back(tok::identifier);
251  ExpectedTokens.push_back(tok::l_paren);
252  ExpectedTokens.push_back(tok::identifier);
253  ExpectedTokens.push_back(tok::r_paren);
254
255  std::vector<Token> toks = CheckLex("#define M(x) x\n"
256                                     "#define C(x) c M(x)\n"
257                                     "C(f(M(i)))",
258                                     ExpectedTokens);
259  EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3]));
260}
261
262TEST_F(LexerTest, LexAPI) {
263  std::vector<tok::TokenKind> ExpectedTokens;
264  ExpectedTokens.push_back(tok::l_square);
265  ExpectedTokens.push_back(tok::identifier);
266  ExpectedTokens.push_back(tok::r_square);
267  ExpectedTokens.push_back(tok::l_square);
268  ExpectedTokens.push_back(tok::identifier);
269  ExpectedTokens.push_back(tok::r_square);
270  ExpectedTokens.push_back(tok::identifier);
271  ExpectedTokens.push_back(tok::identifier);
272  ExpectedTokens.push_back(tok::identifier);
273  ExpectedTokens.push_back(tok::identifier);
274
275  std::vector<Token> toks = CheckLex("#define M(x) [x]\n"
276                                     "#define N(x) x\n"
277                                     "#define INN(x) x\n"
278                                     "#define NOF1 INN(val)\n"
279                                     "#define NOF2 val\n"
280                                     "M(foo) N([bar])\n"
281                                     "N(INN(val)) N(NOF1) N(NOF2) N(val)",
282                                     ExpectedTokens);
283
284  SourceLocation lsqrLoc = toks[0].getLocation();
285  SourceLocation idLoc = toks[1].getLocation();
286  SourceLocation rsqrLoc = toks[2].getLocation();
287  std::pair<SourceLocation,SourceLocation>
288    macroPair = SourceMgr.getExpansionRange(lsqrLoc);
289  SourceRange macroRange = SourceRange(macroPair.first, macroPair.second);
290
291  SourceLocation Loc;
292  EXPECT_TRUE(Lexer::isAtStartOfMacroExpansion(lsqrLoc, SourceMgr, LangOpts, &Loc));
293  EXPECT_EQ(Loc, macroRange.getBegin());
294  EXPECT_FALSE(Lexer::isAtStartOfMacroExpansion(idLoc, SourceMgr, LangOpts));
295  EXPECT_FALSE(Lexer::isAtEndOfMacroExpansion(idLoc, SourceMgr, LangOpts));
296  EXPECT_TRUE(Lexer::isAtEndOfMacroExpansion(rsqrLoc, SourceMgr, LangOpts, &Loc));
297  EXPECT_EQ(Loc, macroRange.getEnd());
298
299  CharSourceRange range = Lexer::makeFileCharRange(
300           CharSourceRange::getTokenRange(lsqrLoc, idLoc), SourceMgr, LangOpts);
301  EXPECT_TRUE(range.isInvalid());
302  range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(idLoc, rsqrLoc),
303                                   SourceMgr, LangOpts);
304  EXPECT_TRUE(range.isInvalid());
305  range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc),
306                                   SourceMgr, LangOpts);
307  EXPECT_TRUE(!range.isTokenRange());
308  EXPECT_EQ(range.getAsRange(),
309            SourceRange(macroRange.getBegin(),
310                        macroRange.getEnd().getLocWithOffset(1)));
311
312  StringRef text = Lexer::getSourceText(
313                               CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc),
314                               SourceMgr, LangOpts);
315  EXPECT_EQ(text, "M(foo)");
316
317  SourceLocation macroLsqrLoc = toks[3].getLocation();
318  SourceLocation macroIdLoc = toks[4].getLocation();
319  SourceLocation macroRsqrLoc = toks[5].getLocation();
320  SourceLocation fileLsqrLoc = SourceMgr.getSpellingLoc(macroLsqrLoc);
321  SourceLocation fileIdLoc = SourceMgr.getSpellingLoc(macroIdLoc);
322  SourceLocation fileRsqrLoc = SourceMgr.getSpellingLoc(macroRsqrLoc);
323
324  range = Lexer::makeFileCharRange(
325      CharSourceRange::getTokenRange(macroLsqrLoc, macroIdLoc),
326      SourceMgr, LangOpts);
327  EXPECT_EQ(SourceRange(fileLsqrLoc, fileIdLoc.getLocWithOffset(3)),
328            range.getAsRange());
329
330  range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(macroIdLoc, macroRsqrLoc),
331                                   SourceMgr, LangOpts);
332  EXPECT_EQ(SourceRange(fileIdLoc, fileRsqrLoc.getLocWithOffset(1)),
333            range.getAsRange());
334
335  macroPair = SourceMgr.getExpansionRange(macroLsqrLoc);
336  range = Lexer::makeFileCharRange(
337                     CharSourceRange::getTokenRange(macroLsqrLoc, macroRsqrLoc),
338                     SourceMgr, LangOpts);
339  EXPECT_EQ(SourceRange(macroPair.first, macroPair.second.getLocWithOffset(1)),
340            range.getAsRange());
341
342  text = Lexer::getSourceText(
343          CharSourceRange::getTokenRange(SourceRange(macroLsqrLoc, macroIdLoc)),
344          SourceMgr, LangOpts);
345  EXPECT_EQ(text, "[bar");
346
347
348  SourceLocation idLoc1 = toks[6].getLocation();
349  SourceLocation idLoc2 = toks[7].getLocation();
350  SourceLocation idLoc3 = toks[8].getLocation();
351  SourceLocation idLoc4 = toks[9].getLocation();
352  EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc1, SourceMgr, LangOpts));
353  EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc2, SourceMgr, LangOpts));
354  EXPECT_EQ("NOF2", Lexer::getImmediateMacroName(idLoc3, SourceMgr, LangOpts));
355  EXPECT_EQ("N", Lexer::getImmediateMacroName(idLoc4, SourceMgr, LangOpts));
356}
357
358} // anonymous namespace
359