1//===- unittests/Lex/LexerTest.cpp ------ Lexer tests ---------------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9
10#include "clang/Lex/Lexer.h"
11#include "clang/Basic/Diagnostic.h"
12#include "clang/Basic/DiagnosticOptions.h"
13#include "clang/Basic/FileManager.h"
14#include "clang/Basic/LangOptions.h"
15#include "clang/Basic/SourceManager.h"
16#include "clang/Basic/TargetInfo.h"
17#include "clang/Basic/TargetOptions.h"
18#include "clang/Lex/HeaderSearch.h"
19#include "clang/Lex/HeaderSearchOptions.h"
20#include "clang/Lex/ModuleLoader.h"
21#include "clang/Lex/Preprocessor.h"
22#include "clang/Lex/PreprocessorOptions.h"
23#include "gtest/gtest.h"
24
25using namespace llvm;
26using namespace clang;
27
28namespace {
29
30class VoidModuleLoader : public ModuleLoader {
31  ModuleLoadResult loadModule(SourceLocation ImportLoc,
32                              ModuleIdPath Path,
33                              Module::NameVisibilityKind Visibility,
34                              bool IsInclusionDirective) override {
35    return ModuleLoadResult();
36  }
37
38  void makeModuleVisible(Module *Mod,
39                         Module::NameVisibilityKind Visibility,
40                         SourceLocation ImportLoc,
41                         bool Complain) override { }
42
43  GlobalModuleIndex *loadGlobalModuleIndex(SourceLocation TriggerLoc) override
44    { return nullptr; }
45  bool lookupMissingImports(StringRef Name, SourceLocation TriggerLoc) override
46    { return 0; };
47};
48
49// The test fixture.
50class LexerTest : public ::testing::Test {
51protected:
52  LexerTest()
53    : FileMgr(FileMgrOpts),
54      DiagID(new DiagnosticIDs()),
55      Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()),
56      SourceMgr(Diags, FileMgr),
57      TargetOpts(new TargetOptions)
58  {
59    TargetOpts->Triple = "x86_64-apple-darwin11.1.0";
60    Target = TargetInfo::CreateTargetInfo(Diags, TargetOpts);
61  }
62
63  std::vector<Token> CheckLex(StringRef Source,
64                              ArrayRef<tok::TokenKind> ExpectedTokens) {
65    MemoryBuffer *buf = MemoryBuffer::getMemBuffer(Source);
66    SourceMgr.setMainFileID(SourceMgr.createFileID(buf));
67
68    VoidModuleLoader ModLoader;
69    HeaderSearch HeaderInfo(new HeaderSearchOptions, SourceMgr, Diags, LangOpts,
70                            Target.get());
71    Preprocessor PP(new PreprocessorOptions(), Diags, LangOpts, SourceMgr,
72                    HeaderInfo, ModLoader, /*IILookup =*/nullptr,
73                    /*OwnsHeaderSearch =*/false);
74    PP.Initialize(*Target);
75    PP.EnterMainSourceFile();
76
77    std::vector<Token> toks;
78    while (1) {
79      Token tok;
80      PP.Lex(tok);
81      if (tok.is(tok::eof))
82        break;
83      toks.push_back(tok);
84    }
85
86    EXPECT_EQ(ExpectedTokens.size(), toks.size());
87    for (unsigned i = 0, e = ExpectedTokens.size(); i != e; ++i) {
88      EXPECT_EQ(ExpectedTokens[i], toks[i].getKind());
89    }
90
91    return toks;
92  }
93
94  std::string getSourceText(Token Begin, Token End) {
95    bool Invalid;
96    StringRef Str =
97        Lexer::getSourceText(CharSourceRange::getTokenRange(SourceRange(
98                                    Begin.getLocation(), End.getLocation())),
99                             SourceMgr, LangOpts, &Invalid);
100    if (Invalid)
101      return "<INVALID>";
102    return Str;
103  }
104
105  FileSystemOptions FileMgrOpts;
106  FileManager FileMgr;
107  IntrusiveRefCntPtr<DiagnosticIDs> DiagID;
108  DiagnosticsEngine Diags;
109  SourceManager SourceMgr;
110  LangOptions LangOpts;
111  std::shared_ptr<TargetOptions> TargetOpts;
112  IntrusiveRefCntPtr<TargetInfo> Target;
113};
114
115TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgument) {
116  std::vector<tok::TokenKind> ExpectedTokens;
117  ExpectedTokens.push_back(tok::identifier);
118  ExpectedTokens.push_back(tok::l_paren);
119  ExpectedTokens.push_back(tok::identifier);
120  ExpectedTokens.push_back(tok::r_paren);
121
122  std::vector<Token> toks = CheckLex("#define M(x) x\n"
123                                     "M(f(M(i)))",
124                                     ExpectedTokens);
125
126  EXPECT_EQ("M(i)", getSourceText(toks[2], toks[2]));
127}
128
129TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgumentForEndOfMacro) {
130  std::vector<tok::TokenKind> ExpectedTokens;
131  ExpectedTokens.push_back(tok::identifier);
132  ExpectedTokens.push_back(tok::identifier);
133
134  std::vector<Token> toks = CheckLex("#define M(x) x\n"
135                                     "M(M(i) c)",
136                                     ExpectedTokens);
137
138  EXPECT_EQ("M(i)", getSourceText(toks[0], toks[0]));
139}
140
141TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForBeginOfMacro) {
142  std::vector<tok::TokenKind> ExpectedTokens;
143  ExpectedTokens.push_back(tok::identifier);
144  ExpectedTokens.push_back(tok::identifier);
145  ExpectedTokens.push_back(tok::identifier);
146
147  std::vector<Token> toks = CheckLex("#define M(x) x\n"
148                                     "M(c c M(i))",
149                                     ExpectedTokens);
150
151  EXPECT_EQ("c M(i)", getSourceText(toks[1], toks[2]));
152}
153
154TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForEndOfMacro) {
155  std::vector<tok::TokenKind> ExpectedTokens;
156  ExpectedTokens.push_back(tok::identifier);
157  ExpectedTokens.push_back(tok::identifier);
158  ExpectedTokens.push_back(tok::identifier);
159
160  std::vector<Token> toks = CheckLex("#define M(x) x\n"
161                                     "M(M(i) c c)",
162                                     ExpectedTokens);
163
164  EXPECT_EQ("M(i) c", getSourceText(toks[0], toks[1]));
165}
166
167TEST_F(LexerTest, GetSourceTextInSeparateFnMacros) {
168  std::vector<tok::TokenKind> ExpectedTokens;
169  ExpectedTokens.push_back(tok::identifier);
170  ExpectedTokens.push_back(tok::identifier);
171  ExpectedTokens.push_back(tok::identifier);
172  ExpectedTokens.push_back(tok::identifier);
173
174  std::vector<Token> toks = CheckLex("#define M(x) x\n"
175                                     "M(c M(i)) M(M(i) c)",
176                                     ExpectedTokens);
177
178  EXPECT_EQ("<INVALID>", getSourceText(toks[1], toks[2]));
179}
180
181TEST_F(LexerTest, GetSourceTextWorksAcrossTokenPastes) {
182  std::vector<tok::TokenKind> ExpectedTokens;
183  ExpectedTokens.push_back(tok::identifier);
184  ExpectedTokens.push_back(tok::l_paren);
185  ExpectedTokens.push_back(tok::identifier);
186  ExpectedTokens.push_back(tok::r_paren);
187
188  std::vector<Token> toks = CheckLex("#define M(x) x\n"
189                                     "#define C(x) M(x##c)\n"
190                                     "M(f(C(i)))",
191                                     ExpectedTokens);
192
193  EXPECT_EQ("C(i)", getSourceText(toks[2], toks[2]));
194}
195
196TEST_F(LexerTest, GetSourceTextExpandsAcrossMultipleMacroCalls) {
197  std::vector<tok::TokenKind> ExpectedTokens;
198  ExpectedTokens.push_back(tok::identifier);
199  ExpectedTokens.push_back(tok::l_paren);
200  ExpectedTokens.push_back(tok::identifier);
201  ExpectedTokens.push_back(tok::r_paren);
202
203  std::vector<Token> toks = CheckLex("#define M(x) x\n"
204                                     "f(M(M(i)))",
205                                     ExpectedTokens);
206  EXPECT_EQ("M(M(i))", getSourceText(toks[2], toks[2]));
207}
208
209TEST_F(LexerTest, GetSourceTextInMiddleOfMacroArgument) {
210  std::vector<tok::TokenKind> ExpectedTokens;
211  ExpectedTokens.push_back(tok::identifier);
212  ExpectedTokens.push_back(tok::l_paren);
213  ExpectedTokens.push_back(tok::identifier);
214  ExpectedTokens.push_back(tok::r_paren);
215
216  std::vector<Token> toks = CheckLex("#define M(x) x\n"
217                                     "M(f(i))",
218                                     ExpectedTokens);
219  EXPECT_EQ("i", getSourceText(toks[2], toks[2]));
220}
221
222TEST_F(LexerTest, GetSourceTextExpandsAroundDifferentMacroCalls) {
223  std::vector<tok::TokenKind> ExpectedTokens;
224  ExpectedTokens.push_back(tok::identifier);
225  ExpectedTokens.push_back(tok::l_paren);
226  ExpectedTokens.push_back(tok::identifier);
227  ExpectedTokens.push_back(tok::r_paren);
228
229  std::vector<Token> toks = CheckLex("#define M(x) x\n"
230                                     "#define C(x) x\n"
231                                     "f(C(M(i)))",
232                                     ExpectedTokens);
233  EXPECT_EQ("C(M(i))", getSourceText(toks[2], toks[2]));
234}
235
236TEST_F(LexerTest, GetSourceTextOnlyExpandsIfFirstTokenInMacro) {
237  std::vector<tok::TokenKind> ExpectedTokens;
238  ExpectedTokens.push_back(tok::identifier);
239  ExpectedTokens.push_back(tok::l_paren);
240  ExpectedTokens.push_back(tok::identifier);
241  ExpectedTokens.push_back(tok::identifier);
242  ExpectedTokens.push_back(tok::r_paren);
243
244  std::vector<Token> toks = CheckLex("#define M(x) x\n"
245                                     "#define C(x) c x\n"
246                                     "f(C(M(i)))",
247                                     ExpectedTokens);
248  EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3]));
249}
250
251TEST_F(LexerTest, GetSourceTextExpandsRecursively) {
252  std::vector<tok::TokenKind> ExpectedTokens;
253  ExpectedTokens.push_back(tok::identifier);
254  ExpectedTokens.push_back(tok::identifier);
255  ExpectedTokens.push_back(tok::l_paren);
256  ExpectedTokens.push_back(tok::identifier);
257  ExpectedTokens.push_back(tok::r_paren);
258
259  std::vector<Token> toks = CheckLex("#define M(x) x\n"
260                                     "#define C(x) c M(x)\n"
261                                     "C(f(M(i)))",
262                                     ExpectedTokens);
263  EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3]));
264}
265
266TEST_F(LexerTest, LexAPI) {
267  std::vector<tok::TokenKind> ExpectedTokens;
268  ExpectedTokens.push_back(tok::l_square);
269  ExpectedTokens.push_back(tok::identifier);
270  ExpectedTokens.push_back(tok::r_square);
271  ExpectedTokens.push_back(tok::l_square);
272  ExpectedTokens.push_back(tok::identifier);
273  ExpectedTokens.push_back(tok::r_square);
274  ExpectedTokens.push_back(tok::identifier);
275  ExpectedTokens.push_back(tok::identifier);
276  ExpectedTokens.push_back(tok::identifier);
277  ExpectedTokens.push_back(tok::identifier);
278
279  std::vector<Token> toks = CheckLex("#define M(x) [x]\n"
280                                     "#define N(x) x\n"
281                                     "#define INN(x) x\n"
282                                     "#define NOF1 INN(val)\n"
283                                     "#define NOF2 val\n"
284                                     "M(foo) N([bar])\n"
285                                     "N(INN(val)) N(NOF1) N(NOF2) N(val)",
286                                     ExpectedTokens);
287
288  SourceLocation lsqrLoc = toks[0].getLocation();
289  SourceLocation idLoc = toks[1].getLocation();
290  SourceLocation rsqrLoc = toks[2].getLocation();
291  std::pair<SourceLocation,SourceLocation>
292    macroPair = SourceMgr.getExpansionRange(lsqrLoc);
293  SourceRange macroRange = SourceRange(macroPair.first, macroPair.second);
294
295  SourceLocation Loc;
296  EXPECT_TRUE(Lexer::isAtStartOfMacroExpansion(lsqrLoc, SourceMgr, LangOpts, &Loc));
297  EXPECT_EQ(Loc, macroRange.getBegin());
298  EXPECT_FALSE(Lexer::isAtStartOfMacroExpansion(idLoc, SourceMgr, LangOpts));
299  EXPECT_FALSE(Lexer::isAtEndOfMacroExpansion(idLoc, SourceMgr, LangOpts));
300  EXPECT_TRUE(Lexer::isAtEndOfMacroExpansion(rsqrLoc, SourceMgr, LangOpts, &Loc));
301  EXPECT_EQ(Loc, macroRange.getEnd());
302
303  CharSourceRange range = Lexer::makeFileCharRange(
304           CharSourceRange::getTokenRange(lsqrLoc, idLoc), SourceMgr, LangOpts);
305  EXPECT_TRUE(range.isInvalid());
306  range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(idLoc, rsqrLoc),
307                                   SourceMgr, LangOpts);
308  EXPECT_TRUE(range.isInvalid());
309  range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc),
310                                   SourceMgr, LangOpts);
311  EXPECT_TRUE(!range.isTokenRange());
312  EXPECT_EQ(range.getAsRange(),
313            SourceRange(macroRange.getBegin(),
314                        macroRange.getEnd().getLocWithOffset(1)));
315
316  StringRef text = Lexer::getSourceText(
317                               CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc),
318                               SourceMgr, LangOpts);
319  EXPECT_EQ(text, "M(foo)");
320
321  SourceLocation macroLsqrLoc = toks[3].getLocation();
322  SourceLocation macroIdLoc = toks[4].getLocation();
323  SourceLocation macroRsqrLoc = toks[5].getLocation();
324  SourceLocation fileLsqrLoc = SourceMgr.getSpellingLoc(macroLsqrLoc);
325  SourceLocation fileIdLoc = SourceMgr.getSpellingLoc(macroIdLoc);
326  SourceLocation fileRsqrLoc = SourceMgr.getSpellingLoc(macroRsqrLoc);
327
328  range = Lexer::makeFileCharRange(
329      CharSourceRange::getTokenRange(macroLsqrLoc, macroIdLoc),
330      SourceMgr, LangOpts);
331  EXPECT_EQ(SourceRange(fileLsqrLoc, fileIdLoc.getLocWithOffset(3)),
332            range.getAsRange());
333
334  range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(macroIdLoc, macroRsqrLoc),
335                                   SourceMgr, LangOpts);
336  EXPECT_EQ(SourceRange(fileIdLoc, fileRsqrLoc.getLocWithOffset(1)),
337            range.getAsRange());
338
339  macroPair = SourceMgr.getExpansionRange(macroLsqrLoc);
340  range = Lexer::makeFileCharRange(
341                     CharSourceRange::getTokenRange(macroLsqrLoc, macroRsqrLoc),
342                     SourceMgr, LangOpts);
343  EXPECT_EQ(SourceRange(macroPair.first, macroPair.second.getLocWithOffset(1)),
344            range.getAsRange());
345
346  text = Lexer::getSourceText(
347          CharSourceRange::getTokenRange(SourceRange(macroLsqrLoc, macroIdLoc)),
348          SourceMgr, LangOpts);
349  EXPECT_EQ(text, "[bar");
350
351
352  SourceLocation idLoc1 = toks[6].getLocation();
353  SourceLocation idLoc2 = toks[7].getLocation();
354  SourceLocation idLoc3 = toks[8].getLocation();
355  SourceLocation idLoc4 = toks[9].getLocation();
356  EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc1, SourceMgr, LangOpts));
357  EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc2, SourceMgr, LangOpts));
358  EXPECT_EQ("NOF2", Lexer::getImmediateMacroName(idLoc3, SourceMgr, LangOpts));
359  EXPECT_EQ("N", Lexer::getImmediateMacroName(idLoc4, SourceMgr, LangOpts));
360}
361
362} // anonymous namespace
363