1//===- unittests/Lex/LexerTest.cpp ------ Lexer tests ---------------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9
10#include "clang/Lex/Lexer.h"
11#include "clang/Basic/Diagnostic.h"
12#include "clang/Basic/DiagnosticOptions.h"
13#include "clang/Basic/FileManager.h"
14#include "clang/Basic/LangOptions.h"
15#include "clang/Basic/SourceManager.h"
16#include "clang/Basic/TargetInfo.h"
17#include "clang/Basic/TargetOptions.h"
18#include "clang/Lex/HeaderSearch.h"
19#include "clang/Lex/HeaderSearchOptions.h"
20#include "clang/Lex/ModuleLoader.h"
21#include "clang/Lex/Preprocessor.h"
22#include "clang/Lex/PreprocessorOptions.h"
23#include "gtest/gtest.h"
24
25using namespace llvm;
26using namespace clang;
27
28namespace {
29
30class VoidModuleLoader : public ModuleLoader {
31  ModuleLoadResult loadModule(SourceLocation ImportLoc,
32                              ModuleIdPath Path,
33                              Module::NameVisibilityKind Visibility,
34                              bool IsInclusionDirective) override {
35    return ModuleLoadResult();
36  }
37
38  void makeModuleVisible(Module *Mod,
39                         Module::NameVisibilityKind Visibility,
40                         SourceLocation ImportLoc) override { }
41
42  GlobalModuleIndex *loadGlobalModuleIndex(SourceLocation TriggerLoc) override
43    { return nullptr; }
44  bool lookupMissingImports(StringRef Name, SourceLocation TriggerLoc) override
45    { return 0; }
46};
47
48// The test fixture.
49class LexerTest : public ::testing::Test {
50protected:
51  LexerTest()
52    : FileMgr(FileMgrOpts),
53      DiagID(new DiagnosticIDs()),
54      Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()),
55      SourceMgr(Diags, FileMgr),
56      TargetOpts(new TargetOptions)
57  {
58    TargetOpts->Triple = "x86_64-apple-darwin11.1.0";
59    Target = TargetInfo::CreateTargetInfo(Diags, TargetOpts);
60  }
61
62  std::vector<Token> CheckLex(StringRef Source,
63                              ArrayRef<tok::TokenKind> ExpectedTokens) {
64    std::unique_ptr<MemoryBuffer> Buf = MemoryBuffer::getMemBuffer(Source);
65    SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf)));
66
67    VoidModuleLoader ModLoader;
68    HeaderSearch HeaderInfo(new HeaderSearchOptions, SourceMgr, Diags, LangOpts,
69                            Target.get());
70    Preprocessor PP(new PreprocessorOptions(), Diags, LangOpts, SourceMgr,
71                    HeaderInfo, ModLoader, /*IILookup =*/nullptr,
72                    /*OwnsHeaderSearch =*/false);
73    PP.Initialize(*Target);
74    PP.EnterMainSourceFile();
75
76    std::vector<Token> toks;
77    while (1) {
78      Token tok;
79      PP.Lex(tok);
80      if (tok.is(tok::eof))
81        break;
82      toks.push_back(tok);
83    }
84
85    EXPECT_EQ(ExpectedTokens.size(), toks.size());
86    for (unsigned i = 0, e = ExpectedTokens.size(); i != e; ++i) {
87      EXPECT_EQ(ExpectedTokens[i], toks[i].getKind());
88    }
89
90    return toks;
91  }
92
93  std::string getSourceText(Token Begin, Token End) {
94    bool Invalid;
95    StringRef Str =
96        Lexer::getSourceText(CharSourceRange::getTokenRange(SourceRange(
97                                    Begin.getLocation(), End.getLocation())),
98                             SourceMgr, LangOpts, &Invalid);
99    if (Invalid)
100      return "<INVALID>";
101    return Str;
102  }
103
104  FileSystemOptions FileMgrOpts;
105  FileManager FileMgr;
106  IntrusiveRefCntPtr<DiagnosticIDs> DiagID;
107  DiagnosticsEngine Diags;
108  SourceManager SourceMgr;
109  LangOptions LangOpts;
110  std::shared_ptr<TargetOptions> TargetOpts;
111  IntrusiveRefCntPtr<TargetInfo> Target;
112};
113
114TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgument) {
115  std::vector<tok::TokenKind> ExpectedTokens;
116  ExpectedTokens.push_back(tok::identifier);
117  ExpectedTokens.push_back(tok::l_paren);
118  ExpectedTokens.push_back(tok::identifier);
119  ExpectedTokens.push_back(tok::r_paren);
120
121  std::vector<Token> toks = CheckLex("#define M(x) x\n"
122                                     "M(f(M(i)))",
123                                     ExpectedTokens);
124
125  EXPECT_EQ("M(i)", getSourceText(toks[2], toks[2]));
126}
127
128TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgumentForEndOfMacro) {
129  std::vector<tok::TokenKind> ExpectedTokens;
130  ExpectedTokens.push_back(tok::identifier);
131  ExpectedTokens.push_back(tok::identifier);
132
133  std::vector<Token> toks = CheckLex("#define M(x) x\n"
134                                     "M(M(i) c)",
135                                     ExpectedTokens);
136
137  EXPECT_EQ("M(i)", getSourceText(toks[0], toks[0]));
138}
139
140TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForBeginOfMacro) {
141  std::vector<tok::TokenKind> ExpectedTokens;
142  ExpectedTokens.push_back(tok::identifier);
143  ExpectedTokens.push_back(tok::identifier);
144  ExpectedTokens.push_back(tok::identifier);
145
146  std::vector<Token> toks = CheckLex("#define M(x) x\n"
147                                     "M(c c M(i))",
148                                     ExpectedTokens);
149
150  EXPECT_EQ("c M(i)", getSourceText(toks[1], toks[2]));
151}
152
153TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForEndOfMacro) {
154  std::vector<tok::TokenKind> ExpectedTokens;
155  ExpectedTokens.push_back(tok::identifier);
156  ExpectedTokens.push_back(tok::identifier);
157  ExpectedTokens.push_back(tok::identifier);
158
159  std::vector<Token> toks = CheckLex("#define M(x) x\n"
160                                     "M(M(i) c c)",
161                                     ExpectedTokens);
162
163  EXPECT_EQ("M(i) c", getSourceText(toks[0], toks[1]));
164}
165
166TEST_F(LexerTest, GetSourceTextInSeparateFnMacros) {
167  std::vector<tok::TokenKind> ExpectedTokens;
168  ExpectedTokens.push_back(tok::identifier);
169  ExpectedTokens.push_back(tok::identifier);
170  ExpectedTokens.push_back(tok::identifier);
171  ExpectedTokens.push_back(tok::identifier);
172
173  std::vector<Token> toks = CheckLex("#define M(x) x\n"
174                                     "M(c M(i)) M(M(i) c)",
175                                     ExpectedTokens);
176
177  EXPECT_EQ("<INVALID>", getSourceText(toks[1], toks[2]));
178}
179
180TEST_F(LexerTest, GetSourceTextWorksAcrossTokenPastes) {
181  std::vector<tok::TokenKind> ExpectedTokens;
182  ExpectedTokens.push_back(tok::identifier);
183  ExpectedTokens.push_back(tok::l_paren);
184  ExpectedTokens.push_back(tok::identifier);
185  ExpectedTokens.push_back(tok::r_paren);
186
187  std::vector<Token> toks = CheckLex("#define M(x) x\n"
188                                     "#define C(x) M(x##c)\n"
189                                     "M(f(C(i)))",
190                                     ExpectedTokens);
191
192  EXPECT_EQ("C(i)", getSourceText(toks[2], toks[2]));
193}
194
195TEST_F(LexerTest, GetSourceTextExpandsAcrossMultipleMacroCalls) {
196  std::vector<tok::TokenKind> ExpectedTokens;
197  ExpectedTokens.push_back(tok::identifier);
198  ExpectedTokens.push_back(tok::l_paren);
199  ExpectedTokens.push_back(tok::identifier);
200  ExpectedTokens.push_back(tok::r_paren);
201
202  std::vector<Token> toks = CheckLex("#define M(x) x\n"
203                                     "f(M(M(i)))",
204                                     ExpectedTokens);
205  EXPECT_EQ("M(M(i))", getSourceText(toks[2], toks[2]));
206}
207
208TEST_F(LexerTest, GetSourceTextInMiddleOfMacroArgument) {
209  std::vector<tok::TokenKind> ExpectedTokens;
210  ExpectedTokens.push_back(tok::identifier);
211  ExpectedTokens.push_back(tok::l_paren);
212  ExpectedTokens.push_back(tok::identifier);
213  ExpectedTokens.push_back(tok::r_paren);
214
215  std::vector<Token> toks = CheckLex("#define M(x) x\n"
216                                     "M(f(i))",
217                                     ExpectedTokens);
218  EXPECT_EQ("i", getSourceText(toks[2], toks[2]));
219}
220
221TEST_F(LexerTest, GetSourceTextExpandsAroundDifferentMacroCalls) {
222  std::vector<tok::TokenKind> ExpectedTokens;
223  ExpectedTokens.push_back(tok::identifier);
224  ExpectedTokens.push_back(tok::l_paren);
225  ExpectedTokens.push_back(tok::identifier);
226  ExpectedTokens.push_back(tok::r_paren);
227
228  std::vector<Token> toks = CheckLex("#define M(x) x\n"
229                                     "#define C(x) x\n"
230                                     "f(C(M(i)))",
231                                     ExpectedTokens);
232  EXPECT_EQ("C(M(i))", getSourceText(toks[2], toks[2]));
233}
234
235TEST_F(LexerTest, GetSourceTextOnlyExpandsIfFirstTokenInMacro) {
236  std::vector<tok::TokenKind> ExpectedTokens;
237  ExpectedTokens.push_back(tok::identifier);
238  ExpectedTokens.push_back(tok::l_paren);
239  ExpectedTokens.push_back(tok::identifier);
240  ExpectedTokens.push_back(tok::identifier);
241  ExpectedTokens.push_back(tok::r_paren);
242
243  std::vector<Token> toks = CheckLex("#define M(x) x\n"
244                                     "#define C(x) c x\n"
245                                     "f(C(M(i)))",
246                                     ExpectedTokens);
247  EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3]));
248}
249
250TEST_F(LexerTest, GetSourceTextExpandsRecursively) {
251  std::vector<tok::TokenKind> ExpectedTokens;
252  ExpectedTokens.push_back(tok::identifier);
253  ExpectedTokens.push_back(tok::identifier);
254  ExpectedTokens.push_back(tok::l_paren);
255  ExpectedTokens.push_back(tok::identifier);
256  ExpectedTokens.push_back(tok::r_paren);
257
258  std::vector<Token> toks = CheckLex("#define M(x) x\n"
259                                     "#define C(x) c M(x)\n"
260                                     "C(f(M(i)))",
261                                     ExpectedTokens);
262  EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3]));
263}
264
265TEST_F(LexerTest, LexAPI) {
266  std::vector<tok::TokenKind> ExpectedTokens;
267  ExpectedTokens.push_back(tok::l_square);
268  ExpectedTokens.push_back(tok::identifier);
269  ExpectedTokens.push_back(tok::r_square);
270  ExpectedTokens.push_back(tok::l_square);
271  ExpectedTokens.push_back(tok::identifier);
272  ExpectedTokens.push_back(tok::r_square);
273  ExpectedTokens.push_back(tok::identifier);
274  ExpectedTokens.push_back(tok::identifier);
275  ExpectedTokens.push_back(tok::identifier);
276  ExpectedTokens.push_back(tok::identifier);
277
278  std::vector<Token> toks = CheckLex("#define M(x) [x]\n"
279                                     "#define N(x) x\n"
280                                     "#define INN(x) x\n"
281                                     "#define NOF1 INN(val)\n"
282                                     "#define NOF2 val\n"
283                                     "M(foo) N([bar])\n"
284                                     "N(INN(val)) N(NOF1) N(NOF2) N(val)",
285                                     ExpectedTokens);
286
287  SourceLocation lsqrLoc = toks[0].getLocation();
288  SourceLocation idLoc = toks[1].getLocation();
289  SourceLocation rsqrLoc = toks[2].getLocation();
290  std::pair<SourceLocation,SourceLocation>
291    macroPair = SourceMgr.getExpansionRange(lsqrLoc);
292  SourceRange macroRange = SourceRange(macroPair.first, macroPair.second);
293
294  SourceLocation Loc;
295  EXPECT_TRUE(Lexer::isAtStartOfMacroExpansion(lsqrLoc, SourceMgr, LangOpts, &Loc));
296  EXPECT_EQ(Loc, macroRange.getBegin());
297  EXPECT_FALSE(Lexer::isAtStartOfMacroExpansion(idLoc, SourceMgr, LangOpts));
298  EXPECT_FALSE(Lexer::isAtEndOfMacroExpansion(idLoc, SourceMgr, LangOpts));
299  EXPECT_TRUE(Lexer::isAtEndOfMacroExpansion(rsqrLoc, SourceMgr, LangOpts, &Loc));
300  EXPECT_EQ(Loc, macroRange.getEnd());
301
302  CharSourceRange range = Lexer::makeFileCharRange(
303           CharSourceRange::getTokenRange(lsqrLoc, idLoc), SourceMgr, LangOpts);
304  EXPECT_TRUE(range.isInvalid());
305  range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(idLoc, rsqrLoc),
306                                   SourceMgr, LangOpts);
307  EXPECT_TRUE(range.isInvalid());
308  range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc),
309                                   SourceMgr, LangOpts);
310  EXPECT_TRUE(!range.isTokenRange());
311  EXPECT_EQ(range.getAsRange(),
312            SourceRange(macroRange.getBegin(),
313                        macroRange.getEnd().getLocWithOffset(1)));
314
315  StringRef text = Lexer::getSourceText(
316                               CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc),
317                               SourceMgr, LangOpts);
318  EXPECT_EQ(text, "M(foo)");
319
320  SourceLocation macroLsqrLoc = toks[3].getLocation();
321  SourceLocation macroIdLoc = toks[4].getLocation();
322  SourceLocation macroRsqrLoc = toks[5].getLocation();
323  SourceLocation fileLsqrLoc = SourceMgr.getSpellingLoc(macroLsqrLoc);
324  SourceLocation fileIdLoc = SourceMgr.getSpellingLoc(macroIdLoc);
325  SourceLocation fileRsqrLoc = SourceMgr.getSpellingLoc(macroRsqrLoc);
326
327  range = Lexer::makeFileCharRange(
328      CharSourceRange::getTokenRange(macroLsqrLoc, macroIdLoc),
329      SourceMgr, LangOpts);
330  EXPECT_EQ(SourceRange(fileLsqrLoc, fileIdLoc.getLocWithOffset(3)),
331            range.getAsRange());
332
333  range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(macroIdLoc, macroRsqrLoc),
334                                   SourceMgr, LangOpts);
335  EXPECT_EQ(SourceRange(fileIdLoc, fileRsqrLoc.getLocWithOffset(1)),
336            range.getAsRange());
337
338  macroPair = SourceMgr.getExpansionRange(macroLsqrLoc);
339  range = Lexer::makeFileCharRange(
340                     CharSourceRange::getTokenRange(macroLsqrLoc, macroRsqrLoc),
341                     SourceMgr, LangOpts);
342  EXPECT_EQ(SourceRange(macroPair.first, macroPair.second.getLocWithOffset(1)),
343            range.getAsRange());
344
345  text = Lexer::getSourceText(
346          CharSourceRange::getTokenRange(SourceRange(macroLsqrLoc, macroIdLoc)),
347          SourceMgr, LangOpts);
348  EXPECT_EQ(text, "[bar");
349
350
351  SourceLocation idLoc1 = toks[6].getLocation();
352  SourceLocation idLoc2 = toks[7].getLocation();
353  SourceLocation idLoc3 = toks[8].getLocation();
354  SourceLocation idLoc4 = toks[9].getLocation();
355  EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc1, SourceMgr, LangOpts));
356  EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc2, SourceMgr, LangOpts));
357  EXPECT_EQ("NOF2", Lexer::getImmediateMacroName(idLoc3, SourceMgr, LangOpts));
358  EXPECT_EQ("N", Lexer::getImmediateMacroName(idLoc4, SourceMgr, LangOpts));
359}
360
361} // anonymous namespace
362