1 //===- unittests/Lex/LexerTest.cpp ------ Lexer tests ---------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9
10 #include "clang/Lex/Lexer.h"
11 #include "clang/Basic/Diagnostic.h"
12 #include "clang/Basic/DiagnosticOptions.h"
13 #include "clang/Basic/FileManager.h"
14 #include "clang/Basic/LangOptions.h"
15 #include "clang/Basic/SourceManager.h"
16 #include "clang/Basic/TargetInfo.h"
17 #include "clang/Basic/TargetOptions.h"
18 #include "clang/Lex/HeaderSearch.h"
19 #include "clang/Lex/HeaderSearchOptions.h"
20 #include "clang/Lex/ModuleLoader.h"
21 #include "clang/Lex/Preprocessor.h"
22 #include "clang/Lex/PreprocessorOptions.h"
23 #include "gtest/gtest.h"
24
25 using namespace llvm;
26 using namespace clang;
27
28 namespace {
29
30 class VoidModuleLoader : public ModuleLoader {
loadModule(SourceLocation ImportLoc,ModuleIdPath Path,Module::NameVisibilityKind Visibility,bool IsInclusionDirective)31 ModuleLoadResult loadModule(SourceLocation ImportLoc,
32 ModuleIdPath Path,
33 Module::NameVisibilityKind Visibility,
34 bool IsInclusionDirective) override {
35 return ModuleLoadResult();
36 }
37
makeModuleVisible(Module * Mod,Module::NameVisibilityKind Visibility,SourceLocation ImportLoc)38 void makeModuleVisible(Module *Mod,
39 Module::NameVisibilityKind Visibility,
40 SourceLocation ImportLoc) override { }
41
loadGlobalModuleIndex(SourceLocation TriggerLoc)42 GlobalModuleIndex *loadGlobalModuleIndex(SourceLocation TriggerLoc) override
43 { return nullptr; }
lookupMissingImports(StringRef Name,SourceLocation TriggerLoc)44 bool lookupMissingImports(StringRef Name, SourceLocation TriggerLoc) override
45 { return 0; }
46 };
47
48 // The test fixture.
49 class LexerTest : public ::testing::Test {
50 protected:
LexerTest()51 LexerTest()
52 : FileMgr(FileMgrOpts),
53 DiagID(new DiagnosticIDs()),
54 Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()),
55 SourceMgr(Diags, FileMgr),
56 TargetOpts(new TargetOptions)
57 {
58 TargetOpts->Triple = "x86_64-apple-darwin11.1.0";
59 Target = TargetInfo::CreateTargetInfo(Diags, TargetOpts);
60 }
61
CheckLex(StringRef Source,ArrayRef<tok::TokenKind> ExpectedTokens)62 std::vector<Token> CheckLex(StringRef Source,
63 ArrayRef<tok::TokenKind> ExpectedTokens) {
64 std::unique_ptr<MemoryBuffer> Buf = MemoryBuffer::getMemBuffer(Source);
65 SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf)));
66
67 VoidModuleLoader ModLoader;
68 HeaderSearch HeaderInfo(new HeaderSearchOptions, SourceMgr, Diags, LangOpts,
69 Target.get());
70 Preprocessor PP(new PreprocessorOptions(), Diags, LangOpts, SourceMgr,
71 HeaderInfo, ModLoader, /*IILookup =*/nullptr,
72 /*OwnsHeaderSearch =*/false);
73 PP.Initialize(*Target);
74 PP.EnterMainSourceFile();
75
76 std::vector<Token> toks;
77 while (1) {
78 Token tok;
79 PP.Lex(tok);
80 if (tok.is(tok::eof))
81 break;
82 toks.push_back(tok);
83 }
84
85 EXPECT_EQ(ExpectedTokens.size(), toks.size());
86 for (unsigned i = 0, e = ExpectedTokens.size(); i != e; ++i) {
87 EXPECT_EQ(ExpectedTokens[i], toks[i].getKind());
88 }
89
90 return toks;
91 }
92
getSourceText(Token Begin,Token End)93 std::string getSourceText(Token Begin, Token End) {
94 bool Invalid;
95 StringRef Str =
96 Lexer::getSourceText(CharSourceRange::getTokenRange(SourceRange(
97 Begin.getLocation(), End.getLocation())),
98 SourceMgr, LangOpts, &Invalid);
99 if (Invalid)
100 return "<INVALID>";
101 return Str;
102 }
103
104 FileSystemOptions FileMgrOpts;
105 FileManager FileMgr;
106 IntrusiveRefCntPtr<DiagnosticIDs> DiagID;
107 DiagnosticsEngine Diags;
108 SourceManager SourceMgr;
109 LangOptions LangOpts;
110 std::shared_ptr<TargetOptions> TargetOpts;
111 IntrusiveRefCntPtr<TargetInfo> Target;
112 };
113
TEST_F(LexerTest,GetSourceTextExpandsToMaximumInMacroArgument)114 TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgument) {
115 std::vector<tok::TokenKind> ExpectedTokens;
116 ExpectedTokens.push_back(tok::identifier);
117 ExpectedTokens.push_back(tok::l_paren);
118 ExpectedTokens.push_back(tok::identifier);
119 ExpectedTokens.push_back(tok::r_paren);
120
121 std::vector<Token> toks = CheckLex("#define M(x) x\n"
122 "M(f(M(i)))",
123 ExpectedTokens);
124
125 EXPECT_EQ("M(i)", getSourceText(toks[2], toks[2]));
126 }
127
TEST_F(LexerTest,GetSourceTextExpandsToMaximumInMacroArgumentForEndOfMacro)128 TEST_F(LexerTest, GetSourceTextExpandsToMaximumInMacroArgumentForEndOfMacro) {
129 std::vector<tok::TokenKind> ExpectedTokens;
130 ExpectedTokens.push_back(tok::identifier);
131 ExpectedTokens.push_back(tok::identifier);
132
133 std::vector<Token> toks = CheckLex("#define M(x) x\n"
134 "M(M(i) c)",
135 ExpectedTokens);
136
137 EXPECT_EQ("M(i)", getSourceText(toks[0], toks[0]));
138 }
139
TEST_F(LexerTest,GetSourceTextExpandsInMacroArgumentForBeginOfMacro)140 TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForBeginOfMacro) {
141 std::vector<tok::TokenKind> ExpectedTokens;
142 ExpectedTokens.push_back(tok::identifier);
143 ExpectedTokens.push_back(tok::identifier);
144 ExpectedTokens.push_back(tok::identifier);
145
146 std::vector<Token> toks = CheckLex("#define M(x) x\n"
147 "M(c c M(i))",
148 ExpectedTokens);
149
150 EXPECT_EQ("c M(i)", getSourceText(toks[1], toks[2]));
151 }
152
TEST_F(LexerTest,GetSourceTextExpandsInMacroArgumentForEndOfMacro)153 TEST_F(LexerTest, GetSourceTextExpandsInMacroArgumentForEndOfMacro) {
154 std::vector<tok::TokenKind> ExpectedTokens;
155 ExpectedTokens.push_back(tok::identifier);
156 ExpectedTokens.push_back(tok::identifier);
157 ExpectedTokens.push_back(tok::identifier);
158
159 std::vector<Token> toks = CheckLex("#define M(x) x\n"
160 "M(M(i) c c)",
161 ExpectedTokens);
162
163 EXPECT_EQ("M(i) c", getSourceText(toks[0], toks[1]));
164 }
165
TEST_F(LexerTest,GetSourceTextInSeparateFnMacros)166 TEST_F(LexerTest, GetSourceTextInSeparateFnMacros) {
167 std::vector<tok::TokenKind> ExpectedTokens;
168 ExpectedTokens.push_back(tok::identifier);
169 ExpectedTokens.push_back(tok::identifier);
170 ExpectedTokens.push_back(tok::identifier);
171 ExpectedTokens.push_back(tok::identifier);
172
173 std::vector<Token> toks = CheckLex("#define M(x) x\n"
174 "M(c M(i)) M(M(i) c)",
175 ExpectedTokens);
176
177 EXPECT_EQ("<INVALID>", getSourceText(toks[1], toks[2]));
178 }
179
TEST_F(LexerTest,GetSourceTextWorksAcrossTokenPastes)180 TEST_F(LexerTest, GetSourceTextWorksAcrossTokenPastes) {
181 std::vector<tok::TokenKind> ExpectedTokens;
182 ExpectedTokens.push_back(tok::identifier);
183 ExpectedTokens.push_back(tok::l_paren);
184 ExpectedTokens.push_back(tok::identifier);
185 ExpectedTokens.push_back(tok::r_paren);
186
187 std::vector<Token> toks = CheckLex("#define M(x) x\n"
188 "#define C(x) M(x##c)\n"
189 "M(f(C(i)))",
190 ExpectedTokens);
191
192 EXPECT_EQ("C(i)", getSourceText(toks[2], toks[2]));
193 }
194
TEST_F(LexerTest,GetSourceTextExpandsAcrossMultipleMacroCalls)195 TEST_F(LexerTest, GetSourceTextExpandsAcrossMultipleMacroCalls) {
196 std::vector<tok::TokenKind> ExpectedTokens;
197 ExpectedTokens.push_back(tok::identifier);
198 ExpectedTokens.push_back(tok::l_paren);
199 ExpectedTokens.push_back(tok::identifier);
200 ExpectedTokens.push_back(tok::r_paren);
201
202 std::vector<Token> toks = CheckLex("#define M(x) x\n"
203 "f(M(M(i)))",
204 ExpectedTokens);
205 EXPECT_EQ("M(M(i))", getSourceText(toks[2], toks[2]));
206 }
207
TEST_F(LexerTest,GetSourceTextInMiddleOfMacroArgument)208 TEST_F(LexerTest, GetSourceTextInMiddleOfMacroArgument) {
209 std::vector<tok::TokenKind> ExpectedTokens;
210 ExpectedTokens.push_back(tok::identifier);
211 ExpectedTokens.push_back(tok::l_paren);
212 ExpectedTokens.push_back(tok::identifier);
213 ExpectedTokens.push_back(tok::r_paren);
214
215 std::vector<Token> toks = CheckLex("#define M(x) x\n"
216 "M(f(i))",
217 ExpectedTokens);
218 EXPECT_EQ("i", getSourceText(toks[2], toks[2]));
219 }
220
TEST_F(LexerTest,GetSourceTextExpandsAroundDifferentMacroCalls)221 TEST_F(LexerTest, GetSourceTextExpandsAroundDifferentMacroCalls) {
222 std::vector<tok::TokenKind> ExpectedTokens;
223 ExpectedTokens.push_back(tok::identifier);
224 ExpectedTokens.push_back(tok::l_paren);
225 ExpectedTokens.push_back(tok::identifier);
226 ExpectedTokens.push_back(tok::r_paren);
227
228 std::vector<Token> toks = CheckLex("#define M(x) x\n"
229 "#define C(x) x\n"
230 "f(C(M(i)))",
231 ExpectedTokens);
232 EXPECT_EQ("C(M(i))", getSourceText(toks[2], toks[2]));
233 }
234
TEST_F(LexerTest,GetSourceTextOnlyExpandsIfFirstTokenInMacro)235 TEST_F(LexerTest, GetSourceTextOnlyExpandsIfFirstTokenInMacro) {
236 std::vector<tok::TokenKind> ExpectedTokens;
237 ExpectedTokens.push_back(tok::identifier);
238 ExpectedTokens.push_back(tok::l_paren);
239 ExpectedTokens.push_back(tok::identifier);
240 ExpectedTokens.push_back(tok::identifier);
241 ExpectedTokens.push_back(tok::r_paren);
242
243 std::vector<Token> toks = CheckLex("#define M(x) x\n"
244 "#define C(x) c x\n"
245 "f(C(M(i)))",
246 ExpectedTokens);
247 EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3]));
248 }
249
TEST_F(LexerTest,GetSourceTextExpandsRecursively)250 TEST_F(LexerTest, GetSourceTextExpandsRecursively) {
251 std::vector<tok::TokenKind> ExpectedTokens;
252 ExpectedTokens.push_back(tok::identifier);
253 ExpectedTokens.push_back(tok::identifier);
254 ExpectedTokens.push_back(tok::l_paren);
255 ExpectedTokens.push_back(tok::identifier);
256 ExpectedTokens.push_back(tok::r_paren);
257
258 std::vector<Token> toks = CheckLex("#define M(x) x\n"
259 "#define C(x) c M(x)\n"
260 "C(f(M(i)))",
261 ExpectedTokens);
262 EXPECT_EQ("M(i)", getSourceText(toks[3], toks[3]));
263 }
264
TEST_F(LexerTest,LexAPI)265 TEST_F(LexerTest, LexAPI) {
266 std::vector<tok::TokenKind> ExpectedTokens;
267 ExpectedTokens.push_back(tok::l_square);
268 ExpectedTokens.push_back(tok::identifier);
269 ExpectedTokens.push_back(tok::r_square);
270 ExpectedTokens.push_back(tok::l_square);
271 ExpectedTokens.push_back(tok::identifier);
272 ExpectedTokens.push_back(tok::r_square);
273 ExpectedTokens.push_back(tok::identifier);
274 ExpectedTokens.push_back(tok::identifier);
275 ExpectedTokens.push_back(tok::identifier);
276 ExpectedTokens.push_back(tok::identifier);
277
278 std::vector<Token> toks = CheckLex("#define M(x) [x]\n"
279 "#define N(x) x\n"
280 "#define INN(x) x\n"
281 "#define NOF1 INN(val)\n"
282 "#define NOF2 val\n"
283 "M(foo) N([bar])\n"
284 "N(INN(val)) N(NOF1) N(NOF2) N(val)",
285 ExpectedTokens);
286
287 SourceLocation lsqrLoc = toks[0].getLocation();
288 SourceLocation idLoc = toks[1].getLocation();
289 SourceLocation rsqrLoc = toks[2].getLocation();
290 std::pair<SourceLocation,SourceLocation>
291 macroPair = SourceMgr.getExpansionRange(lsqrLoc);
292 SourceRange macroRange = SourceRange(macroPair.first, macroPair.second);
293
294 SourceLocation Loc;
295 EXPECT_TRUE(Lexer::isAtStartOfMacroExpansion(lsqrLoc, SourceMgr, LangOpts, &Loc));
296 EXPECT_EQ(Loc, macroRange.getBegin());
297 EXPECT_FALSE(Lexer::isAtStartOfMacroExpansion(idLoc, SourceMgr, LangOpts));
298 EXPECT_FALSE(Lexer::isAtEndOfMacroExpansion(idLoc, SourceMgr, LangOpts));
299 EXPECT_TRUE(Lexer::isAtEndOfMacroExpansion(rsqrLoc, SourceMgr, LangOpts, &Loc));
300 EXPECT_EQ(Loc, macroRange.getEnd());
301
302 CharSourceRange range = Lexer::makeFileCharRange(
303 CharSourceRange::getTokenRange(lsqrLoc, idLoc), SourceMgr, LangOpts);
304 EXPECT_TRUE(range.isInvalid());
305 range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(idLoc, rsqrLoc),
306 SourceMgr, LangOpts);
307 EXPECT_TRUE(range.isInvalid());
308 range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc),
309 SourceMgr, LangOpts);
310 EXPECT_TRUE(!range.isTokenRange());
311 EXPECT_EQ(range.getAsRange(),
312 SourceRange(macroRange.getBegin(),
313 macroRange.getEnd().getLocWithOffset(1)));
314
315 StringRef text = Lexer::getSourceText(
316 CharSourceRange::getTokenRange(lsqrLoc, rsqrLoc),
317 SourceMgr, LangOpts);
318 EXPECT_EQ(text, "M(foo)");
319
320 SourceLocation macroLsqrLoc = toks[3].getLocation();
321 SourceLocation macroIdLoc = toks[4].getLocation();
322 SourceLocation macroRsqrLoc = toks[5].getLocation();
323 SourceLocation fileLsqrLoc = SourceMgr.getSpellingLoc(macroLsqrLoc);
324 SourceLocation fileIdLoc = SourceMgr.getSpellingLoc(macroIdLoc);
325 SourceLocation fileRsqrLoc = SourceMgr.getSpellingLoc(macroRsqrLoc);
326
327 range = Lexer::makeFileCharRange(
328 CharSourceRange::getTokenRange(macroLsqrLoc, macroIdLoc),
329 SourceMgr, LangOpts);
330 EXPECT_EQ(SourceRange(fileLsqrLoc, fileIdLoc.getLocWithOffset(3)),
331 range.getAsRange());
332
333 range = Lexer::makeFileCharRange(CharSourceRange::getTokenRange(macroIdLoc, macroRsqrLoc),
334 SourceMgr, LangOpts);
335 EXPECT_EQ(SourceRange(fileIdLoc, fileRsqrLoc.getLocWithOffset(1)),
336 range.getAsRange());
337
338 macroPair = SourceMgr.getExpansionRange(macroLsqrLoc);
339 range = Lexer::makeFileCharRange(
340 CharSourceRange::getTokenRange(macroLsqrLoc, macroRsqrLoc),
341 SourceMgr, LangOpts);
342 EXPECT_EQ(SourceRange(macroPair.first, macroPair.second.getLocWithOffset(1)),
343 range.getAsRange());
344
345 text = Lexer::getSourceText(
346 CharSourceRange::getTokenRange(SourceRange(macroLsqrLoc, macroIdLoc)),
347 SourceMgr, LangOpts);
348 EXPECT_EQ(text, "[bar");
349
350
351 SourceLocation idLoc1 = toks[6].getLocation();
352 SourceLocation idLoc2 = toks[7].getLocation();
353 SourceLocation idLoc3 = toks[8].getLocation();
354 SourceLocation idLoc4 = toks[9].getLocation();
355 EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc1, SourceMgr, LangOpts));
356 EXPECT_EQ("INN", Lexer::getImmediateMacroName(idLoc2, SourceMgr, LangOpts));
357 EXPECT_EQ("NOF2", Lexer::getImmediateMacroName(idLoc3, SourceMgr, LangOpts));
358 EXPECT_EQ("N", Lexer::getImmediateMacroName(idLoc4, SourceMgr, LangOpts));
359 }
360
361 } // anonymous namespace
362