1 //===- TokensTest.cpp -----------------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "clang/Tooling/Syntax/Tokens.h"
10 #include "clang/AST/ASTConsumer.h"
11 #include "clang/AST/Expr.h"
12 #include "clang/Basic/Diagnostic.h"
13 #include "clang/Basic/DiagnosticIDs.h"
14 #include "clang/Basic/DiagnosticOptions.h"
15 #include "clang/Basic/FileManager.h"
16 #include "clang/Basic/FileSystemOptions.h"
17 #include "clang/Basic/LLVM.h"
18 #include "clang/Basic/LangOptions.h"
19 #include "clang/Basic/SourceLocation.h"
20 #include "clang/Basic/SourceManager.h"
21 #include "clang/Basic/TokenKinds.def"
22 #include "clang/Basic/TokenKinds.h"
23 #include "clang/Frontend/CompilerInstance.h"
24 #include "clang/Frontend/FrontendAction.h"
25 #include "clang/Frontend/Utils.h"
26 #include "clang/Lex/Lexer.h"
27 #include "clang/Lex/PreprocessorOptions.h"
28 #include "clang/Lex/Token.h"
29 #include "clang/Tooling/Tooling.h"
30 #include "llvm/ADT/ArrayRef.h"
31 #include "llvm/ADT/IntrusiveRefCntPtr.h"
32 #include "llvm/ADT/None.h"
33 #include "llvm/ADT/Optional.h"
34 #include "llvm/ADT/STLExtras.h"
35 #include "llvm/ADT/StringRef.h"
36 #include "llvm/Support/FormatVariadic.h"
37 #include "llvm/Support/MemoryBuffer.h"
38 #include "llvm/Support/VirtualFileSystem.h"
39 #include "llvm/Support/raw_os_ostream.h"
40 #include "llvm/Support/raw_ostream.h"
41 #include "llvm/Testing/Support/Annotations.h"
42 #include "llvm/Testing/Support/SupportHelpers.h"
43 #include "gmock/gmock.h"
44 #include <cassert>
45 #include <cstdlib>
46 #include <gmock/gmock.h>
47 #include <gtest/gtest.h>
48 #include <memory>
49 #include <ostream>
50 #include <string>
51 
52 using namespace clang;
53 using namespace clang::syntax;
54 
55 using llvm::ValueIs;
56 using ::testing::_;
57 using ::testing::AllOf;
58 using ::testing::Contains;
59 using ::testing::ElementsAre;
60 using ::testing::Field;
61 using ::testing::IsEmpty;
62 using ::testing::Matcher;
63 using ::testing::Not;
64 using ::testing::Pointee;
65 using ::testing::StartsWith;
66 
67 namespace {
68 // Checks the passed ArrayRef<T> has the same begin() and end() iterators as the
69 // argument.
70 MATCHER_P(SameRange, A, "") {
71   return A.begin() == arg.begin() && A.end() == arg.end();
72 }
73 
74 Matcher<TokenBuffer::Expansion>
IsExpansion(Matcher<llvm::ArrayRef<syntax::Token>> Spelled,Matcher<llvm::ArrayRef<syntax::Token>> Expanded)75 IsExpansion(Matcher<llvm::ArrayRef<syntax::Token>> Spelled,
76             Matcher<llvm::ArrayRef<syntax::Token>> Expanded) {
77   return AllOf(Field(&TokenBuffer::Expansion::Spelled, Spelled),
78                Field(&TokenBuffer::Expansion::Expanded, Expanded));
79 }
80 // Matchers for syntax::Token.
81 MATCHER_P(Kind, K, "") { return arg.kind() == K; }
82 MATCHER_P2(HasText, Text, SourceMgr, "") {
83   return arg.text(*SourceMgr) == Text;
84 }
85 /// Checks the start and end location of a token are equal to SourceRng.
86 MATCHER_P(RangeIs, SourceRng, "") {
87   return arg.location() == SourceRng.first &&
88          arg.endLocation() == SourceRng.second;
89 }
90 
91 class TokenCollectorTest : public ::testing::Test {
92 public:
93   /// Run the clang frontend, collect the preprocessed tokens from the frontend
94   /// invocation and store them in this->Buffer.
95   /// This also clears SourceManager before running the compiler.
recordTokens(llvm::StringRef Code)96   void recordTokens(llvm::StringRef Code) {
97     class RecordTokens : public ASTFrontendAction {
98     public:
99       explicit RecordTokens(TokenBuffer &Result) : Result(Result) {}
100 
101       bool BeginSourceFileAction(CompilerInstance &CI) override {
102         assert(!Collector && "expected only a single call to BeginSourceFile");
103         Collector.emplace(CI.getPreprocessor());
104         return true;
105       }
106       void EndSourceFileAction() override {
107         assert(Collector && "BeginSourceFileAction was never called");
108         Result = std::move(*Collector).consume();
109       }
110 
111       std::unique_ptr<ASTConsumer>
112       CreateASTConsumer(CompilerInstance &CI, StringRef InFile) override {
113         return std::make_unique<ASTConsumer>();
114       }
115 
116     private:
117       TokenBuffer &Result;
118       llvm::Optional<TokenCollector> Collector;
119     };
120 
121     constexpr const char *FileName = "./input.cpp";
122     FS->addFile(FileName, time_t(), llvm::MemoryBuffer::getMemBufferCopy(""));
123     // Prepare to run a compiler.
124     if (!Diags->getClient())
125       Diags->setClient(new IgnoringDiagConsumer);
126     std::vector<const char *> Args = {"tok-test", "-std=c++03", "-fsyntax-only",
127                                       FileName};
128     auto CI = createInvocationFromCommandLine(Args, Diags, FS);
129     assert(CI);
130     CI->getFrontendOpts().DisableFree = false;
131     CI->getPreprocessorOpts().addRemappedFile(
132         FileName, llvm::MemoryBuffer::getMemBufferCopy(Code).release());
133     CompilerInstance Compiler;
134     Compiler.setInvocation(std::move(CI));
135     Compiler.setDiagnostics(Diags.get());
136     Compiler.setFileManager(FileMgr.get());
137     Compiler.setSourceManager(SourceMgr.get());
138 
139     this->Buffer = TokenBuffer(*SourceMgr);
140     RecordTokens Recorder(this->Buffer);
141     ASSERT_TRUE(Compiler.ExecuteAction(Recorder))
142         << "failed to run the frontend";
143   }
144 
145   /// Record the tokens and return a test dump of the resulting buffer.
collectAndDump(llvm::StringRef Code)146   std::string collectAndDump(llvm::StringRef Code) {
147     recordTokens(Code);
148     return Buffer.dumpForTests();
149   }
150 
151   // Adds a file to the test VFS.
addFile(llvm::StringRef Path,llvm::StringRef Contents)152   void addFile(llvm::StringRef Path, llvm::StringRef Contents) {
153     if (!FS->addFile(Path, time_t(),
154                      llvm::MemoryBuffer::getMemBufferCopy(Contents))) {
155       ADD_FAILURE() << "could not add a file to VFS: " << Path;
156     }
157   }
158 
159   /// Add a new file, run syntax::tokenize() on the range if any, run it on the
160   /// whole file otherwise and return the results.
tokenize(llvm::StringRef Text)161   std::vector<syntax::Token> tokenize(llvm::StringRef Text) {
162     llvm::Annotations Annot(Text);
163     auto FID = SourceMgr->createFileID(
164         llvm::MemoryBuffer::getMemBufferCopy(Annot.code()));
165     // FIXME: pass proper LangOptions.
166     if (Annot.ranges().empty())
167       return syntax::tokenize(FID, *SourceMgr, LangOptions());
168     return syntax::tokenize(
169         syntax::FileRange(FID, Annot.range().Begin, Annot.range().End),
170         *SourceMgr, LangOptions());
171   }
172 
173   // Specialized versions of matchers that hide the SourceManager from clients.
HasText(std::string Text) const174   Matcher<syntax::Token> HasText(std::string Text) const {
175     return ::HasText(Text, SourceMgr.get());
176   }
RangeIs(llvm::Annotations::Range R) const177   Matcher<syntax::Token> RangeIs(llvm::Annotations::Range R) const {
178     std::pair<SourceLocation, SourceLocation> Ls;
179     Ls.first = SourceMgr->getLocForStartOfFile(SourceMgr->getMainFileID())
180                    .getLocWithOffset(R.Begin);
181     Ls.second = SourceMgr->getLocForStartOfFile(SourceMgr->getMainFileID())
182                     .getLocWithOffset(R.End);
183     return ::RangeIs(Ls);
184   }
185 
186   /// Finds a subrange in O(n * m).
187   template <class T, class U, class Eq>
findSubrange(llvm::ArrayRef<U> Subrange,llvm::ArrayRef<T> Range,Eq F)188   llvm::ArrayRef<T> findSubrange(llvm::ArrayRef<U> Subrange,
189                                  llvm::ArrayRef<T> Range, Eq F) {
190     assert(Subrange.size() >= 1);
191     if (Range.size() < Subrange.size())
192       return llvm::makeArrayRef(Range.end(), Range.end());
193     for (auto Begin = Range.begin(), Last = Range.end() - Subrange.size();
194          Begin <= Last; ++Begin) {
195       auto It = Begin;
196       for (auto ItSub = Subrange.begin(); ItSub != Subrange.end();
197            ++ItSub, ++It) {
198         if (!F(*ItSub, *It))
199           goto continue_outer;
200       }
201       return llvm::makeArrayRef(Begin, It);
202     continue_outer:;
203     }
204     return llvm::makeArrayRef(Range.end(), Range.end());
205   }
206 
207   /// Finds a subrange in \p Tokens that match the tokens specified in \p Query.
208   /// The match should be unique. \p Query is a whitespace-separated list of
209   /// tokens to search for.
210   llvm::ArrayRef<syntax::Token>
findTokenRange(llvm::StringRef Query,llvm::ArrayRef<syntax::Token> Tokens)211   findTokenRange(llvm::StringRef Query, llvm::ArrayRef<syntax::Token> Tokens) {
212     llvm::SmallVector<llvm::StringRef, 8> QueryTokens;
213     Query.split(QueryTokens, ' ', /*MaxSplit=*/-1, /*KeepEmpty=*/false);
214     if (QueryTokens.empty()) {
215       ADD_FAILURE() << "will not look for an empty list of tokens";
216       std::abort();
217     }
218     // An equality test for search.
219     auto TextMatches = [this](llvm::StringRef Q, const syntax::Token &T) {
220       return Q == T.text(*SourceMgr);
221     };
222     // Find a match.
223     auto Found =
224         findSubrange(llvm::makeArrayRef(QueryTokens), Tokens, TextMatches);
225     if (Found.begin() == Tokens.end()) {
226       ADD_FAILURE() << "could not find the subrange for " << Query;
227       std::abort();
228     }
229     // Check that the match is unique.
230     if (findSubrange(llvm::makeArrayRef(QueryTokens),
231                      llvm::makeArrayRef(Found.end(), Tokens.end()), TextMatches)
232             .begin() != Tokens.end()) {
233       ADD_FAILURE() << "match is not unique for " << Query;
234       std::abort();
235     }
236     return Found;
237   };
238 
239   // Specialized versions of findTokenRange for expanded and spelled tokens.
findExpanded(llvm::StringRef Query)240   llvm::ArrayRef<syntax::Token> findExpanded(llvm::StringRef Query) {
241     return findTokenRange(Query, Buffer.expandedTokens());
242   }
findSpelled(llvm::StringRef Query,FileID File=FileID ())243   llvm::ArrayRef<syntax::Token> findSpelled(llvm::StringRef Query,
244                                             FileID File = FileID()) {
245     if (!File.isValid())
246       File = SourceMgr->getMainFileID();
247     return findTokenRange(Query, Buffer.spelledTokens(File));
248   }
249 
250   // Data fields.
251   llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
252       new DiagnosticsEngine(new DiagnosticIDs, new DiagnosticOptions);
253   IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> FS =
254       new llvm::vfs::InMemoryFileSystem;
255   llvm::IntrusiveRefCntPtr<FileManager> FileMgr =
256       new FileManager(FileSystemOptions(), FS);
257   llvm::IntrusiveRefCntPtr<SourceManager> SourceMgr =
258       new SourceManager(*Diags, *FileMgr);
259   /// Contains last result of calling recordTokens().
260   TokenBuffer Buffer = TokenBuffer(*SourceMgr);
261 };
262 
TEST_F(TokenCollectorTest,RawMode)263 TEST_F(TokenCollectorTest, RawMode) {
264   EXPECT_THAT(tokenize("int main() {}"),
265               ElementsAre(Kind(tok::kw_int),
266                           AllOf(HasText("main"), Kind(tok::identifier)),
267                           Kind(tok::l_paren), Kind(tok::r_paren),
268                           Kind(tok::l_brace), Kind(tok::r_brace)));
269   // Comments are ignored for now.
270   EXPECT_THAT(tokenize("/* foo */int a; // more comments"),
271               ElementsAre(Kind(tok::kw_int),
272                           AllOf(HasText("a"), Kind(tok::identifier)),
273                           Kind(tok::semi)));
274   EXPECT_THAT(tokenize("int [[main() {]]}"),
275               ElementsAre(AllOf(HasText("main"), Kind(tok::identifier)),
276                           Kind(tok::l_paren), Kind(tok::r_paren),
277                           Kind(tok::l_brace)));
278   EXPECT_THAT(tokenize("int [[main() {   ]]}"),
279               ElementsAre(AllOf(HasText("main"), Kind(tok::identifier)),
280                           Kind(tok::l_paren), Kind(tok::r_paren),
281                           Kind(tok::l_brace)));
282   // First token is partially parsed, last token is fully included even though
283   // only a part of it is contained in the range.
284   EXPECT_THAT(tokenize("int m[[ain() {ret]]urn 0;}"),
285               ElementsAre(AllOf(HasText("ain"), Kind(tok::identifier)),
286                           Kind(tok::l_paren), Kind(tok::r_paren),
287                           Kind(tok::l_brace), Kind(tok::kw_return)));
288 }
289 
TEST_F(TokenCollectorTest,Basic)290 TEST_F(TokenCollectorTest, Basic) {
291   std::pair</*Input*/ std::string, /*Expected*/ std::string> TestCases[] = {
292       {"int main() {}",
293        R"(expanded tokens:
294   int main ( ) { }
295 file './input.cpp'
296   spelled tokens:
297     int main ( ) { }
298   no mappings.
299 )"},
300       // All kinds of whitespace are ignored.
301       {"\t\n  int\t\n  main\t\n  (\t\n  )\t\n{\t\n  }\t\n",
302        R"(expanded tokens:
303   int main ( ) { }
304 file './input.cpp'
305   spelled tokens:
306     int main ( ) { }
307   no mappings.
308 )"},
309       // Annotation tokens are ignored.
310       {R"cpp(
311         #pragma GCC visibility push (public)
312         #pragma GCC visibility pop
313       )cpp",
314        R"(expanded tokens:
315   <empty>
316 file './input.cpp'
317   spelled tokens:
318     # pragma GCC visibility push ( public ) # pragma GCC visibility pop
319   mappings:
320     ['#'_0, '<eof>'_13) => ['<eof>'_0, '<eof>'_0)
321 )"},
322       // Empty files should not crash.
323       {R"cpp()cpp", R"(expanded tokens:
324   <empty>
325 file './input.cpp'
326   spelled tokens:
327     <empty>
328   no mappings.
329 )"},
330       // Should not crash on errors inside '#define' directives. Error is that
331       // stringification (#B) does not refer to a macro parameter.
332       {
333           R"cpp(
334 a
335 #define MACRO() A #B
336 )cpp",
337           R"(expanded tokens:
338   a
339 file './input.cpp'
340   spelled tokens:
341     a # define MACRO ( ) A # B
342   mappings:
343     ['#'_1, '<eof>'_9) => ['<eof>'_1, '<eof>'_1)
344 )"}};
345   for (auto &Test : TestCases)
346     EXPECT_EQ(collectAndDump(Test.first), Test.second)
347         << collectAndDump(Test.first);
348 }
349 
TEST_F(TokenCollectorTest,Locations)350 TEST_F(TokenCollectorTest, Locations) {
351   // Check locations of the tokens.
352   llvm::Annotations Code(R"cpp(
353     $r1[[int]] $r2[[a]] $r3[[=]] $r4[["foo bar baz"]] $r5[[;]]
354   )cpp");
355   recordTokens(Code.code());
356   // Check expanded tokens.
357   EXPECT_THAT(
358       Buffer.expandedTokens(),
359       ElementsAre(AllOf(Kind(tok::kw_int), RangeIs(Code.range("r1"))),
360                   AllOf(Kind(tok::identifier), RangeIs(Code.range("r2"))),
361                   AllOf(Kind(tok::equal), RangeIs(Code.range("r3"))),
362                   AllOf(Kind(tok::string_literal), RangeIs(Code.range("r4"))),
363                   AllOf(Kind(tok::semi), RangeIs(Code.range("r5"))),
364                   Kind(tok::eof)));
365   // Check spelled tokens.
366   EXPECT_THAT(
367       Buffer.spelledTokens(SourceMgr->getMainFileID()),
368       ElementsAre(AllOf(Kind(tok::kw_int), RangeIs(Code.range("r1"))),
369                   AllOf(Kind(tok::identifier), RangeIs(Code.range("r2"))),
370                   AllOf(Kind(tok::equal), RangeIs(Code.range("r3"))),
371                   AllOf(Kind(tok::string_literal), RangeIs(Code.range("r4"))),
372                   AllOf(Kind(tok::semi), RangeIs(Code.range("r5")))));
373 
374   auto StartLoc = SourceMgr->getLocForStartOfFile(SourceMgr->getMainFileID());
375   for (auto &R : Code.ranges()) {
376     EXPECT_THAT(Buffer.spelledTokenAt(StartLoc.getLocWithOffset(R.Begin)),
377                 Pointee(RangeIs(R)));
378   }
379 }
380 
TEST_F(TokenCollectorTest,MacroDirectives)381 TEST_F(TokenCollectorTest, MacroDirectives) {
382   // Macro directives are not stored anywhere at the moment.
383   std::string Code = R"cpp(
384     #define FOO a
385     #include "unresolved_file.h"
386     #undef FOO
387     #ifdef X
388     #else
389     #endif
390     #ifndef Y
391     #endif
392     #if 1
393     #elif 2
394     #else
395     #endif
396     #pragma once
397     #pragma something lalala
398 
399     int a;
400   )cpp";
401   std::string Expected =
402       "expanded tokens:\n"
403       "  int a ;\n"
404       "file './input.cpp'\n"
405       "  spelled tokens:\n"
406       "    # define FOO a # include \"unresolved_file.h\" # undef FOO "
407       "# ifdef X # else # endif # ifndef Y # endif # if 1 # elif 2 # else "
408       "# endif # pragma once # pragma something lalala int a ;\n"
409       "  mappings:\n"
410       "    ['#'_0, 'int'_39) => ['int'_0, 'int'_0)\n";
411   EXPECT_EQ(collectAndDump(Code), Expected);
412 }
413 
TEST_F(TokenCollectorTest,MacroReplacements)414 TEST_F(TokenCollectorTest, MacroReplacements) {
415   std::pair</*Input*/ std::string, /*Expected*/ std::string> TestCases[] = {
416       // A simple object-like macro.
417       {R"cpp(
418     #define INT int const
419     INT a;
420   )cpp",
421        R"(expanded tokens:
422   int const a ;
423 file './input.cpp'
424   spelled tokens:
425     # define INT int const INT a ;
426   mappings:
427     ['#'_0, 'INT'_5) => ['int'_0, 'int'_0)
428     ['INT'_5, 'a'_6) => ['int'_0, 'a'_2)
429 )"},
430       // A simple function-like macro.
431       {R"cpp(
432     #define INT(a) const int
433     INT(10+10) a;
434   )cpp",
435        R"(expanded tokens:
436   const int a ;
437 file './input.cpp'
438   spelled tokens:
439     # define INT ( a ) const int INT ( 10 + 10 ) a ;
440   mappings:
441     ['#'_0, 'INT'_8) => ['const'_0, 'const'_0)
442     ['INT'_8, 'a'_14) => ['const'_0, 'a'_2)
443 )"},
444       // Recursive macro replacements.
445       {R"cpp(
446     #define ID(X) X
447     #define INT int const
448     ID(ID(INT)) a;
449   )cpp",
450        R"(expanded tokens:
451   int const a ;
452 file './input.cpp'
453   spelled tokens:
454     # define ID ( X ) X # define INT int const ID ( ID ( INT ) ) a ;
455   mappings:
456     ['#'_0, 'ID'_12) => ['int'_0, 'int'_0)
457     ['ID'_12, 'a'_19) => ['int'_0, 'a'_2)
458 )"},
459       // A little more complicated recursive macro replacements.
460       {R"cpp(
461     #define ADD(X, Y) X+Y
462     #define MULT(X, Y) X*Y
463 
464     int a = ADD(MULT(1,2), MULT(3,ADD(4,5)));
465   )cpp",
466        "expanded tokens:\n"
467        "  int a = 1 * 2 + 3 * 4 + 5 ;\n"
468        "file './input.cpp'\n"
469        "  spelled tokens:\n"
470        "    # define ADD ( X , Y ) X + Y # define MULT ( X , Y ) X * Y int "
471        "a = ADD ( MULT ( 1 , 2 ) , MULT ( 3 , ADD ( 4 , 5 ) ) ) ;\n"
472        "  mappings:\n"
473        "    ['#'_0, 'int'_22) => ['int'_0, 'int'_0)\n"
474        "    ['ADD'_25, ';'_46) => ['1'_3, ';'_12)\n"},
475       // Empty macro replacement.
476       // FIXME: the #define directives should not be glued together.
477       {R"cpp(
478     #define EMPTY
479     #define EMPTY_FUNC(X)
480     EMPTY
481     EMPTY_FUNC(1+2+3)
482     )cpp",
483        R"(expanded tokens:
484   <empty>
485 file './input.cpp'
486   spelled tokens:
487     # define EMPTY # define EMPTY_FUNC ( X ) EMPTY EMPTY_FUNC ( 1 + 2 + 3 )
488   mappings:
489     ['#'_0, 'EMPTY'_9) => ['<eof>'_0, '<eof>'_0)
490     ['EMPTY'_9, 'EMPTY_FUNC'_10) => ['<eof>'_0, '<eof>'_0)
491     ['EMPTY_FUNC'_10, '<eof>'_18) => ['<eof>'_0, '<eof>'_0)
492 )"},
493       // File ends with a macro replacement.
494       {R"cpp(
495     #define FOO 10+10;
496     int a = FOO
497     )cpp",
498        R"(expanded tokens:
499   int a = 10 + 10 ;
500 file './input.cpp'
501   spelled tokens:
502     # define FOO 10 + 10 ; int a = FOO
503   mappings:
504     ['#'_0, 'int'_7) => ['int'_0, 'int'_0)
505     ['FOO'_10, '<eof>'_11) => ['10'_3, '<eof>'_7)
506 )"},
507       {R"cpp(
508          #define NUM 42
509          #define ID(a) a
510          #define M 1 + ID
511          M(NUM)
512        )cpp",
513        R"(expanded tokens:
514   1 + 42
515 file './input.cpp'
516   spelled tokens:
517     # define NUM 42 # define ID ( a ) a # define M 1 + ID M ( NUM )
518   mappings:
519     ['#'_0, 'M'_17) => ['1'_0, '1'_0)
520     ['M'_17, '<eof>'_21) => ['1'_0, '<eof>'_3)
521 )"},
522   };
523 
524   for (auto &Test : TestCases) {
525     std::string Dump = collectAndDump(Test.first);
526     EXPECT_EQ(Test.second, Dump) << Dump;
527   }
528 }
529 
TEST_F(TokenCollectorTest,SpecialTokens)530 TEST_F(TokenCollectorTest, SpecialTokens) {
531   // Tokens coming from concatenations.
532   recordTokens(R"cpp(
533     #define CONCAT(a, b) a ## b
534     int a = CONCAT(1, 2);
535   )cpp");
536   EXPECT_THAT(std::vector<syntax::Token>(Buffer.expandedTokens()),
537               Contains(HasText("12")));
538   // Multi-line tokens with slashes at the end.
539   recordTokens("i\\\nn\\\nt");
540   EXPECT_THAT(Buffer.expandedTokens(),
541               ElementsAre(AllOf(Kind(tok::kw_int), HasText("i\\\nn\\\nt")),
542                           Kind(tok::eof)));
543   // FIXME: test tokens with digraphs and UCN identifiers.
544 }
545 
TEST_F(TokenCollectorTest,LateBoundTokens)546 TEST_F(TokenCollectorTest, LateBoundTokens) {
547   // The parser eventually breaks the first '>>' into two tokens ('>' and '>'),
548   // but we choose to record them as a single token (for now).
549   llvm::Annotations Code(R"cpp(
550     template <class T>
551     struct foo { int a; };
552     int bar = foo<foo<int$br[[>>]]().a;
553     int baz = 10 $op[[>>]] 2;
554   )cpp");
555   recordTokens(Code.code());
556   EXPECT_THAT(std::vector<syntax::Token>(Buffer.expandedTokens()),
557               AllOf(Contains(AllOf(Kind(tok::greatergreater),
558                                    RangeIs(Code.range("br")))),
559                     Contains(AllOf(Kind(tok::greatergreater),
560                                    RangeIs(Code.range("op"))))));
561 }
562 
TEST_F(TokenCollectorTest,DelayedParsing)563 TEST_F(TokenCollectorTest, DelayedParsing) {
564   llvm::StringLiteral Code = R"cpp(
565     struct Foo {
566       int method() {
567         // Parser will visit method bodies and initializers multiple times, but
568         // TokenBuffer should only record the first walk over the tokens;
569         return 100;
570       }
571       int a = 10;
572 
573       struct Subclass {
574         void foo() {
575           Foo().method();
576         }
577       };
578     };
579   )cpp";
580   std::string ExpectedTokens =
581       "expanded tokens:\n"
582       "  struct Foo { int method ( ) { return 100 ; } int a = 10 ; struct "
583       "Subclass { void foo ( ) { Foo ( ) . method ( ) ; } } ; } ;\n";
584   EXPECT_THAT(collectAndDump(Code), StartsWith(ExpectedTokens));
585 }
586 
TEST_F(TokenCollectorTest,MultiFile)587 TEST_F(TokenCollectorTest, MultiFile) {
588   addFile("./foo.h", R"cpp(
589     #define ADD(X, Y) X+Y
590     int a = 100;
591     #include "bar.h"
592   )cpp");
593   addFile("./bar.h", R"cpp(
594     int b = ADD(1, 2);
595     #define MULT(X, Y) X*Y
596   )cpp");
597   llvm::StringLiteral Code = R"cpp(
598     #include "foo.h"
599     int c = ADD(1, MULT(2,3));
600   )cpp";
601 
602   std::string Expected = R"(expanded tokens:
603   int a = 100 ; int b = 1 + 2 ; int c = 1 + 2 * 3 ;
604 file './input.cpp'
605   spelled tokens:
606     # include "foo.h" int c = ADD ( 1 , MULT ( 2 , 3 ) ) ;
607   mappings:
608     ['#'_0, 'int'_3) => ['int'_12, 'int'_12)
609     ['ADD'_6, ';'_17) => ['1'_15, ';'_20)
610 file './foo.h'
611   spelled tokens:
612     # define ADD ( X , Y ) X + Y int a = 100 ; # include "bar.h"
613   mappings:
614     ['#'_0, 'int'_11) => ['int'_0, 'int'_0)
615     ['#'_16, '<eof>'_19) => ['int'_5, 'int'_5)
616 file './bar.h'
617   spelled tokens:
618     int b = ADD ( 1 , 2 ) ; # define MULT ( X , Y ) X * Y
619   mappings:
620     ['ADD'_3, ';'_9) => ['1'_8, ';'_11)
621     ['#'_10, '<eof>'_21) => ['int'_12, 'int'_12)
622 )";
623 
624   EXPECT_EQ(Expected, collectAndDump(Code))
625       << "input: " << Code << "\nresults: " << collectAndDump(Code);
626 }
627 
628 class TokenBufferTest : public TokenCollectorTest {};
629 
TEST_F(TokenBufferTest,SpelledByExpanded)630 TEST_F(TokenBufferTest, SpelledByExpanded) {
631   recordTokens(R"cpp(
632     a1 a2 a3 b1 b2
633   )cpp");
634 
635   // Sanity check: expanded and spelled tokens are stored separately.
636   EXPECT_THAT(findExpanded("a1 a2"), Not(SameRange(findSpelled("a1 a2"))));
637   // Searching for subranges of expanded tokens should give the corresponding
638   // spelled ones.
639   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1 a2 a3 b1 b2")),
640               ValueIs(SameRange(findSpelled("a1 a2 a3 b1 b2"))));
641   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1 a2 a3")),
642               ValueIs(SameRange(findSpelled("a1 a2 a3"))));
643   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("b1 b2")),
644               ValueIs(SameRange(findSpelled("b1 b2"))));
645 
646   // Test search on simple macro expansions.
647   recordTokens(R"cpp(
648     #define A a1 a2 a3
649     #define B b1 b2
650 
651     A split B
652   )cpp");
653   // Ranges going across expansion boundaries.
654   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1 a2 a3 split b1 b2")),
655               ValueIs(SameRange(findSpelled("A split B"))));
656   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1 a2 a3")),
657               ValueIs(SameRange(findSpelled("A split").drop_back())));
658   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("b1 b2")),
659               ValueIs(SameRange(findSpelled("split B").drop_front())));
660   // Ranges not fully covering macro invocations should fail.
661   EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("a1 a2")), llvm::None);
662   EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("b2")), llvm::None);
663   EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("a2 a3 split b1 b2")),
664             llvm::None);
665 
666   // Recursive macro invocations.
667   recordTokens(R"cpp(
668     #define ID(x) x
669     #define B b1 b2
670 
671     ID(ID(ID(a1) a2 a3)) split ID(B)
672   )cpp");
673 
674   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("b1 b2")),
675               ValueIs(SameRange(findSpelled("( B").drop_front())));
676   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1 a2 a3 split b1 b2")),
677               ValueIs(SameRange(findSpelled(
678                   "ID ( ID ( ID ( a1 ) a2 a3 ) ) split ID ( B )"))));
679   // Mixed ranges with expanded and spelled tokens.
680   EXPECT_THAT(
681       Buffer.spelledForExpanded(findExpanded("a1 a2 a3 split")),
682       ValueIs(SameRange(findSpelled("ID ( ID ( ID ( a1 ) a2 a3 ) ) split"))));
683   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("split b1 b2")),
684               ValueIs(SameRange(findSpelled("split ID ( B )"))));
685   // Macro arguments
686   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1")),
687               ValueIs(SameRange(findSpelled("a1"))));
688   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a2")),
689               ValueIs(SameRange(findSpelled("a2"))));
690   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a3")),
691               ValueIs(SameRange(findSpelled("a3"))));
692   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1 a2")),
693               ValueIs(SameRange(findSpelled("ID ( a1 ) a2"))));
694   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a1 a2 a3")),
695               ValueIs(SameRange(findSpelled("ID ( a1 ) a2 a3"))));
696 
697   // Empty macro expansions.
698   recordTokens(R"cpp(
699     #define EMPTY
700     #define ID(X) X
701 
702     EMPTY EMPTY ID(1 2 3) EMPTY EMPTY split1
703     EMPTY EMPTY ID(4 5 6) split2
704     ID(7 8 9) EMPTY EMPTY
705   )cpp");
706   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("1 2 3")),
707               ValueIs(SameRange(findSpelled("1 2 3"))));
708   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("4 5 6")),
709               ValueIs(SameRange(findSpelled("4 5 6"))));
710   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("7 8 9")),
711               ValueIs(SameRange(findSpelled("7 8 9"))));
712 
713   // Empty mappings coming from various directives.
714   recordTokens(R"cpp(
715     #define ID(X) X
716     ID(1)
717     #pragma lalala
718     not_mapped
719   )cpp");
720   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("not_mapped")),
721               ValueIs(SameRange(findSpelled("not_mapped"))));
722 
723   // Multiple macro arguments
724   recordTokens(R"cpp(
725     #define ID(X) X
726     #define ID2(X, Y) X Y
727 
728     ID2(ID(a1), ID(a2) a3) ID2(a4, a5 a6 a7)
729   )cpp");
730   // Should fail, spans multiple arguments.
731   EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("a1 a2")), llvm::None);
732   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a2 a3")),
733               ValueIs(SameRange(findSpelled("ID ( a2 ) a3"))));
734   EXPECT_THAT(
735       Buffer.spelledForExpanded(findExpanded("a1 a2 a3")),
736       ValueIs(SameRange(findSpelled("ID2 ( ID ( a1 ) , ID ( a2 ) a3 )"))));
737   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a5 a6")),
738               ValueIs(SameRange(findSpelled("a5 a6"))));
739   EXPECT_THAT(Buffer.spelledForExpanded(findExpanded("a4 a5 a6 a7")),
740               ValueIs(SameRange(findSpelled("ID2 ( a4 , a5 a6 a7 )"))));
741   // Should fail, spans multiple invocations.
742   EXPECT_EQ(Buffer.spelledForExpanded(findExpanded("a1 a2 a3 a4")), llvm::None);
743 }
744 
TEST_F(TokenBufferTest,ExpandedTokensForRange)745 TEST_F(TokenBufferTest, ExpandedTokensForRange) {
746   recordTokens(R"cpp(
747     #define SIGN(X) X##_washere
748     A SIGN(B) C SIGN(D) E SIGN(F) G
749   )cpp");
750 
751   SourceRange R(findExpanded("C").front().location(),
752                 findExpanded("F_washere").front().location());
753   // Sanity check: expanded and spelled tokens are stored separately.
754   EXPECT_THAT(Buffer.expandedTokens(R),
755               SameRange(findExpanded("C D_washere E F_washere")));
756   EXPECT_THAT(Buffer.expandedTokens(SourceRange()), testing::IsEmpty());
757 }
758 
TEST_F(TokenBufferTest,ExpansionsOverlapping)759 TEST_F(TokenBufferTest, ExpansionsOverlapping) {
760   // Object-like macro expansions.
761   recordTokens(R"cpp(
762     #define FOO 3+4
763     int a = FOO 1;
764     int b = FOO 2;
765   )cpp");
766 
767   llvm::ArrayRef<syntax::Token> Foo1 = findSpelled("FOO 1");
768   EXPECT_THAT(
769       Buffer.expansionStartingAt(Foo1.data()),
770       ValueIs(IsExpansion(SameRange(Foo1.drop_back()),
771                           SameRange(findExpanded("3 + 4 1").drop_back()))));
772   EXPECT_THAT(
773       Buffer.expansionsOverlapping(Foo1),
774       ElementsAre(IsExpansion(SameRange(Foo1.drop_back()),
775                               SameRange(findExpanded("3 + 4 1").drop_back()))));
776 
777   llvm::ArrayRef<syntax::Token> Foo2 = findSpelled("FOO 2");
778   EXPECT_THAT(
779       Buffer.expansionStartingAt(Foo2.data()),
780       ValueIs(IsExpansion(SameRange(Foo2.drop_back()),
781                           SameRange(findExpanded("3 + 4 2").drop_back()))));
782   EXPECT_THAT(Buffer.expansionsOverlapping(
783                   llvm::makeArrayRef(Foo1.begin(), Foo2.end())),
784               ElementsAre(IsExpansion(SameRange(Foo1.drop_back()), _),
785                           IsExpansion(SameRange(Foo2.drop_back()), _)));
786 
787   // Function-like macro expansions.
788   recordTokens(R"cpp(
789     #define ID(X) X
790     int a = ID(1+2+3);
791     int b = ID(ID(2+3+4));
792   )cpp");
793 
794   llvm::ArrayRef<syntax::Token> ID1 = findSpelled("ID ( 1 + 2 + 3 )");
795   EXPECT_THAT(Buffer.expansionStartingAt(&ID1.front()),
796               ValueIs(IsExpansion(SameRange(ID1),
797                                   SameRange(findExpanded("1 + 2 + 3")))));
798   // Only the first spelled token should be found.
799   for (const auto &T : ID1.drop_front())
800     EXPECT_EQ(Buffer.expansionStartingAt(&T), llvm::None);
801 
802   llvm::ArrayRef<syntax::Token> ID2 = findSpelled("ID ( ID ( 2 + 3 + 4 ) )");
803   EXPECT_THAT(Buffer.expansionStartingAt(&ID2.front()),
804               ValueIs(IsExpansion(SameRange(ID2),
805                                   SameRange(findExpanded("2 + 3 + 4")))));
806   // Only the first spelled token should be found.
807   for (const auto &T : ID2.drop_front())
808     EXPECT_EQ(Buffer.expansionStartingAt(&T), llvm::None);
809 
810   EXPECT_THAT(Buffer.expansionsOverlapping(llvm::makeArrayRef(
811                   findSpelled("1 + 2").data(), findSpelled("4").data())),
812               ElementsAre(IsExpansion(SameRange(ID1), _),
813                           IsExpansion(SameRange(ID2), _)));
814 
815   // PP directives.
816   recordTokens(R"cpp(
817 #define FOO 1
818 int a = FOO;
819 #pragma once
820 int b = 1;
821   )cpp");
822 
823   llvm::ArrayRef<syntax::Token> DefineFoo = findSpelled("# define FOO 1");
824   EXPECT_THAT(
825       Buffer.expansionStartingAt(&DefineFoo.front()),
826       ValueIs(IsExpansion(SameRange(DefineFoo),
827                           SameRange(findExpanded("int a").take_front(0)))));
828   // Only the first spelled token should be found.
829   for (const auto &T : DefineFoo.drop_front())
830     EXPECT_EQ(Buffer.expansionStartingAt(&T), llvm::None);
831 
832   llvm::ArrayRef<syntax::Token> PragmaOnce = findSpelled("# pragma once");
833   EXPECT_THAT(
834       Buffer.expansionStartingAt(&PragmaOnce.front()),
835       ValueIs(IsExpansion(SameRange(PragmaOnce),
836                           SameRange(findExpanded("int b").take_front(0)))));
837   // Only the first spelled token should be found.
838   for (const auto &T : PragmaOnce.drop_front())
839     EXPECT_EQ(Buffer.expansionStartingAt(&T), llvm::None);
840 
841   EXPECT_THAT(
842       Buffer.expansionsOverlapping(findSpelled("FOO ; # pragma")),
843       ElementsAre(IsExpansion(SameRange(findSpelled("FOO ;").drop_back()), _),
844                   IsExpansion(SameRange(PragmaOnce), _)));
845 }
846 
TEST_F(TokenBufferTest,TokensToFileRange)847 TEST_F(TokenBufferTest, TokensToFileRange) {
848   addFile("./foo.h", "token_from_header");
849   llvm::Annotations Code(R"cpp(
850     #define FOO token_from_expansion
851     #include "./foo.h"
852     $all[[$i[[int]] a = FOO;]]
853   )cpp");
854   recordTokens(Code.code());
855 
856   auto &SM = *SourceMgr;
857 
858   // Two simple examples.
859   auto Int = findExpanded("int").front();
860   auto Semi = findExpanded(";").front();
861   EXPECT_EQ(Int.range(SM), FileRange(SM.getMainFileID(), Code.range("i").Begin,
862                                      Code.range("i").End));
863   EXPECT_EQ(syntax::Token::range(SM, Int, Semi),
864             FileRange(SM.getMainFileID(), Code.range("all").Begin,
865                       Code.range("all").End));
866   // We don't test assertion failures because death tests are slow.
867 }
868 
TEST_F(TokenBufferTest,MacroExpansions)869 TEST_F(TokenBufferTest, MacroExpansions) {
870   llvm::Annotations Code(R"cpp(
871     #define FOO B
872     #define FOO2 BA
873     #define CALL(X) int X
874     #define G CALL(FOO2)
875     int B;
876     $macro[[FOO]];
877     $macro[[CALL]](A);
878     $macro[[G]];
879   )cpp");
880   recordTokens(Code.code());
881   auto &SM = *SourceMgr;
882   auto Expansions = Buffer.macroExpansions(SM.getMainFileID());
883   std::vector<FileRange> ExpectedMacroRanges;
884   for (auto Range : Code.ranges("macro"))
885     ExpectedMacroRanges.push_back(
886         FileRange(SM.getMainFileID(), Range.Begin, Range.End));
887   std::vector<FileRange> ActualMacroRanges;
888   for (auto Expansion : Expansions)
889     ActualMacroRanges.push_back(Expansion->range(SM));
890   EXPECT_EQ(ExpectedMacroRanges, ActualMacroRanges);
891 }
892 
TEST_F(TokenBufferTest,Touching)893 TEST_F(TokenBufferTest, Touching) {
894   llvm::Annotations Code("^i^nt^ ^a^b^=^1;^");
895   recordTokens(Code.code());
896 
897   auto Touching = [&](int Index) {
898     SourceLocation Loc = SourceMgr->getComposedLoc(SourceMgr->getMainFileID(),
899                                                    Code.points()[Index]);
900     return spelledTokensTouching(Loc, Buffer);
901   };
902   auto Identifier = [&](int Index) {
903     SourceLocation Loc = SourceMgr->getComposedLoc(SourceMgr->getMainFileID(),
904                                                    Code.points()[Index]);
905     const syntax::Token *Tok = spelledIdentifierTouching(Loc, Buffer);
906     return Tok ? Tok->text(*SourceMgr) : "";
907   };
908 
909   EXPECT_THAT(Touching(0), SameRange(findSpelled("int")));
910   EXPECT_EQ(Identifier(0), "");
911   EXPECT_THAT(Touching(1), SameRange(findSpelled("int")));
912   EXPECT_EQ(Identifier(1), "");
913   EXPECT_THAT(Touching(2), SameRange(findSpelled("int")));
914   EXPECT_EQ(Identifier(2), "");
915 
916   EXPECT_THAT(Touching(3), SameRange(findSpelled("ab")));
917   EXPECT_EQ(Identifier(3), "ab");
918   EXPECT_THAT(Touching(4), SameRange(findSpelled("ab")));
919   EXPECT_EQ(Identifier(4), "ab");
920 
921   EXPECT_THAT(Touching(5), SameRange(findSpelled("ab =")));
922   EXPECT_EQ(Identifier(5), "ab");
923 
924   EXPECT_THAT(Touching(6), SameRange(findSpelled("= 1")));
925   EXPECT_EQ(Identifier(6), "");
926 
927   EXPECT_THAT(Touching(7), SameRange(findSpelled(";")));
928   EXPECT_EQ(Identifier(7), "");
929 
930   ASSERT_EQ(Code.points().size(), 8u);
931 }
932 
TEST_F(TokenBufferTest,ExpandedBySpelled)933 TEST_F(TokenBufferTest, ExpandedBySpelled) {
934   recordTokens(R"cpp(
935     a1 a2 a3 b1 b2
936   )cpp");
937   // Sanity check: expanded and spelled tokens are stored separately.
938   EXPECT_THAT(findExpanded("a1 a2"), Not(SameRange(findSpelled("a1 a2"))));
939   // Searching for subranges of expanded tokens should give the corresponding
940   // spelled ones.
941   EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("a1 a2 a3 b1 b2")),
942               ElementsAre(SameRange(findExpanded("a1 a2 a3 b1 b2"))));
943   EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("a1 a2 a3")),
944               ElementsAre(SameRange(findExpanded("a1 a2 a3"))));
945   EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("b1 b2")),
946               ElementsAre(SameRange(findExpanded("b1 b2"))));
947 
948   // Test search on simple macro expansions.
949   recordTokens(R"cpp(
950     #define A a1 a2 a3
951     #define B b1 b2
952 
953     A split B
954   )cpp");
955   EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("A split B")),
956               ElementsAre(SameRange(findExpanded("a1 a2 a3 split b1 b2"))));
957   EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("A split").drop_back()),
958               ElementsAre(SameRange(findExpanded("a1 a2 a3"))));
959   EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("split B").drop_front()),
960               ElementsAre(SameRange(findExpanded("b1 b2"))));
961 
962   // Ranges not fully covering macro expansions should fail.
963   recordTokens(R"cpp(
964     #define ID(x) x
965 
966     ID(a)
967   )cpp");
968   // Spelled don't cover entire mapping (missing ID token) -> empty result
969   EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("( a )")), IsEmpty());
970   // Spelled don't cover entire mapping (missing ) token) -> empty result
971   EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("ID ( a")), IsEmpty());
972 
973   // Recursive macro invocations.
974   recordTokens(R"cpp(
975     #define ID(x) x
976     #define B b1 b2
977 
978     ID(ID(ID(a1) a2 a3)) split ID(B)
979   )cpp");
980 
981   EXPECT_THAT(
982       Buffer.expandedForSpelled(findSpelled("ID ( ID ( ID ( a1 ) a2 a3 ) )")),
983       ElementsAre(SameRange(findExpanded("a1 a2 a3"))));
984   EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("ID ( B )")),
985               ElementsAre(SameRange(findExpanded("b1 b2"))));
986   EXPECT_THAT(Buffer.expandedForSpelled(
987                   findSpelled("ID ( ID ( ID ( a1 ) a2 a3 ) ) split ID ( B )")),
988               ElementsAre(SameRange(findExpanded("a1 a2 a3 split b1 b2"))));
989   // FIXME: these should succeed, but we do not support macro arguments yet.
990   EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("a1")), IsEmpty());
991   EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("ID ( a1 ) a2")),
992               IsEmpty());
993 
994   // Empty macro expansions.
995   recordTokens(R"cpp(
996     #define EMPTY
997     #define ID(X) X
998 
999     EMPTY EMPTY ID(1 2 3) EMPTY EMPTY split1
1000     EMPTY EMPTY ID(4 5 6) split2
1001     ID(7 8 9) EMPTY EMPTY
1002   )cpp");
1003   // Covered by empty expansions on one of both of the sides.
1004   EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("ID ( 1 2 3 )")),
1005               ElementsAre(SameRange(findExpanded("1 2 3"))));
1006   EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("ID ( 4 5 6 )")),
1007               ElementsAre(SameRange(findExpanded("4 5 6"))));
1008   EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("ID ( 7 8 9 )")),
1009               ElementsAre(SameRange(findExpanded("7 8 9"))));
1010   // Including the empty macro expansions on the side.
1011   EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("EMPTY ID ( 1 2 3 )")),
1012               ElementsAre(SameRange(findExpanded("1 2 3"))));
1013   EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("ID ( 1 2 3 ) EMPTY")),
1014               ElementsAre(SameRange(findExpanded("1 2 3"))));
1015   EXPECT_THAT(
1016       Buffer.expandedForSpelled(findSpelled("EMPTY ID ( 1 2 3 ) EMPTY")),
1017       ElementsAre(SameRange(findExpanded("1 2 3"))));
1018 
1019   // Empty mappings coming from various directives.
1020   recordTokens(R"cpp(
1021     #define ID(X) X
1022     ID(1)
1023     #pragma lalala
1024     not_mapped
1025   )cpp");
1026   EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("# define ID ( X ) X")),
1027               IsEmpty());
1028   EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("# pragma lalala")),
1029               IsEmpty());
1030 
1031   // Empty macro expansion.
1032   recordTokens(R"cpp(
1033     #define EMPTY
1034     EMPTY int a = 100;
1035   )cpp");
1036   EXPECT_THAT(Buffer.expandedForSpelled(findSpelled("EMPTY int").drop_back()),
1037               IsEmpty());
1038 }
1039 
1040 } // namespace
1041