1 //===--- PreambleTests.cpp --------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "Annotations.h"
10 #include "Compiler.h"
11 #include "Headers.h"
12 #include "Hover.h"
13 #include "Preamble.h"
14 #include "SourceCode.h"
15 #include "TestFS.h"
16 #include "TestTU.h"
17 #include "XRefs.h"
18 #include "clang/Format/Format.h"
19 #include "clang/Frontend/PrecompiledPreamble.h"
20 #include "clang/Lex/PreprocessorOptions.h"
21 #include "llvm/ADT/STLExtras.h"
22 #include "llvm/ADT/StringExtras.h"
23 #include "llvm/ADT/StringRef.h"
24 #include "llvm/Support/Error.h"
25 #include "llvm/Support/MemoryBuffer.h"
26 #include "llvm/Support/VirtualFileSystem.h"
27 #include "gmock/gmock.h"
28 #include "gtest/gtest.h"
29 #include <clang/Frontend/FrontendActions.h>
30 #include <memory>
31 #include <string>
32 #include <vector>
33 
34 using testing::Contains;
35 using testing::Field;
36 using testing::Matcher;
37 using testing::MatchesRegex;
38 
39 namespace clang {
40 namespace clangd {
41 namespace {
42 
43 MATCHER_P2(Distance, File, D, "") {
44   return arg.first() == File && arg.second == D;
45 }
46 
47 // Builds a preamble for BaselineContents, patches it for ModifiedContents and
48 // returns the includes in the patch.
49 IncludeStructure
collectPatchedIncludes(llvm::StringRef ModifiedContents,llvm::StringRef BaselineContents,llvm::StringRef MainFileName="main.cpp")50 collectPatchedIncludes(llvm::StringRef ModifiedContents,
51                        llvm::StringRef BaselineContents,
52                        llvm::StringRef MainFileName = "main.cpp") {
53   MockFS FS;
54   auto TU = TestTU::withCode(BaselineContents);
55   TU.Filename = MainFileName.str();
56   // ms-compatibility changes meaning of #import, make sure it is turned off.
57   TU.ExtraArgs = {"-fno-ms-compatibility"};
58   auto BaselinePreamble = TU.preamble();
59   // Create the patch.
60   TU.Code = ModifiedContents.str();
61   auto PI = TU.inputs(FS);
62   auto PP = PreamblePatch::create(testPath(TU.Filename), PI, *BaselinePreamble);
63   // Collect patch contents.
64   IgnoreDiagnostics Diags;
65   auto CI = buildCompilerInvocation(PI, Diags);
66   PP.apply(*CI);
67   // Run preprocessor over the modified contents with patched Invocation. We
68   // provide a preamble and trim contents to ensure only the implicit header
69   // introduced by the patch is parsed and nothing else.
70   // We don't run PP directly over the patch cotents to test production
71   // behaviour.
72   auto Bounds = Lexer::ComputePreamble(ModifiedContents, *CI->getLangOpts());
73   auto Clang =
74       prepareCompilerInstance(std::move(CI), &BaselinePreamble->Preamble,
75                               llvm::MemoryBuffer::getMemBufferCopy(
76                                   ModifiedContents.slice(0, Bounds.Size).str()),
77                               PI.TFS->view(PI.CompileCommand.Directory), Diags);
78   PreprocessOnlyAction Action;
79   if (!Action.BeginSourceFile(*Clang, Clang->getFrontendOpts().Inputs[0])) {
80     ADD_FAILURE() << "failed begin source file";
81     return {};
82   }
83   IncludeStructure Includes;
84   Clang->getPreprocessor().addPPCallbacks(
85       collectIncludeStructureCallback(Clang->getSourceManager(), &Includes));
86   if (llvm::Error Err = Action.Execute()) {
87     ADD_FAILURE() << "failed to execute action: " << std::move(Err);
88     return {};
89   }
90   Action.EndSourceFile();
91   return Includes;
92 }
93 
94 // Check preamble lexing logic by building an empty preamble and patching it
95 // with all the contents.
TEST(PreamblePatchTest,IncludeParsing)96 TEST(PreamblePatchTest, IncludeParsing) {
97   // We expect any line with a point to show up in the patch.
98   llvm::StringRef Cases[] = {
99       // Only preamble
100       R"cpp(^#include "a.h")cpp",
101       // Both preamble and mainfile
102       R"cpp(
103         ^#include "a.h"
104         garbage, finishes preamble
105         #include "a.h")cpp",
106       // Mixed directives
107       R"cpp(
108         ^#include "a.h"
109         #pragma directive
110         // some comments
111         ^#include_next <a.h>
112         #ifdef skipped
113         ^#import "a.h"
114         #endif)cpp",
115       // Broken directives
116       R"cpp(
117         #include "a
118         ^#include "a.h"
119         #include <b
120         ^#include <b.h>)cpp",
121       // Directive is not part of preamble if it is not the token immediately
122       // followed by the hash (#).
123       R"cpp(
124         ^#include "a.h"
125         #/**/include <b.h>)cpp",
126   };
127 
128   for (const auto &Case : Cases) {
129     Annotations Test(Case);
130     const auto Code = Test.code();
131     SCOPED_TRACE(Code);
132 
133     auto Includes =
134         collectPatchedIncludes(Code, /*BaselineContents=*/"").MainFileIncludes;
135     auto Points = Test.points();
136     ASSERT_EQ(Includes.size(), Points.size());
137     for (size_t I = 0, E = Includes.size(); I != E; ++I)
138       EXPECT_EQ(Includes[I].HashLine, Points[I].line);
139   }
140 }
141 
142 TEST(PreamblePatchTest, ContainsNewIncludes) {
143   constexpr llvm::StringLiteral BaselineContents = R"cpp(
144     #include <a.h>
145     #include <b.h> // This will be removed
146     #include <c.h>
147   )cpp";
148   constexpr llvm::StringLiteral ModifiedContents = R"cpp(
149     #include <a.h>
150     #include <c.h> // This has changed a line.
151     #include <c.h> // This is a duplicate.
152     #include <d.h> // This is newly introduced.
153   )cpp";
154   auto Includes = collectPatchedIncludes(ModifiedContents, BaselineContents)
155                       .MainFileIncludes;
156   EXPECT_THAT(Includes, ElementsAre(AllOf(Field(&Inclusion::Written, "<d.h>"),
157                                           Field(&Inclusion::HashLine, 4))));
158 }
159 
160 TEST(PreamblePatchTest, MainFileIsEscaped) {
161   auto Includes = collectPatchedIncludes("#include <a.h>", "", "file\"name.cpp")
162                       .MainFileIncludes;
163   EXPECT_THAT(Includes, ElementsAre(AllOf(Field(&Inclusion::Written, "<a.h>"),
164                                           Field(&Inclusion::HashLine, 0))));
165 }
166 
167 TEST(PreamblePatchTest, PatchesPreambleIncludes) {
168   MockFS FS;
169   IgnoreDiagnostics Diags;
170   auto TU = TestTU::withCode(R"cpp(
171     #include "a.h"
172     #include "c.h"
173   )cpp");
174   TU.AdditionalFiles["a.h"] = "#include \"b.h\"";
175   TU.AdditionalFiles["b.h"] = "";
176   TU.AdditionalFiles["c.h"] = "";
177   auto PI = TU.inputs(FS);
178   auto BaselinePreamble = buildPreamble(
179       TU.Filename, *buildCompilerInvocation(PI, Diags), PI, true, nullptr);
180   // We drop c.h from modified and add a new header. Since the latter is patched
181   // we should only get a.h in preamble includes.
182   TU.Code = R"cpp(
183     #include "a.h"
184     #include "b.h"
185   )cpp";
186   auto PP = PreamblePatch::create(testPath(TU.Filename), TU.inputs(FS),
187                                   *BaselinePreamble);
188   // Only a.h should exists in the preamble, as c.h has been dropped and b.h was
189   // newly introduced.
190   EXPECT_THAT(PP.preambleIncludes(),
191               ElementsAre(AllOf(Field(&Inclusion::Written, "\"a.h\""),
192                                 Field(&Inclusion::Resolved, testPath("a.h")))));
193 }
194 
195 llvm::Optional<ParsedAST> createPatchedAST(llvm::StringRef Baseline,
196                                            llvm::StringRef Modified) {
197   auto BaselinePreamble = TestTU::withCode(Baseline).preamble();
198   if (!BaselinePreamble) {
199     ADD_FAILURE() << "Failed to build baseline preamble";
200     return llvm::None;
201   }
202 
203   IgnoreDiagnostics Diags;
204   MockFS FS;
205   auto TU = TestTU::withCode(Modified);
206   auto CI = buildCompilerInvocation(TU.inputs(FS), Diags);
207   if (!CI) {
208     ADD_FAILURE() << "Failed to build compiler invocation";
209     return llvm::None;
210   }
211   return ParsedAST::build(testPath(TU.Filename), TU.inputs(FS), std::move(CI),
212                           {}, BaselinePreamble);
213 }
214 
215 std::string getPreamblePatch(llvm::StringRef Baseline,
216                              llvm::StringRef Modified) {
217   auto BaselinePreamble = TestTU::withCode(Baseline).preamble();
218   if (!BaselinePreamble) {
219     ADD_FAILURE() << "Failed to build baseline preamble";
220     return "";
221   }
222   MockFS FS;
223   auto TU = TestTU::withCode(Modified);
224   return PreamblePatch::create(testPath("main.cpp"), TU.inputs(FS),
225                                *BaselinePreamble)
226       .text()
227       .str();
228 }
229 
230 TEST(PreamblePatchTest, Define) {
231   // BAR should be defined while parsing the AST.
232   struct {
233     const char *const Contents;
234     const char *const ExpectedPatch;
235   } Cases[] = {
236       {
237           R"cpp(
238         #define BAR
239         [[BAR]])cpp",
240           R"cpp(#line 0 ".*main.cpp"
241 #line 2
242 #define         BAR
243 )cpp",
244       },
245       // multiline macro
246       {
247           R"cpp(
248         #define BAR \
249 
250         [[BAR]])cpp",
251           R"cpp(#line 0 ".*main.cpp"
252 #line 2
253 #define         BAR
254 )cpp",
255       },
256       // multiline macro
257       {
258           R"cpp(
259         #define \
260                 BAR
261         [[BAR]])cpp",
262           R"cpp(#line 0 ".*main.cpp"
263 #line 3
264 #define         BAR
265 )cpp",
266       },
267   };
268 
269   for (const auto &Case : Cases) {
270     SCOPED_TRACE(Case.Contents);
271     Annotations Modified(Case.Contents);
272     EXPECT_THAT(getPreamblePatch("", Modified.code()),
273                 MatchesRegex(Case.ExpectedPatch));
274 
275     auto AST = createPatchedAST("", Modified.code());
276     ASSERT_TRUE(AST);
277     EXPECT_THAT(AST->getDiagnostics(),
278                 Not(Contains(Field(&Diag::Range, Modified.range()))));
279   }
280 }
281 
282 TEST(PreamblePatchTest, OrderingPreserved) {
283   llvm::StringLiteral Baseline = "#define BAR(X) X";
284   Annotations Modified(R"cpp(
285     #define BAR(X, Y) X Y
286     #define BAR(X) X
287     [[BAR]](int y);
288   )cpp");
289 
290   llvm::StringLiteral ExpectedPatch(R"cpp(#line 0 ".*main.cpp"
291 #line 2
292 #define     BAR\(X, Y\) X Y
293 #line 3
294 #define     BAR\(X\) X
295 )cpp");
296   EXPECT_THAT(getPreamblePatch(Baseline, Modified.code()),
297               MatchesRegex(ExpectedPatch.str()));
298 
299   auto AST = createPatchedAST(Baseline, Modified.code());
300   ASSERT_TRUE(AST);
301   EXPECT_THAT(AST->getDiagnostics(),
302               Not(Contains(Field(&Diag::Range, Modified.range()))));
303 }
304 
305 TEST(PreamblePatchTest, LocateMacroAtWorks) {
306   struct {
307     const char *const Baseline;
308     const char *const Modified;
309   } Cases[] = {
310       // Addition of new directive
311       {
312           "",
313           R"cpp(
314             #define $def^FOO
315             $use^FOO)cpp",
316       },
317       // Available inside preamble section
318       {
319           "",
320           R"cpp(
321             #define $def^FOO
322             #undef $use^FOO)cpp",
323       },
324       // Available after undef, as we don't patch those
325       {
326           "",
327           R"cpp(
328             #define $def^FOO
329             #undef FOO
330             $use^FOO)cpp",
331       },
332       // Identifier on a different line
333       {
334           "",
335           R"cpp(
336             #define \
337               $def^FOO
338             $use^FOO)cpp",
339       },
340       // In presence of comment tokens
341       {
342           "",
343           R"cpp(
344             #\
345               define /* FOO */\
346               /* FOO */ $def^FOO
347             $use^FOO)cpp",
348       },
349       // Moved around
350       {
351           "#define FOO",
352           R"cpp(
353             #define BAR
354             #define $def^FOO
355             $use^FOO)cpp",
356       },
357   };
358   for (const auto &Case : Cases) {
359     SCOPED_TRACE(Case.Modified);
360     llvm::Annotations Modified(Case.Modified);
361     auto AST = createPatchedAST(Case.Baseline, Modified.code());
362     ASSERT_TRUE(AST);
363 
364     const auto &SM = AST->getSourceManager();
365     auto *MacroTok = AST->getTokens().spelledTokenAt(
366         SM.getComposedLoc(SM.getMainFileID(), Modified.point("use")));
367     ASSERT_TRUE(MacroTok);
368 
369     auto FoundMacro = locateMacroAt(*MacroTok, AST->getPreprocessor());
370     ASSERT_TRUE(FoundMacro);
371     EXPECT_THAT(FoundMacro->Name, "FOO");
372 
373     auto MacroLoc = FoundMacro->NameLoc;
374     EXPECT_EQ(SM.getFileID(MacroLoc), SM.getMainFileID());
375     EXPECT_EQ(SM.getFileOffset(MacroLoc), Modified.point("def"));
376   }
377 }
378 
379 TEST(PreamblePatchTest, LocateMacroAtDeletion) {
380   {
381     // We don't patch deleted define directives, make sure we don't crash.
382     llvm::StringLiteral Baseline = "#define FOO";
383     llvm::Annotations Modified("^FOO");
384 
385     auto AST = createPatchedAST(Baseline, Modified.code());
386     ASSERT_TRUE(AST);
387 
388     const auto &SM = AST->getSourceManager();
389     auto *MacroTok = AST->getTokens().spelledTokenAt(
390         SM.getComposedLoc(SM.getMainFileID(), Modified.point()));
391     ASSERT_TRUE(MacroTok);
392 
393     auto FoundMacro = locateMacroAt(*MacroTok, AST->getPreprocessor());
394     ASSERT_TRUE(FoundMacro);
395     EXPECT_THAT(FoundMacro->Name, "FOO");
396     auto HI =
397         getHover(*AST, offsetToPosition(Modified.code(), Modified.point()),
398                  format::getLLVMStyle(), nullptr);
399     ASSERT_TRUE(HI);
400     EXPECT_THAT(HI->Definition, testing::IsEmpty());
401   }
402 
403   {
404     // Offset is valid, but underlying text is different.
405     llvm::StringLiteral Baseline = "#define FOO";
406     Annotations Modified(R"cpp(#define BAR
407     ^FOO")cpp");
408 
409     auto AST = createPatchedAST(Baseline, Modified.code());
410     ASSERT_TRUE(AST);
411 
412     auto HI = getHover(*AST, Modified.point(), format::getLLVMStyle(), nullptr);
413     ASSERT_TRUE(HI);
414     EXPECT_THAT(HI->Definition, "#define BAR");
415   }
416 }
417 
418 TEST(PreamblePatchTest, RefsToMacros) {
419   struct {
420     const char *const Baseline;
421     const char *const Modified;
422   } Cases[] = {
423       // Newly added
424       {
425           "",
426           R"cpp(
427             #define ^FOO
428             ^[[FOO]])cpp",
429       },
430       // Moved around
431       {
432           "#define FOO",
433           R"cpp(
434             #define BAR
435             #define ^FOO
436             ^[[FOO]])cpp",
437       },
438       // Ref in preamble section
439       {
440           "",
441           R"cpp(
442             #define ^FOO
443             #undef ^FOO)cpp",
444       },
445   };
446 
447   for (const auto &Case : Cases) {
448     Annotations Modified(Case.Modified);
449     auto AST = createPatchedAST("", Modified.code());
450     ASSERT_TRUE(AST);
451 
452     const auto &SM = AST->getSourceManager();
453     std::vector<Matcher<Location>> ExpectedLocations;
454     for (const auto &R : Modified.ranges())
455       ExpectedLocations.push_back(Field(&Location::range, R));
456 
457     for (const auto &P : Modified.points()) {
458       auto *MacroTok = AST->getTokens().spelledTokenAt(SM.getComposedLoc(
459           SM.getMainFileID(),
460           llvm::cantFail(positionToOffset(Modified.code(), P))));
461       ASSERT_TRUE(MacroTok);
462       EXPECT_THAT(findReferences(*AST, P, 0).References,
463                   testing::ElementsAreArray(ExpectedLocations));
464     }
465   }
466 }
467 
468 TEST(TranslatePreamblePatchLocation, Simple) {
469   auto TU = TestTU::withHeaderCode(R"cpp(
470     #line 3 "main.cpp"
471     int foo();)cpp");
472   // Presumed line/col needs to be valid in the main file.
473   TU.Code = R"cpp(// line 1
474     // line 2
475     // line 3
476     // line 4)cpp";
477   TU.Filename = "main.cpp";
478   TU.HeaderFilename = "__preamble_patch__.h";
479   TU.ImplicitHeaderGuard = false;
480 
481   auto AST = TU.build();
482   auto &SM = AST.getSourceManager();
483   auto &ND = findDecl(AST, "foo");
484   EXPECT_NE(SM.getFileID(ND.getLocation()), SM.getMainFileID());
485 
486   auto TranslatedLoc = translatePreamblePatchLocation(ND.getLocation(), SM);
487   auto DecompLoc = SM.getDecomposedLoc(TranslatedLoc);
488   EXPECT_EQ(DecompLoc.first, SM.getMainFileID());
489   EXPECT_EQ(SM.getLineNumber(DecompLoc.first, DecompLoc.second), 3U);
490 }
491 
492 TEST(PreamblePatch, ModifiedBounds) {
493   struct {
494     const char *const Baseline;
495     const char *const Modified;
496   } Cases[] = {
497       // Size increased
498       {
499           "",
500           R"cpp(
501             #define FOO
502             FOO)cpp",
503       },
504       // Stayed same
505       {"#define FOO", "#define BAR"},
506       // Got smaller
507       {
508           R"cpp(
509             #define FOO
510             #undef FOO)cpp",
511           "#define FOO"},
512   };
513 
514   for (const auto &Case : Cases) {
515     auto TU = TestTU::withCode(Case.Baseline);
516     auto BaselinePreamble = TU.preamble();
517     ASSERT_TRUE(BaselinePreamble);
518 
519     Annotations Modified(Case.Modified);
520     TU.Code = Modified.code().str();
521     MockFS FS;
522     auto PP = PreamblePatch::create(testPath(TU.Filename), TU.inputs(FS),
523                                     *BaselinePreamble);
524 
525     IgnoreDiagnostics Diags;
526     auto CI = buildCompilerInvocation(TU.inputs(FS), Diags);
527     ASSERT_TRUE(CI);
528 
529     const auto ExpectedBounds =
530         Lexer::ComputePreamble(Case.Modified, *CI->getLangOpts());
531     EXPECT_EQ(PP.modifiedBounds().Size, ExpectedBounds.Size);
532     EXPECT_EQ(PP.modifiedBounds().PreambleEndsAtStartOfLine,
533               ExpectedBounds.PreambleEndsAtStartOfLine);
534   }
535 }
536 } // namespace
537 } // namespace clangd
538 } // namespace clang
539