1 //===-- QualityTests.cpp ----------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Evaluating scoring functions isn't a great fit for assert-based tests.
10 // For interesting cases, both exact scores and "X beats Y" are too brittle to
11 // make good hard assertions.
12 //
13 // Here we test the signal extraction and sanity-check that signals point in
14 // the right direction. This should be supplemented by quality metrics which
15 // we can compute from a corpus of queries and preferred rankings.
16 //
17 //===----------------------------------------------------------------------===//
18 
19 #include "FileDistance.h"
20 #include "Quality.h"
21 #include "TestFS.h"
22 #include "TestTU.h"
23 #include "clang/AST/Decl.h"
24 #include "clang/AST/DeclCXX.h"
25 #include "clang/AST/Type.h"
26 #include "clang/Sema/CodeCompleteConsumer.h"
27 #include "llvm/Support/Casting.h"
28 #include "gmock/gmock.h"
29 #include "gtest/gtest.h"
30 #include <vector>
31 
32 namespace clang {
33 namespace clangd {
34 
35 // Force the unittest URI scheme to be linked,
36 static int LLVM_ATTRIBUTE_UNUSED UnittestSchemeAnchorDest =
37     UnittestSchemeAnchorSource;
38 
39 namespace {
40 
TEST(QualityTests,SymbolQualitySignalExtraction)41 TEST(QualityTests, SymbolQualitySignalExtraction) {
42   auto Header = TestTU::withHeaderCode(R"cpp(
43     int _X;
44 
45     [[deprecated]]
46     int _f() { return _X; }
47 
48     #define DECL_NAME(x, y) x##_##y##_Decl
49     #define DECL(x, y) class DECL_NAME(x, y) {};
50     DECL(X, Y); // X_Y_Decl
51   )cpp");
52 
53   auto Symbols = Header.headerSymbols();
54   auto AST = Header.build();
55 
56   SymbolQualitySignals Quality;
57   Quality.merge(findSymbol(Symbols, "_X"));
58   EXPECT_FALSE(Quality.Deprecated);
59   EXPECT_FALSE(Quality.ImplementationDetail);
60   EXPECT_TRUE(Quality.ReservedName);
61   EXPECT_EQ(Quality.References, SymbolQualitySignals().References);
62   EXPECT_EQ(Quality.Category, SymbolQualitySignals::Variable);
63 
64   Quality.merge(findSymbol(Symbols, "X_Y_Decl"));
65   EXPECT_TRUE(Quality.ImplementationDetail);
66 
67   Symbol F = findSymbol(Symbols, "_f");
68   F.References = 24; // TestTU doesn't count references, so fake it.
69   Quality = {};
70   Quality.merge(F);
71   EXPECT_TRUE(Quality.Deprecated);
72   EXPECT_FALSE(Quality.ReservedName);
73   EXPECT_EQ(Quality.References, 24u);
74   EXPECT_EQ(Quality.Category, SymbolQualitySignals::Function);
75 
76   Quality = {};
77   Quality.merge(CodeCompletionResult(&findDecl(AST, "_f"), /*Priority=*/42));
78   EXPECT_TRUE(Quality.Deprecated);
79   EXPECT_FALSE(Quality.ReservedName);
80   EXPECT_EQ(Quality.References, SymbolQualitySignals().References);
81   EXPECT_EQ(Quality.Category, SymbolQualitySignals::Function);
82 
83   Quality = {};
84   Quality.merge(CodeCompletionResult("if"));
85   EXPECT_EQ(Quality.Category, SymbolQualitySignals::Keyword);
86 }
87 
TEST(QualityTests,SymbolRelevanceSignalExtraction)88 TEST(QualityTests, SymbolRelevanceSignalExtraction) {
89   TestTU Test;
90   Test.HeaderCode = R"cpp(
91   int header();
92   int header_main();
93 
94   namespace hdr { class Bar {}; } // namespace hdr
95 
96   #define DEFINE_FLAG(X) \
97   namespace flags { \
98   int FLAGS_##X; \
99   } \
100 
101   DEFINE_FLAG(FOO)
102   )cpp";
103   Test.Code = R"cpp(
104   using hdr::Bar;
105 
106   using flags::FLAGS_FOO;
107 
108   int ::header_main() {}
109   int main();
110 
111   [[deprecated]]
112   int deprecated() { return 0; }
113 
114   namespace { struct X { void y() { int z; } }; }
115   struct S{};
116   )cpp";
117   auto AST = Test.build();
118 
119   SymbolRelevanceSignals Relevance;
120   Relevance.merge(CodeCompletionResult(&findDecl(AST, "deprecated"),
121                                        /*Priority=*/42, nullptr, false,
122                                        /*Accessible=*/false));
123   EXPECT_EQ(Relevance.NameMatch, SymbolRelevanceSignals().NameMatch);
124   EXPECT_TRUE(Relevance.Forbidden);
125   EXPECT_EQ(Relevance.Scope, SymbolRelevanceSignals::GlobalScope);
126 
127   Relevance = {};
128   Relevance.merge(CodeCompletionResult(&findDecl(AST, "main"), 42));
129   EXPECT_FLOAT_EQ(Relevance.SemaFileProximityScore, 1.0f)
130       << "Decl in current file";
131   Relevance = {};
132   Relevance.merge(CodeCompletionResult(&findDecl(AST, "header"), 42));
133   EXPECT_FLOAT_EQ(Relevance.SemaFileProximityScore, 0.6f) << "Decl from header";
134   Relevance = {};
135   Relevance.merge(CodeCompletionResult(&findDecl(AST, "header_main"), 42));
136   EXPECT_FLOAT_EQ(Relevance.SemaFileProximityScore, 1.0f)
137       << "Current file and header";
138 
139   auto constructShadowDeclCompletionResult = [&](const std::string DeclName) {
140     auto *Shadow =
141         *dyn_cast<UsingDecl>(&findDecl(AST, [&](const NamedDecl &ND) {
142            if (const UsingDecl *Using = dyn_cast<UsingDecl>(&ND))
143              if (Using->shadow_size() &&
144                  Using->getQualifiedNameAsString() == DeclName)
145                return true;
146            return false;
147          }))->shadow_begin();
148     CodeCompletionResult Result(Shadow->getTargetDecl(), 42);
149     Result.ShadowDecl = Shadow;
150     return Result;
151   };
152 
153   Relevance = {};
154   Relevance.merge(constructShadowDeclCompletionResult("Bar"));
155   EXPECT_FLOAT_EQ(Relevance.SemaFileProximityScore, 1.0f)
156       << "Using declaration in main file";
157   Relevance.merge(constructShadowDeclCompletionResult("FLAGS_FOO"));
158   EXPECT_FLOAT_EQ(Relevance.SemaFileProximityScore, 1.0f)
159       << "Using declaration in main file";
160 
161   Relevance = {};
162   Relevance.merge(CodeCompletionResult(&findUnqualifiedDecl(AST, "X"), 42));
163   EXPECT_EQ(Relevance.Scope, SymbolRelevanceSignals::FileScope);
164   Relevance = {};
165   Relevance.merge(CodeCompletionResult(&findUnqualifiedDecl(AST, "y"), 42));
166   EXPECT_EQ(Relevance.Scope, SymbolRelevanceSignals::ClassScope);
167   Relevance = {};
168   Relevance.merge(CodeCompletionResult(&findUnqualifiedDecl(AST, "z"), 42));
169   EXPECT_EQ(Relevance.Scope, SymbolRelevanceSignals::FunctionScope);
170   // The injected class name is treated as the outer class name.
171   Relevance = {};
172   Relevance.merge(CodeCompletionResult(&findDecl(AST, "S::S"), 42));
173   EXPECT_EQ(Relevance.Scope, SymbolRelevanceSignals::GlobalScope);
174 
175   Relevance = {};
176   EXPECT_FALSE(Relevance.InBaseClass);
177   auto BaseMember = CodeCompletionResult(&findUnqualifiedDecl(AST, "y"), 42);
178   BaseMember.InBaseClass = true;
179   Relevance.merge(BaseMember);
180   EXPECT_TRUE(Relevance.InBaseClass);
181 
182   auto Index = Test.index();
183   FuzzyFindRequest Req;
184   Req.Query = "X";
185   Req.AnyScope = true;
186   bool Matched = false;
187   Index->fuzzyFind(Req, [&](const Symbol &S) {
188     Matched = true;
189     Relevance = {};
190     Relevance.merge(S);
191     EXPECT_EQ(Relevance.Scope, SymbolRelevanceSignals::FileScope);
192   });
193   EXPECT_TRUE(Matched);
194 }
195 
196 // Do the signals move the scores in the direction we expect?
TEST(QualityTests,SymbolQualitySignalsSanity)197 TEST(QualityTests, SymbolQualitySignalsSanity) {
198   SymbolQualitySignals Default;
199   EXPECT_EQ(Default.evaluateHeuristics(), 1);
200 
201   SymbolQualitySignals Deprecated;
202   Deprecated.Deprecated = true;
203   EXPECT_LT(Deprecated.evaluateHeuristics(), Default.evaluateHeuristics());
204 
205   SymbolQualitySignals ReservedName;
206   ReservedName.ReservedName = true;
207   EXPECT_LT(ReservedName.evaluateHeuristics(), Default.evaluateHeuristics());
208 
209   SymbolQualitySignals ImplementationDetail;
210   ImplementationDetail.ImplementationDetail = true;
211   EXPECT_LT(ImplementationDetail.evaluateHeuristics(),
212             Default.evaluateHeuristics());
213 
214   SymbolQualitySignals WithReferences, ManyReferences;
215   WithReferences.References = 20;
216   ManyReferences.References = 1000;
217   EXPECT_GT(WithReferences.evaluateHeuristics(), Default.evaluateHeuristics());
218   EXPECT_GT(ManyReferences.evaluateHeuristics(),
219             WithReferences.evaluateHeuristics());
220 
221   SymbolQualitySignals Keyword, Variable, Macro, Constructor, Function,
222       Destructor, Operator;
223   Keyword.Category = SymbolQualitySignals::Keyword;
224   Variable.Category = SymbolQualitySignals::Variable;
225   Macro.Category = SymbolQualitySignals::Macro;
226   Constructor.Category = SymbolQualitySignals::Constructor;
227   Destructor.Category = SymbolQualitySignals::Destructor;
228   Destructor.Category = SymbolQualitySignals::Destructor;
229   Operator.Category = SymbolQualitySignals::Operator;
230   Function.Category = SymbolQualitySignals::Function;
231   EXPECT_GT(Variable.evaluateHeuristics(), Default.evaluateHeuristics());
232   EXPECT_GT(Keyword.evaluateHeuristics(), Variable.evaluateHeuristics());
233   EXPECT_LT(Macro.evaluateHeuristics(), Default.evaluateHeuristics());
234   EXPECT_LT(Operator.evaluateHeuristics(), Default.evaluateHeuristics());
235   EXPECT_LT(Constructor.evaluateHeuristics(), Function.evaluateHeuristics());
236   EXPECT_LT(Destructor.evaluateHeuristics(), Constructor.evaluateHeuristics());
237 }
238 
TEST(QualityTests,SymbolRelevanceSignalsSanity)239 TEST(QualityTests, SymbolRelevanceSignalsSanity) {
240   SymbolRelevanceSignals Default;
241   EXPECT_EQ(Default.evaluateHeuristics(), 1);
242 
243   SymbolRelevanceSignals Forbidden;
244   Forbidden.Forbidden = true;
245   EXPECT_LT(Forbidden.evaluateHeuristics(), Default.evaluateHeuristics());
246 
247   SymbolRelevanceSignals PoorNameMatch;
248   PoorNameMatch.NameMatch = 0.2f;
249   EXPECT_LT(PoorNameMatch.evaluateHeuristics(), Default.evaluateHeuristics());
250 
251   SymbolRelevanceSignals WithSemaFileProximity;
252   WithSemaFileProximity.SemaFileProximityScore = 0.2f;
253   EXPECT_GT(WithSemaFileProximity.evaluateHeuristics(),
254             Default.evaluateHeuristics());
255 
256   ScopeDistance ScopeProximity({"x::y::"});
257 
258   SymbolRelevanceSignals WithSemaScopeProximity;
259   WithSemaScopeProximity.ScopeProximityMatch = &ScopeProximity;
260   WithSemaScopeProximity.SemaSaysInScope = true;
261   EXPECT_GT(WithSemaScopeProximity.evaluateHeuristics(),
262             Default.evaluateHeuristics());
263 
264   SymbolRelevanceSignals WithIndexScopeProximity;
265   WithIndexScopeProximity.ScopeProximityMatch = &ScopeProximity;
266   WithIndexScopeProximity.SymbolScope = "x::";
267   EXPECT_GT(WithSemaScopeProximity.evaluateHeuristics(),
268             Default.evaluateHeuristics());
269 
270   SymbolRelevanceSignals IndexProximate;
271   IndexProximate.SymbolURI = "unittest:/foo/bar.h";
272   llvm::StringMap<SourceParams> ProxSources;
273   ProxSources.try_emplace(testPath("foo/baz.h"));
274   URIDistance Distance(ProxSources);
275   IndexProximate.FileProximityMatch = &Distance;
276   EXPECT_GT(IndexProximate.evaluateHeuristics(), Default.evaluateHeuristics());
277   SymbolRelevanceSignals IndexDistant = IndexProximate;
278   IndexDistant.SymbolURI = "unittest:/elsewhere/path.h";
279   EXPECT_GT(IndexProximate.evaluateHeuristics(),
280             IndexDistant.evaluateHeuristics())
281       << IndexProximate << IndexDistant;
282   EXPECT_GT(IndexDistant.evaluateHeuristics(), Default.evaluateHeuristics());
283 
284   SymbolRelevanceSignals Scoped;
285   Scoped.Scope = SymbolRelevanceSignals::FileScope;
286   EXPECT_LT(Scoped.evaluateHeuristics(), Default.evaluateHeuristics());
287   Scoped.Query = SymbolRelevanceSignals::CodeComplete;
288   EXPECT_GT(Scoped.evaluateHeuristics(), Default.evaluateHeuristics());
289 
290   SymbolRelevanceSignals Instance;
291   Instance.IsInstanceMember = false;
292   EXPECT_EQ(Instance.evaluateHeuristics(), Default.evaluateHeuristics());
293   Instance.Context = CodeCompletionContext::CCC_DotMemberAccess;
294   EXPECT_LT(Instance.evaluateHeuristics(), Default.evaluateHeuristics());
295   Instance.IsInstanceMember = true;
296   EXPECT_EQ(Instance.evaluateHeuristics(), Default.evaluateHeuristics());
297 
298   SymbolRelevanceSignals InBaseClass;
299   InBaseClass.InBaseClass = true;
300   EXPECT_LT(InBaseClass.evaluateHeuristics(), Default.evaluateHeuristics());
301 
302   llvm::StringSet<> Words = {"one", "two", "three"};
303   SymbolRelevanceSignals WithoutMatchingWord;
304   WithoutMatchingWord.ContextWords = &Words;
305   WithoutMatchingWord.Name = "four";
306   EXPECT_EQ(WithoutMatchingWord.evaluateHeuristics(),
307             Default.evaluateHeuristics());
308   SymbolRelevanceSignals WithMatchingWord;
309   WithMatchingWord.ContextWords = &Words;
310   WithMatchingWord.Name = "TheTwoTowers";
311   EXPECT_GT(WithMatchingWord.evaluateHeuristics(),
312             Default.evaluateHeuristics());
313 }
314 
TEST(QualityTests,ScopeProximity)315 TEST(QualityTests, ScopeProximity) {
316   SymbolRelevanceSignals Relevance;
317   ScopeDistance ScopeProximity({"x::y::z::", "x::", "llvm::", ""});
318   Relevance.ScopeProximityMatch = &ScopeProximity;
319 
320   Relevance.SymbolScope = "other::";
321   float NotMatched = Relevance.evaluateHeuristics();
322 
323   Relevance.SymbolScope = "";
324   float Global = Relevance.evaluateHeuristics();
325   EXPECT_GT(Global, NotMatched);
326 
327   Relevance.SymbolScope = "llvm::";
328   float NonParent = Relevance.evaluateHeuristics();
329   EXPECT_GT(NonParent, Global);
330 
331   Relevance.SymbolScope = "x::";
332   float GrandParent = Relevance.evaluateHeuristics();
333   EXPECT_GT(GrandParent, Global);
334 
335   Relevance.SymbolScope = "x::y::";
336   float Parent = Relevance.evaluateHeuristics();
337   EXPECT_GT(Parent, GrandParent);
338 
339   Relevance.SymbolScope = "x::y::z::";
340   float Enclosing = Relevance.evaluateHeuristics();
341   EXPECT_GT(Enclosing, Parent);
342 }
343 
TEST(QualityTests,SortText)344 TEST(QualityTests, SortText) {
345   EXPECT_LT(sortText(std::numeric_limits<float>::infinity()),
346             sortText(1000.2f));
347   EXPECT_LT(sortText(1000.2f), sortText(1));
348   EXPECT_LT(sortText(1), sortText(0.3f));
349   EXPECT_LT(sortText(0.3f), sortText(0));
350   EXPECT_LT(sortText(0), sortText(-10));
351   EXPECT_LT(sortText(-10), sortText(-std::numeric_limits<float>::infinity()));
352 
353   EXPECT_LT(sortText(1, "z"), sortText(0, "a"));
354   EXPECT_LT(sortText(0, "a"), sortText(0, "z"));
355 }
356 
TEST(QualityTests,NoBoostForClassConstructor)357 TEST(QualityTests, NoBoostForClassConstructor) {
358   auto Header = TestTU::withHeaderCode(R"cpp(
359     class Foo {
360     public:
361       Foo(int);
362     };
363   )cpp");
364   auto Symbols = Header.headerSymbols();
365   auto AST = Header.build();
366 
367   const NamedDecl *Foo = &findDecl(AST, "Foo");
368   SymbolRelevanceSignals Cls;
369   Cls.merge(CodeCompletionResult(Foo, /*Priority=*/0));
370 
371   const NamedDecl *CtorDecl = &findDecl(AST, [](const NamedDecl &ND) {
372     return (ND.getQualifiedNameAsString() == "Foo::Foo") &&
373            isa<CXXConstructorDecl>(&ND);
374   });
375   SymbolRelevanceSignals Ctor;
376   Ctor.merge(CodeCompletionResult(CtorDecl, /*Priority=*/0));
377 
378   EXPECT_EQ(Cls.Scope, SymbolRelevanceSignals::GlobalScope);
379   EXPECT_EQ(Ctor.Scope, SymbolRelevanceSignals::GlobalScope);
380 }
381 
TEST(QualityTests,IsInstanceMember)382 TEST(QualityTests, IsInstanceMember) {
383   auto Header = TestTU::withHeaderCode(R"cpp(
384     class Foo {
385     public:
386       static void foo() {}
387 
388       template <typename T> void tpl(T *t) {}
389 
390       void bar() {}
391     };
392   )cpp");
393   auto Symbols = Header.headerSymbols();
394 
395   SymbolRelevanceSignals Rel;
396   const Symbol &FooSym = findSymbol(Symbols, "Foo::foo");
397   Rel.merge(FooSym);
398   EXPECT_FALSE(Rel.IsInstanceMember);
399   const Symbol &BarSym = findSymbol(Symbols, "Foo::bar");
400   Rel.merge(BarSym);
401   EXPECT_TRUE(Rel.IsInstanceMember);
402 
403   Rel.IsInstanceMember = false;
404   const Symbol &TplSym = findSymbol(Symbols, "Foo::tpl");
405   Rel.merge(TplSym);
406   EXPECT_TRUE(Rel.IsInstanceMember);
407 
408   auto AST = Header.build();
409   const NamedDecl *Foo = &findDecl(AST, "Foo::foo");
410   const NamedDecl *Bar = &findDecl(AST, "Foo::bar");
411   const NamedDecl *Tpl = &findDecl(AST, "Foo::tpl");
412 
413   Rel.IsInstanceMember = false;
414   Rel.merge(CodeCompletionResult(Foo, /*Priority=*/0));
415   EXPECT_FALSE(Rel.IsInstanceMember);
416   Rel.merge(CodeCompletionResult(Bar, /*Priority=*/0));
417   EXPECT_TRUE(Rel.IsInstanceMember);
418   Rel.IsInstanceMember = false;
419   Rel.merge(CodeCompletionResult(Tpl, /*Priority=*/0));
420   EXPECT_TRUE(Rel.IsInstanceMember);
421 }
422 
TEST(QualityTests,ConstructorDestructor)423 TEST(QualityTests, ConstructorDestructor) {
424   auto Header = TestTU::withHeaderCode(R"cpp(
425     class Foo {
426     public:
427       Foo(int);
428       ~Foo();
429     };
430   )cpp");
431   auto Symbols = Header.headerSymbols();
432   auto AST = Header.build();
433 
434   const NamedDecl *CtorDecl = &findDecl(AST, [](const NamedDecl &ND) {
435     return (ND.getQualifiedNameAsString() == "Foo::Foo") &&
436            isa<CXXConstructorDecl>(&ND);
437   });
438   const NamedDecl *DtorDecl = &findDecl(AST, [](const NamedDecl &ND) {
439     return (ND.getQualifiedNameAsString() == "Foo::~Foo") &&
440            isa<CXXDestructorDecl>(&ND);
441   });
442 
443   SymbolQualitySignals CtorQ;
444   CtorQ.merge(CodeCompletionResult(CtorDecl, /*Priority=*/0));
445   EXPECT_EQ(CtorQ.Category, SymbolQualitySignals::Constructor);
446 
447   CtorQ.Category = SymbolQualitySignals::Unknown;
448   const Symbol &CtorSym = findSymbol(Symbols, "Foo::Foo");
449   CtorQ.merge(CtorSym);
450   EXPECT_EQ(CtorQ.Category, SymbolQualitySignals::Constructor);
451 
452   SymbolQualitySignals DtorQ;
453   DtorQ.merge(CodeCompletionResult(DtorDecl, /*Priority=*/0));
454   EXPECT_EQ(DtorQ.Category, SymbolQualitySignals::Destructor);
455 }
456 
TEST(QualityTests,Operator)457 TEST(QualityTests, Operator) {
458   auto Header = TestTU::withHeaderCode(R"cpp(
459     class Foo {
460     public:
461       bool operator<(const Foo& f1);
462     };
463   )cpp");
464   auto AST = Header.build();
465 
466   const NamedDecl *Operator = &findDecl(AST, [](const NamedDecl &ND) {
467     if (const auto *OD = dyn_cast<FunctionDecl>(&ND))
468       if (OD->isOverloadedOperator())
469         return true;
470     return false;
471   });
472   SymbolQualitySignals Q;
473   Q.merge(CodeCompletionResult(Operator, /*Priority=*/0));
474   EXPECT_EQ(Q.Category, SymbolQualitySignals::Operator);
475 }
476 
TEST(QualityTests,ItemWithFixItsRankedDown)477 TEST(QualityTests, ItemWithFixItsRankedDown) {
478   CodeCompleteOptions Opts;
479   Opts.IncludeFixIts = true;
480 
481   auto Header = TestTU::withHeaderCode(R"cpp(
482         int x;
483       )cpp");
484   auto AST = Header.build();
485 
486   SymbolRelevanceSignals RelevanceWithFixIt;
487   RelevanceWithFixIt.merge(CodeCompletionResult(&findDecl(AST, "x"), 0, nullptr,
488                                                 false, true, {FixItHint{}}));
489   EXPECT_TRUE(RelevanceWithFixIt.NeedsFixIts);
490 
491   SymbolRelevanceSignals RelevanceWithoutFixIt;
492   RelevanceWithoutFixIt.merge(
493       CodeCompletionResult(&findDecl(AST, "x"), 0, nullptr, false, true, {}));
494   EXPECT_FALSE(RelevanceWithoutFixIt.NeedsFixIts);
495 
496   EXPECT_LT(RelevanceWithFixIt.evaluateHeuristics(),
497             RelevanceWithoutFixIt.evaluateHeuristics());
498 }
499 
500 } // namespace
501 } // namespace clangd
502 } // namespace clang
503