1 //===- InjectTLIMAppings.cpp - TLI to VFABI attribute injection ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Populates the VFABI attribute with the scalar-to-vector mappings
10 // from the TargetLibraryInfo.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "llvm/Transforms/Utils/InjectTLIMappings.h"
15 #include "llvm/ADT/Statistic.h"
16 #include "llvm/Analysis/DemandedBits.h"
17 #include "llvm/Analysis/GlobalsModRef.h"
18 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
19 #include "llvm/Analysis/TargetLibraryInfo.h"
20 #include "llvm/Analysis/VectorUtils.h"
21 #include "llvm/IR/InstIterator.h"
22 #include "llvm/IR/IntrinsicInst.h"
23 #include "llvm/Transforms/Utils.h"
24 #include "llvm/Transforms/Utils/ModuleUtils.h"
25
26 using namespace llvm;
27
28 #define DEBUG_TYPE "inject-tli-mappings"
29
30 STATISTIC(NumCallInjected,
31 "Number of calls in which the mappings have been injected.");
32
33 STATISTIC(NumVFDeclAdded,
34 "Number of function declarations that have been added.");
35 STATISTIC(NumCompUsedAdded,
36 "Number of `@llvm.compiler.used` operands that have been added.");
37
38 /// A helper function that adds the vector function declaration that
39 /// vectorizes the CallInst CI with a vectorization factor of VF
40 /// lanes. The TLI assumes that all parameters and the return type of
41 /// CI (other than void) need to be widened to a VectorType of VF
42 /// lanes.
addVariantDeclaration(CallInst & CI,const unsigned VF,const StringRef VFName)43 static void addVariantDeclaration(CallInst &CI, const unsigned VF,
44 const StringRef VFName) {
45 Module *M = CI.getModule();
46
47 // Add function declaration.
48 Type *RetTy = ToVectorTy(CI.getType(), VF);
49 SmallVector<Type *, 4> Tys;
50 for (Value *ArgOperand : CI.arg_operands())
51 Tys.push_back(ToVectorTy(ArgOperand->getType(), VF));
52 assert(!CI.getFunctionType()->isVarArg() &&
53 "VarArg functions are not supported.");
54 FunctionType *FTy = FunctionType::get(RetTy, Tys, /*isVarArg=*/false);
55 Function *VectorF =
56 Function::Create(FTy, Function::ExternalLinkage, VFName, M);
57 VectorF->copyAttributesFrom(CI.getCalledFunction());
58 ++NumVFDeclAdded;
59 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added to the module: `" << VFName
60 << "` of type " << *(VectorF->getType()) << "\n");
61
62 // Make function declaration (without a body) "sticky" in the IR by
63 // listing it in the @llvm.compiler.used intrinsic.
64 assert(!VectorF->size() && "VFABI attribute requires `@llvm.compiler.used` "
65 "only on declarations.");
66 appendToCompilerUsed(*M, {VectorF});
67 LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << VFName
68 << "` to `@llvm.compiler.used`.\n");
69 ++NumCompUsedAdded;
70 }
71
addMappingsFromTLI(const TargetLibraryInfo & TLI,CallInst & CI)72 static void addMappingsFromTLI(const TargetLibraryInfo &TLI, CallInst &CI) {
73 // This is needed to make sure we don't query the TLI for calls to
74 // bitcast of function pointers, like `%call = call i32 (i32*, ...)
75 // bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull %i)`,
76 // as such calls make the `isFunctionVectorizable` raise an
77 // exception.
78 if (CI.isNoBuiltin() || !CI.getCalledFunction())
79 return;
80
81 StringRef ScalarName = CI.getCalledFunction()->getName();
82
83 // Nothing to be done if the TLI thinks the function is not
84 // vectorizable.
85 if (!TLI.isFunctionVectorizable(ScalarName))
86 return;
87 SmallVector<std::string, 8> Mappings;
88 VFABI::getVectorVariantNames(CI, Mappings);
89 Module *M = CI.getModule();
90 const SetVector<StringRef> OriginalSetOfMappings(Mappings.begin(),
91 Mappings.end());
92 // All VFs in the TLI are powers of 2.
93 for (unsigned VF = 2, WidestVF = TLI.getWidestVF(ScalarName); VF <= WidestVF;
94 VF *= 2) {
95 const std::string TLIName =
96 std::string(TLI.getVectorizedFunction(ScalarName, VF));
97 if (!TLIName.empty()) {
98 std::string MangledName = VFABI::mangleTLIVectorName(
99 TLIName, ScalarName, CI.getNumArgOperands(), VF);
100 if (!OriginalSetOfMappings.count(MangledName)) {
101 Mappings.push_back(MangledName);
102 ++NumCallInjected;
103 }
104 Function *VariantF = M->getFunction(TLIName);
105 if (!VariantF)
106 addVariantDeclaration(CI, VF, TLIName);
107 }
108 }
109
110 VFABI::setVectorVariantNames(&CI, Mappings);
111 }
112
runImpl(const TargetLibraryInfo & TLI,Function & F)113 static bool runImpl(const TargetLibraryInfo &TLI, Function &F) {
114 for (auto &I : instructions(F))
115 if (auto CI = dyn_cast<CallInst>(&I))
116 addMappingsFromTLI(TLI, *CI);
117 // Even if the pass adds IR attributes, the analyses are preserved.
118 return false;
119 }
120
121 ////////////////////////////////////////////////////////////////////////////////
122 // New pass manager implementation.
123 ////////////////////////////////////////////////////////////////////////////////
run(Function & F,FunctionAnalysisManager & AM)124 PreservedAnalyses InjectTLIMappings::run(Function &F,
125 FunctionAnalysisManager &AM) {
126 const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(F);
127 runImpl(TLI, F);
128 // Even if the pass adds IR attributes, the analyses are preserved.
129 return PreservedAnalyses::all();
130 }
131
132 ////////////////////////////////////////////////////////////////////////////////
133 // Legacy PM Implementation.
134 ////////////////////////////////////////////////////////////////////////////////
runOnFunction(Function & F)135 bool InjectTLIMappingsLegacy::runOnFunction(Function &F) {
136 const TargetLibraryInfo &TLI =
137 getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
138 return runImpl(TLI, F);
139 }
140
getAnalysisUsage(AnalysisUsage & AU) const141 void InjectTLIMappingsLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
142 AU.setPreservesCFG();
143 AU.addRequired<TargetLibraryInfoWrapperPass>();
144 AU.addPreserved<TargetLibraryInfoWrapperPass>();
145 AU.addPreserved<ScalarEvolutionWrapperPass>();
146 AU.addPreserved<AAResultsWrapperPass>();
147 AU.addPreserved<LoopAccessLegacyAnalysis>();
148 AU.addPreserved<DemandedBitsWrapperPass>();
149 AU.addPreserved<OptimizationRemarkEmitterWrapperPass>();
150 AU.addPreserved<GlobalsAAWrapperPass>();
151 }
152
153 ////////////////////////////////////////////////////////////////////////////////
154 // Legacy Pass manager initialization
155 ////////////////////////////////////////////////////////////////////////////////
156 char InjectTLIMappingsLegacy::ID = 0;
157
158 INITIALIZE_PASS_BEGIN(InjectTLIMappingsLegacy, DEBUG_TYPE,
159 "Inject TLI Mappings", false, false)
INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)160 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
161 INITIALIZE_PASS_END(InjectTLIMappingsLegacy, DEBUG_TYPE, "Inject TLI Mappings",
162 false, false)
163
164 FunctionPass *llvm::createInjectTLIMappingsLegacyPass() {
165 return new InjectTLIMappingsLegacy();
166 }
167