1 //===-- AMDGPULowerIntrinsics.cpp -----------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "AMDGPU.h"
10 #include "AMDGPUSubtarget.h"
11 #include "llvm/CodeGen/TargetPassConfig.h"
12 #include "llvm/Analysis/TargetTransformInfo.h"
13 #include "llvm/IR/Constants.h"
14 #include "llvm/IR/Instructions.h"
15 #include "llvm/IR/IntrinsicInst.h"
16 #include "llvm/IR/Module.h"
17 #include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
18
19 #define DEBUG_TYPE "amdgpu-lower-intrinsics"
20
21 using namespace llvm;
22
23 namespace {
24
25 static int MaxStaticSize;
26
27 static cl::opt<int, true> MemIntrinsicExpandSizeThresholdOpt(
28 "amdgpu-mem-intrinsic-expand-size",
29 cl::desc("Set minimum mem intrinsic size to expand in IR"),
30 cl::location(MaxStaticSize),
31 cl::init(1024),
32 cl::Hidden);
33
34
35 class AMDGPULowerIntrinsics : public ModulePass {
36 private:
37 bool makeLIDRangeMetadata(Function &F) const;
38
39 public:
40 static char ID;
41
AMDGPULowerIntrinsics()42 AMDGPULowerIntrinsics() : ModulePass(ID) {}
43
44 bool runOnModule(Module &M) override;
45 bool expandMemIntrinsicUses(Function &F);
getPassName() const46 StringRef getPassName() const override {
47 return "AMDGPU Lower Intrinsics";
48 }
49
getAnalysisUsage(AnalysisUsage & AU) const50 void getAnalysisUsage(AnalysisUsage &AU) const override {
51 AU.addRequired<TargetTransformInfoWrapperPass>();
52 }
53 };
54
55 }
56
57 char AMDGPULowerIntrinsics::ID = 0;
58
59 char &llvm::AMDGPULowerIntrinsicsID = AMDGPULowerIntrinsics::ID;
60
61 INITIALIZE_PASS(AMDGPULowerIntrinsics, DEBUG_TYPE, "Lower intrinsics", false,
62 false)
63
64 // TODO: Should refine based on estimated number of accesses (e.g. does it
65 // require splitting based on alignment)
shouldExpandOperationWithSize(Value * Size)66 static bool shouldExpandOperationWithSize(Value *Size) {
67 ConstantInt *CI = dyn_cast<ConstantInt>(Size);
68 return !CI || (CI->getSExtValue() > MaxStaticSize);
69 }
70
expandMemIntrinsicUses(Function & F)71 bool AMDGPULowerIntrinsics::expandMemIntrinsicUses(Function &F) {
72 Intrinsic::ID ID = F.getIntrinsicID();
73 bool Changed = false;
74
75 for (auto I = F.user_begin(), E = F.user_end(); I != E;) {
76 Instruction *Inst = cast<Instruction>(*I);
77 ++I;
78
79 switch (ID) {
80 case Intrinsic::memcpy: {
81 auto *Memcpy = cast<MemCpyInst>(Inst);
82 if (shouldExpandOperationWithSize(Memcpy->getLength())) {
83 Function *ParentFunc = Memcpy->getParent()->getParent();
84 const TargetTransformInfo &TTI =
85 getAnalysis<TargetTransformInfoWrapperPass>().getTTI(*ParentFunc);
86 expandMemCpyAsLoop(Memcpy, TTI);
87 Changed = true;
88 Memcpy->eraseFromParent();
89 }
90
91 break;
92 }
93 case Intrinsic::memmove: {
94 auto *Memmove = cast<MemMoveInst>(Inst);
95 if (shouldExpandOperationWithSize(Memmove->getLength())) {
96 expandMemMoveAsLoop(Memmove);
97 Changed = true;
98 Memmove->eraseFromParent();
99 }
100
101 break;
102 }
103 case Intrinsic::memset: {
104 auto *Memset = cast<MemSetInst>(Inst);
105 if (shouldExpandOperationWithSize(Memset->getLength())) {
106 expandMemSetAsLoop(Memset);
107 Changed = true;
108 Memset->eraseFromParent();
109 }
110
111 break;
112 }
113 default:
114 break;
115 }
116 }
117
118 return Changed;
119 }
120
makeLIDRangeMetadata(Function & F) const121 bool AMDGPULowerIntrinsics::makeLIDRangeMetadata(Function &F) const {
122 auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
123 if (!TPC)
124 return false;
125
126 const TargetMachine &TM = TPC->getTM<TargetMachine>();
127 bool Changed = false;
128
129 for (auto *U : F.users()) {
130 auto *CI = dyn_cast<CallInst>(U);
131 if (!CI)
132 continue;
133
134 Function *Caller = CI->getParent()->getParent();
135 const AMDGPUSubtarget &ST = AMDGPUSubtarget::get(TM, *Caller);
136 Changed |= ST.makeLIDRangeMetadata(CI);
137 }
138 return Changed;
139 }
140
runOnModule(Module & M)141 bool AMDGPULowerIntrinsics::runOnModule(Module &M) {
142 bool Changed = false;
143
144 for (Function &F : M) {
145 if (!F.isDeclaration())
146 continue;
147
148 switch (F.getIntrinsicID()) {
149 case Intrinsic::memcpy:
150 case Intrinsic::memmove:
151 case Intrinsic::memset:
152 if (expandMemIntrinsicUses(F))
153 Changed = true;
154 break;
155
156 case Intrinsic::amdgcn_workitem_id_x:
157 case Intrinsic::r600_read_tidig_x:
158 case Intrinsic::amdgcn_workitem_id_y:
159 case Intrinsic::r600_read_tidig_y:
160 case Intrinsic::amdgcn_workitem_id_z:
161 case Intrinsic::r600_read_tidig_z:
162 case Intrinsic::r600_read_local_size_x:
163 case Intrinsic::r600_read_local_size_y:
164 case Intrinsic::r600_read_local_size_z:
165 Changed |= makeLIDRangeMetadata(F);
166 break;
167
168 default:
169 break;
170 }
171 }
172
173 return Changed;
174 }
175
createAMDGPULowerIntrinsicsPass()176 ModulePass *llvm::createAMDGPULowerIntrinsicsPass() {
177 return new AMDGPULowerIntrinsics();
178 }
179