1 /*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the
6 * "Software"), to deal in the Software without restriction, including
7 * without limitation the rights to use, copy, modify, merge, publish,
8 * distribute, sub license, and/or sell copies of the Software, and to
9 * permit persons to whom the Software is furnished to do so, subject to
10 * the following conditions:
11 *
12 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
13 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
14 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
15 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
16 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
17 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
18 * USE OR OTHER DEALINGS IN THE SOFTWARE.
19 *
20 * The above copyright notice and this permission notice (including the
21 * next paragraph) shall be included in all copies or substantial portions
22 * of the Software.
23 *
24 */
25 /* based on pieces from si_pipe.c and radeon_llvm_emit.c */
26 #include "ac_llvm_util.h"
27
28 #include "ac_llvm_build.h"
29 #include "c11/threads.h"
30 #include "gallivm/lp_bld_misc.h"
31 #include "util/bitscan.h"
32 #include "util/u_math.h"
33 #include <llvm-c/Core.h>
34 #include <llvm-c/Support.h>
35 #include <llvm-c/Transforms/IPO.h>
36 #include <llvm-c/Transforms/Scalar.h>
37 #include <llvm-c/Transforms/Utils.h>
38
39 #include <assert.h>
40 #include <stdio.h>
41 #include <string.h>
42
ac_init_llvm_target()43 static void ac_init_llvm_target()
44 {
45 LLVMInitializeAMDGPUTargetInfo();
46 LLVMInitializeAMDGPUTarget();
47 LLVMInitializeAMDGPUTargetMC();
48 LLVMInitializeAMDGPUAsmPrinter();
49
50 /* For inline assembly. */
51 LLVMInitializeAMDGPUAsmParser();
52
53 /* For ACO disassembly. */
54 LLVMInitializeAMDGPUDisassembler();
55
56 /* Workaround for bug in llvm 4.0 that causes image intrinsics
57 * to disappear.
58 * https://reviews.llvm.org/D26348
59 *
60 * "mesa" is the prefix for error messages.
61 *
62 * -global-isel-abort=2 is a no-op unless global isel has been enabled.
63 * This option tells the backend to fall-back to SelectionDAG and print
64 * a diagnostic message if global isel fails.
65 */
66 const char *argv[] = {
67 "mesa",
68 "-simplifycfg-sink-common=false",
69 "-global-isel-abort=2",
70 #if LLVM_VERSION_MAJOR >= 10
71 /* Atomic optimizations require LLVM 10.0 for gfx10 support. */
72 "-amdgpu-atomic-optimizations=true",
73 #endif
74 #if LLVM_VERSION_MAJOR >= 11
75 /* This was disabled by default in: https://reviews.llvm.org/D77228 */
76 "-structurizecfg-skip-uniform-regions",
77 #endif
78 };
79 LLVMParseCommandLineOptions(ARRAY_SIZE(argv), argv, NULL);
80 }
81
ac_init_shared_llvm_once(void)82 PUBLIC void ac_init_shared_llvm_once(void)
83 {
84 static once_flag ac_init_llvm_target_once_flag = ONCE_FLAG_INIT;
85 call_once(&ac_init_llvm_target_once_flag, ac_init_llvm_target);
86 }
87
88 #if !LLVM_IS_SHARED
89 static once_flag ac_init_static_llvm_target_once_flag = ONCE_FLAG_INIT;
ac_init_static_llvm_once(void)90 static void ac_init_static_llvm_once(void)
91 {
92 call_once(&ac_init_static_llvm_target_once_flag, ac_init_llvm_target);
93 }
94 #endif
95
ac_init_llvm_once(void)96 void ac_init_llvm_once(void)
97 {
98 #if LLVM_IS_SHARED
99 ac_init_shared_llvm_once();
100 #else
101 ac_init_static_llvm_once();
102 #endif
103 }
104
ac_get_llvm_target(const char * triple)105 static LLVMTargetRef ac_get_llvm_target(const char *triple)
106 {
107 LLVMTargetRef target = NULL;
108 char *err_message = NULL;
109
110 if (LLVMGetTargetFromTriple(triple, &target, &err_message)) {
111 fprintf(stderr, "Cannot find target for triple %s ", triple);
112 if (err_message) {
113 fprintf(stderr, "%s\n", err_message);
114 }
115 LLVMDisposeMessage(err_message);
116 return NULL;
117 }
118 return target;
119 }
120
ac_get_llvm_processor_name(enum radeon_family family)121 const char *ac_get_llvm_processor_name(enum radeon_family family)
122 {
123 switch (family) {
124 case CHIP_TAHITI:
125 return "tahiti";
126 case CHIP_PITCAIRN:
127 return "pitcairn";
128 case CHIP_VERDE:
129 return "verde";
130 case CHIP_OLAND:
131 return "oland";
132 case CHIP_HAINAN:
133 return "hainan";
134 case CHIP_BONAIRE:
135 return "bonaire";
136 case CHIP_KABINI:
137 return "kabini";
138 case CHIP_KAVERI:
139 return "kaveri";
140 case CHIP_HAWAII:
141 return "hawaii";
142 case CHIP_TONGA:
143 return "tonga";
144 case CHIP_ICELAND:
145 return "iceland";
146 case CHIP_CARRIZO:
147 return "carrizo";
148 case CHIP_FIJI:
149 return "fiji";
150 case CHIP_STONEY:
151 return "stoney";
152 case CHIP_POLARIS10:
153 return "polaris10";
154 case CHIP_POLARIS11:
155 case CHIP_POLARIS12:
156 case CHIP_VEGAM:
157 return "polaris11";
158 case CHIP_VEGA10:
159 return "gfx900";
160 case CHIP_RAVEN:
161 return "gfx902";
162 case CHIP_VEGA12:
163 return "gfx904";
164 case CHIP_VEGA20:
165 return "gfx906";
166 case CHIP_RAVEN2:
167 case CHIP_RENOIR:
168 return "gfx909";
169 case CHIP_ARCTURUS:
170 return "gfx908";
171 case CHIP_NAVI10:
172 return "gfx1010";
173 case CHIP_NAVI12:
174 return "gfx1011";
175 case CHIP_NAVI14:
176 return "gfx1012";
177 case CHIP_SIENNA_CICHLID:
178 case CHIP_NAVY_FLOUNDER:
179 case CHIP_DIMGREY_CAVEFISH:
180 case CHIP_VANGOGH:
181 return "gfx1030";
182 default:
183 return "";
184 }
185 }
186
ac_create_target_machine(enum radeon_family family,enum ac_target_machine_options tm_options,LLVMCodeGenOptLevel level,const char ** out_triple)187 static LLVMTargetMachineRef ac_create_target_machine(enum radeon_family family,
188 enum ac_target_machine_options tm_options,
189 LLVMCodeGenOptLevel level,
190 const char **out_triple)
191 {
192 assert(family >= CHIP_TAHITI);
193 char features[256];
194 const char *triple = (tm_options & AC_TM_SUPPORTS_SPILL) ? "amdgcn-mesa-mesa3d" : "amdgcn--";
195 LLVMTargetRef target = ac_get_llvm_target(triple);
196
197 snprintf(features, sizeof(features), "+DumpCode%s%s%s%s%s",
198 LLVM_VERSION_MAJOR >= 11 ? "" : ",-fp32-denormals,+fp64-denormals",
199 family >= CHIP_NAVI10 && !(tm_options & AC_TM_WAVE32)
200 ? ",+wavefrontsize64,-wavefrontsize32"
201 : "",
202 family <= CHIP_NAVI14 && tm_options & AC_TM_FORCE_ENABLE_XNACK ? ",+xnack" : "",
203 family <= CHIP_NAVI14 && tm_options & AC_TM_FORCE_DISABLE_XNACK ? ",-xnack" : "",
204 tm_options & AC_TM_PROMOTE_ALLOCA_TO_SCRATCH ? ",-promote-alloca" : "");
205
206 LLVMTargetMachineRef tm =
207 LLVMCreateTargetMachine(target, triple, ac_get_llvm_processor_name(family), features, level,
208 LLVMRelocDefault, LLVMCodeModelDefault);
209
210 if (out_triple)
211 *out_triple = triple;
212 if (tm_options & AC_TM_ENABLE_GLOBAL_ISEL)
213 ac_enable_global_isel(tm);
214 return tm;
215 }
216
ac_create_passmgr(LLVMTargetLibraryInfoRef target_library_info,bool check_ir)217 static LLVMPassManagerRef ac_create_passmgr(LLVMTargetLibraryInfoRef target_library_info,
218 bool check_ir)
219 {
220 LLVMPassManagerRef passmgr = LLVMCreatePassManager();
221 if (!passmgr)
222 return NULL;
223
224 if (target_library_info)
225 LLVMAddTargetLibraryInfo(target_library_info, passmgr);
226
227 if (check_ir)
228 LLVMAddVerifierPass(passmgr);
229 LLVMAddAlwaysInlinerPass(passmgr);
230 /* Normally, the pass manager runs all passes on one function before
231 * moving onto another. Adding a barrier no-op pass forces the pass
232 * manager to run the inliner on all functions first, which makes sure
233 * that the following passes are only run on the remaining non-inline
234 * function, so it removes useless work done on dead inline functions.
235 */
236 ac_llvm_add_barrier_noop_pass(passmgr);
237 /* This pass should eliminate all the load and store instructions. */
238 LLVMAddPromoteMemoryToRegisterPass(passmgr);
239 LLVMAddScalarReplAggregatesPass(passmgr);
240 LLVMAddLICMPass(passmgr);
241 LLVMAddAggressiveDCEPass(passmgr);
242 LLVMAddCFGSimplificationPass(passmgr);
243 /* This is recommended by the instruction combining pass. */
244 LLVMAddEarlyCSEMemSSAPass(passmgr);
245 LLVMAddInstructionCombiningPass(passmgr);
246 return passmgr;
247 }
248
attr_to_str(enum ac_func_attr attr)249 static const char *attr_to_str(enum ac_func_attr attr)
250 {
251 switch (attr) {
252 case AC_FUNC_ATTR_ALWAYSINLINE:
253 return "alwaysinline";
254 case AC_FUNC_ATTR_INREG:
255 return "inreg";
256 case AC_FUNC_ATTR_NOALIAS:
257 return "noalias";
258 case AC_FUNC_ATTR_NOUNWIND:
259 return "nounwind";
260 case AC_FUNC_ATTR_READNONE:
261 return "readnone";
262 case AC_FUNC_ATTR_READONLY:
263 return "readonly";
264 case AC_FUNC_ATTR_WRITEONLY:
265 return "writeonly";
266 case AC_FUNC_ATTR_INACCESSIBLE_MEM_ONLY:
267 return "inaccessiblememonly";
268 case AC_FUNC_ATTR_CONVERGENT:
269 return "convergent";
270 default:
271 fprintf(stderr, "Unhandled function attribute: %x\n", attr);
272 return 0;
273 }
274 }
275
ac_add_function_attr(LLVMContextRef ctx,LLVMValueRef function,int attr_idx,enum ac_func_attr attr)276 void ac_add_function_attr(LLVMContextRef ctx, LLVMValueRef function, int attr_idx,
277 enum ac_func_attr attr)
278 {
279 const char *attr_name = attr_to_str(attr);
280 unsigned kind_id = LLVMGetEnumAttributeKindForName(attr_name, strlen(attr_name));
281 LLVMAttributeRef llvm_attr = LLVMCreateEnumAttribute(ctx, kind_id, 0);
282
283 if (LLVMIsAFunction(function))
284 LLVMAddAttributeAtIndex(function, attr_idx, llvm_attr);
285 else
286 LLVMAddCallSiteAttribute(function, attr_idx, llvm_attr);
287 }
288
ac_add_func_attributes(LLVMContextRef ctx,LLVMValueRef function,unsigned attrib_mask)289 void ac_add_func_attributes(LLVMContextRef ctx, LLVMValueRef function, unsigned attrib_mask)
290 {
291 attrib_mask |= AC_FUNC_ATTR_NOUNWIND;
292 attrib_mask &= ~AC_FUNC_ATTR_LEGACY;
293
294 while (attrib_mask) {
295 enum ac_func_attr attr = 1u << u_bit_scan(&attrib_mask);
296 ac_add_function_attr(ctx, function, -1, attr);
297 }
298 }
299
ac_dump_module(LLVMModuleRef module)300 void ac_dump_module(LLVMModuleRef module)
301 {
302 char *str = LLVMPrintModuleToString(module);
303 fprintf(stderr, "%s", str);
304 LLVMDisposeMessage(str);
305 }
306
ac_llvm_add_target_dep_function_attr(LLVMValueRef F,const char * name,unsigned value)307 void ac_llvm_add_target_dep_function_attr(LLVMValueRef F, const char *name, unsigned value)
308 {
309 char str[16];
310
311 snprintf(str, sizeof(str), "0x%x", value);
312 LLVMAddTargetDependentFunctionAttr(F, name, str);
313 }
314
ac_llvm_set_workgroup_size(LLVMValueRef F,unsigned size)315 void ac_llvm_set_workgroup_size(LLVMValueRef F, unsigned size)
316 {
317 if (!size)
318 return;
319
320 char str[32];
321 snprintf(str, sizeof(str), "%u,%u", size, size);
322 LLVMAddTargetDependentFunctionAttr(F, "amdgpu-flat-work-group-size", str);
323 }
324
ac_count_scratch_private_memory(LLVMValueRef function)325 unsigned ac_count_scratch_private_memory(LLVMValueRef function)
326 {
327 unsigned private_mem_vgprs = 0;
328
329 /* Process all LLVM instructions. */
330 LLVMBasicBlockRef bb = LLVMGetFirstBasicBlock(function);
331 while (bb) {
332 LLVMValueRef next = LLVMGetFirstInstruction(bb);
333
334 while (next) {
335 LLVMValueRef inst = next;
336 next = LLVMGetNextInstruction(next);
337
338 if (LLVMGetInstructionOpcode(inst) != LLVMAlloca)
339 continue;
340
341 LLVMTypeRef type = LLVMGetElementType(LLVMTypeOf(inst));
342 /* No idea why LLVM aligns allocas to 4 elements. */
343 unsigned alignment = LLVMGetAlignment(inst);
344 unsigned dw_size = align(ac_get_type_size(type) / 4, alignment);
345 private_mem_vgprs += dw_size;
346 }
347 bb = LLVMGetNextBasicBlock(bb);
348 }
349
350 return private_mem_vgprs;
351 }
352
ac_init_llvm_compiler(struct ac_llvm_compiler * compiler,enum radeon_family family,enum ac_target_machine_options tm_options)353 bool ac_init_llvm_compiler(struct ac_llvm_compiler *compiler, enum radeon_family family,
354 enum ac_target_machine_options tm_options)
355 {
356 const char *triple;
357 memset(compiler, 0, sizeof(*compiler));
358
359 compiler->tm = ac_create_target_machine(family, tm_options, LLVMCodeGenLevelDefault, &triple);
360 if (!compiler->tm)
361 return false;
362
363 if (tm_options & AC_TM_CREATE_LOW_OPT) {
364 compiler->low_opt_tm =
365 ac_create_target_machine(family, tm_options, LLVMCodeGenLevelLess, NULL);
366 if (!compiler->low_opt_tm)
367 goto fail;
368 }
369
370 if (family >= CHIP_NAVI10) {
371 assert(!(tm_options & AC_TM_CREATE_LOW_OPT));
372 compiler->tm_wave32 =
373 ac_create_target_machine(family, tm_options | AC_TM_WAVE32, LLVMCodeGenLevelDefault, NULL);
374 if (!compiler->tm_wave32)
375 goto fail;
376 }
377
378 compiler->target_library_info = ac_create_target_library_info(triple);
379 if (!compiler->target_library_info)
380 goto fail;
381
382 compiler->passmgr =
383 ac_create_passmgr(compiler->target_library_info, tm_options & AC_TM_CHECK_IR);
384 if (!compiler->passmgr)
385 goto fail;
386
387 return true;
388 fail:
389 ac_destroy_llvm_compiler(compiler);
390 return false;
391 }
392
ac_destroy_llvm_compiler(struct ac_llvm_compiler * compiler)393 void ac_destroy_llvm_compiler(struct ac_llvm_compiler *compiler)
394 {
395 ac_destroy_llvm_passes(compiler->passes);
396 ac_destroy_llvm_passes(compiler->passes_wave32);
397 ac_destroy_llvm_passes(compiler->low_opt_passes);
398
399 if (compiler->passmgr)
400 LLVMDisposePassManager(compiler->passmgr);
401 if (compiler->target_library_info)
402 ac_dispose_target_library_info(compiler->target_library_info);
403 if (compiler->low_opt_tm)
404 LLVMDisposeTargetMachine(compiler->low_opt_tm);
405 if (compiler->tm)
406 LLVMDisposeTargetMachine(compiler->tm);
407 if (compiler->tm_wave32)
408 LLVMDisposeTargetMachine(compiler->tm_wave32);
409 }
410