1 /*
2  * Copyright 2016 Bas Nieuwenhuizen
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the
6  * "Software"), to deal in the Software without restriction, including
7  * without limitation the rights to use, copy, modify, merge, publish,
8  * distribute, sub license, and/or sell copies of the Software, and to
9  * permit persons to whom the Software is furnished to do so, subject to
10  * the following conditions:
11  *
12  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
13  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
14  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
15  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
16  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
17  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
18  * USE OR OTHER DEALINGS IN THE SOFTWARE.
19  *
20  * The above copyright notice and this permission notice (including the
21  * next paragraph) shall be included in all copies or substantial portions
22  * of the Software.
23  *
24  */
25 #ifndef AC_LLVM_BUILD_H
26 #define AC_LLVM_BUILD_H
27 
28 #include "ac_shader_abi.h"
29 #include "ac_shader_args.h"
30 #include "ac_shader_util.h"
31 #include "amd_family.h"
32 #include "compiler/nir/nir.h"
33 #include <llvm-c/Core.h>
34 
35 #include <stdbool.h>
36 
37 #ifdef __cplusplus
38 extern "C" {
39 #endif
40 
41 enum
42 {
43    AC_ADDR_SPACE_FLAT = 0, /* Slower than global. */
44    AC_ADDR_SPACE_GLOBAL = 1,
45    AC_ADDR_SPACE_GDS = 2,
46    AC_ADDR_SPACE_LDS = 3,
47    AC_ADDR_SPACE_CONST = 4,       /* Global allowing SMEM. */
48    AC_ADDR_SPACE_CONST_32BIT = 6, /* same as CONST, but the pointer type has 32 bits */
49 };
50 
51 #define AC_WAIT_LGKM   (1 << 0) /* LDS, GDS, constant, message */
52 #define AC_WAIT_VLOAD  (1 << 1) /* VMEM load/sample instructions */
53 #define AC_WAIT_VSTORE (1 << 2) /* VMEM store instructions */
54 
55 struct ac_llvm_flow;
56 struct ac_llvm_compiler;
57 enum ac_float_mode;
58 
59 struct ac_llvm_flow_state {
60    struct ac_llvm_flow *stack;
61    unsigned depth_max;
62    unsigned depth;
63 };
64 
65 struct ac_llvm_context {
66    LLVMContextRef context;
67    LLVMModuleRef module;
68    LLVMBuilderRef builder;
69 
70    LLVMValueRef main_function;
71 
72    LLVMTypeRef voidt;
73    LLVMTypeRef i1;
74    LLVMTypeRef i8;
75    LLVMTypeRef i16;
76    LLVMTypeRef i32;
77    LLVMTypeRef i64;
78    LLVMTypeRef i128;
79    LLVMTypeRef intptr;
80    LLVMTypeRef f16;
81    LLVMTypeRef f32;
82    LLVMTypeRef f64;
83    LLVMTypeRef v2i16;
84    LLVMTypeRef v4i16;
85    LLVMTypeRef v2f16;
86    LLVMTypeRef v4f16;
87    LLVMTypeRef v2i32;
88    LLVMTypeRef v3i32;
89    LLVMTypeRef v4i32;
90    LLVMTypeRef v2f32;
91    LLVMTypeRef v3f32;
92    LLVMTypeRef v4f32;
93    LLVMTypeRef v8i32;
94    LLVMTypeRef iN_wavemask;
95    LLVMTypeRef iN_ballotmask;
96 
97    LLVMValueRef i8_0;
98    LLVMValueRef i8_1;
99    LLVMValueRef i16_0;
100    LLVMValueRef i16_1;
101    LLVMValueRef i32_0;
102    LLVMValueRef i32_1;
103    LLVMValueRef i64_0;
104    LLVMValueRef i64_1;
105    LLVMValueRef i128_0;
106    LLVMValueRef i128_1;
107    LLVMValueRef f16_0;
108    LLVMValueRef f16_1;
109    LLVMValueRef f32_0;
110    LLVMValueRef f32_1;
111    LLVMValueRef f64_0;
112    LLVMValueRef f64_1;
113    LLVMValueRef i1true;
114    LLVMValueRef i1false;
115 
116    /* Temporary helper to implement demote_to_helper:
117     * True = live lanes
118     * False = demoted lanes
119     */
120    LLVMValueRef postponed_kill;
121 
122    /* Since ac_nir_translate makes a local copy of ac_llvm_context, there
123     * are two ac_llvm_contexts. Declare a pointer here, so that the control
124     * flow stack is shared by both ac_llvm_contexts.
125     */
126    struct ac_llvm_flow_state *flow;
127 
128    unsigned range_md_kind;
129    unsigned invariant_load_md_kind;
130    unsigned uniform_md_kind;
131    LLVMValueRef empty_md;
132 
133    enum chip_class chip_class;
134    enum radeon_family family;
135 
136    unsigned wave_size;
137    unsigned ballot_mask_bits;
138 
139    unsigned float_mode;
140 
141    LLVMValueRef lds;
142 };
143 
144 void ac_llvm_context_init(struct ac_llvm_context *ctx, struct ac_llvm_compiler *compiler,
145                           enum chip_class chip_class, enum radeon_family family,
146                           enum ac_float_mode float_mode, unsigned wave_size,
147                           unsigned ballot_mask_bits);
148 
149 void ac_llvm_context_dispose(struct ac_llvm_context *ctx);
150 
151 int ac_get_llvm_num_components(LLVMValueRef value);
152 
153 int ac_get_elem_bits(struct ac_llvm_context *ctx, LLVMTypeRef type);
154 
155 LLVMValueRef ac_llvm_extract_elem(struct ac_llvm_context *ac, LLVMValueRef value, int index);
156 
157 unsigned ac_get_type_size(LLVMTypeRef type);
158 
159 LLVMTypeRef ac_to_integer_type(struct ac_llvm_context *ctx, LLVMTypeRef t);
160 LLVMValueRef ac_to_integer(struct ac_llvm_context *ctx, LLVMValueRef v);
161 LLVMValueRef ac_to_integer_or_pointer(struct ac_llvm_context *ctx, LLVMValueRef v);
162 LLVMTypeRef ac_to_float_type(struct ac_llvm_context *ctx, LLVMTypeRef t);
163 LLVMValueRef ac_to_float(struct ac_llvm_context *ctx, LLVMValueRef v);
164 
165 LLVMValueRef ac_build_intrinsic(struct ac_llvm_context *ctx, const char *name,
166                                 LLVMTypeRef return_type, LLVMValueRef *params, unsigned param_count,
167                                 unsigned attrib_mask);
168 
169 void ac_build_type_name_for_intr(LLVMTypeRef type, char *buf, unsigned bufsize);
170 
171 LLVMValueRef ac_build_phi(struct ac_llvm_context *ctx, LLVMTypeRef type, unsigned count_incoming,
172                           LLVMValueRef *values, LLVMBasicBlockRef *blocks);
173 
174 void ac_build_s_barrier(struct ac_llvm_context *ctx);
175 void ac_build_optimization_barrier(struct ac_llvm_context *ctx, LLVMValueRef *pvgpr);
176 
177 LLVMValueRef ac_build_shader_clock(struct ac_llvm_context *ctx, nir_scope scope);
178 
179 LLVMValueRef ac_build_ballot(struct ac_llvm_context *ctx, LLVMValueRef value);
180 LLVMValueRef ac_get_i1_sgpr_mask(struct ac_llvm_context *ctx, LLVMValueRef value);
181 
182 LLVMValueRef ac_build_vote_all(struct ac_llvm_context *ctx, LLVMValueRef value);
183 
184 LLVMValueRef ac_build_vote_any(struct ac_llvm_context *ctx, LLVMValueRef value);
185 
186 LLVMValueRef ac_build_vote_eq(struct ac_llvm_context *ctx, LLVMValueRef value);
187 
188 LLVMValueRef ac_build_varying_gather_values(struct ac_llvm_context *ctx, LLVMValueRef *values,
189                                             unsigned value_count, unsigned component);
190 
191 LLVMValueRef ac_build_gather_values_extended(struct ac_llvm_context *ctx, LLVMValueRef *values,
192                                              unsigned value_count, unsigned value_stride, bool load,
193                                              bool always_vector);
194 LLVMValueRef ac_build_gather_values(struct ac_llvm_context *ctx, LLVMValueRef *values,
195                                     unsigned value_count);
196 
197 LLVMValueRef ac_extract_components(struct ac_llvm_context *ctx, LLVMValueRef value, unsigned start,
198                                    unsigned channels);
199 
200 LLVMValueRef ac_build_expand_to_vec4(struct ac_llvm_context *ctx, LLVMValueRef value,
201                                      unsigned num_channels);
202 LLVMValueRef ac_build_round(struct ac_llvm_context *ctx, LLVMValueRef value);
203 
204 LLVMValueRef ac_build_fdiv(struct ac_llvm_context *ctx, LLVMValueRef num, LLVMValueRef den);
205 
206 LLVMValueRef ac_build_fast_udiv(struct ac_llvm_context *ctx, LLVMValueRef num,
207                                 LLVMValueRef multiplier, LLVMValueRef pre_shift,
208                                 LLVMValueRef post_shift, LLVMValueRef increment);
209 LLVMValueRef ac_build_fast_udiv_nuw(struct ac_llvm_context *ctx, LLVMValueRef num,
210                                     LLVMValueRef multiplier, LLVMValueRef pre_shift,
211                                     LLVMValueRef post_shift, LLVMValueRef increment);
212 LLVMValueRef ac_build_fast_udiv_u31_d_not_one(struct ac_llvm_context *ctx, LLVMValueRef num,
213                                               LLVMValueRef multiplier, LLVMValueRef post_shift);
214 
215 void ac_prepare_cube_coords(struct ac_llvm_context *ctx, bool is_deriv, bool is_array, bool is_lod,
216                             LLVMValueRef *coords_arg, LLVMValueRef *derivs_arg);
217 
218 LLVMValueRef ac_build_fs_interp(struct ac_llvm_context *ctx, LLVMValueRef llvm_chan,
219                                 LLVMValueRef attr_number, LLVMValueRef params, LLVMValueRef i,
220                                 LLVMValueRef j);
221 
222 LLVMValueRef ac_build_fs_interp_f16(struct ac_llvm_context *ctx, LLVMValueRef llvm_chan,
223                                     LLVMValueRef attr_number, LLVMValueRef params, LLVMValueRef i,
224                                     LLVMValueRef j);
225 
226 LLVMValueRef ac_build_fs_interp_mov(struct ac_llvm_context *ctx, LLVMValueRef parameter,
227                                     LLVMValueRef llvm_chan, LLVMValueRef attr_number,
228                                     LLVMValueRef params);
229 
230 LLVMValueRef ac_build_gep_ptr(struct ac_llvm_context *ctx, LLVMValueRef base_ptr,
231                               LLVMValueRef index);
232 
233 LLVMValueRef ac_build_gep0(struct ac_llvm_context *ctx, LLVMValueRef base_ptr, LLVMValueRef index);
234 LLVMValueRef ac_build_pointer_add(struct ac_llvm_context *ctx, LLVMValueRef ptr,
235                                   LLVMValueRef index);
236 
237 void ac_build_indexed_store(struct ac_llvm_context *ctx, LLVMValueRef base_ptr, LLVMValueRef index,
238                             LLVMValueRef value);
239 
240 LLVMValueRef ac_build_load(struct ac_llvm_context *ctx, LLVMValueRef base_ptr, LLVMValueRef index);
241 LLVMValueRef ac_build_load_invariant(struct ac_llvm_context *ctx, LLVMValueRef base_ptr,
242                                      LLVMValueRef index);
243 LLVMValueRef ac_build_load_to_sgpr(struct ac_llvm_context *ctx, LLVMValueRef base_ptr,
244                                    LLVMValueRef index);
245 LLVMValueRef ac_build_load_to_sgpr_uint_wraparound(struct ac_llvm_context *ctx,
246                                                    LLVMValueRef base_ptr, LLVMValueRef index);
247 
248 void ac_build_buffer_store_dword(struct ac_llvm_context *ctx, LLVMValueRef rsrc, LLVMValueRef vdata,
249                                  unsigned num_channels, LLVMValueRef voffset, LLVMValueRef soffset,
250                                  unsigned inst_offset, unsigned cache_policy);
251 
252 void ac_build_buffer_store_format(struct ac_llvm_context *ctx, LLVMValueRef rsrc, LLVMValueRef data,
253                                   LLVMValueRef vindex, LLVMValueRef voffset, unsigned cache_policy);
254 
255 LLVMValueRef ac_build_buffer_load(struct ac_llvm_context *ctx, LLVMValueRef rsrc, int num_channels,
256                                   LLVMValueRef vindex, LLVMValueRef voffset, LLVMValueRef soffset,
257                                   unsigned inst_offset, unsigned cache_policy, bool can_speculate,
258                                   bool allow_smem);
259 
260 LLVMValueRef ac_build_buffer_load_format(struct ac_llvm_context *ctx, LLVMValueRef rsrc,
261                                          LLVMValueRef vindex, LLVMValueRef voffset,
262                                          unsigned num_channels, unsigned cache_policy,
263                                          bool can_speculate, bool d16);
264 
265 LLVMValueRef ac_build_tbuffer_load_short(struct ac_llvm_context *ctx, LLVMValueRef rsrc,
266                                          LLVMValueRef voffset, LLVMValueRef soffset,
267                                          LLVMValueRef immoffset, unsigned cache_policy);
268 
269 LLVMValueRef ac_build_tbuffer_load_byte(struct ac_llvm_context *ctx, LLVMValueRef rsrc,
270                                         LLVMValueRef voffset, LLVMValueRef soffset,
271                                         LLVMValueRef immoffset, unsigned cache_policy);
272 
273 LLVMValueRef ac_build_struct_tbuffer_load(struct ac_llvm_context *ctx, LLVMValueRef rsrc,
274                                           LLVMValueRef vindex, LLVMValueRef voffset,
275                                           LLVMValueRef soffset, LLVMValueRef immoffset,
276                                           unsigned num_channels, unsigned dfmt, unsigned nfmt,
277                                           unsigned cache_policy, bool can_speculate);
278 
279 LLVMValueRef ac_build_raw_tbuffer_load(struct ac_llvm_context *ctx, LLVMValueRef rsrc,
280                                        LLVMValueRef voffset, LLVMValueRef soffset,
281                                        LLVMValueRef immoffset, unsigned num_channels, unsigned dfmt,
282                                        unsigned nfmt, unsigned cache_policy, bool can_speculate);
283 
284 
285 LLVMValueRef ac_build_opencoded_load_format(struct ac_llvm_context *ctx, unsigned log_size,
286                                             unsigned num_channels, unsigned format, bool reverse,
287                                             bool known_aligned, LLVMValueRef rsrc,
288                                             LLVMValueRef vindex, LLVMValueRef voffset,
289                                             LLVMValueRef soffset, unsigned cache_policy,
290                                             bool can_speculate);
291 
292 void ac_build_tbuffer_store_short(struct ac_llvm_context *ctx, LLVMValueRef rsrc,
293                                   LLVMValueRef vdata, LLVMValueRef voffset, LLVMValueRef soffset,
294                                   unsigned cache_policy);
295 
296 void ac_build_tbuffer_store_byte(struct ac_llvm_context *ctx, LLVMValueRef rsrc, LLVMValueRef vdata,
297                                  LLVMValueRef voffset, LLVMValueRef soffset, unsigned cache_policy);
298 
299 void ac_build_struct_tbuffer_store(struct ac_llvm_context *ctx, LLVMValueRef rsrc,
300                                    LLVMValueRef vdata, LLVMValueRef vindex, LLVMValueRef voffset,
301                                    LLVMValueRef soffset, LLVMValueRef immoffset,
302                                    unsigned num_channels, unsigned dfmt, unsigned nfmt,
303                                    unsigned cache_policy);
304 
305 void ac_build_raw_tbuffer_store(struct ac_llvm_context *ctx, LLVMValueRef rsrc, LLVMValueRef vdata,
306                                 LLVMValueRef voffset, LLVMValueRef soffset, LLVMValueRef immoffset,
307                                 unsigned num_channels, unsigned dfmt, unsigned nfmt,
308                                 unsigned cache_policy);
309 
310 LLVMValueRef ac_get_thread_id(struct ac_llvm_context *ctx);
311 
312 #define AC_TID_MASK_TOP_LEFT 0xfffffffc
313 #define AC_TID_MASK_TOP      0xfffffffd
314 #define AC_TID_MASK_LEFT     0xfffffffe
315 
316 LLVMValueRef ac_build_ddxy(struct ac_llvm_context *ctx, uint32_t mask, int idx, LLVMValueRef val);
317 
318 #define AC_SENDMSG_GS           2
319 #define AC_SENDMSG_GS_DONE      3
320 #define AC_SENDMSG_GS_ALLOC_REQ 9
321 
322 #define AC_SENDMSG_GS_OP_NOP      (0 << 4)
323 #define AC_SENDMSG_GS_OP_CUT      (1 << 4)
324 #define AC_SENDMSG_GS_OP_EMIT     (2 << 4)
325 #define AC_SENDMSG_GS_OP_EMIT_CUT (3 << 4)
326 
327 void ac_build_sendmsg(struct ac_llvm_context *ctx, uint32_t msg, LLVMValueRef wave_id);
328 
329 LLVMValueRef ac_build_imsb(struct ac_llvm_context *ctx, LLVMValueRef arg, LLVMTypeRef dst_type);
330 
331 LLVMValueRef ac_build_umsb(struct ac_llvm_context *ctx, LLVMValueRef arg, LLVMTypeRef dst_type);
332 LLVMValueRef ac_build_fmin(struct ac_llvm_context *ctx, LLVMValueRef a, LLVMValueRef b);
333 LLVMValueRef ac_build_fmax(struct ac_llvm_context *ctx, LLVMValueRef a, LLVMValueRef b);
334 LLVMValueRef ac_build_imin(struct ac_llvm_context *ctx, LLVMValueRef a, LLVMValueRef b);
335 LLVMValueRef ac_build_imax(struct ac_llvm_context *ctx, LLVMValueRef a, LLVMValueRef b);
336 LLVMValueRef ac_build_umin(struct ac_llvm_context *ctx, LLVMValueRef a, LLVMValueRef b);
337 LLVMValueRef ac_build_umax(struct ac_llvm_context *ctx, LLVMValueRef a, LLVMValueRef b);
338 LLVMValueRef ac_build_clamp(struct ac_llvm_context *ctx, LLVMValueRef value);
339 
340 struct ac_export_args {
341    LLVMValueRef out[4];
342    unsigned target;
343    unsigned enabled_channels;
344    bool compr;
345    bool done;
346    bool valid_mask;
347 };
348 
349 void ac_build_export(struct ac_llvm_context *ctx, struct ac_export_args *a);
350 
351 void ac_build_export_null(struct ac_llvm_context *ctx);
352 
353 enum ac_image_opcode
354 {
355    ac_image_sample,
356    ac_image_gather4,
357    ac_image_load,
358    ac_image_load_mip,
359    ac_image_store,
360    ac_image_store_mip,
361    ac_image_get_lod,
362    ac_image_get_resinfo,
363    ac_image_atomic,
364    ac_image_atomic_cmpswap,
365 };
366 
367 enum ac_atomic_op
368 {
369    ac_atomic_swap,
370    ac_atomic_add,
371    ac_atomic_sub,
372    ac_atomic_smin,
373    ac_atomic_umin,
374    ac_atomic_smax,
375    ac_atomic_umax,
376    ac_atomic_and,
377    ac_atomic_or,
378    ac_atomic_xor,
379    ac_atomic_inc_wrap,
380    ac_atomic_dec_wrap,
381 };
382 
383 /* These cache policy bits match the definitions used by the LLVM intrinsics. */
384 enum ac_image_cache_policy
385 {
386    ac_glc = 1 << 0,      /* per-CU cache control */
387    ac_slc = 1 << 1,      /* global L2 cache control */
388    ac_dlc = 1 << 2,      /* per-shader-array cache control */
389    ac_swizzled = 1 << 3, /* the access is swizzled, disabling load/store merging */
390 };
391 
392 struct ac_image_args {
393    enum ac_image_opcode opcode : 4;
394    enum ac_atomic_op atomic : 4; /* for the ac_image_atomic opcode */
395    enum ac_image_dim dim : 3;
396    unsigned dmask : 4;
397    unsigned cache_policy : 3;
398    bool unorm : 1;
399    bool level_zero : 1;
400    bool d16 : 1;        /* data and return values are 16-bit, requires GFX8+ */
401    unsigned attributes; /* additional call-site specific AC_FUNC_ATTRs */
402 
403    LLVMValueRef resource;
404    LLVMValueRef sampler;
405    LLVMValueRef data[2]; /* data[0] is source data (vector); data[1] is cmp for cmpswap */
406    LLVMValueRef offset;
407    LLVMValueRef bias;
408    LLVMValueRef compare;
409    LLVMValueRef derivs[6];
410    LLVMValueRef coords[4];
411    LLVMValueRef lod; // also used by ac_image_get_resinfo
412    LLVMValueRef min_lod;
413 };
414 
415 LLVMValueRef ac_build_image_opcode(struct ac_llvm_context *ctx, struct ac_image_args *a);
416 LLVMValueRef ac_build_image_get_sample_count(struct ac_llvm_context *ctx, LLVMValueRef rsrc);
417 LLVMValueRef ac_build_cvt_pkrtz_f16(struct ac_llvm_context *ctx, LLVMValueRef args[2]);
418 LLVMValueRef ac_build_cvt_pknorm_i16(struct ac_llvm_context *ctx, LLVMValueRef args[2]);
419 LLVMValueRef ac_build_cvt_pknorm_u16(struct ac_llvm_context *ctx, LLVMValueRef args[2]);
420 LLVMValueRef ac_build_cvt_pknorm_i16_f16(struct ac_llvm_context *ctx, LLVMValueRef args[2]);
421 LLVMValueRef ac_build_cvt_pknorm_u16_f16(struct ac_llvm_context *ctx, LLVMValueRef args[2]);
422 LLVMValueRef ac_build_cvt_pk_i16(struct ac_llvm_context *ctx, LLVMValueRef args[2], unsigned bits,
423                                  bool hi);
424 LLVMValueRef ac_build_cvt_pk_u16(struct ac_llvm_context *ctx, LLVMValueRef args[2], unsigned bits,
425                                  bool hi);
426 LLVMValueRef ac_build_wqm_vote(struct ac_llvm_context *ctx, LLVMValueRef i1);
427 void ac_build_kill_if_false(struct ac_llvm_context *ctx, LLVMValueRef i1);
428 LLVMValueRef ac_build_bfe(struct ac_llvm_context *ctx, LLVMValueRef input, LLVMValueRef offset,
429                           LLVMValueRef width, bool is_signed);
430 LLVMValueRef ac_build_imad(struct ac_llvm_context *ctx, LLVMValueRef s0, LLVMValueRef s1,
431                            LLVMValueRef s2);
432 LLVMValueRef ac_build_fmad(struct ac_llvm_context *ctx, LLVMValueRef s0, LLVMValueRef s1,
433                            LLVMValueRef s2);
434 
435 void ac_build_waitcnt(struct ac_llvm_context *ctx, unsigned wait_flags);
436 
437 LLVMValueRef ac_build_fract(struct ac_llvm_context *ctx, LLVMValueRef src0, unsigned bitsize);
438 LLVMValueRef ac_const_uint_vec(struct ac_llvm_context *ctx, LLVMTypeRef type, uint64_t value);
439 LLVMValueRef ac_build_isign(struct ac_llvm_context *ctx, LLVMValueRef src0);
440 LLVMValueRef ac_build_fsign(struct ac_llvm_context *ctx, LLVMValueRef src);
441 LLVMValueRef ac_build_bit_count(struct ac_llvm_context *ctx, LLVMValueRef src0);
442 
443 LLVMValueRef ac_build_fsat(struct ac_llvm_context *ctx, LLVMValueRef src,
444                            LLVMTypeRef type);
445 
446 LLVMValueRef ac_build_bitfield_reverse(struct ac_llvm_context *ctx, LLVMValueRef src0);
447 
448 void ac_optimize_vs_outputs(struct ac_llvm_context *ac, LLVMValueRef main_fn,
449                             uint8_t *vs_output_param_offset, uint32_t num_outputs,
450                             uint32_t skip_output_mask, uint8_t *num_param_exports);
451 void ac_init_exec_full_mask(struct ac_llvm_context *ctx);
452 
453 void ac_declare_lds_as_pointer(struct ac_llvm_context *ac);
454 LLVMValueRef ac_lds_load(struct ac_llvm_context *ctx, LLVMValueRef dw_addr);
455 void ac_lds_store(struct ac_llvm_context *ctx, LLVMValueRef dw_addr, LLVMValueRef value);
456 
457 LLVMValueRef ac_find_lsb(struct ac_llvm_context *ctx, LLVMTypeRef dst_type, LLVMValueRef src0);
458 
459 LLVMTypeRef ac_array_in_const_addr_space(LLVMTypeRef elem_type);
460 LLVMTypeRef ac_array_in_const32_addr_space(LLVMTypeRef elem_type);
461 
462 void ac_build_bgnloop(struct ac_llvm_context *ctx, int lable_id);
463 void ac_build_break(struct ac_llvm_context *ctx);
464 void ac_build_continue(struct ac_llvm_context *ctx);
465 void ac_build_else(struct ac_llvm_context *ctx, int lable_id);
466 void ac_build_endif(struct ac_llvm_context *ctx, int lable_id);
467 void ac_build_endloop(struct ac_llvm_context *ctx, int lable_id);
468 void ac_build_ifcc(struct ac_llvm_context *ctx, LLVMValueRef cond, int label_id);
469 
470 LLVMValueRef ac_build_alloca(struct ac_llvm_context *ac, LLVMTypeRef type, const char *name);
471 LLVMValueRef ac_build_alloca_undef(struct ac_llvm_context *ac, LLVMTypeRef type, const char *name);
472 
473 LLVMValueRef ac_cast_ptr(struct ac_llvm_context *ctx, LLVMValueRef ptr, LLVMTypeRef type);
474 
475 LLVMValueRef ac_trim_vector(struct ac_llvm_context *ctx, LLVMValueRef value, unsigned count);
476 
477 LLVMValueRef ac_unpack_param(struct ac_llvm_context *ctx, LLVMValueRef param, unsigned rshift,
478                              unsigned bitwidth);
479 
480 void ac_apply_fmask_to_sample(struct ac_llvm_context *ac, LLVMValueRef fmask, LLVMValueRef *addr,
481                               bool is_array_tex);
482 
483 LLVMValueRef ac_build_ds_swizzle(struct ac_llvm_context *ctx, LLVMValueRef src, unsigned mask);
484 
485 LLVMValueRef ac_build_readlane_no_opt_barrier(struct ac_llvm_context *ctx, LLVMValueRef src,
486                                               LLVMValueRef lane);
487 
488 LLVMValueRef ac_build_readlane(struct ac_llvm_context *ctx, LLVMValueRef src, LLVMValueRef lane);
489 
490 LLVMValueRef ac_build_writelane(struct ac_llvm_context *ctx, LLVMValueRef src, LLVMValueRef value,
491                                 LLVMValueRef lane);
492 
493 LLVMValueRef ac_build_mbcnt(struct ac_llvm_context *ctx, LLVMValueRef mask);
494 
495 LLVMValueRef ac_build_inclusive_scan(struct ac_llvm_context *ctx, LLVMValueRef src, nir_op op);
496 
497 LLVMValueRef ac_build_exclusive_scan(struct ac_llvm_context *ctx, LLVMValueRef src, nir_op op);
498 
499 LLVMValueRef ac_build_reduce(struct ac_llvm_context *ctx, LLVMValueRef src, nir_op op,
500                              unsigned cluster_size);
501 
502 /**
503  * Common arguments for a scan/reduce operation that accumulates per-wave
504  * values across an entire workgroup, while respecting the order of waves.
505  */
506 struct ac_wg_scan {
507    bool enable_reduce;
508    bool enable_exclusive;
509    bool enable_inclusive;
510    nir_op op;
511    LLVMValueRef src; /* clobbered! */
512    LLVMValueRef result_reduce;
513    LLVMValueRef result_exclusive;
514    LLVMValueRef result_inclusive;
515    LLVMValueRef extra;
516    LLVMValueRef waveidx;
517    LLVMValueRef numwaves; /* only needed for "reduce" operations */
518 
519    /* T addrspace(LDS) pointer to the same type as value, at least maxwaves entries */
520    LLVMValueRef scratch;
521    unsigned maxwaves;
522 };
523 
524 void ac_build_wg_wavescan_top(struct ac_llvm_context *ctx, struct ac_wg_scan *ws);
525 void ac_build_wg_wavescan_bottom(struct ac_llvm_context *ctx, struct ac_wg_scan *ws);
526 void ac_build_wg_wavescan(struct ac_llvm_context *ctx, struct ac_wg_scan *ws);
527 
528 void ac_build_wg_scan_top(struct ac_llvm_context *ctx, struct ac_wg_scan *ws);
529 void ac_build_wg_scan_bottom(struct ac_llvm_context *ctx, struct ac_wg_scan *ws);
530 void ac_build_wg_scan(struct ac_llvm_context *ctx, struct ac_wg_scan *ws);
531 
532 LLVMValueRef ac_build_quad_swizzle(struct ac_llvm_context *ctx, LLVMValueRef src, unsigned lane0,
533                                    unsigned lane1, unsigned lane2, unsigned lane3);
534 
535 LLVMValueRef ac_build_shuffle(struct ac_llvm_context *ctx, LLVMValueRef src, LLVMValueRef index);
536 
537 LLVMValueRef ac_build_frexp_exp(struct ac_llvm_context *ctx, LLVMValueRef src0, unsigned bitsize);
538 
539 LLVMValueRef ac_build_frexp_mant(struct ac_llvm_context *ctx, LLVMValueRef src0, unsigned bitsize);
540 
541 LLVMValueRef ac_build_canonicalize(struct ac_llvm_context *ctx, LLVMValueRef src0,
542                                    unsigned bitsize);
543 
544 LLVMValueRef ac_build_ddxy_interp(struct ac_llvm_context *ctx, LLVMValueRef interp_ij);
545 
546 LLVMValueRef ac_build_load_helper_invocation(struct ac_llvm_context *ctx);
547 
548 LLVMValueRef ac_build_is_helper_invocation(struct ac_llvm_context *ctx);
549 
550 LLVMValueRef ac_build_call(struct ac_llvm_context *ctx, LLVMValueRef func, LLVMValueRef *args,
551                            unsigned num_args);
552 
553 LLVMValueRef ac_build_atomic_rmw(struct ac_llvm_context *ctx, LLVMAtomicRMWBinOp op,
554                                  LLVMValueRef ptr, LLVMValueRef val, const char *sync_scope);
555 
556 LLVMValueRef ac_build_atomic_cmp_xchg(struct ac_llvm_context *ctx, LLVMValueRef ptr,
557                                       LLVMValueRef cmp, LLVMValueRef val, const char *sync_scope);
558 
559 void ac_export_mrt_z(struct ac_llvm_context *ctx, LLVMValueRef depth, LLVMValueRef stencil,
560                      LLVMValueRef samplemask, struct ac_export_args *args);
561 
562 void ac_build_sendmsg_gs_alloc_req(struct ac_llvm_context *ctx, LLVMValueRef wave_id,
563                                    LLVMValueRef vtx_cnt, LLVMValueRef prim_cnt);
564 
565 struct ac_ngg_prim {
566    unsigned num_vertices;
567    LLVMValueRef isnull;
568    LLVMValueRef index[3];
569    LLVMValueRef edgeflag[3];
570    LLVMValueRef passthrough;
571 };
572 
573 LLVMValueRef ac_pack_prim_export(struct ac_llvm_context *ctx, const struct ac_ngg_prim *prim);
574 void ac_build_export_prim(struct ac_llvm_context *ctx, const struct ac_ngg_prim *prim);
575 
ac_get_arg(struct ac_llvm_context * ctx,struct ac_arg arg)576 static inline LLVMValueRef ac_get_arg(struct ac_llvm_context *ctx, struct ac_arg arg)
577 {
578    assert(arg.used);
579    return LLVMGetParam(ctx->main_function, arg.arg_index);
580 }
581 
582 enum ac_llvm_calling_convention
583 {
584    AC_LLVM_AMDGPU_VS = 87,
585    AC_LLVM_AMDGPU_GS = 88,
586    AC_LLVM_AMDGPU_PS = 89,
587    AC_LLVM_AMDGPU_CS = 90,
588    AC_LLVM_AMDGPU_HS = 93,
589 };
590 
591 LLVMValueRef ac_build_main(const struct ac_shader_args *args, struct ac_llvm_context *ctx,
592                            enum ac_llvm_calling_convention convention, const char *name,
593                            LLVMTypeRef ret_type, LLVMModuleRef module);
594 void ac_build_s_endpgm(struct ac_llvm_context *ctx);
595 
596 LLVMValueRef ac_prefix_bitcount(struct ac_llvm_context *ctx, LLVMValueRef mask, LLVMValueRef index);
597 LLVMValueRef ac_prefix_bitcount_2x64(struct ac_llvm_context *ctx, LLVMValueRef mask[2],
598                                      LLVMValueRef index);
599 void ac_build_triangle_strip_indices_to_triangle(struct ac_llvm_context *ctx, LLVMValueRef is_odd,
600                                                  LLVMValueRef flatshade_first,
601                                                  LLVMValueRef index[3]);
602 
603 #ifdef __cplusplus
604 }
605 #endif
606 
607 #endif
608