1 /*
2 * Copyright 2013 Vadim Girlin <vadimgirlin@gmail.com>
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 *
23 * Authors:
24 * Vadim Girlin
25 */
26
27 #define BCP_DEBUG 0
28
29 #if BCP_DEBUG
30 #define BCP_DUMP(q) do { q } while (0)
31 #else
32 #define BCP_DUMP(q)
33 #endif
34
35 #include "r600_pipe.h"
36 #include "r600_shader.h"
37 #include "eg_sq.h" // CM_V_SQ_MOVA_DST_CF_IDX0/1
38
39 #include <stack>
40
41 #include "sb_bc.h"
42 #include "sb_shader.h"
43 #include "sb_pass.h"
44 #include "util/macros.h"
45
46 namespace r600_sb {
47
decode()48 int bc_parser::decode() {
49
50 dw = bc->bytecode;
51 bc_ndw = bc->ndw;
52 max_cf = 0;
53
54 dec = new bc_decoder(ctx, dw, bc_ndw);
55
56 shader_target t = TARGET_UNKNOWN;
57
58 if (pshader) {
59 switch (bc->type) {
60 case PIPE_SHADER_FRAGMENT: t = TARGET_PS; break;
61 case PIPE_SHADER_VERTEX:
62 t = pshader->vs_as_ls ? TARGET_LS : (pshader->vs_as_es ? TARGET_ES : TARGET_VS);
63 break;
64 case PIPE_SHADER_GEOMETRY: t = TARGET_GS; break;
65 case PIPE_SHADER_COMPUTE: t = TARGET_COMPUTE; break;
66 case PIPE_SHADER_TESS_CTRL: t = TARGET_HS; break;
67 case PIPE_SHADER_TESS_EVAL: t = pshader->tes_as_es ? TARGET_ES : TARGET_VS; break;
68 default: assert(!"unknown shader target"); return -1; break;
69 }
70 } else {
71 if (bc->type == PIPE_SHADER_COMPUTE)
72 t = TARGET_COMPUTE;
73 else
74 t = TARGET_FETCH;
75 }
76
77 sh = new shader(ctx, t, bc->debug_id);
78 sh->safe_math = sb_context::safe_math || (t == TARGET_COMPUTE || bc->precise);
79
80 int r = decode_shader();
81
82 delete dec;
83
84 sh->ngpr = bc->ngpr;
85 sh->nstack = bc->nstack;
86
87 return r;
88 }
89
decode_shader()90 int bc_parser::decode_shader() {
91 int r = 0;
92 unsigned i = 0;
93 bool eop = false;
94
95 sh->init();
96
97 do {
98 eop = false;
99 if ((r = decode_cf(i, eop)))
100 return r;
101
102 } while (!eop || (i >> 1) < max_cf);
103
104 return 0;
105 }
106
prepare()107 int bc_parser::prepare() {
108 int r = 0;
109 if ((r = parse_decls()))
110 return r;
111 if ((r = prepare_ir()))
112 return r;
113 return 0;
114 }
115
parse_decls()116 int bc_parser::parse_decls() {
117
118 if (!pshader) {
119 if (gpr_reladdr)
120 sh->add_gpr_array(0, bc->ngpr, 0x0F);
121
122 // compute shaders have some values preloaded in R0, R1
123 sh->add_input(0 /* GPR */, true /* preloaded */, 0x0F /* mask */);
124 sh->add_input(1 /* GPR */, true /* preloaded */, 0x0F /* mask */);
125 return 0;
126 }
127
128 if (pshader->indirect_files & ~((1 << TGSI_FILE_CONSTANT) | (1 << TGSI_FILE_SAMPLER))) {
129
130 assert(pshader->num_arrays);
131
132 if (pshader->num_arrays) {
133 for (unsigned i = 0; i < pshader->num_arrays; ++i) {
134 r600_shader_array &a = pshader->arrays[i];
135 sh->add_gpr_array(a.gpr_start, a.gpr_count, a.comp_mask);
136 }
137 } else {
138 sh->add_gpr_array(0, pshader->bc.ngpr, 0x0F);
139 }
140 }
141
142 // GS inputs can add indirect addressing
143 if (sh->target == TARGET_GS) {
144 if (pshader->num_arrays) {
145 for (unsigned i = 0; i < pshader->num_arrays; ++i) {
146 r600_shader_array &a = pshader->arrays[i];
147 sh->add_gpr_array(a.gpr_start, a.gpr_count, a.comp_mask);
148 }
149 }
150 }
151
152 if (sh->target == TARGET_VS || sh->target == TARGET_ES || sh->target == TARGET_HS || sh->target == TARGET_LS)
153 sh->add_input(0, 1, 0x0F);
154 else if (sh->target == TARGET_GS) {
155 sh->add_input(0, 1, 0x0F);
156 sh->add_input(1, 1, 0x0F);
157 } else if (sh->target == TARGET_COMPUTE) {
158 sh->add_input(0, 1, 0x0F);
159 sh->add_input(1, 1, 0x0F);
160 }
161
162 bool ps_interp = ctx.hw_class >= HW_CLASS_EVERGREEN
163 && sh->target == TARGET_PS;
164
165 bool ij_interpolators[6];
166 memset(ij_interpolators, 0, sizeof(ij_interpolators));
167
168 for (unsigned i = 0; i < pshader->ninput; ++i) {
169 r600_shader_io & in = pshader->input[i];
170 bool preloaded = sh->target == TARGET_PS && !(ps_interp && in.spi_sid);
171 sh->add_input(in.gpr, preloaded, /*in.write_mask*/ 0x0F);
172 if (ps_interp && in.spi_sid) {
173 int k = eg_get_interpolator_index(in.interpolate, in.interpolate_location);
174 if (k >= 0) {
175 ij_interpolators[k] |= true;
176 if (in.uses_interpolate_at_centroid) {
177 k = eg_get_interpolator_index(in.interpolate, TGSI_INTERPOLATE_LOC_CENTROID);
178 ij_interpolators[k] |= true;
179 }
180 }
181 }
182 }
183
184 if (ps_interp) {
185 /* add the egcm ij interpolators to live inputs */
186 unsigned num_ij = 0;
187 for (unsigned i = 0; i < ARRAY_SIZE(ij_interpolators); i++) {
188 num_ij += ij_interpolators[i];
189 }
190
191 unsigned mask = (1 << (2 * num_ij)) - 1;
192 unsigned gpr = 0;
193
194 while (mask) {
195 sh->add_input(gpr, true, mask & 0x0F);
196 ++gpr;
197 mask >>= 4;
198 }
199 }
200
201 return 0;
202 }
203
decode_cf(unsigned & i,bool & eop)204 int bc_parser::decode_cf(unsigned &i, bool &eop) {
205
206 int r;
207
208 cf_node *cf = sh->create_cf();
209 sh->root->push_back(cf);
210
211 unsigned id = i >> 1;
212
213 cf->bc.id = id;
214
215 if (cf_map.size() < id + 1)
216 cf_map.resize(id + 1);
217
218 cf_map[id] = cf;
219
220 if ((r = dec->decode_cf(i, cf->bc)))
221 return r;
222
223 cf_op_flags flags = (cf_op_flags)cf->bc.op_ptr->flags;
224
225 if (flags & CF_ALU) {
226 if ((r = decode_alu_clause(cf)))
227 return r;
228 } else if (flags & CF_FETCH) {
229 if ((r = decode_fetch_clause(cf)))
230 return r;
231 } else if (flags & CF_EXP) {
232 if (cf->bc.rw_rel)
233 gpr_reladdr = true;
234 assert(!cf->bc.rw_rel);
235 } else if (flags & CF_MEM) {
236 if (cf->bc.rw_rel)
237 gpr_reladdr = true;
238 assert(!cf->bc.rw_rel);
239 } else if (flags & CF_BRANCH) {
240 if (cf->bc.addr > max_cf)
241 max_cf = cf->bc.addr;
242 }
243
244 eop = cf->bc.end_of_program || cf->bc.op == CF_OP_CF_END ||
245 cf->bc.op == CF_OP_RET;
246 return 0;
247 }
248
decode_alu_clause(cf_node * cf)249 int bc_parser::decode_alu_clause(cf_node* cf) {
250 unsigned i = cf->bc.addr << 1, cnt = cf->bc.count + 1, gcnt;
251
252 cf->subtype = NST_ALU_CLAUSE;
253
254 cgroup = 0;
255 memset(slots[0], 0, 5*sizeof(slots[0][0]));
256
257 unsigned ng = 0;
258
259 do {
260 decode_alu_group(cf, i, gcnt);
261 assert(gcnt <= cnt);
262 cnt -= gcnt;
263 ng++;
264 } while (cnt);
265
266 return 0;
267 }
268
decode_alu_group(cf_node * cf,unsigned & i,unsigned & gcnt)269 int bc_parser::decode_alu_group(cf_node* cf, unsigned &i, unsigned &gcnt) {
270 int r;
271 alu_node *n;
272 alu_group_node *g = sh->create_alu_group();
273
274 cgroup = !cgroup;
275 memset(slots[cgroup], 0, 5*sizeof(slots[0][0]));
276 gcnt = 0;
277
278 unsigned literal_mask = 0;
279
280 do {
281 n = sh->create_alu();
282 g->push_back(n);
283
284 if ((r = dec->decode_alu(i, n->bc)))
285 return r;
286
287 if (!sh->assign_slot(n, slots[cgroup])) {
288 assert(!"alu slot assignment failed");
289 return -1;
290 }
291
292 gcnt++;
293
294 } while (gcnt <= 5 && !n->bc.last);
295
296 assert(n->bc.last);
297
298 for (node_iterator I = g->begin(), E = g->end(); I != E; ++I) {
299 n = static_cast<alu_node*>(*I);
300
301 if (n->bc.dst_rel)
302 gpr_reladdr = true;
303
304 for (int k = 0; k < n->bc.op_ptr->src_count; ++k) {
305 bc_alu_src &src = n->bc.src[k];
306 if (src.rel)
307 gpr_reladdr = true;
308 if (src.sel == ALU_SRC_LITERAL) {
309 literal_mask |= (1 << src.chan);
310 src.value.u = dw[i + src.chan];
311 }
312 }
313 }
314
315 unsigned literal_ndw = 0;
316 while (literal_mask) {
317 g->literals.push_back(dw[i + literal_ndw]);
318 literal_ndw += 1;
319 literal_mask >>= 1;
320 }
321
322 literal_ndw = (literal_ndw + 1) & ~1u;
323
324 i += literal_ndw;
325 gcnt += literal_ndw >> 1;
326
327 cf->push_back(g);
328 return 0;
329 }
330
prepare_alu_clause(cf_node * cf)331 int bc_parser::prepare_alu_clause(cf_node* cf) {
332
333 // loop over alu groups
334 for (node_iterator I = cf->begin(), E = cf->end(); I != E; ++I) {
335 assert(I->subtype == NST_ALU_GROUP);
336 alu_group_node *g = static_cast<alu_group_node*>(*I);
337 prepare_alu_group(cf, g);
338 }
339
340 return 0;
341 }
342
save_set_cf_index(value * val,unsigned idx)343 void bc_parser::save_set_cf_index(value *val, unsigned idx)
344 {
345 assert(idx <= 1);
346 assert(val);
347 cf_index_value[idx] = val;
348 }
get_cf_index_value(unsigned idx)349 value *bc_parser::get_cf_index_value(unsigned idx)
350 {
351 assert(idx <= 1);
352 assert(cf_index_value[idx]);
353 return cf_index_value[idx];
354 }
save_mova(alu_node * mova)355 void bc_parser::save_mova(alu_node *mova)
356 {
357 assert(mova);
358 this->mova = mova;
359 }
get_mova()360 alu_node *bc_parser::get_mova()
361 {
362 assert(mova);
363 return mova;
364 }
365
prepare_alu_group(cf_node * cf,alu_group_node * g)366 int bc_parser::prepare_alu_group(cf_node* cf, alu_group_node *g) {
367
368 alu_node *n;
369
370 cgroup = !cgroup;
371 memset(slots[cgroup], 0, 5*sizeof(slots[0][0]));
372
373 for (node_iterator I = g->begin(), E = g->end();
374 I != E; ++I) {
375 n = static_cast<alu_node*>(*I);
376 bool ubo_indexing[2] = {};
377
378 if (!sh->assign_slot(n, slots[cgroup])) {
379 assert(!"alu slot assignment failed");
380 return -1;
381 }
382
383 unsigned src_count = n->bc.op_ptr->src_count;
384
385 if (ctx.alu_slots(n->bc.op) & AF_4SLOT)
386 n->flags |= NF_ALU_4SLOT;
387
388 n->src.resize(src_count);
389
390 unsigned flags = n->bc.op_ptr->flags;
391
392 if (flags & AF_LDS) {
393 bool need_rw = false, need_oqa = false, need_oqb = false;
394 int ndst = 0, ncount = 0;
395
396 /* all non-read operations have side effects */
397 if (n->bc.op != LDS_OP2_LDS_READ2_RET &&
398 n->bc.op != LDS_OP1_LDS_READ_REL_RET &&
399 n->bc.op != LDS_OP1_LDS_READ_RET) {
400 n->flags |= NF_DONT_KILL;
401 ndst++;
402 need_rw = true;
403 }
404
405 if (n->bc.op >= LDS_OP2_LDS_ADD_RET && n->bc.op <= LDS_OP1_LDS_USHORT_READ_RET) {
406 need_oqa = true;
407 ndst++;
408 }
409
410 if (n->bc.op == LDS_OP2_LDS_READ2_RET || n->bc.op == LDS_OP1_LDS_READ_REL_RET) {
411 need_oqb = true;
412 ndst++;
413 }
414
415 n->dst.resize(ndst);
416 if (need_oqa)
417 n->dst[ncount++] = sh->get_special_value(SV_LDS_OQA);
418 if (need_oqb)
419 n->dst[ncount++] = sh->get_special_value(SV_LDS_OQB);
420 if (need_rw)
421 n->dst[ncount++] = sh->get_special_value(SV_LDS_RW);
422
423 n->flags |= NF_DONT_MOVE | NF_DONT_HOIST;
424
425 } else if (flags & AF_PRED) {
426 n->dst.resize(3);
427 if (n->bc.update_pred)
428 n->dst[1] = sh->get_special_value(SV_ALU_PRED);
429 if (n->bc.update_exec_mask)
430 n->dst[2] = sh->get_special_value(SV_EXEC_MASK);
431
432 n->flags |= NF_DONT_HOIST;
433
434 } else if (flags & AF_KILL) {
435
436 n->dst.resize(2);
437 n->dst[1] = sh->get_special_value(SV_VALID_MASK);
438 sh->set_uses_kill();
439
440 n->flags |= NF_DONT_HOIST | NF_DONT_MOVE |
441 NF_DONT_KILL | NF_SCHEDULE_EARLY;
442
443 } else {
444 n->dst.resize(1);
445 }
446
447 if (n->bc.op == ALU_OP0_SET_CF_IDX0 || n->bc.op == ALU_OP0_SET_CF_IDX1) {
448 // Move CF_IDX value into tex instruction operands, scheduler will later re-emit setting of CF_IDX
449 // DCE will kill this op
450 save_set_cf_index(get_mova()->src[0], n->bc.op == ALU_OP0_SET_CF_IDX1);
451 } else if (flags & AF_MOVA) {
452
453 n->dst[0] = sh->get_special_value(SV_AR_INDEX);
454 save_mova(n);
455
456 n->flags |= NF_DONT_HOIST;
457
458 } else if ((n->bc.op_ptr->src_count == 3 || n->bc.write_mask) && !(flags & AF_LDS)) {
459 assert(!n->bc.dst_rel || n->bc.index_mode == INDEX_AR_X);
460
461 value *v = sh->get_gpr_value(false, n->bc.dst_gpr, n->bc.dst_chan,
462 n->bc.dst_rel);
463
464 n->dst[0] = v;
465 }
466
467 if (n->bc.pred_sel) {
468 sh->has_alu_predication = true;
469 n->pred = sh->get_special_value(SV_ALU_PRED);
470 }
471
472 for (unsigned s = 0; s < src_count; ++s) {
473 bc_alu_src &src = n->bc.src[s];
474
475 if (src.sel == ALU_SRC_LITERAL) {
476 n->src[s] = sh->get_const_value(src.value);
477 } else if (src.sel == ALU_SRC_PS || src.sel == ALU_SRC_PV) {
478 unsigned pgroup = !cgroup, prev_slot = src.sel == ALU_SRC_PS ?
479 ((unsigned)SLOT_TRANS) : src.chan;
480
481 // XXX shouldn't happen but llvm backend uses PS on cayman
482 if (prev_slot == SLOT_TRANS && ctx.is_cayman())
483 prev_slot = SLOT_X;
484
485 alu_node *prev_alu = slots[pgroup][prev_slot];
486
487 assert(prev_alu);
488
489 if (!prev_alu->dst[0]) {
490 value * t = sh->create_temp_value();
491 prev_alu->dst[0] = t;
492 }
493
494 value *d = prev_alu->dst[0];
495
496 if (d->is_rel()) {
497 d = sh->get_gpr_value(true, prev_alu->bc.dst_gpr,
498 prev_alu->bc.dst_chan,
499 prev_alu->bc.dst_rel);
500 }
501
502 n->src[s] = d;
503 } else if (ctx.is_kcache_sel(src.sel)) {
504 unsigned sel = src.sel, kc_addr;
505 unsigned kc_set = ((sel >> 7) & 2) + ((sel >> 5) & 1);
506
507 bc_kcache &kc = cf->bc.kc[kc_set];
508 kc_addr = (kc.addr << 4) + (sel & 0x1F);
509 n->src[s] = sh->get_kcache_value(kc.bank, kc_addr, src.chan, (alu_kcache_index_mode)kc.index_mode);
510
511 if (kc.index_mode != KC_INDEX_NONE) {
512 assert(kc.index_mode != KC_LOCK_LOOP);
513 ubo_indexing[kc.index_mode - KC_INDEX_0] = true;
514 }
515 } else if (src.sel < MAX_GPR) {
516 value *v = sh->get_gpr_value(true, src.sel, src.chan, src.rel);
517
518 n->src[s] = v;
519
520 } else if (src.sel >= ALU_SRC_PARAM_OFFSET) {
521 // using slot for value channel because in fact the slot
522 // determines the channel that is loaded by INTERP_LOAD_P0
523 // (and maybe some others).
524 // otherwise GVN will consider INTERP_LOAD_P0s with the same
525 // param index as equal instructions and leave only one of them
526 n->src[s] = sh->get_special_ro_value(sel_chan(src.sel,
527 n->bc.slot));
528 } else if (ctx.is_lds_oq(src.sel)) {
529 switch (src.sel) {
530 case ALU_SRC_LDS_OQ_A:
531 case ALU_SRC_LDS_OQ_B:
532 assert(!"Unsupported LDS queue access in SB");
533 break;
534 case ALU_SRC_LDS_OQ_A_POP:
535 n->src[s] = sh->get_special_value(SV_LDS_OQA);
536 break;
537 case ALU_SRC_LDS_OQ_B_POP:
538 n->src[s] = sh->get_special_value(SV_LDS_OQB);
539 break;
540 }
541 n->flags |= NF_DONT_HOIST | NF_DONT_MOVE;
542
543 } else {
544 switch (src.sel) {
545 case ALU_SRC_0:
546 n->src[s] = sh->get_const_value(0);
547 break;
548 case ALU_SRC_0_5:
549 n->src[s] = sh->get_const_value(0.5f);
550 break;
551 case ALU_SRC_1:
552 n->src[s] = sh->get_const_value(1.0f);
553 break;
554 case ALU_SRC_1_INT:
555 n->src[s] = sh->get_const_value(1);
556 break;
557 case ALU_SRC_M_1_INT:
558 n->src[s] = sh->get_const_value(-1);
559 break;
560 default:
561 n->src[s] = sh->get_special_ro_value(src.sel);
562 break;
563 }
564 }
565 }
566
567 // add UBO index values if any as dependencies
568 if (ubo_indexing[0]) {
569 n->src.push_back(get_cf_index_value(0));
570 }
571 if (ubo_indexing[1]) {
572 n->src.push_back(get_cf_index_value(1));
573 }
574
575 if ((flags & AF_MOVA) && (n->bc.dst_gpr == CM_V_SQ_MOVA_DST_CF_IDX0 || n->bc.dst_gpr == CM_V_SQ_MOVA_DST_CF_IDX1) &&
576 ctx.is_cayman())
577 // Move CF_IDX value into tex instruction operands, scheduler will later re-emit setting of CF_IDX
578 save_set_cf_index(n->src[0], n->bc.dst_gpr == CM_V_SQ_MOVA_DST_CF_IDX1);
579 }
580
581 // pack multislot instructions into alu_packed_node
582
583 alu_packed_node *p = NULL;
584 for (node_iterator N, I = g->begin(), E = g->end(); I != E; I = N) {
585 N = I + 1;
586 alu_node *a = static_cast<alu_node*>(*I);
587 unsigned sflags = a->bc.slot_flags;
588
589 if (sflags == AF_4V || (ctx.is_cayman() && sflags == AF_S)) {
590 if (!p)
591 p = sh->create_alu_packed();
592
593 a->remove();
594 p->push_back(a);
595 }
596 }
597
598 if (p) {
599 g->push_front(p);
600
601 if (p->count() == 3 && ctx.is_cayman()) {
602 // cayman's scalar instruction that can use 3 or 4 slots
603
604 // FIXME for simplicity we'll always add 4th slot,
605 // but probably we might want to always remove 4th slot and make
606 // sure that regalloc won't choose 'w' component for dst
607
608 alu_node *f = static_cast<alu_node*>(p->first);
609 alu_node *a = sh->create_alu();
610 a->src = f->src;
611 a->dst.resize(f->dst.size());
612 a->bc = f->bc;
613 a->bc.slot = SLOT_W;
614 p->push_back(a);
615 }
616 }
617
618 return 0;
619 }
620
decode_fetch_clause(cf_node * cf)621 int bc_parser::decode_fetch_clause(cf_node* cf) {
622 int r;
623 unsigned i = cf->bc.addr << 1, cnt = cf->bc.count + 1;
624
625 if (cf->bc.op_ptr->flags & FF_GDS)
626 cf->subtype = NST_GDS_CLAUSE;
627 else
628 cf->subtype = NST_TEX_CLAUSE;
629
630 while (cnt--) {
631 fetch_node *n = sh->create_fetch();
632 cf->push_back(n);
633 if ((r = dec->decode_fetch(i, n->bc)))
634 return r;
635 if (n->bc.src_rel || n->bc.dst_rel)
636 gpr_reladdr = true;
637
638 }
639 return 0;
640 }
641
prepare_fetch_clause(cf_node * cf)642 int bc_parser::prepare_fetch_clause(cf_node *cf) {
643
644 vvec grad_v, grad_h, texture_offsets;
645
646 for (node_iterator I = cf->begin(), E = cf->end(); I != E; ++I) {
647
648 fetch_node *n = static_cast<fetch_node*>(*I);
649 assert(n->is_valid());
650
651 unsigned flags = n->bc.op_ptr->flags;
652
653 unsigned vtx = flags & FF_VTX;
654 unsigned gds = flags & FF_GDS;
655 unsigned num_src = gds ? 2 : vtx ? ctx.vtx_src_num : 4;
656
657 n->dst.resize(4);
658
659 if (gds) {
660 n->flags |= NF_DONT_HOIST | NF_DONT_MOVE | NF_DONT_KILL;
661 }
662 if (flags & (FF_SETGRAD | FF_USEGRAD | FF_GETGRAD)) {
663 sh->uses_gradients = true;
664 }
665
666 if (flags & (FF_SETGRAD | FF_SET_TEXTURE_OFFSETS)) {
667
668 vvec *grad = NULL;
669
670 switch (n->bc.op) {
671 case FETCH_OP_SET_GRADIENTS_V:
672 grad = &grad_v;
673 break;
674 case FETCH_OP_SET_GRADIENTS_H:
675 grad = &grad_h;
676 break;
677 case FETCH_OP_SET_TEXTURE_OFFSETS:
678 grad = &texture_offsets;
679 break;
680 default:
681 assert(!"unexpected SET_GRAD instruction");
682 return -1;
683 }
684
685 if (grad->empty())
686 grad->resize(4);
687
688 for(unsigned s = 0; s < 4; ++s) {
689 unsigned sw = n->bc.src_sel[s];
690 if (sw <= SEL_W)
691 (*grad)[s] = sh->get_gpr_value(true, n->bc.src_gpr,
692 sw, false);
693 else if (sw == SEL_0)
694 (*grad)[s] = sh->get_const_value(0.0f);
695 else if (sw == SEL_1)
696 (*grad)[s] = sh->get_const_value(1.0f);
697 }
698 } else {
699 // Fold source values for instructions with hidden target values in to the instructions
700 // using them. The set instructions are later re-emitted by bc_finalizer
701 if (flags & FF_USEGRAD) {
702 n->src.resize(12);
703 std::copy(grad_v.begin(), grad_v.end(), n->src.begin() + 4);
704 std::copy(grad_h.begin(), grad_h.end(), n->src.begin() + 8);
705 } else if (flags & FF_USE_TEXTURE_OFFSETS) {
706 n->src.resize(8);
707 std::copy(texture_offsets.begin(), texture_offsets.end(), n->src.begin() + 4);
708 } else {
709 n->src.resize(4);
710 }
711
712 for(int s = 0; s < 4; ++s) {
713 if (n->bc.dst_sel[s] != SEL_MASK)
714 n->dst[s] = sh->get_gpr_value(false, n->bc.dst_gpr, s, false);
715 // NOTE: it doesn't matter here which components of the result we
716 // are using, but original n->bc.dst_sel should be taken into
717 // account when building the bytecode
718 }
719 for(unsigned s = 0; s < num_src; ++s) {
720 if (n->bc.src_sel[s] <= SEL_W)
721 n->src[s] = sh->get_gpr_value(true, n->bc.src_gpr,
722 n->bc.src_sel[s], false);
723 }
724
725 // Scheduler will emit the appropriate instructions to set CF_IDX0/1
726 if (n->bc.sampler_index_mode != V_SQ_CF_INDEX_NONE) {
727 n->src.push_back(get_cf_index_value(n->bc.sampler_index_mode == V_SQ_CF_INDEX_1));
728 }
729 if (n->bc.resource_index_mode != V_SQ_CF_INDEX_NONE) {
730 n->src.push_back(get_cf_index_value(n->bc.resource_index_mode == V_SQ_CF_INDEX_1));
731 }
732 }
733
734 if (n->bc.op == FETCH_OP_READ_SCRATCH) {
735 n->src.push_back(sh->get_special_value(SV_SCRATCH));
736 n->dst.push_back(sh->get_special_value(SV_SCRATCH));
737 }
738 }
739
740 return 0;
741 }
742
prepare_ir()743 int bc_parser::prepare_ir() {
744
745 for(id_cf_map::iterator I = cf_map.begin(), E = cf_map.end(); I != E; ++I) {
746 cf_node *c = *I;
747
748 if (!c)
749 continue;
750
751 unsigned flags = c->bc.op_ptr->flags;
752
753 if (flags & CF_ALU) {
754 prepare_alu_clause(c);
755 } else if (flags & CF_FETCH) {
756 prepare_fetch_clause(c);
757 } else if (c->bc.op == CF_OP_CALL_FS) {
758 sh->init_call_fs(c);
759 c->flags |= NF_SCHEDULE_EARLY | NF_DONT_MOVE;
760 } else if (flags & CF_LOOP_START) {
761 prepare_loop(c);
762 } else if (c->bc.op == CF_OP_JUMP) {
763 prepare_if(c);
764 } else if (c->bc.op == CF_OP_LOOP_END) {
765 loop_stack.pop();
766 } else if (c->bc.op == CF_OP_LOOP_CONTINUE) {
767 assert(!loop_stack.empty());
768 repeat_node *rep = sh->create_repeat(loop_stack.top());
769 if (c->parent->first != c)
770 rep->move(c->parent->first, c);
771 c->replace_with(rep);
772 sh->simplify_dep_rep(rep);
773 } else if (c->bc.op == CF_OP_LOOP_BREAK) {
774 assert(!loop_stack.empty());
775 depart_node *dep = sh->create_depart(loop_stack.top());
776 if (c->parent->first != c)
777 dep->move(c->parent->first, c);
778 c->replace_with(dep);
779 sh->simplify_dep_rep(dep);
780 } else if (flags & CF_EXP) {
781
782 // unroll burst exports
783
784 assert(c->bc.op == CF_OP_EXPORT || c->bc.op == CF_OP_EXPORT_DONE);
785
786 c->bc.set_op(CF_OP_EXPORT);
787
788 unsigned burst_count = c->bc.burst_count;
789 unsigned eop = c->bc.end_of_program;
790
791 c->bc.end_of_program = 0;
792 c->bc.burst_count = 0;
793
794 do {
795 c->src.resize(4);
796
797 for(int s = 0; s < 4; ++s) {
798 switch (c->bc.sel[s]) {
799 case SEL_0:
800 c->src[s] = sh->get_const_value(0.0f);
801 break;
802 case SEL_1:
803 c->src[s] = sh->get_const_value(1.0f);
804 break;
805 case SEL_MASK:
806 break;
807 default:
808 if (c->bc.sel[s] <= SEL_W)
809 c->src[s] = sh->get_gpr_value(true, c->bc.rw_gpr,
810 c->bc.sel[s], false);
811 else
812 assert(!"invalid src_sel for export");
813 }
814 }
815
816 if (!burst_count--)
817 break;
818
819 cf_node *cf_next = sh->create_cf();
820 cf_next->bc = c->bc;
821 ++cf_next->bc.rw_gpr;
822 ++cf_next->bc.array_base;
823
824 c->insert_after(cf_next);
825 c = cf_next;
826
827 } while (1);
828
829 c->bc.end_of_program = eop;
830 } else if (flags & CF_MEM) {
831
832 unsigned burst_count = c->bc.burst_count;
833 unsigned eop = c->bc.end_of_program;
834
835 c->bc.end_of_program = 0;
836 c->bc.burst_count = 0;
837
838 do {
839
840 if (ctx.hw_class == HW_CLASS_R600 && c->bc.op == CF_OP_MEM_SCRATCH &&
841 (c->bc.type == 2 || c->bc.type == 3)) {
842 c->dst.resize(4);
843 for(int s = 0; s < 4; ++s) {
844 if (c->bc.comp_mask & (1 << s))
845 c->dst[s] =
846 sh->get_gpr_value(true, c->bc.rw_gpr, s, false);
847 }
848 } else {
849 c->src.resize(4);
850
851
852 for(int s = 0; s < 4; ++s) {
853 if (c->bc.comp_mask & (1 << s))
854 c->src[s] =
855 sh->get_gpr_value(true, c->bc.rw_gpr, s, false);
856 }
857 }
858
859 if (((flags & CF_RAT) || (!(flags & CF_STRM))) && (c->bc.type & 1)) { // indexed write
860 c->src.resize(8);
861 for(int s = 0; s < 3; ++s) {
862 c->src[4 + s] =
863 sh->get_gpr_value(true, c->bc.index_gpr, s, false);
864 }
865
866 // FIXME probably we can relax it a bit
867 c->flags |= NF_DONT_HOIST | NF_DONT_MOVE;
868 }
869
870 if (flags & CF_EMIT) {
871 // Instruction implicitly depends on prior [EMIT_][CUT]_VERTEX
872 c->src.push_back(sh->get_special_value(SV_GEOMETRY_EMIT));
873 c->dst.push_back(sh->get_special_value(SV_GEOMETRY_EMIT));
874 if (sh->target == TARGET_ES) {
875 // For ES shaders this is an export
876 c->flags |= NF_DONT_KILL;
877 }
878 }
879 else if (c->bc.op == CF_OP_MEM_SCRATCH) {
880 c->src.push_back(sh->get_special_value(SV_SCRATCH));
881 c->dst.push_back(sh->get_special_value(SV_SCRATCH));
882 }
883
884 if (!burst_count--)
885 break;
886
887 cf_node *cf_next = sh->create_cf();
888 cf_next->bc = c->bc;
889 ++cf_next->bc.rw_gpr;
890
891 // FIXME is it correct?
892 cf_next->bc.array_base += cf_next->bc.elem_size + 1;
893
894 c->insert_after(cf_next);
895 c = cf_next;
896 } while (1);
897
898 c->bc.end_of_program = eop;
899
900 } else if (flags & CF_EMIT) {
901 /* quick peephole */
902 cf_node *prev = static_cast<cf_node *>(c->prev);
903 if (c->bc.op == CF_OP_CUT_VERTEX &&
904 prev && prev->is_valid() &&
905 prev->bc.op == CF_OP_EMIT_VERTEX &&
906 c->bc.count == prev->bc.count) {
907 prev->bc.set_op(CF_OP_EMIT_CUT_VERTEX);
908 prev->bc.end_of_program = c->bc.end_of_program;
909 c->remove();
910 }
911 else {
912 c->flags |= NF_DONT_KILL | NF_DONT_HOIST | NF_DONT_MOVE;
913
914 c->src.push_back(sh->get_special_value(SV_GEOMETRY_EMIT));
915 c->dst.push_back(sh->get_special_value(SV_GEOMETRY_EMIT));
916 }
917 } else if (c->bc.op == CF_OP_WAIT_ACK) {
918 c->src.push_back(sh->get_special_value(SV_SCRATCH));
919 c->dst.push_back(sh->get_special_value(SV_SCRATCH));
920 }
921 }
922
923 assert(loop_stack.empty());
924 return 0;
925 }
926
prepare_loop(cf_node * c)927 int bc_parser::prepare_loop(cf_node* c) {
928 assert(c->bc.addr-1 < cf_map.size());
929
930 cf_node *end = cf_map[c->bc.addr - 1];
931 assert(end->bc.op == CF_OP_LOOP_END);
932 assert(c->parent == end->parent);
933
934 region_node *reg = sh->create_region();
935 repeat_node *rep = sh->create_repeat(reg);
936
937 reg->push_back(rep);
938 c->insert_before(reg);
939 rep->move(c, end->next);
940
941 reg->src_loop = true;
942
943 loop_stack.push(reg);
944 return 0;
945 }
946
prepare_if(cf_node * c)947 int bc_parser::prepare_if(cf_node* c) {
948 assert(c->bc.addr-1 < cf_map.size());
949 cf_node *c_else = NULL, *end = cf_map[c->bc.addr];
950
951 if (!end)
952 return 0; // not quite sure how this happens, malformed input?
953
954 BCP_DUMP(
955 sblog << "parsing JUMP @" << c->bc.id;
956 sblog << "\n";
957 );
958
959 if (end->bc.op == CF_OP_ELSE) {
960 BCP_DUMP(
961 sblog << " found ELSE : ";
962 dump::dump_op(end);
963 sblog << "\n";
964 );
965
966 c_else = end;
967 end = cf_map[c_else->bc.addr];
968 } else {
969 BCP_DUMP(
970 sblog << " no else\n";
971 );
972
973 c_else = end;
974 }
975
976 if (c_else->parent != c->parent)
977 c_else = NULL;
978
979 if (end && end->parent != c->parent)
980 end = NULL;
981
982 region_node *reg = sh->create_region();
983
984 depart_node *dep2 = sh->create_depart(reg);
985 depart_node *dep = sh->create_depart(reg);
986 if_node *n_if = sh->create_if();
987
988 c->insert_before(reg);
989
990 if (c_else != end)
991 dep->move(c_else, end);
992 dep2->move(c, end);
993
994 reg->push_back(dep);
995 dep->push_front(n_if);
996 n_if->push_back(dep2);
997
998 n_if->cond = sh->get_special_value(SV_EXEC_MASK);
999
1000 return 0;
1001 }
1002
1003
1004 } // namespace r600_sb
1005