Lines Matching refs:instr
43 #define di(instr, fmt, ...) do { if (SCHED_DEBUG) { \ argument
45 ir3_print_instr(instr); \
106 struct ir3_instruction *instr; member
148 static void sched_node_init(struct ir3_sched_ctx *ctx, struct ir3_instruction *instr);
149 static void sched_node_add_dep(struct ir3_instruction *instr, struct ir3_instruction *src, int i);
151 static bool is_scheduled(struct ir3_instruction *instr) in is_scheduled() argument
153 return !!(instr->flags & IR3_INSTR_MARK); in is_scheduled()
157 schedule(struct ir3_sched_ctx *ctx, struct ir3_instruction *instr) in schedule() argument
159 debug_assert(ctx->block == instr->block); in schedule()
163 list_delinit(&instr->node); in schedule()
165 if (writes_addr0(instr)) { in schedule()
167 ctx->addr0 = instr; in schedule()
170 if (writes_addr1(instr)) { in schedule()
172 ctx->addr1 = instr; in schedule()
175 if (writes_pred(instr)) { in schedule()
177 ctx->pred = instr; in schedule()
180 instr->flags |= IR3_INSTR_MARK; in schedule()
182 di(instr, "schedule"); in schedule()
184 list_addtail(&instr->node, &instr->block->instr_list); in schedule()
185 ctx->scheduled = instr; in schedule()
187 if (is_kill(instr)){ in schedule()
192 struct ir3_sched_node *n = instr->data; in schedule()
199 if (src->block != instr->block) in schedule()
208 if (is_meta(instr) && (instr->opc != OPC_META_TEX_PREFETCH)) in schedule()
211 if (is_sfu(instr)) { in schedule()
213 } else if (check_src_cond(instr, is_sfu)) { in schedule()
219 if (is_tex_or_prefetch(instr)) { in schedule()
228 } else if (check_src_cond(instr, is_tex_or_prefetch)) { in schedule()
248 could_sched(struct ir3_instruction *instr, struct ir3_instruction *src) in could_sched() argument
250 foreach_ssa_src (other_src, instr) { in could_sched()
264 struct ir3_instruction *instr) in check_instr() argument
266 debug_assert(!is_scheduled(instr)); in check_instr()
268 if (ctx->remaining_kills && (is_tex(instr) || is_mem(instr))) { in check_instr()
275 struct ir3_sched_node *n = instr->data; in check_instr()
287 if (writes_addr0(instr)) { in check_instr()
288 struct ir3 *ir = instr->block->shader; in check_instr()
294 if (indirect->address != instr) in check_instr()
296 ready = could_sched(indirect, instr); in check_instr()
304 if (writes_addr1(instr)) { in check_instr()
305 struct ir3 *ir = instr->block->shader; in check_instr()
311 if (indirect->address != instr) in check_instr()
313 ready = could_sched(indirect, instr); in check_instr()
325 if (writes_addr0(instr) && ctx->addr0) { in check_instr()
326 debug_assert(ctx->addr0 != instr); in check_instr()
331 if (writes_addr1(instr) && ctx->addr1) { in check_instr()
332 debug_assert(ctx->addr1 != instr); in check_instr()
337 if (writes_pred(instr) && ctx->pred) { in check_instr()
338 debug_assert(ctx->pred != instr); in check_instr()
353 if (is_kill(instr)) { in check_instr()
354 struct ir3 *ir = instr->block->shader; in check_instr()
381 nearest_use(struct ir3_instruction *instr) in nearest_use() argument
384 foreach_ssa_use (use, instr) in nearest_use()
393 if (is_input(instr)) in nearest_use()
400 use_count(struct ir3_instruction *instr) in use_count() argument
403 foreach_ssa_use (use, instr) in use_count()
411 live_effect(struct ir3_instruction *instr) in live_effect() argument
413 struct ir3_sched_node *n = instr->data; in live_effect()
414 int new_live = n->partially_live ? 0 : dest_regs(instr); in live_effect()
423 foreach_ssa_src_n (src, n, instr) { in live_effect()
424 if (__is_false_dep(instr, n)) in live_effect()
427 if (instr->block != src->block) in live_effect()
443 would_sync(struct ir3_sched_ctx *ctx, struct ir3_instruction *instr) in would_sync() argument
446 if (check_src_cond(instr, is_sfu)) in would_sync()
456 if (check_src_cond(instr, is_tex_or_prefetch)) in would_sync()
483 if (avoid_sync && would_sync(ctx, n->instr)) in choose_instr_dec()
486 unsigned d = ir3_delay_calc(ctx->block, n->instr, false, false); in choose_instr_dec()
491 if (live_effect(n->instr) > -1) in choose_instr_dec()
494 if (!check_instr(ctx, notes, n->instr)) in choose_instr_dec()
503 di(chosen->instr, "dec%s: chose (freed+ready)", mode); in choose_instr_dec()
509 if (avoid_sync && would_sync(ctx, n->instr)) in choose_instr_dec()
512 if (live_effect(n->instr) > -1) in choose_instr_dec()
515 if (!check_instr(ctx, notes, n->instr)) in choose_instr_dec()
524 di(chosen->instr, "dec%s: chose (freed)", mode); in choose_instr_dec()
536 if (avoid_sync && would_sync(ctx, n->instr)) in choose_instr_dec()
539 unsigned d = ir3_delay_calc(ctx->block, n->instr, false, false); in choose_instr_dec()
544 if (live_effect(n->instr) > 0) in choose_instr_dec()
547 if (!check_instr(ctx, notes, n->instr)) in choose_instr_dec()
555 di(chosen->instr, "dec%s: chose (neutral+ready)", mode); in choose_instr_dec()
560 if (avoid_sync && would_sync(ctx, n->instr)) in choose_instr_dec()
563 if (live_effect(n->instr) > 0) in choose_instr_dec()
566 if (!check_instr(ctx, notes, n->instr)) in choose_instr_dec()
574 di(chosen->instr, "dec%s: chose (neutral)", mode); in choose_instr_dec()
604 if (avoid_sync && would_sync(ctx, n->instr)) in choose_instr_inc()
607 unsigned d = ir3_delay_calc(ctx->block, n->instr, false, false); in choose_instr_inc()
612 if (!check_instr(ctx, notes, n->instr)) in choose_instr_inc()
615 unsigned distance = nearest_use(n->instr); in choose_instr_inc()
624 di(chosen->instr, "inc%s: chose (distance+ready)", mode); in choose_instr_inc()
633 if (avoid_sync && would_sync(ctx, n->instr)) in choose_instr_inc()
636 if (!check_instr(ctx, notes, n->instr)) in choose_instr_inc()
639 unsigned distance = nearest_use(n->instr); in choose_instr_inc()
648 di(chosen->instr, "inc%s: chose (distance)", mode); in choose_instr_inc()
664 if (!is_meta(n->instr)) in choose_instr_prio()
672 di(chosen->instr, "prio: chose (meta)"); in choose_instr_prio()
686 di(n->instr, "maxdel=%3d le=%d del=%u ", in dump_state()
687 n->max_delay, live_effect(n->instr), in dump_state()
688 ir3_delay_calc(ctx->block, n->instr, false, false)); in dump_state()
693 di(child->instr, " -> (%d parents) ", child->dag.parent_count); in dump_state()
708 return chosen->instr; in choose_instr()
712 return chosen->instr; in choose_instr()
716 return chosen->instr; in choose_instr()
720 return chosen->instr; in choose_instr()
815 predicated->regs[1]->instr = new_pred; in split_pred()
831 sched_node_init(struct ir3_sched_ctx *ctx, struct ir3_instruction *instr) in sched_node_init() argument
837 n->instr = instr; in sched_node_init()
838 instr->data = n; in sched_node_init()
842 sched_node_add_dep(struct ir3_instruction *instr, struct ir3_instruction *src, int i) in sched_node_add_dep() argument
845 if (src->block != instr->block) in sched_node_add_dep()
850 debug_assert(__is_false_dep(instr, i)); in sched_node_add_dep()
854 struct ir3_sched_node *n = instr->data; in sched_node_add_dep()
861 if (instr->opc == OPC_META_COLLECT) in sched_node_add_dep()
862 sn->collect = instr; in sched_node_add_dep()
866 unsigned d = ir3_delayslots(src, instr, i, true); in sched_node_add_dep()
871 mark_kill_path(struct ir3_instruction *instr) in mark_kill_path() argument
873 struct ir3_sched_node *n = instr->data; in mark_kill_path()
876 foreach_ssa_src (src, instr) { in mark_kill_path()
877 if (src->block != instr->block) in mark_kill_path()
885 is_output_collect(struct ir3_instruction *instr) in is_output_collect() argument
887 struct ir3 *ir = instr->block->shader; in is_output_collect()
892 if (instr == collect) in is_output_collect()
901 is_output_only(struct ir3_instruction *instr) in is_output_only() argument
903 if (!writes_gpr(instr)) in is_output_only()
906 if (!(instr->regs[0]->flags & IR3_REG_SSA)) in is_output_only()
909 foreach_ssa_use (use, instr) in is_output_only()
917 sched_node_add_deps(struct ir3_instruction *instr) in sched_node_add_deps() argument
922 foreach_ssa_src_n (src, i, instr) { in sched_node_add_deps()
923 sched_node_add_dep(instr, src, i); in sched_node_add_deps()
929 if (is_kill(instr) || is_input(instr)) { in sched_node_add_deps()
930 mark_kill_path(instr); in sched_node_add_deps()
933 if (is_output_only(instr)) { in sched_node_add_deps()
934 struct ir3_sched_node *n = instr->data; in sched_node_add_deps()
958 foreach_instr (instr, &ctx->unscheduled_list) { in sched_dag_init()
959 sched_node_init(ctx, instr); in sched_dag_init()
960 sched_node_add_deps(instr); in sched_dag_init()
996 foreach_instr_safe (instr, &ctx->unscheduled_list) { in sched_block()
997 if (is_kill(instr)) in sched_block()
999 if (is_tex_or_prefetch(instr)) in sched_block()
1012 foreach_instr_safe (instr, &ctx->unscheduled_list) in sched_block()
1013 if (instr->opc == OPC_META_INPUT) in sched_block()
1014 schedule(ctx, instr); in sched_block()
1016 foreach_instr_safe (instr, &ctx->unscheduled_list) in sched_block()
1017 if (instr->opc == OPC_META_TEX_PREFETCH) in sched_block()
1018 schedule(ctx, instr); in sched_block()
1022 struct ir3_instruction *instr; in sched_block() local
1024 instr = choose_instr(ctx, ¬es); in sched_block()
1025 if (instr) { in sched_block()
1026 unsigned delay = ir3_delay_calc(ctx->block, instr, false, false); in sched_block()
1038 schedule(ctx, instr); in sched_block()
1058 foreach_instr (instr, &ctx->unscheduled_list) in sched_block()
1059 di(instr, "unscheduled: "); in sched_block()
1080 foreach_instr (instr, &block->instr_list) { in ir3_sched()
1081 instr->data = NULL; in ir3_sched()
1101 get_array_id(struct ir3_instruction *instr) in get_array_id() argument
1107 for (unsigned i = 0; i < instr->regs_count; i++) in get_array_id()
1108 if (instr->regs[i]->flags & IR3_REG_ARRAY) in get_array_id()
1109 return instr->regs[i]->array.id; in get_array_id()
1116 depends_on(struct ir3_instruction *instr, struct ir3_instruction *prior) in depends_on() argument
1123 if (((instr->barrier_class & IR3_BARRIER_EVERYTHING) && prior->barrier_class) || in depends_on()
1124 ((prior->barrier_class & IR3_BARRIER_EVERYTHING) && instr->barrier_class)) in depends_on()
1127 if (instr->barrier_class & prior->barrier_conflict) { in depends_on()
1128 if (!(instr->barrier_class & ~(IR3_BARRIER_ARRAY_R | IR3_BARRIER_ARRAY_W))) { in depends_on()
1133 if (get_array_id(instr) != get_array_id(prior)) { in depends_on()
1145 add_barrier_deps(struct ir3_block *block, struct ir3_instruction *instr) in add_barrier_deps() argument
1147 struct list_head *prev = instr->node.prev; in add_barrier_deps()
1148 struct list_head *next = instr->node.next; in add_barrier_deps()
1162 if (instr->barrier_class == pi->barrier_class) { in add_barrier_deps()
1163 ir3_instr_add_dep(instr, pi); in add_barrier_deps()
1167 if (depends_on(instr, pi)) in add_barrier_deps()
1168 ir3_instr_add_dep(instr, pi); in add_barrier_deps()
1183 if (instr->barrier_class == ni->barrier_class) { in add_barrier_deps()
1184 ir3_instr_add_dep(ni, instr); in add_barrier_deps()
1188 if (depends_on(ni, instr)) in add_barrier_deps()
1189 ir3_instr_add_dep(ni, instr); in add_barrier_deps()
1208 foreach_instr (instr, &block->instr_list) { in ir3_sched_add_deps()
1209 if (instr->barrier_class) { in ir3_sched_add_deps()
1210 add_barrier_deps(block, instr); in ir3_sched_add_deps()