Lines Matching refs:instr
180 int get_wait_states(aco_ptr<Instruction>& instr) in get_wait_states() argument
182 if (instr->opcode == aco_opcode::s_nop) in get_wait_states()
183 return static_cast<SOPP_instruction*>(instr.get())->imm + 1; in get_wait_states()
184 else if (instr->opcode == aco_opcode::p_constaddr) in get_wait_states()
289 aco_ptr<Instruction>& instr, int *NOPs) in handle_smem_clause_hazards() argument
295 …if (ctx.smem_write || instr->definitions.empty() || instr_info.is_atomic[(unsigned)instr->opcode])… in handle_smem_clause_hazards()
298 for (Operand op : instr->operands) { in handle_smem_clause_hazards()
305 Definition def = instr->definitions[0]; in handle_smem_clause_hazards()
314 … aco_ptr<Instruction>& instr, std::vector<aco_ptr<Instruction>>& new_instructions) in handle_instruction_gfx6() argument
319 if (instr->format == Format::SMEM) { in handle_instruction_gfx6()
325 for (unsigned i = 0; i < instr->operands.size(); i++) { in handle_instruction_gfx6()
326 Operand op = instr->operands[i]; in handle_instruction_gfx6()
338 handle_smem_clause_hazards(program, ctx, instr, &NOPs); in handle_instruction_gfx6()
339 } else if (instr->isSALU()) { in handle_instruction_gfx6()
340 …if (instr->opcode == aco_opcode::s_setreg_b32 || instr->opcode == aco_opcode::s_setreg_imm32_b32 || in handle_instruction_gfx6()
341 instr->opcode == aco_opcode::s_getreg_b32) { in handle_instruction_gfx6()
346 … if (instr->opcode == aco_opcode::s_movrels_b32 || instr->opcode == aco_opcode::s_movrels_b64 || in handle_instruction_gfx6()
347 … instr->opcode == aco_opcode::s_movreld_b32 || instr->opcode == aco_opcode::s_movreld_b64) { in handle_instruction_gfx6()
352 if (instr->opcode == aco_opcode::s_sendmsg || instr->opcode == aco_opcode::s_ttracedata) in handle_instruction_gfx6()
354 } else if (instr->format == Format::DS && static_cast<DS_instruction *>(instr.get())->gds) { in handle_instruction_gfx6()
356 } else if (instr->isVALU() || instr->format == Format::VINTRP) { in handle_instruction_gfx6()
357 for (Operand op : instr->operands) { in handle_instruction_gfx6()
364 if (instr->isDPP()) { in handle_instruction_gfx6()
366 handle_valu_then_read_hazard(program, cur_block, &NOPs, 2, instr->operands[0]); in handle_instruction_gfx6()
369 for (Definition def : instr->definitions) { in handle_instruction_gfx6()
376 if ((instr->opcode == aco_opcode::v_readlane_b32 || in handle_instruction_gfx6()
377 instr->opcode == aco_opcode::v_readlane_b32_e64 || in handle_instruction_gfx6()
378 instr->opcode == aco_opcode::v_writelane_b32 || in handle_instruction_gfx6()
379 instr->opcode == aco_opcode::v_writelane_b32_e64) && in handle_instruction_gfx6()
380 !instr->operands[1].isConstant()) { in handle_instruction_gfx6()
381 handle_valu_then_read_hazard(program, cur_block, &NOPs, 4, instr->operands[1]); in handle_instruction_gfx6()
390 (instr->opcode == aco_opcode::v_readlane_b32 || /* GFX6 doesn't have v_readlane_b32_e64 */ in handle_instruction_gfx6()
391 instr->opcode == aco_opcode::v_readfirstlane_b32)) { in handle_instruction_gfx6()
392 handle_vintrp_then_read_hazard(program, cur_block, &NOPs, 1, instr->operands[0]); in handle_instruction_gfx6()
395 … if (instr->opcode == aco_opcode::v_div_fmas_f32 || instr->opcode == aco_opcode::v_div_fmas_f64) in handle_instruction_gfx6()
397 } else if (instr->isVMEM() || instr->isFlatOrGlobal() || instr->format == Format::SCRATCH) { in handle_instruction_gfx6()
399 for (Operand op : instr->operands) { in handle_instruction_gfx6()
405 if (!instr->isSALU() && instr->format != Format::SMEM) in handle_instruction_gfx6()
409 … bool lds_scratch_global = (instr->format == Format::SCRATCH || instr->format == Format::GLOBAL) && in handle_instruction_gfx6()
410 static_cast<FLAT_instruction *>(instr.get())->lds; in handle_instruction_gfx6()
411 if (instr->format == Format::VINTRP || in handle_instruction_gfx6()
412 instr->opcode == aco_opcode::ds_read_addtid_b32 || in handle_instruction_gfx6()
413 instr->opcode == aco_opcode::ds_write_addtid_b32 || in handle_instruction_gfx6()
414 instr->opcode == aco_opcode::buffer_store_lds_dword || in handle_instruction_gfx6()
420 ctx.add_wait_states(NOPs + get_wait_states(instr)); in handle_instruction_gfx6()
432 if ((ctx.smem_clause || ctx.smem_write) && (NOPs || instr->format != Format::SMEM)) { in handle_instruction_gfx6()
442 if (instr->format == Format::SMEM) { in handle_instruction_gfx6()
443 if (instr->definitions.empty() || instr_info.is_atomic[(unsigned)instr->opcode]) { in handle_instruction_gfx6()
449 for (Operand op : instr->operands) { in handle_instruction_gfx6()
455 Definition def = instr->definitions[0]; in handle_instruction_gfx6()
460 } else if (instr->isVALU()) { in handle_instruction_gfx6()
461 for (Definition def : instr->definitions) { in handle_instruction_gfx6()
473 } else if (instr->isSALU() && !instr->definitions.empty()) { in handle_instruction_gfx6()
474 if (!instr->definitions.empty()) { in handle_instruction_gfx6()
476 Definition def = instr->definitions[0]; in handle_instruction_gfx6()
482 …} else if (instr->opcode == aco_opcode::s_setreg_b32 || instr->opcode == aco_opcode::s_setreg_imm3… in handle_instruction_gfx6()
483 SOPK_instruction *sopk = static_cast<SOPK_instruction *>(instr.get()); in handle_instruction_gfx6()
492 } else if (instr->isVMEM() || instr->isFlatOrGlobal() || instr->format == Format::SCRATCH) { in handle_instruction_gfx6()
494 bool consider_buf = (instr->format == Format::MUBUF || instr->format == Format::MTBUF) && in handle_instruction_gfx6()
495 instr->operands.size() == 4 && in handle_instruction_gfx6()
496 instr->operands[3].size() > 2 && in handle_instruction_gfx6()
497 instr->operands[2].physReg() >= 128; in handle_instruction_gfx6()
499 bool consider_mimg = instr->format == Format::MIMG && in handle_instruction_gfx6()
500 instr->operands[1].regClass().type() == RegType::vgpr && in handle_instruction_gfx6()
501 instr->operands[1].size() > 2 && in handle_instruction_gfx6()
502 instr->operands[0].size() == 4; in handle_instruction_gfx6()
504 bool consider_flat = (instr->isFlatOrGlobal() || instr->format == Format::SCRATCH) && in handle_instruction_gfx6()
505 instr->operands.size() == 3 && in handle_instruction_gfx6()
506 instr->operands[2].size() > 2; in handle_instruction_gfx6()
508 PhysReg wrdata = instr->operands[consider_flat ? 2 : 3].physReg(); in handle_instruction_gfx6()
509 unsigned size = instr->operands[consider_flat ? 2 : 3].size(); in handle_instruction_gfx6()
517 bool check_written_regs(const aco_ptr<Instruction> &instr, const std::bitset<N> &check_regs) in check_written_regs() argument
519 …return std::any_of(instr->definitions.begin(), instr->definitions.end(), [&check_regs](const Defin… in check_written_regs()
530 void mark_read_regs(const aco_ptr<Instruction> &instr, std::bitset<N> ®_reads) in mark_read_regs() argument
532 for (const Operand &op : instr->operands) { in mark_read_regs()
541 bool VALU_writes_sgpr(aco_ptr<Instruction>& instr) in VALU_writes_sgpr() argument
543 if ((uint32_t) instr->format & (uint32_t) Format::VOPC) in VALU_writes_sgpr()
545 if (instr->isVOP3() && instr->definitions.size() == 2) in VALU_writes_sgpr()
547 if (instr->opcode == aco_opcode::v_readfirstlane_b32 || in VALU_writes_sgpr()
548 instr->opcode == aco_opcode::v_readlane_b32 || in VALU_writes_sgpr()
549 instr->opcode == aco_opcode::v_readlane_b32_e64) in VALU_writes_sgpr()
554 bool instr_writes_exec(const aco_ptr<Instruction>& instr) in instr_writes_exec() argument
556 …return std::any_of(instr->definitions.begin(), instr->definitions.end(), [](const Definition &def)… in instr_writes_exec()
561 bool instr_writes_sgpr(const aco_ptr<Instruction>& instr) in instr_writes_sgpr() argument
563 …return std::any_of(instr->definitions.begin(), instr->definitions.end(), [](const Definition &def)… in instr_writes_sgpr()
568 inline bool instr_is_branch(const aco_ptr<Instruction>& instr) in instr_is_branch() argument
570 return instr->opcode == aco_opcode::s_branch || in instr_is_branch()
571 instr->opcode == aco_opcode::s_cbranch_scc0 || in instr_is_branch()
572 instr->opcode == aco_opcode::s_cbranch_scc1 || in instr_is_branch()
573 instr->opcode == aco_opcode::s_cbranch_vccz || in instr_is_branch()
574 instr->opcode == aco_opcode::s_cbranch_vccnz || in instr_is_branch()
575 instr->opcode == aco_opcode::s_cbranch_execz || in instr_is_branch()
576 instr->opcode == aco_opcode::s_cbranch_execnz || in instr_is_branch()
577 instr->opcode == aco_opcode::s_cbranch_cdbgsys || in instr_is_branch()
578 instr->opcode == aco_opcode::s_cbranch_cdbguser || in instr_is_branch()
579 instr->opcode == aco_opcode::s_cbranch_cdbgsys_or_user || in instr_is_branch()
580 instr->opcode == aco_opcode::s_cbranch_cdbgsys_and_user || in instr_is_branch()
581 instr->opcode == aco_opcode::s_subvector_loop_begin || in instr_is_branch()
582 instr->opcode == aco_opcode::s_subvector_loop_end || in instr_is_branch()
583 instr->opcode == aco_opcode::s_setpc_b64 || in instr_is_branch()
584 instr->opcode == aco_opcode::s_swappc_b64 || in instr_is_branch()
585 instr->opcode == aco_opcode::s_getpc_b64 || in instr_is_branch()
586 instr->opcode == aco_opcode::s_call_b64; in instr_is_branch()
590 … aco_ptr<Instruction>& instr, std::vector<aco_ptr<Instruction>>& new_instructions) in handle_instruction_gfx10() argument
597 if (instr->isVMEM() || instr->format == Format::FLAT || instr->format == Format::GLOBAL || in handle_instruction_gfx10()
598 instr->format == Format::SCRATCH || instr->format == Format::DS) { in handle_instruction_gfx10()
600 mark_read_regs(instr, ctx.sgprs_read_by_VMEM); in handle_instruction_gfx10()
604 } else if (instr->isSALU() || instr->format == Format::SMEM) { in handle_instruction_gfx10()
605 if (instr->opcode == aco_opcode::s_waitcnt) { in handle_instruction_gfx10()
607 uint16_t imm = static_cast<SOPP_instruction*>(instr.get())->imm; in handle_instruction_gfx10()
611 } else if (instr->opcode == aco_opcode::s_waitcnt_depctr) { in handle_instruction_gfx10()
613 const SOPP_instruction *sopp = static_cast<const SOPP_instruction *>(instr.get()); in handle_instruction_gfx10()
619 if (check_written_regs(instr, ctx.sgprs_read_by_VMEM)) { in handle_instruction_gfx10()
628 } else if (instr->isVALU()) { in handle_instruction_gfx10()
636 if (instr->format == Format::VOPC) { in handle_instruction_gfx10()
639 (instr->opcode == aco_opcode::v_permlane16_b32 || in handle_instruction_gfx10()
640 instr->opcode == aco_opcode::v_permlanex16_b32)) { in handle_instruction_gfx10()
645 v_mov->definitions[0] = Definition(instr->operands[0].physReg(), v1); in handle_instruction_gfx10()
646 v_mov->operands[0] = Operand(instr->operands[0].physReg(), v1); in handle_instruction_gfx10()
648 } else if (instr->isVALU() && instr->opcode != aco_opcode::v_nop) { in handle_instruction_gfx10()
655 if (!instr->isVALU() && instr->reads_exec()) { in handle_instruction_gfx10()
657 } else if (instr->isVALU()) { in handle_instruction_gfx10()
658 if (instr_writes_exec(instr)) { in handle_instruction_gfx10()
666 } else if (instr_writes_sgpr(instr)) { in handle_instruction_gfx10()
670 } else if (instr->opcode == aco_opcode::s_waitcnt_depctr) { in handle_instruction_gfx10()
672 const SOPP_instruction *sopp = static_cast<const SOPP_instruction *>(instr.get()); in handle_instruction_gfx10()
680 if (instr->format == Format::SMEM) { in handle_instruction_gfx10()
682 mark_read_regs(instr, ctx.sgprs_read_by_SMEM); in handle_instruction_gfx10()
683 } else if (VALU_writes_sgpr(instr)) { in handle_instruction_gfx10()
685 if (check_written_regs(instr, ctx.sgprs_read_by_SMEM)) { in handle_instruction_gfx10()
694 } else if (instr->isSALU()) { in handle_instruction_gfx10()
695 if (instr->format != Format::SOPP) { in handle_instruction_gfx10()
700 const SOPP_instruction *sopp = static_cast<const SOPP_instruction *>(instr.get()); in handle_instruction_gfx10()
715 if (instr->isVMEM() || instr->format == Format::GLOBAL || instr->format == Format::SCRATCH) { in handle_instruction_gfx10()
720 } else if (instr->format == Format::DS) { in handle_instruction_gfx10()
725 } else if (instr_is_branch(instr)) { in handle_instruction_gfx10()
728 } else if (instr->opcode == aco_opcode::s_waitcnt_vscnt) { in handle_instruction_gfx10()
730 const SOPK_instruction *sopk = static_cast<const SOPK_instruction *>(instr.get()); in handle_instruction_gfx10()
759 for (aco_ptr<Instruction>& instr : old_instructions) { in handle_block()
760 Handle(program, &block, ctx, instr, block.instructions); in handle_block()
761 block.instructions.emplace_back(std::move(instr)); in handle_block()