Lines Matching refs:instr
79 …auto check = [&program, &is_valid](bool check, const char * msg, aco::Instruction * instr) -> void… in validate_ir() argument
88 aco_print_instr(instr, memf); in validate_ir()
106 for (aco_ptr<Instruction>& instr : block.instructions) { in validate_ir()
109 Format base_format = instr->format; in validate_ir()
119 if (instr->opcode == aco_opcode::v_interp_p1ll_f16 || in validate_ir()
120 instr->opcode == aco_opcode::v_interp_p1lv_f16 || in validate_ir()
121 instr->opcode == aco_opcode::v_interp_p2_legacy_f16 || in validate_ir()
122 instr->opcode == aco_opcode::v_interp_p2_f16) { in validate_ir()
131 …check(base_format == instr_info.format[(int)instr->opcode], "Wrong base format for instruction", i… in validate_ir()
134 if (((uint32_t)instr->format & (uint32_t)Format::VOP3) && instr->format != Format::VOP3) { in validate_ir()
139 "Format cannot have VOP3A/VOP3B applied", instr.get()); in validate_ir()
143 if (instr->isSDWA()) { in validate_ir()
147 "Format cannot have SDWA applied", instr.get()); in validate_ir()
149 check(program->chip_class >= GFX8, "SDWA is GFX8+ only", instr.get()); in validate_ir()
151 SDWA_instruction *sdwa = static_cast<SDWA_instruction*>(instr.get()); in validate_ir()
152 …(sdwa->omod == 0 || program->chip_class >= GFX9, "SDWA omod only supported on GFX9+", instr.get()); in validate_ir()
154 …mp == false || program->chip_class == GFX8, "SDWA VOPC clamp only supported on GFX8", instr.get()); in validate_ir()
155 check((instr->definitions[0].isFixed() && instr->definitions[0].physReg() == vcc) || in validate_ir()
157 "SDWA+VOPC definition must be fixed to vcc on GFX8", instr.get()); in validate_ir()
160 if (instr->operands.size() >= 3) { in validate_ir()
161 check(instr->operands[2].isFixed() && instr->operands[2].physReg() == vcc, in validate_ir()
162 "3rd operand must be fixed to vcc with SDWA", instr.get()); in validate_ir()
164 if (instr->definitions.size() >= 2) { in validate_ir()
165 check(instr->definitions[1].isFixed() && instr->definitions[1].physReg() == vcc, in validate_ir()
166 "2nd definition must be fixed to vcc with SDWA", instr.get()); in validate_ir()
169 check(instr->opcode != aco_opcode::v_madmk_f32 && in validate_ir()
170 instr->opcode != aco_opcode::v_madak_f32 && in validate_ir()
171 instr->opcode != aco_opcode::v_madmk_f16 && in validate_ir()
172 instr->opcode != aco_opcode::v_madak_f16 && in validate_ir()
173 instr->opcode != aco_opcode::v_readfirstlane_b32 && in validate_ir()
174 instr->opcode != aco_opcode::v_clrexcp && in validate_ir()
175 instr->opcode != aco_opcode::v_swap_b32, in validate_ir()
176 "SDWA can't be used with this opcode", instr.get()); in validate_ir()
178 check(instr->opcode != aco_opcode::v_mac_f32 && in validate_ir()
179 instr->opcode != aco_opcode::v_mac_f16 && in validate_ir()
180 instr->opcode != aco_opcode::v_fmac_f32 && in validate_ir()
181 instr->opcode != aco_opcode::v_fmac_f16, in validate_ir()
182 "SDWA can't be used with this opcode", instr.get()); in validate_ir()
185 for (unsigned i = 0; i < MIN2(instr->operands.size(), 2); i++) { in validate_ir()
186 if (instr->operands[i].hasRegClass() && instr->operands[i].regClass().is_subdword()) in validate_ir()
187 …el[i] & sdwa_asuint) == (sdwa_isra | instr->operands[i].bytes()), "Unexpected SDWA sel for sub-dwo… in validate_ir()
189 if (instr->definitions[0].regClass().is_subdword()) in validate_ir()
190 …el & sdwa_asuint) == (sdwa_isra | instr->definitions[0].bytes()), "Unexpected SDWA sel for sub-dwo… in validate_ir()
194 if (instr->isVOP3()) { in validate_ir()
195 VOP3A_instruction *vop3 = static_cast<VOP3A_instruction*>(instr.get()); in validate_ir()
196 …(vop3->opsel == 0 || program->chip_class >= GFX9, "Opsel is only supported on GFX9+", instr.get()); in validate_ir()
199 if (i >= instr->operands.size() || in validate_ir()
200 …(instr->operands[i].hasRegClass() && instr->operands[i].regClass().is_subdword() && !instr->operan… in validate_ir()
201 check((vop3->opsel & (1 << i)) == 0, "Unexpected opsel for operand", instr.get()); in validate_ir()
203 if (instr->definitions[0].regClass().is_subdword() && !instr->definitions[0].isFixed()) in validate_ir()
204 … check((vop3->opsel & (1 << 3)) == 0, "Unexpected opsel for sub-dword definition", instr.get()); in validate_ir()
208 for (unsigned i = 0; i < instr->operands.size(); i++) { in validate_ir()
209 if (instr->operands[i].isUndefined()) { in validate_ir()
210 …bool flat = instr->format == Format::FLAT || instr->format == Format::SCRATCH || instr->format == … in validate_ir()
211 bool can_be_undef = is_phi(instr) || instr->format == Format::EXP || in validate_ir()
212 instr->format == Format::PSEUDO_REDUCTION || in validate_ir()
213 instr->opcode == aco_opcode::p_create_vector || in validate_ir()
214 (flat && i == 1) || (instr->format == Format::MIMG && i == 1) || in validate_ir()
215 … ((instr->format == Format::MUBUF || instr->format == Format::MTBUF) && i == 1); in validate_ir()
216 check(can_be_undef, "Undefs can only be used in certain operands", instr.get()); in validate_ir()
218 …check(instr->operands[i].isFixed() || instr->operands[i].isTemp() || instr->operands[i].isConstant… in validate_ir()
223 for (unsigned i = 0; i < instr->definitions.size(); i++) { in validate_ir()
224 if (instr->definitions[i].regClass().is_subdword()) in validate_ir()
225 …instr->format == Format::PSEUDO || instr->definitions[i].bytes() <= 4, "Only Pseudo instructions c… in validate_ir()
228 if (instr->isSALU() || instr->isVALU()) { in validate_ir()
231 for (unsigned i = 0; i < instr->operands.size(); i++) in validate_ir()
233 Operand op = instr->operands[i]; in validate_ir()
237 check(instr->format == Format::SOP1 || in validate_ir()
238 instr->format == Format::SOP2 || in validate_ir()
239 instr->format == Format::SOPC || in validate_ir()
240 instr->format == Format::VOP1 || in validate_ir()
241 instr->format == Format::VOP2 || in validate_ir()
242 instr->format == Format::VOPC || in validate_ir()
243 (instr->isVOP3() && program->chip_class >= GFX10), in validate_ir()
244 "Literal applied on wrong instruction format", instr.get()); in validate_ir()
246 …p.size() && literal.constantValue() == op.constantValue()), "Only 1 Literal allowed", instr.get()); in validate_ir()
248 …check(!instr->isVALU() || instr->isVOP3() || i == 0 || i == 2, "Wrong source position for Literal … in validate_ir()
252 if (instr->isVALU()) { in validate_ir()
253 bool is_shift64 = instr->opcode == aco_opcode::v_lshlrev_b64 || in validate_ir()
254 instr->opcode == aco_opcode::v_lshrrev_b64 || in validate_ir()
255 instr->opcode == aco_opcode::v_ashrrev_i64; in validate_ir()
260 uint32_t scalar_mask = instr->isVOP3() ? 0x7 : 0x5; in validate_ir()
261 if (instr->isSDWA()) in validate_ir()
264 if ((int) instr->format & (int) Format::VOPC || in validate_ir()
265 instr->opcode == aco_opcode::v_readfirstlane_b32 || in validate_ir()
266 instr->opcode == aco_opcode::v_readlane_b32 || in validate_ir()
267 instr->opcode == aco_opcode::v_readlane_b32_e64) { in validate_ir()
268 check(instr->definitions[0].getTemp().type() == RegType::sgpr, in validate_ir()
269 "Wrong Definition type for VALU instruction", instr.get()); in validate_ir()
271 check(instr->definitions[0].getTemp().type() == RegType::vgpr, in validate_ir()
272 "Wrong Definition type for VALU instruction", instr.get()); in validate_ir()
277 for (unsigned i = 0; i < instr->operands.size(); i++) in validate_ir()
279 Operand op = instr->operands[i]; in validate_ir()
280 if (instr->opcode == aco_opcode::v_readfirstlane_b32 || in validate_ir()
281 instr->opcode == aco_opcode::v_readlane_b32 || in validate_ir()
282 instr->opcode == aco_opcode::v_readlane_b32_e64) { in validate_ir()
286 "Must be a SGPR or a constant", instr.get()); in validate_ir()
289 "Wrong Operand type for VALU instruction", instr.get()); in validate_ir()
293 if (instr->opcode == aco_opcode::v_writelane_b32 || in validate_ir()
294 instr->opcode == aco_opcode::v_writelane_b32_e64) { in validate_ir()
297 "Wrong Operand type for VALU instruction", instr.get()); in validate_ir()
301 "Must be a SGPR or a constant", instr.get()); in validate_ir()
304 if (op.isTemp() && instr->operands[i].regClass().type() == RegType::sgpr) { in validate_ir()
305 … check(scalar_mask & (1 << i), "Wrong source position for SGPR argument", instr.get()); in validate_ir()
314 … check(scalar_mask & (1 << i), "Wrong source position for constant argument", instr.get()); in validate_ir()
316 …gprs + (literal.isUndefined() ? 0 : 1) <= const_bus_limit, "Too many SGPRs/literals", instr.get()); in validate_ir()
319 if (instr->format == Format::SOP1 || instr->format == Format::SOP2) { in validate_ir()
320 …check(instr->definitions[0].getTemp().type() == RegType::sgpr, "Wrong Definition type for SALU ins… in validate_ir()
321 for (const Operand& op : instr->operands) { in validate_ir()
323 "Wrong Operand type for SALU instruction", instr.get()); in validate_ir()
328 switch (instr->format) { in validate_ir()
330 if (instr->opcode == aco_opcode::p_parallelcopy) { in validate_ir()
331 for (unsigned i = 0; i < instr->operands.size(); i++) { in validate_ir()
332 if (!instr->definitions[i].regClass().is_subdword()) in validate_ir()
334 Operand op = instr->operands[i]; in validate_ir()
335 …gram->chip_class >= GFX9 || !op.isLiteral(), "Sub-dword copies cannot take literals", instr.get()); in validate_ir()
337 …s >= GFX9, "Sub-dword pseudo instructions can only take constants or SGPRs on GFX9+", instr.get()); in validate_ir()
343 for (Definition def : instr->definitions) in validate_ir()
345 for (unsigned i = 0; i < instr->operands.size(); i++) { in validate_ir()
346 if (instr->opcode == aco_opcode::p_extract_vector && i == 1) in validate_ir()
348 Operand op = instr->operands[i]; in validate_ir()
355 … "Sub-dword pseudo instructions can only take constants or SGPRs on GFX9+", instr.get()); in validate_ir()
358 if (instr->opcode == aco_opcode::p_create_vector) { in validate_ir()
360 for (const Operand& op : instr->operands) { in validate_ir()
363 …check(size == instr->definitions[0].bytes(), "Definition size does not match operand sizes", instr… in validate_ir()
364 if (instr->definitions[0].getTemp().type() == RegType::sgpr) { in validate_ir()
365 for (const Operand& op : instr->operands) { in validate_ir()
367 "Wrong Operand type for scalar vector", instr.get()); in validate_ir()
370 } else if (instr->opcode == aco_opcode::p_extract_vector) { in validate_ir()
371 …check((instr->operands[0].isTemp()) && instr->operands[1].isConstant(), "Wrong Operand types", ins… in validate_ir()
372 …check((instr->operands[1].constantValue() + 1) * instr->definitions[0].bytes() <= instr->operands[… in validate_ir()
373 …check(instr->definitions[0].getTemp().type() == RegType::vgpr || instr->operands[0].regClass().typ… in validate_ir()
374 "Cannot extract SGPR value from VGPR vector", instr.get()); in validate_ir()
375 } else if (instr->opcode == aco_opcode::p_split_vector) { in validate_ir()
376 check(instr->operands[0].isTemp(), "Operand must be a temporary", instr.get()); in validate_ir()
378 for (const Definition& def : instr->definitions) { in validate_ir()
381 …check(size == instr->operands[0].bytes(), "Operand size does not match definition sizes", instr.ge… in validate_ir()
382 if (instr->operands[0].getTemp().type() == RegType::vgpr) { in validate_ir()
383 for (const Definition& def : instr->definitions) in validate_ir()
384 …ef.regClass().type() == RegType::vgpr, "Wrong Definition type for VGPR split_vector", instr.get()); in validate_ir()
386 } else if (instr->opcode == aco_opcode::p_parallelcopy) { in validate_ir()
387 …check(instr->definitions.size() == instr->operands.size(), "Number of Operands does not match numb… in validate_ir()
388 for (unsigned i = 0; i < instr->operands.size(); i++) { in validate_ir()
389 if (instr->operands[i].isTemp()) in validate_ir()
390 … check((instr->definitions[i].getTemp().type() == instr->operands[i].regClass().type()) || in validate_ir()
391 …(instr->definitions[i].getTemp().type() == RegType::vgpr && instr->operands[i].regClass().type() =… in validate_ir()
392 "Operand and Definition types do not match", instr.get()); in validate_ir()
394 } else if (instr->opcode == aco_opcode::p_phi) { in validate_ir()
395 …check(instr->operands.size() == block.logical_preds.size(), "Number of Operands does not match num… in validate_ir()
396 …check(instr->definitions[0].getTemp().type() == RegType::vgpr, "Logical Phi Definition must be vgp… in validate_ir()
397 } else if (instr->opcode == aco_opcode::p_linear_phi) { in validate_ir()
398 for (const Operand& op : instr->operands) in validate_ir()
399 … check(!op.isTemp() || op.getTemp().is_linear(), "Wrong Operand type", instr.get()); in validate_ir()
400 …check(instr->operands.size() == block.linear_preds.size(), "Number of Operands does not match numb… in validate_ir()
405 for (const Operand &op : instr->operands) in validate_ir()
406 … == RegType::vgpr, "All operands of PSEUDO_REDUCTION instructions must be in VGPRs.", instr.get()); in validate_ir()
408 … unsigned cluster_size = static_cast<Pseudo_reduction_instruction *>(instr.get())->cluster_size; in validate_ir()
410 if (instr->opcode == aco_opcode::p_reduce && cluster_size == program->wave_size) in validate_ir()
411 …check(instr->definitions[0].regClass().type() == RegType::sgpr, "The result of unclustered reducti… in validate_ir()
413 …check(instr->definitions[0].regClass().type() == RegType::vgpr, "The result of scans and clustered… in validate_ir()
418 if (instr->operands.size() >= 1) in validate_ir()
419 check((instr->operands[0].isFixed() && !instr->operands[0].isConstant()) || in validate_ir()
420 …(instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::sgpr), "SMEM oper… in validate_ir()
421 if (instr->operands.size() >= 2) in validate_ir()
422 …check(instr->operands[1].isConstant() || (instr->operands[1].isTemp() && instr->operands[1].regCla… in validate_ir()
423 "SMEM offset must be constant or sgpr", instr.get()); in validate_ir()
424 if (!instr->definitions.empty()) in validate_ir()
425 …check(instr->definitions[0].getTemp().type() == RegType::sgpr, "SMEM result must be sgpr", instr.g… in validate_ir()
430 …check(instr->operands.size() > 1, "VMEM instructions must have at least one operand", instr.get()); in validate_ir()
431 … check(instr->operands[1].hasRegClass() && instr->operands[1].regClass().type() == RegType::vgpr, in validate_ir()
432 "VADDR must be in vgpr for VMEM instructions", instr.get()); in validate_ir()
433 …check(instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::sgpr, "VMEM … in validate_ir()
434 …instr->operands.size() < 4 || (instr->operands[3].isTemp() && instr->operands[3].regClass().type()… in validate_ir()
438 … check(instr->operands.size() == 3, "MIMG instructions must have exactly 3 operands", instr.get()); in validate_ir()
439 …check(instr->operands[0].hasRegClass() && (instr->operands[0].regClass() == s4 || instr->operands[… in validate_ir()
440 "MIMG operands[0] (resource constant) must be in 4 or 8 SGPRs", instr.get()); in validate_ir()
441 … if (instr->operands[1].hasRegClass() && instr->operands[1].regClass().type() == RegType::sgpr) in validate_ir()
442 …check(instr->operands[1].regClass() == s4, "MIMG operands[1] (sampler constant) must be 4 SGPRs", … in validate_ir()
443 …else if (instr->operands[1].hasRegClass() && instr->operands[1].regClass().type() == RegType::vgpr) in validate_ir()
444 …check((instr->definitions.empty() || instr->definitions[0].regClass() == instr->operands[1].regCla… in validate_ir()
445 …instr->opcode == aco_opcode::image_atomic_cmpswap || instr->opcode == aco_opcode::image_atomic_fcm… in validate_ir()
446 … "MIMG operands[1] (VDATA) must be the same as definitions[0] for atomics", instr.get()); in validate_ir()
447 … check(instr->operands[2].hasRegClass() && instr->operands[2].regClass().type() == RegType::vgpr, in validate_ir()
448 "MIMG operands[2] (VADDR) must be VGPR", instr.get()); in validate_ir()
449 …check(instr->definitions.empty() || (instr->definitions[0].isTemp() && instr->definitions[0].regCl… in validate_ir()
450 "MIMG definitions[0] (VDATA) must be VGPR", instr.get()); in validate_ir()
454 for (const Operand& op : instr->operands) { in validate_ir()
456 "Only VGPRs are valid DS instruction operands", instr.get()); in validate_ir()
458 if (!instr->definitions.empty()) in validate_ir()
459 …check(instr->definitions[0].getTemp().type() == RegType::vgpr, "DS instruction must return VGPR", … in validate_ir()
464 … check(instr->operands[i].hasRegClass() && instr->operands[i].regClass().type() == RegType::vgpr, in validate_ir()
465 "Only VGPRs are valid Export arguments", instr.get()); in validate_ir()
469 … check(instr->operands[1].isUndefined(), "Flat instructions don't support SADDR", instr.get()); in validate_ir()
473 …check(instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::vgpr, "FLAT/… in validate_ir()
474 … check(instr->operands[1].hasRegClass() && instr->operands[1].regClass().type() == RegType::sgpr, in validate_ir()
475 "FLAT/GLOBAL/SCRATCH sgpr address must be undefined or sgpr", instr.get()); in validate_ir()
476 if (!instr->definitions.empty()) in validate_ir()
477 …check(instr->definitions[0].getTemp().type() == RegType::vgpr, "FLAT/GLOBAL/SCRATCH result must be… in validate_ir()
479 …check(instr->operands[2].regClass().type() == RegType::vgpr, "FLAT/GLOBAL/SCRATCH data must be vgp… in validate_ir()
519 Location() : block(NULL), instr(NULL) {} in Location()
522 Instruction *instr; //NULL if it's the block's live-in member
545 if (loc.instr) { in ra_fail()
546 aco_print_instr(loc.instr, memf); in ra_fail()
553 aco_print_instr(loc2.instr, memf); in ra_fail()
564 bool validate_subdword_operand(chip_class chip, const aco_ptr<Instruction>& instr, unsigned index) in validate_subdword_operand() argument
566 Operand op = instr->operands[index]; in validate_subdword_operand()
569 if (instr->opcode == aco_opcode::p_as_uniform) in validate_subdword_operand()
571 if (instr->format == Format::PSEUDO && chip >= GFX8) in validate_subdword_operand()
573 …if (instr->isSDWA() && (static_cast<SDWA_instruction *>(instr.get())->sel[index] & sdwa_asuint) ==… in validate_subdword_operand()
575 if (byte == 2 && can_use_opsel(chip, instr->opcode, index, 1)) in validate_subdword_operand()
578 switch (instr->opcode) { in validate_subdword_operand()
616 bool validate_subdword_definition(chip_class chip, const aco_ptr<Instruction>& instr) in validate_subdword_definition() argument
618 Definition def = instr->definitions[0]; in validate_subdword_definition()
621 if (instr->format == Format::PSEUDO && chip >= GFX8) in validate_subdword_definition()
623 …if (instr->isSDWA() && static_cast<SDWA_instruction *>(instr.get())->dst_sel == (sdwa_isra | def.b… in validate_subdword_definition()
625 if (byte == 2 && can_use_opsel(chip, instr->opcode, -1, 1)) in validate_subdword_definition()
628 switch (instr->opcode) { in validate_subdword_definition()
647 unsigned get_subdword_bytes_written(Program *program, const aco_ptr<Instruction>& instr, unsigned i… in get_subdword_bytes_written() argument
650 Definition def = instr->definitions[index]; in get_subdword_bytes_written()
652 if (instr->format == Format::PSEUDO) in get_subdword_bytes_written()
654 …if (instr->isSDWA() && static_cast<SDWA_instruction *>(instr.get())->dst_sel == (sdwa_isra | def.b… in get_subdword_bytes_written()
657 switch (instr->opcode) { in get_subdword_bytes_written()
691 … return MAX2(chip >= GFX10 ? def.bytes() : 4, instr_info.definition_size[(int)instr->opcode] / 8u); in get_subdword_bytes_written()
708 for (aco_ptr<Instruction>& instr : block.instructions) { in validate_ra()
709 if (instr->opcode == aco_opcode::p_phi) { in validate_ra()
710 for (unsigned i = 0; i < instr->operands.size(); i++) { in validate_ra()
711 if (instr->operands[i].isTemp() && in validate_ra()
712 instr->operands[i].getTemp().type() == RegType::sgpr && in validate_ra()
713 instr->operands[i].isFirstKill()) in validate_ra()
714 phi_sgpr_ops[block.logical_preds[i]].emplace_back(instr->operands[i].getTemp()); in validate_ra()
718 loc.instr = instr.get(); in validate_ra()
719 for (unsigned i = 0; i < instr->operands.size(); i++) { in validate_ra()
720 Operand& op = instr->operands[i]; in validate_ra()
732 … if (op.regClass().is_subdword() && !validate_subdword_operand(program->chip_class, instr, i)) in validate_ra()
740 for (unsigned i = 0; i < instr->definitions.size(); i++) { in validate_ra()
741 Definition& def = instr->definitions[i]; in validate_ra()
753 … if (def.regClass().is_subdword() && !validate_subdword_definition(program->chip_class, instr)) in validate_ra()
790 aco_ptr<Instruction>& instr = *it; in validate_ra() local
793 if (instr->opcode == aco_opcode::p_logical_end) { in validate_ra()
804 for (const Definition& def : instr->definitions) { in validate_ra()
812 if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) { in validate_ra()
813 for (const Operand& op : instr->operands) { in validate_ra()
827 for (aco_ptr<Instruction>& instr : block.instructions) { in validate_ra()
828 loc.instr = instr.get(); in validate_ra()
831 if (instr->opcode == aco_opcode::p_logical_end) { in validate_ra()
839 if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) { in validate_ra()
840 for (const Operand& op : instr->operands) { in validate_ra()
850 for (unsigned i = 0; i < instr->definitions.size(); i++) { in validate_ra()
851 Definition& def = instr->definitions[i]; in validate_ra()
862 unsigned written = get_subdword_bytes_written(program, instr, i); in validate_ra()
872 for (const Definition& def : instr->definitions) { in validate_ra()
881 if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) { in validate_ra()
882 for (const Operand& op : instr->operands) { in validate_ra()