1 2template = """\ 3/* 4 * Copyright (c) 2019 Valve Corporation 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the next 14 * paragraph) shall be included in all copies or substantial portions of the 15 * Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 23 * IN THE SOFTWARE. 24 * 25 * This file was generated by aco_builder_h.py 26 */ 27 28#ifndef _ACO_BUILDER_ 29#define _ACO_BUILDER_ 30 31#include "aco_ir.h" 32#include "util/u_math.h" 33#include "util/bitscan.h" 34 35namespace aco { 36enum dpp_ctrl { 37 _dpp_quad_perm = 0x000, 38 _dpp_row_sl = 0x100, 39 _dpp_row_sr = 0x110, 40 _dpp_row_rr = 0x120, 41 dpp_wf_sl1 = 0x130, 42 dpp_wf_rl1 = 0x134, 43 dpp_wf_sr1 = 0x138, 44 dpp_wf_rr1 = 0x13C, 45 dpp_row_mirror = 0x140, 46 dpp_row_half_mirror = 0x141, 47 dpp_row_bcast15 = 0x142, 48 dpp_row_bcast31 = 0x143 49}; 50 51inline dpp_ctrl 52dpp_quad_perm(unsigned lane0, unsigned lane1, unsigned lane2, unsigned lane3) 53{ 54 assert(lane0 < 4 && lane1 < 4 && lane2 < 4 && lane3 < 4); 55 return (dpp_ctrl)(lane0 | (lane1 << 2) | (lane2 << 4) | (lane3 << 6)); 56} 57 58inline dpp_ctrl 59dpp_row_sl(unsigned amount) 60{ 61 assert(amount > 0 && amount < 16); 62 return (dpp_ctrl)(((unsigned) _dpp_row_sl) | amount); 63} 64 65inline dpp_ctrl 66dpp_row_sr(unsigned amount) 67{ 68 assert(amount > 0 && amount < 16); 69 return (dpp_ctrl)(((unsigned) _dpp_row_sr) | amount); 70} 71 72inline dpp_ctrl 73dpp_row_rr(unsigned amount) 74{ 75 assert(amount > 0 && amount < 16); 76 return (dpp_ctrl)(((unsigned) _dpp_row_rr) | amount); 77} 78 79inline unsigned 80ds_pattern_bitmode(unsigned and_mask, unsigned or_mask, unsigned xor_mask) 81{ 82 assert(and_mask < 32 && or_mask < 32 && xor_mask < 32); 83 return and_mask | (or_mask << 5) | (xor_mask << 10); 84} 85 86aco_ptr<Instruction> create_s_mov(Definition dst, Operand src); 87 88enum sendmsg { 89 sendmsg_none = 0, 90 _sendmsg_gs = 2, 91 _sendmsg_gs_done = 3, 92 sendmsg_save_wave = 4, 93 sendmsg_stall_wave_gen = 5, 94 sendmsg_halt_waves = 6, 95 sendmsg_ordered_ps_done = 7, 96 sendmsg_early_prim_dealloc = 8, 97 sendmsg_gs_alloc_req = 9, 98 sendmsg_id_mask = 0xf, 99}; 100 101inline sendmsg 102sendmsg_gs(bool cut, bool emit, unsigned stream) 103{ 104 assert(stream < 4); 105 return (sendmsg)((unsigned)_sendmsg_gs | (cut << 4) | (emit << 5) | (stream << 8)); 106} 107 108inline sendmsg 109sendmsg_gs_done(bool cut, bool emit, unsigned stream) 110{ 111 assert(stream < 4); 112 return (sendmsg)((unsigned)_sendmsg_gs_done | (cut << 4) | (emit << 5) | (stream << 8)); 113} 114 115class Builder { 116public: 117 struct Result { 118 Instruction *instr; 119 120 Result(Instruction *instr) : instr(instr) {} 121 122 operator Instruction *() const { 123 return instr; 124 } 125 126 operator Temp() const { 127 return instr->definitions[0].getTemp(); 128 } 129 130 operator Operand() const { 131 return Operand((Temp)*this); 132 } 133 134 Definition& def(unsigned index) const { 135 return instr->definitions[index]; 136 } 137 138 aco_ptr<Instruction> get_ptr() const { 139 return aco_ptr<Instruction>(instr); 140 } 141 }; 142 143 struct Op { 144 Operand op; 145 Op(Temp tmp) : op(tmp) {} 146 Op(Operand op_) : op(op_) {} 147 Op(Result res) : op((Temp)res) {} 148 }; 149 150 enum WaveSpecificOpcode { 151 s_cselect = (unsigned) aco_opcode::s_cselect_b64, 152 s_cmp_lg = (unsigned) aco_opcode::s_cmp_lg_u64, 153 s_and = (unsigned) aco_opcode::s_and_b64, 154 s_andn2 = (unsigned) aco_opcode::s_andn2_b64, 155 s_or = (unsigned) aco_opcode::s_or_b64, 156 s_orn2 = (unsigned) aco_opcode::s_orn2_b64, 157 s_not = (unsigned) aco_opcode::s_not_b64, 158 s_mov = (unsigned) aco_opcode::s_mov_b64, 159 s_wqm = (unsigned) aco_opcode::s_wqm_b64, 160 s_and_saveexec = (unsigned) aco_opcode::s_and_saveexec_b64, 161 s_or_saveexec = (unsigned) aco_opcode::s_or_saveexec_b64, 162 s_xnor = (unsigned) aco_opcode::s_xnor_b64, 163 s_xor = (unsigned) aco_opcode::s_xor_b64, 164 s_bcnt1_i32 = (unsigned) aco_opcode::s_bcnt1_i32_b64, 165 s_bitcmp1 = (unsigned) aco_opcode::s_bitcmp1_b64, 166 s_ff1_i32 = (unsigned) aco_opcode::s_ff1_i32_b64, 167 s_flbit_i32 = (unsigned) aco_opcode::s_flbit_i32_b64, 168 s_lshl = (unsigned) aco_opcode::s_lshl_b64, 169 }; 170 171 Program *program; 172 bool use_iterator; 173 bool start; // only when use_iterator == false 174 RegClass lm; 175 176 std::vector<aco_ptr<Instruction>> *instructions; 177 std::vector<aco_ptr<Instruction>>::iterator it; 178 bool is_precise = false; 179 bool is_nuw = false; 180 181 Builder(Program *pgm) : program(pgm), use_iterator(false), start(false), lm(pgm ? pgm->lane_mask : s2), instructions(NULL) {} 182 Builder(Program *pgm, Block *block) : program(pgm), use_iterator(false), start(false), lm(pgm ? pgm->lane_mask : s2), instructions(&block->instructions) {} 183 Builder(Program *pgm, std::vector<aco_ptr<Instruction>> *instrs) : program(pgm), use_iterator(false), start(false), lm(pgm ? pgm->lane_mask : s2), instructions(instrs) {} 184 185 Builder precise() const { 186 Builder res = *this; 187 res.is_precise = true; 188 return res; 189 }; 190 191 Builder nuw() const { 192 Builder res = *this; 193 res.is_nuw = true; 194 return res; 195 } 196 197 void moveEnd(Block *block) { 198 instructions = &block->instructions; 199 } 200 201 void reset() { 202 use_iterator = false; 203 start = false; 204 instructions = NULL; 205 } 206 207 void reset(Block *block) { 208 use_iterator = false; 209 start = false; 210 instructions = &block->instructions; 211 } 212 213 void reset(std::vector<aco_ptr<Instruction>> *instrs) { 214 use_iterator = false; 215 start = false; 216 instructions = instrs; 217 } 218 219 void reset(std::vector<aco_ptr<Instruction>> *instrs, std::vector<aco_ptr<Instruction>>::iterator instr_it) { 220 use_iterator = true; 221 start = false; 222 instructions = instrs; 223 it = instr_it; 224 } 225 226 Result insert(aco_ptr<Instruction> instr) { 227 Instruction *instr_ptr = instr.get(); 228 if (instructions) { 229 if (use_iterator) { 230 it = instructions->emplace(it, std::move(instr)); 231 it = std::next(it); 232 } else if (!start) { 233 instructions->emplace_back(std::move(instr)); 234 } else { 235 instructions->emplace(instructions->begin(), std::move(instr)); 236 } 237 } 238 return Result(instr_ptr); 239 } 240 241 Result insert(Instruction* instr) { 242 if (instructions) { 243 if (use_iterator) { 244 it = instructions->emplace(it, aco_ptr<Instruction>(instr)); 245 it = std::next(it); 246 } else if (!start) { 247 instructions->emplace_back(aco_ptr<Instruction>(instr)); 248 } else { 249 instructions->emplace(instructions->begin(), aco_ptr<Instruction>(instr)); 250 } 251 } 252 return Result(instr); 253 } 254 255 Temp tmp(RegClass rc) { 256 return program->allocateTmp(rc); 257 } 258 259 Temp tmp(RegType type, unsigned size) { 260 return tmp(RegClass(type, size)); 261 } 262 263 Definition def(RegClass rc) { 264 return Definition(program->allocateTmp(rc)); 265 } 266 267 Definition def(RegType type, unsigned size) { 268 return def(RegClass(type, size)); 269 } 270 271 Definition def(RegClass rc, PhysReg reg) { 272 return Definition(program->allocateId(rc), reg, rc); 273 } 274 275 inline aco_opcode w64or32(WaveSpecificOpcode opcode) const { 276 if (program->wave_size == 64) 277 return (aco_opcode) opcode; 278 279 switch (opcode) { 280 case s_cselect: 281 return aco_opcode::s_cselect_b32; 282 case s_cmp_lg: 283 return aco_opcode::s_cmp_lg_u32; 284 case s_and: 285 return aco_opcode::s_and_b32; 286 case s_andn2: 287 return aco_opcode::s_andn2_b32; 288 case s_or: 289 return aco_opcode::s_or_b32; 290 case s_orn2: 291 return aco_opcode::s_orn2_b32; 292 case s_not: 293 return aco_opcode::s_not_b32; 294 case s_mov: 295 return aco_opcode::s_mov_b32; 296 case s_wqm: 297 return aco_opcode::s_wqm_b32; 298 case s_and_saveexec: 299 return aco_opcode::s_and_saveexec_b32; 300 case s_or_saveexec: 301 return aco_opcode::s_or_saveexec_b32; 302 case s_xnor: 303 return aco_opcode::s_xnor_b32; 304 case s_xor: 305 return aco_opcode::s_xor_b32; 306 case s_bcnt1_i32: 307 return aco_opcode::s_bcnt1_i32_b32; 308 case s_bitcmp1: 309 return aco_opcode::s_bitcmp1_b32; 310 case s_ff1_i32: 311 return aco_opcode::s_ff1_i32_b32; 312 case s_flbit_i32: 313 return aco_opcode::s_flbit_i32_b32; 314 case s_lshl: 315 return aco_opcode::s_lshl_b32; 316 default: 317 unreachable("Unsupported wave specific opcode."); 318 } 319 } 320 321% for fixed in ['m0', 'vcc', 'exec', 'scc']: 322 Operand ${fixed}(Temp tmp) { 323 % if fixed == 'vcc' or fixed == 'exec': 324 //vcc_hi and exec_hi can still be used in wave32 325 assert(tmp.type() == RegType::sgpr && tmp.bytes() <= 8); 326 % endif 327 Operand op(tmp); 328 op.setFixed(aco::${fixed}); 329 return op; 330 } 331 332 Definition ${fixed}(Definition def) { 333 % if fixed == 'vcc' or fixed == 'exec': 334 //vcc_hi and exec_hi can still be used in wave32 335 assert(def.regClass().type() == RegType::sgpr && def.bytes() <= 8); 336 % endif 337 def.setFixed(aco::${fixed}); 338 return def; 339 } 340 341 Definition hint_${fixed}(Definition def) { 342 % if fixed == 'vcc' or fixed == 'exec': 343 //vcc_hi and exec_hi can still be used in wave32 344 assert(def.regClass().type() == RegType::sgpr && def.bytes() <= 8); 345 % endif 346 def.setHint(aco::${fixed}); 347 return def; 348 } 349 350 Definition hint_${fixed}(RegClass rc) { 351 return hint_${fixed}(def(rc)); 352 } 353 354% endfor 355 /* hand-written helpers */ 356 Temp as_uniform(Op op) 357 { 358 assert(op.op.isTemp()); 359 if (op.op.getTemp().type() == RegType::vgpr) 360 return pseudo(aco_opcode::p_as_uniform, def(RegType::sgpr, op.op.size()), op); 361 else 362 return op.op.getTemp(); 363 } 364 365 Result v_mul_imm(Definition dst, Temp tmp, uint32_t imm, bool bits24=false) 366 { 367 assert(tmp.type() == RegType::vgpr); 368 if (imm == 0) { 369 return vop1(aco_opcode::v_mov_b32, dst, Operand(0u)); 370 } else if (imm == 1) { 371 return copy(dst, Operand(tmp)); 372 } else if (util_is_power_of_two_or_zero(imm)) { 373 return vop2(aco_opcode::v_lshlrev_b32, dst, Operand((uint32_t)ffs(imm) - 1u), tmp); 374 } else if (bits24) { 375 return vop2(aco_opcode::v_mul_u32_u24, dst, Operand(imm), tmp); 376 } else { 377 Temp imm_tmp = copy(def(v1), Operand(imm)); 378 return vop3(aco_opcode::v_mul_lo_u32, dst, imm_tmp, tmp); 379 } 380 } 381 382 Result v_mul24_imm(Definition dst, Temp tmp, uint32_t imm) 383 { 384 return v_mul_imm(dst, tmp, imm, true); 385 } 386 387 Result copy(Definition dst, Op op) { 388 return pseudo(aco_opcode::p_parallelcopy, dst, op); 389 } 390 391 Result vadd32(Definition dst, Op a, Op b, bool carry_out=false, Op carry_in=Op(Operand(s2)), bool post_ra=false) { 392 if (!b.op.isTemp() || b.op.regClass().type() != RegType::vgpr) 393 std::swap(a, b); 394 if (!post_ra && (!b.op.hasRegClass() || b.op.regClass().type() == RegType::sgpr)) 395 b = copy(def(v1), b); 396 397 if (!carry_in.op.isUndefined()) 398 return vop2(aco_opcode::v_addc_co_u32, Definition(dst), hint_vcc(def(lm)), a, b, carry_in); 399 else if (program->chip_class >= GFX10 && carry_out) 400 return vop3(aco_opcode::v_add_co_u32_e64, Definition(dst), def(lm), a, b); 401 else if (program->chip_class < GFX9 || carry_out) 402 return vop2(aco_opcode::v_add_co_u32, Definition(dst), hint_vcc(def(lm)), a, b); 403 else 404 return vop2(aco_opcode::v_add_u32, Definition(dst), a, b); 405 } 406 407 Result vsub32(Definition dst, Op a, Op b, bool carry_out=false, Op borrow=Op(Operand(s2))) 408 { 409 if (!borrow.op.isUndefined() || program->chip_class < GFX9) 410 carry_out = true; 411 412 bool reverse = !b.op.isTemp() || b.op.regClass().type() != RegType::vgpr; 413 if (reverse) 414 std::swap(a, b); 415 if (!b.op.hasRegClass() || b.op.regClass().type() == RegType::sgpr) 416 b = copy(def(v1), b); 417 418 aco_opcode op; 419 Temp carry; 420 if (carry_out) { 421 carry = tmp(s2); 422 if (borrow.op.isUndefined()) 423 op = reverse ? aco_opcode::v_subrev_co_u32 : aco_opcode::v_sub_co_u32; 424 else 425 op = reverse ? aco_opcode::v_subbrev_co_u32 : aco_opcode::v_subb_co_u32; 426 } else { 427 op = reverse ? aco_opcode::v_subrev_u32 : aco_opcode::v_sub_u32; 428 } 429 bool vop3 = false; 430 if (program->chip_class >= GFX10 && op == aco_opcode::v_subrev_co_u32) { 431 vop3 = true; 432 op = aco_opcode::v_subrev_co_u32_e64; 433 } else if (program->chip_class >= GFX10 && op == aco_opcode::v_sub_co_u32) { 434 vop3 = true; 435 op = aco_opcode::v_sub_co_u32_e64; 436 } 437 438 int num_ops = borrow.op.isUndefined() ? 2 : 3; 439 int num_defs = carry_out ? 2 : 1; 440 aco_ptr<Instruction> sub; 441 if (vop3) 442 sub.reset(create_instruction<VOP3A_instruction>(op, Format::VOP3B, num_ops, num_defs)); 443 else 444 sub.reset(create_instruction<VOP2_instruction>(op, Format::VOP2, num_ops, num_defs)); 445 sub->operands[0] = a.op; 446 sub->operands[1] = b.op; 447 if (!borrow.op.isUndefined()) 448 sub->operands[2] = borrow.op; 449 sub->definitions[0] = dst; 450 if (carry_out) { 451 sub->definitions[1] = Definition(carry); 452 sub->definitions[1].setHint(aco::vcc); 453 } 454 return insert(std::move(sub)); 455 } 456 457 Result readlane(Definition dst, Op vsrc, Op lane) 458 { 459 if (program->chip_class >= GFX8) 460 return vop3(aco_opcode::v_readlane_b32_e64, dst, vsrc, lane); 461 else 462 return vop2(aco_opcode::v_readlane_b32, dst, vsrc, lane); 463 } 464 Result writelane(Definition dst, Op val, Op lane, Op vsrc) { 465 if (program->chip_class >= GFX8) 466 return vop3(aco_opcode::v_writelane_b32_e64, dst, val, lane, vsrc); 467 else 468 return vop2(aco_opcode::v_writelane_b32, dst, val, lane, vsrc); 469 } 470<% 471import itertools 472formats = [("pseudo", [Format.PSEUDO], 'Pseudo_instruction', list(itertools.product(range(5), range(5))) + [(8, 1), (1, 8)]), 473 ("sop1", [Format.SOP1], 'SOP1_instruction', [(0, 1), (1, 0), (1, 1), (2, 1), (3, 2)]), 474 ("sop2", [Format.SOP2], 'SOP2_instruction', itertools.product([1, 2], [2, 3])), 475 ("sopk", [Format.SOPK], 'SOPK_instruction', itertools.product([0, 1, 2], [0, 1])), 476 ("sopp", [Format.SOPP], 'SOPP_instruction', itertools.product([0, 1], [0, 1])), 477 ("sopc", [Format.SOPC], 'SOPC_instruction', [(1, 2)]), 478 ("smem", [Format.SMEM], 'SMEM_instruction', [(0, 4), (0, 3), (1, 0), (1, 3), (1, 2), (0, 0)]), 479 ("ds", [Format.DS], 'DS_instruction', [(1, 1), (1, 2), (0, 3), (0, 4)]), 480 ("mubuf", [Format.MUBUF], 'MUBUF_instruction', [(0, 4), (1, 3)]), 481 ("mtbuf", [Format.MTBUF], 'MTBUF_instruction', [(0, 4), (1, 3)]), 482 ("mimg", [Format.MIMG], 'MIMG_instruction', [(0, 3), (1, 3)]), 483 ("exp", [Format.EXP], 'Export_instruction', [(0, 4)]), 484 ("branch", [Format.PSEUDO_BRANCH], 'Pseudo_branch_instruction', itertools.product([1], [0, 1])), 485 ("barrier", [Format.PSEUDO_BARRIER], 'Pseudo_barrier_instruction', [(0, 0)]), 486 ("reduction", [Format.PSEUDO_REDUCTION], 'Pseudo_reduction_instruction', [(3, 2)]), 487 ("vop1", [Format.VOP1], 'VOP1_instruction', [(0, 0), (1, 1), (2, 2)]), 488 ("vop1_sdwa", [Format.VOP1, Format.SDWA], 'SDWA_instruction', [(1, 1)]), 489 ("vop2", [Format.VOP2], 'VOP2_instruction', itertools.product([1, 2], [2, 3])), 490 ("vop2_sdwa", [Format.VOP2, Format.SDWA], 'SDWA_instruction', itertools.product([1, 2], [2, 3])), 491 ("vopc", [Format.VOPC], 'VOPC_instruction', itertools.product([1, 2], [2])), 492 ("vop3", [Format.VOP3A], 'VOP3A_instruction', [(1, 3), (1, 2), (1, 1), (2, 2)]), 493 ("vintrp", [Format.VINTRP], 'Interp_instruction', [(1, 2), (1, 3)]), 494 ("vop1_dpp", [Format.VOP1, Format.DPP], 'DPP_instruction', [(1, 1)]), 495 ("vop2_dpp", [Format.VOP2, Format.DPP], 'DPP_instruction', itertools.product([1, 2], [2, 3])), 496 ("vopc_dpp", [Format.VOPC, Format.DPP], 'DPP_instruction', itertools.product([1, 2], [2])), 497 ("vop1_e64", [Format.VOP1, Format.VOP3A], 'VOP3A_instruction', itertools.product([1], [1])), 498 ("vop2_e64", [Format.VOP2, Format.VOP3A], 'VOP3A_instruction', itertools.product([1, 2], [2, 3])), 499 ("vopc_e64", [Format.VOPC, Format.VOP3A], 'VOP3A_instruction', itertools.product([1, 2], [2])), 500 ("flat", [Format.FLAT], 'FLAT_instruction', [(0, 3), (1, 2)]), 501 ("global", [Format.GLOBAL], 'FLAT_instruction', [(0, 3), (1, 2)])] 502formats = [(f if len(f) == 5 else f + ('',)) for f in formats] 503%>\\ 504% for name, formats, struct, shapes, extra_field_setup in formats: 505 % for num_definitions, num_operands in shapes: 506 <% 507 args = ['aco_opcode opcode'] 508 for i in range(num_definitions): 509 args.append('Definition def%d' % i) 510 for i in range(num_operands): 511 args.append('Op op%d' % i) 512 for f in formats: 513 args += f.get_builder_field_decls() 514 %>\\ 515 516 Result ${name}(${', '.join(args)}) 517 { 518 ${struct} *instr = create_instruction<${struct}>(opcode, (Format)(${'|'.join('(int)Format::%s' % f.name for f in formats)}), ${num_operands}, ${num_definitions}); 519 % for i in range(num_definitions): 520 instr->definitions[${i}] = def${i}; 521 instr->definitions[${i}].setPrecise(is_precise); 522 instr->definitions[${i}].setNUW(is_nuw); 523 % endfor 524 % for i in range(num_operands): 525 instr->operands[${i}] = op${i}.op; 526 % endfor 527 % for f in formats: 528 % for dest, field_name in zip(f.get_builder_field_dests(), f.get_builder_field_names()): 529 instr->${dest} = ${field_name}; 530 % endfor 531 ${f.get_builder_initialization(num_operands)} 532 % endfor 533 ${extra_field_setup} 534 return insert(instr); 535 } 536 537 % if name == 'sop1' or name == 'sop2' or name == 'sopc': 538 <% 539 args[0] = 'WaveSpecificOpcode opcode' 540 params = [] 541 for i in range(num_definitions): 542 params.append('def%d' % i) 543 for i in range(num_operands): 544 params.append('op%d' % i) 545 %>\\ 546 547 inline Result ${name}(${', '.join(args)}) 548 { 549 return ${name}(w64or32(opcode), ${', '.join(params)}); 550 } 551 552 % endif 553 % endfor 554% endfor 555}; 556 557} 558#endif /* _ACO_BUILDER_ */""" 559 560from aco_opcodes import opcodes, Format 561from mako.template import Template 562 563print(Template(template).render(opcodes=opcodes, Format=Format)) 564