Lines Matching refs:VTI
321 multiclass MVE_TwoOpPattern<MVEVectorVTInfo VTI, PatFrag Op, Intrinsic PredInt,
325 def : Pat<(VTI.Vec (Op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn))),
326 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
329 if !ne(VTI.Size, 0b11) then {
330 def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$mask),
331 (VTI.Vec (Op (VTI.Vec MQPR:$Qm),
332 (VTI.Vec MQPR:$Qn))),
333 (VTI.Vec MQPR:$inactive))),
334 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
335 ARMVCCThen, (VTI.Pred VCCR:$mask),
336 (VTI.Vec MQPR:$inactive)))>;
339 def : Pat<(VTI.Vec (Op (VTI.Vec MQPR:$Qm),
340 (VTI.Vec (vselect (VTI.Pred VCCR:$mask),
341 (VTI.Vec MQPR:$Qn),
342 (VTI.Vec IdentityVec))))),
343 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
344 ARMVCCThen, (VTI.Pred VCCR:$mask),
345 (VTI.Vec MQPR:$Qm)))>;
349 def : Pat<(VTI.Vec !con((PredInt (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)),
351 (? (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive)))),
352 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
353 ARMVCCThen, (VTI.Pred VCCR:$mask),
354 (VTI.Vec MQPR:$inactive)))>;
357 multiclass MVE_TwoOpPatternDup<MVEVectorVTInfo VTI, PatFrag Op, Intrinsic PredInt,
361 def : Pat<(VTI.Vec (Op (VTI.Vec MQPR:$Qm), (VTI.Vec (ARMvdup rGPR:$Rn)))),
362 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), rGPR:$Rn))>;
365 if !ne(VTI.Size, 0b11) then {
366 def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$mask),
367 (VTI.Vec (Op (VTI.Vec MQPR:$Qm),
368 (VTI.Vec (ARMvdup rGPR:$Rn)))),
369 (VTI.Vec MQPR:$inactive))),
370 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), rGPR:$Rn,
371 ARMVCCThen, (VTI.Pred VCCR:$mask),
372 (VTI.Vec MQPR:$inactive)))>;
375 def : Pat<(VTI.Vec (Op (VTI.Vec MQPR:$Qm),
376 (VTI.Vec (vselect (VTI.Pred VCCR:$mask),
378 (VTI.Vec IdentityVec))))),
379 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), rGPR:$Rn,
380 ARMVCCThen, (VTI.Pred VCCR:$mask),
381 (VTI.Vec MQPR:$Qm)))>;
385 def : Pat<(VTI.Vec !con((PredInt (VTI.Vec MQPR:$Qm), (VTI.Vec (ARMvdup rGPR:$Rn))),
387 (? (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive)))),
388 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), rGPR:$Rn,
389 ARMVCCThen, (VTI.Pred VCCR:$mask),
390 (VTI.Vec MQPR:$inactive)))>;
636 multiclass MVE_VABAV_m<MVEVectorVTInfo VTI> {
637 def "" : MVE_VABAV<VTI.Suffix, VTI.Unsigned, VTI.Size>;
642 (i32 VTI.Unsigned),
644 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
646 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm)))>;
649 (i32 VTI.Unsigned),
651 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
652 (VTI.Pred VCCR:$mask))),
654 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
655 ARMVCCThen, (VTI.Pred VCCR:$mask)))>;
695 multiclass MVE_VADDV_A<MVEVectorVTInfo VTI> {
696 def acc : MVE_VADDV<"vaddva", VTI.Suffix,
698 0b1, VTI.Unsigned, VTI.Size>;
699 def no_acc : MVE_VADDV<"vaddv", VTI.Suffix,
701 0b0, VTI.Unsigned, VTI.Size>;
707 if VTI.Unsigned then {
708 def : Pat<(i32 (vecreduce_add (VTI.Vec MQPR:$vec))),
710 def : Pat<(i32 (vecreduce_add (VTI.Vec (vselect (VTI.Pred VCCR:$pred),
711 (VTI.Vec MQPR:$vec),
712 (VTI.Vec ARMimmAllZerosV))))),
714 def : Pat<(i32 (ARMVADDVu (VTI.Vec MQPR:$vec))),
716 def : Pat<(i32 (ARMVADDVpu (VTI.Vec MQPR:$vec), (VTI.Pred VCCR:$pred))),
718 def : Pat<(i32 (add (i32 (vecreduce_add (VTI.Vec MQPR:$vec))),
721 def : Pat<(i32 (add (i32 (vecreduce_add (VTI.Vec (vselect (VTI.Pred VCCR:$pred),
722 (VTI.Vec MQPR:$vec),
723 (VTI.Vec ARMimmAllZerosV))))),
726 def : Pat<(i32 (add (i32 (ARMVADDVu (VTI.Vec MQPR:$vec))),
729 def : Pat<(i32 (add (i32 (ARMVADDVpu (VTI.Vec MQPR:$vec), (VTI.Pred VCCR:$pred))),
733 def : Pat<(i32 (ARMVADDVs (VTI.Vec MQPR:$vec))),
735 def : Pat<(i32 (add (i32 (ARMVADDVs (VTI.Vec MQPR:$vec))),
738 def : Pat<(i32 (ARMVADDVps (VTI.Vec MQPR:$vec), (VTI.Pred VCCR:$pred))),
740 def : Pat<(i32 (add (i32 (ARMVADDVps (VTI.Vec MQPR:$vec), (VTI.Pred VCCR:$pred))),
745 def : Pat<(i32 (int_arm_mve_addv_predicated (VTI.Vec MQPR:$vec),
746 (i32 VTI.Unsigned),
747 (VTI.Pred VCCR:$pred))),
749 def : Pat<(i32 (add (int_arm_mve_addv_predicated (VTI.Vec MQPR:$vec),
750 (i32 VTI.Unsigned),
751 (VTI.Pred VCCR:$pred)),
800 multiclass MVE_VADDLV_A<MVEVectorVTInfo VTI> {
801 def acc : MVE_VADDLV<"vaddlva", VTI.Suffix,
804 0b1, VTI.Unsigned>;
805 def no_acc : MVE_VADDLV<"vaddlv", VTI.Suffix,
807 0b0, VTI.Unsigned>;
812 defvar letter = VTI.SuffixLetter;
823 def : Pat<(ARMVADDLVp (v4i32 MQPR:$vec), (VTI.Pred VCCR:$pred)),
824 (InstN (v4i32 MQPR:$vec), ARMVCCThen, (VTI.Pred VCCR:$pred))>;
826 (VTI.Pred VCCR:$pred)),
828 ARMVCCThen, (VTI.Pred VCCR:$pred))>;
861 MVEVectorVTInfo VTI, string intrBaseName,
863 def "": MVE_VMINMAXNMV<iname, VTI.Suffix, VTI.Size{0}, notAbs, isMin>;
870 (VTI.Vec MQPR:$vec))),
872 (VTI.Vec MQPR:$vec)),
875 (VTI.Vec MQPR:$vec),
876 (VTI.Pred VCCR:$pred))),
878 (VTI.Vec MQPR:$vec),
879 ARMVCCThen, (VTI.Pred VCCR:$pred)),
919 MVEVectorVTInfo VTI, string intrBaseName> {
920 def "": MVE_VMINMAXV<iname, VTI.Suffix, VTI.Unsigned, VTI.Size,
925 defvar base_args = (? (i32 rGPR:$prev), (VTI.Vec MQPR:$vec));
926 defvar args = !if(notAbs, !con(base_args, (? (i32 VTI.Unsigned))),
931 (i32 (Inst (i32 rGPR:$prev), (VTI.Vec MQPR:$vec)))>;
932 def : Pat<(i32 !con(args, (pred_intr (VTI.Pred VCCR:$pred)))),
933 (i32 (Inst (i32 rGPR:$prev), (VTI.Vec MQPR:$vec),
934 ARMVCCThen, (VTI.Pred VCCR:$pred)))>;
1049 multiclass MVE_VMLAMLSDAV_A<string iname, string x, MVEVectorVTInfo VTI,
1051 def ""#x#VTI.Suffix : MVE_VMLAMLSDAV<iname # x, VTI.Suffix,
1054 def "a"#x#VTI.Suffix : MVE_VMLAMLSDAV<iname # "a" # x, VTI.Suffix,
1060 (i32 VTI.Unsigned),
1064 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
1065 (i32 (!cast<Instruction>(NAME # x # VTI.Suffix)
1066 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm)))>;
1069 (i32 VTI.Unsigned),
1073 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
1074 (VTI.Pred VCCR:$mask))),
1075 (i32 (!cast<Instruction>(NAME # x # VTI.Suffix)
1076 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
1077 ARMVCCThen, (VTI.Pred VCCR:$mask)))>;
1080 (i32 VTI.Unsigned),
1084 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
1085 (i32 (!cast<Instruction>(NAME # "a" # x # VTI.Suffix)
1087 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm)))>;
1090 (i32 VTI.Unsigned),
1094 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
1095 (VTI.Pred VCCR:$mask))),
1096 (i32 (!cast<Instruction>(NAME # "a" # x # VTI.Suffix)
1098 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
1099 ARMVCCThen, (VTI.Pred VCCR:$mask)))>;
1103 multiclass MVE_VMLAMLSDAV_AX<string iname, MVEVectorVTInfo VTI, bit sz,
1105 defm "" : MVE_VMLAMLSDAV_A<iname, "", VTI, sz, bit_28,
1107 defm "" : MVE_VMLAMLSDAV_A<iname, "x", VTI, sz, bit_28,
1119 multiclass MVE_VMLSDAV_multi<MVEVectorVTInfo VTI, bit sz, bit bit_28> {
1120 defm "" : MVE_VMLAMLSDAV_AX<"vmlsdav", VTI,
1445 multiclass MVE_VMINMAXNM_m<string iname, bit bit_4, MVEVectorVTInfo VTI, SDNode Op, Intrinsic PredI…
1446 def "" : MVE_VMINMAXNM<iname, VTI.Suffix, VTI.Size{0}, bit_4>;
1449 defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 0)), !cast<Instruction>(NAME)>;
1474 multiclass MVE_VMINMAX_m<string iname, bit bit_4, MVEVectorVTInfo VTI,
1476 def "" : MVE_VMINMAX<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, bit_4>;
1479 defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)), !cast<Instruction>(NAME)>;
1483 multiclass MVE_VMAX<MVEVectorVTInfo VTI>
1484 : MVE_VMINMAX_m<"vmax", 0b0, VTI, !if(VTI.Unsigned, umax, smax), int_arm_mve_max_predicated>;
1485 multiclass MVE_VMIN<MVEVectorVTInfo VTI>
1486 : MVE_VMINMAX_m<"vmin", 0b1, VTI, !if(VTI.Unsigned, umin, smin), int_arm_mve_min_predicated>;
1571 foreach VTI = VTIs in {
1572 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$src))),
1573 (VTI.Vec (Inst (VTI.Vec MQPR:$src)))>;
1574 def : Pat<(VTI.Vec (int_arm_mve_vrev_predicated (VTI.Vec MQPR:$src),
1575 revbits, (VTI.Pred VCCR:$pred), (VTI.Vec MQPR:$inactive))),
1576 (VTI.Vec (Inst (VTI.Vec MQPR:$src), ARMVCCThen,
1577 (VTI.Pred VCCR:$pred), (VTI.Vec MQPR:$inactive)))>;
1604 foreach VTI = [ MVE_v16i8, MVE_v8i16, MVE_v4i32, MVE_v2i64 ] in {
1605 def : Pat<(VTI.Vec (vnotq (VTI.Vec MQPR:$val1))),
1606 (VTI.Vec (MVE_VMVN (VTI.Vec MQPR:$val1)))>;
1607 def : Pat<(VTI.Vec (int_arm_mve_mvn_predicated (VTI.Vec MQPR:$val1),
1608 (VTI.Pred VCCR:$pred), (VTI.Vec MQPR:$inactive))),
1609 (VTI.Vec (MVE_VMVN (VTI.Vec MQPR:$val1), ARMVCCThen,
1610 (VTI.Pred VCCR:$pred), (VTI.Vec MQPR:$inactive)))>;
1710 MVEVectorVTInfo VTI, Operand imm_type, SDNode op> {
1711 def "" : MVE_bit_cmode<iname, VTI.Suffix, VTI.Size{0},
1718 defvar UnpredPat = (VTI.Vec (op (VTI.Vec MQPR:$src), timm:$simm));
1722 (VTI.Vec (Inst (VTI.Vec MQPR:$src), imm_type:$simm))>;
1723 def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$pred),
1724 UnpredPat, (VTI.Vec MQPR:$src))),
1725 (VTI.Vec (Inst (VTI.Vec MQPR:$src), imm_type:$simm,
1726 ARMVCCThen, (VTI.Pred VCCR:$pred)))>;
1730 multiclass MVE_VORRimm<MVEVectorVTInfo VTI, Operand imm_type> {
1731 defm "": MVE_bit_cmode_p<"vorr", 0, VTI, imm_type, ARMvorrImm>;
1733 multiclass MVE_VBICimm<MVEVectorVTInfo VTI, Operand imm_type> {
1734 defm "": MVE_bit_cmode_p<"vbic", 1, VTI, imm_type, ARMvbicImm>;
1932 multiclass MVE_VMUL_m<MVEVectorVTInfo VTI> {
1933 def "" : MVE_VMULt1<"vmul", VTI.Suffix, VTI.Size>;
1936 defm : MVE_TwoOpPattern<VTI, mul, int_arm_mve_mul_predicated, (? ),
1960 multiclass MVE_VQxDMULH_m<string iname, MVEVectorVTInfo VTI,
1963 def "" : MVE_VQxDMULH_Base<iname, VTI.Suffix, VTI.Size, rounding>;
1967 defm : MVE_TwoOpPattern<VTI, Op, pred_int, (? ), Inst>;
1970 def : Pat<(VTI.Vec (unpred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn))),
1971 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
1975 multiclass MVE_VQxDMULH<string iname, MVEVectorVTInfo VTI, bit rounding>
1976 : MVE_VQxDMULH_m<iname, VTI, !if(rounding, null_frag,
2005 multiclass MVE_VADDSUB_m<string iname, MVEVectorVTInfo VTI, bit subtract,
2007 def "" : MVE_VADDSUB<iname, VTI.Suffix, VTI.Size, subtract>;
2011 defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? ), !cast<Instruction>(NAME), ARMimmAllZerosV>;
2015 multiclass MVE_VADD<MVEVectorVTInfo VTI>
2016 : MVE_VADDSUB_m<"vadd", VTI, 0b0, add, int_arm_mve_add_predicated>;
2017 multiclass MVE_VSUB<MVEVectorVTInfo VTI>
2018 : MVE_VADDSUB_m<"vsub", VTI, 0b1, sub, int_arm_mve_sub_predicated>;
2048 multiclass MVE_VQADD_m<MVEVectorVTInfo VTI,
2050 def "" : MVE_VQADD_<VTI.Suffix, VTI.Unsigned, VTI.Size>;
2054 defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)),
2059 multiclass MVE_VQADD<MVEVectorVTInfo VTI, SDNode unpred_op>
2060 : MVE_VQADD_m<VTI, unpred_op, int_arm_mve_qadd_predicated>;
2069 multiclass MVE_VQSUB_m<MVEVectorVTInfo VTI,
2071 def "" : MVE_VQSUB_<VTI.Suffix, VTI.Unsigned, VTI.Size>;
2075 defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? (i32 VTI.Unsigned)),
2080 multiclass MVE_VQSUB<MVEVectorVTInfo VTI, SDNode unpred_op>
2081 : MVE_VQSUB_m<VTI, unpred_op, int_arm_mve_qsub_predicated>;
2103 multiclass MVE_VABD_m<MVEVectorVTInfo VTI,
2105 def "" : MVE_VABD_int<VTI.Suffix, VTI.Unsigned, VTI.Size>;
2110 def : Pat<(VTI.Vec (unpred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2111 (i32 VTI.Unsigned))),
2112 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
2115 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2116 (i32 VTI.Unsigned), (VTI.Pred VCCR:$mask),
2117 (VTI.Vec MQPR:$inactive))),
2118 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2119 ARMVCCThen, (VTI.Pred VCCR:$mask),
2120 (VTI.Vec MQPR:$inactive)))>;
2124 multiclass MVE_VABD<MVEVectorVTInfo VTI>
2125 : MVE_VABD_m<VTI, int_arm_mve_vabd, int_arm_mve_abd_predicated>;
2166 multiclass MVE_VRHADD_m<MVEVectorVTInfo VTI,
2168 def "" : MVE_VRHADD_Base<VTI.Suffix, VTI.Unsigned, VTI.Size>;
2173 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2174 (i32 VTI.Unsigned))),
2175 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
2178 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2179 (i32 VTI.Unsigned), (VTI.Pred VCCR:$mask),
2180 (VTI.Vec MQPR:$inactive))),
2181 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2182 ARMVCCThen, (VTI.Pred VCCR:$mask),
2183 (VTI.Vec MQPR:$inactive)))>;
2187 multiclass MVE_VRHADD<MVEVectorVTInfo VTI>
2188 : MVE_VRHADD_m<VTI, int_arm_mve_vrhadd, int_arm_mve_rhadd_predicated>;
2252 multiclass MVE_VHADD_m<MVEVectorVTInfo VTI,
2255 def "" : MVE_VHADD_<VTI.Suffix, VTI.Unsigned, VTI.Size>;
2260 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn), (i32 VTI.Unsigned))),
2261 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
2263 def : Pat<(VTI.Vec (shift_op (add_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)), (i32 1))),
2267 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn), (i32 VTI.Unsigned),
2268 (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive))),
2269 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2270 ARMVCCThen, (VTI.Pred VCCR:$mask),
2271 (VTI.Vec MQPR:$inactive)))>;
2275 multiclass MVE_VHADD<MVEVectorVTInfo VTI, PatFrag add_op, SDNode shift_op>
2276 : MVE_VHADD_m<VTI, int_arm_mve_vhadd, int_arm_mve_hadd_predicated, add_op,
2290 multiclass MVE_VHSUB_m<MVEVectorVTInfo VTI,
2293 def "" : MVE_VHSUB_<VTI.Suffix, VTI.Unsigned, VTI.Size>;
2298 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2299 (i32 VTI.Unsigned))),
2300 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
2302 def : Pat<(VTI.Vec (shift_op (sub_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)), (i32 1))),
2307 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2308 (i32 VTI.Unsigned), (VTI.Pred VCCR:$mask),
2309 (VTI.Vec MQPR:$inactive))),
2310 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
2311 ARMVCCThen, (VTI.Pred VCCR:$mask),
2312 (VTI.Vec MQPR:$inactive)))>;
2316 multiclass MVE_VHSUB<MVEVectorVTInfo VTI, PatFrag sub_op, SDNode shift_op>
2317 : MVE_VHSUB_m<VTI, int_arm_mve_vhsub, int_arm_mve_hsub_predicated, sub_op,
2424 multiclass MVE_VCLSCLZ_p<string opname, bit opcode, MVEVectorVTInfo VTI,
2426 def "": MVE_VCLSCLZ<"v"#opname, VTI.Suffix, VTI.Size, opcode>;
2432 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$val))),
2433 (VTI.Vec (Inst (VTI.Vec MQPR:$val)))>;
2434 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$val), (VTI.Pred VCCR:$pred),
2435 (VTI.Vec MQPR:$inactive))),
2436 (VTI.Vec (Inst (VTI.Vec MQPR:$val), ARMVCCThen,
2437 (VTI.Pred VCCR:$pred), (VTI.Vec MQPR:$inactive)))>;
2470 MVEVectorVTInfo VTI> {
2471 def "" : MVE_VABSNEG_int<iname, VTI.Suffix, VTI.Size, negate, saturate>;
2477 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$v))),
2478 (VTI.Vec (Inst $v))>;
2481 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$v), (VTI.Pred VCCR:$mask),
2482 (VTI.Vec MQPR:$inactive))),
2483 (VTI.Vec (Inst $v, ARMVCCThen, $mask, $inactive))>;
2487 foreach VTI = [ MVE_v16s8, MVE_v8s16, MVE_v4s32 ] in {
2488 defm "MVE_VABS" # VTI.Suffix : MVE_VABSNEG_int_m<
2489 "vabs", 0, 0, abs, int_arm_mve_abs_predicated, VTI>;
2490 defm "MVE_VQABS" # VTI.Suffix : MVE_VABSNEG_int_m<
2491 "vqabs", 0, 1, ?, int_arm_mve_qabs_predicated, VTI>;
2492 defm "MVE_VNEG" # VTI.Suffix : MVE_VABSNEG_int_m<
2493 "vneg", 1, 0, vnegq, int_arm_mve_neg_predicated, VTI>;
2494 defm "MVE_VQNEG" # VTI.Suffix : MVE_VABSNEG_int_m<
2495 "vqneg", 1, 1, ?, int_arm_mve_qneg_predicated, VTI>;
2498 // int_min/int_max: vector containing INT_MIN/INT_MAX VTI.Size times
2500 multiclass vqabsneg_pattern<MVEVectorVTInfo VTI, dag int_min, dag int_max,
2507 def : Pat<(VTI.Vec (vselect
2508 (VTI.Pred (ARMvcmpz (VTI.Vec MQPR:$reg), ARMCCgt)),
2509 (VTI.Vec MQPR:$reg),
2510 (VTI.Vec (vselect
2511 (VTI.Pred (ARMvcmp (VTI.Vec MQPR:$reg), int_min, ARMCCeq)),
2513 (sub (VTI.Vec zero_vec), (VTI.Vec MQPR:$reg)))))),
2514 (VTI.Vec (vqabs_instruction (VTI.Vec MQPR:$reg)))>;
2517 def : Pat<(VTI.Vec (vselect
2518 (VTI.Pred (ARMvcmp (VTI.Vec MQPR:$reg), int_min, ARMCCeq)),
2520 (sub (VTI.Vec zero_vec), (VTI.Vec MQPR:$reg)))),
2521 (VTI.Vec (vqneg_instruction (VTI.Vec MQPR:$reg)))>;
2638 multiclass MVE_VMINMAXA_m<string iname, MVEVectorVTInfo VTI,
2640 def "" : MVE_VMINMAXA<iname, VTI.Suffix, VTI.Size, bit_12>;
2645 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qd), (abs (VTI.Vec MQPR:$Qm)))),
2646 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm)))>;
2649 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm),
2650 (VTI.Pred VCCR:$mask))),
2651 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm),
2652 ARMVCCThen, (VTI.Pred VCCR:$mask)))>;
2656 multiclass MVE_VMINA<MVEVectorVTInfo VTI>
2657 : MVE_VMINMAXA_m<"vmina", VTI, umin, int_arm_mve_vmina_predicated, 0b1>;
2663 multiclass MVE_VMAXA<MVEVectorVTInfo VTI>
2664 : MVE_VMINMAXA_m<"vmaxa", VTI, umax, int_arm_mve_vmaxa_predicated, 0b0>;
2851 multiclass MVE_VSHLL_patterns<MVEVectorVTInfo VTI, int top> {
2852 defvar suffix = !strconcat(VTI.Suffix, !if(top, "th", "bh"));
2859 def : Pat<(VTI.DblVec (unpred_int (VTI.Vec MQPR:$src), imm:$imm,
2860 (i32 VTI.Unsigned), (i32 top))),
2861 (VTI.DblVec (inst_imm (VTI.Vec MQPR:$src), imm:$imm))>;
2862 def : Pat<(VTI.DblVec (unpred_int (VTI.Vec MQPR:$src), (i32 VTI.LaneBits),
2863 (i32 VTI.Unsigned), (i32 top))),
2864 (VTI.DblVec (inst_lw (VTI.Vec MQPR:$src)))>;
2866 def : Pat<(VTI.DblVec (pred_int (VTI.Vec MQPR:$src), imm:$imm,
2867 (i32 VTI.Unsigned), (i32 top),
2868 (VTI.DblPred VCCR:$mask),
2869 (VTI.DblVec MQPR:$inactive))),
2870 (VTI.DblVec (inst_imm (VTI.Vec MQPR:$src), imm:$imm,
2871 ARMVCCThen, (VTI.DblPred VCCR:$mask),
2872 (VTI.DblVec MQPR:$inactive)))>;
2873 def : Pat<(VTI.DblVec (pred_int (VTI.Vec MQPR:$src), (i32 VTI.LaneBits),
2874 (i32 VTI.Unsigned), (i32 top),
2875 (VTI.DblPred VCCR:$mask),
2876 (VTI.DblVec MQPR:$inactive))),
2877 (VTI.DblVec (inst_lw (VTI.Vec MQPR:$src), ARMVCCThen,
2878 (VTI.DblPred VCCR:$mask),
2879 (VTI.DblVec MQPR:$inactive)))>;
2882 foreach VTI = [MVE_v16s8, MVE_v8s16, MVE_v16u8, MVE_v8u16] in
2884 defm : MVE_VSHLL_patterns<VTI, top>;
3115 multiclass MVE_shift_by_vec_p<string iname, MVEVectorVTInfo VTI, bit q, bit r> {
3116 def "" : MVE_shift_by_vec<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, q, r>;
3119 def : Pat<(VTI.Vec (int_arm_mve_vshl_vector
3120 (VTI.Vec MQPR:$in), (VTI.Vec MQPR:$sh),
3121 (i32 q), (i32 r), (i32 VTI.Unsigned))),
3122 (VTI.Vec (Inst (VTI.Vec MQPR:$in), (VTI.Vec MQPR:$sh)))>;
3124 def : Pat<(VTI.Vec (int_arm_mve_vshl_vector_predicated
3125 (VTI.Vec MQPR:$in), (VTI.Vec MQPR:$sh),
3126 (i32 q), (i32 r), (i32 VTI.Unsigned),
3127 (VTI.Pred VCCR:$mask), (VTI.Vec MQPR:$inactive))),
3128 (VTI.Vec (Inst (VTI.Vec MQPR:$in), (VTI.Vec MQPR:$sh),
3129 ARMVCCThen, (VTI.Pred VCCR:$mask),
3130 (VTI.Vec MQPR:$inactive)))>;
3182 MVEVectorVTInfo VTI;
3229 MVEVectorVTInfo VTI> {
3230 defvar inparams = (? (VTI.Vec MQPR:$QdSrc), (VTI.Vec MQPR:$Qm),
3232 defvar outparams = (inst (VTI.Vec MQPR:$QdSrc), (VTI.Vec MQPR:$Qm),
3237 def : Pat<(VTI.Vec !setdagop(inparams, unpred_int)),
3238 (VTI.Vec outparams)>;
3239 def : Pat<(VTI.Vec !con(inparams, (pred_int (VTI.Pred VCCR:$pred)))),
3240 (VTI.Vec !con(outparams, (? ARMVCCThen, VCCR:$pred)))>;
3261 let VTI = VTI_;
3263 let unsignedFlag = (? (i32 VTI.Unsigned));
3301 let VTI = VTI_;
3331 let VTI = VTI_;
3333 let unsignedFlag = (? (i32 VTI.Unsigned));
3364 def : Pat<(inst.VTI.Vec !con((inst.unpred_int (inst.VTI.Vec MQPR:$src),
3367 (inst.VTI.Vec (inst (inst.VTI.Vec MQPR:$src),
3370 def : Pat<(inst.VTI.Vec !con((inst.pred_int (inst.VTI.Vec MQPR:$src),
3373 (? (inst.VTI.Pred VCCR:$mask),
3374 (inst.VTI.Vec MQPR:$inactive)))),
3375 (inst.VTI.Vec (inst (inst.VTI.Vec MQPR:$src),
3377 ARMVCCThen, (inst.VTI.Pred VCCR:$mask),
3378 (inst.VTI.Vec MQPR:$inactive)))>;
3463 MVEVectorVTInfo VTI, Operand imm_operand_type, SDNode unpred_op,
3466 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$src), imm_operand_type:$imm)),
3467 (VTI.Vec (inst (VTI.Vec MQPR:$src), imm_operand_type:$imm))>;
3469 def : Pat<(VTI.Vec !con((pred_int (VTI.Vec MQPR:$src), imm_operand_type:$imm),
3471 (pred_int (VTI.Pred VCCR:$mask),
3472 (VTI.Vec MQPR:$inactive)))),
3473 (VTI.Vec (inst (VTI.Vec MQPR:$src), imm_operand_type:$imm,
3474 ARMVCCThen, (VTI.Pred VCCR:$mask),
3475 (VTI.Vec MQPR:$inactive)))>;
3478 multiclass MVE_immediate_shift_patterns<MVEVectorVTInfo VTI,
3480 defm : MVE_immediate_shift_patterns_inner<VTI, imm_operand_type,
3482 !cast<Instruction>("MVE_VSHL_immi" # VTI.BitsSuffix)>;
3483 defm : MVE_immediate_shift_patterns_inner<VTI, imm_operand_type,
3485 !cast<Instruction>("MVE_VSHR_immu" # VTI.BitsSuffix), [1]>;
3486 defm : MVE_immediate_shift_patterns_inner<VTI, imm_operand_type,
3488 !cast<Instruction>("MVE_VSHR_imms" # VTI.BitsSuffix), [0]>;
3533 multiclass MVE_VRINT_m<MVEVectorVTInfo VTI, string suffix, bits<3> opcode,
3535 def "": MVE_VRINT<suffix, opcode, VTI.Suffix, VTI.Size>;
3540 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$val))),
3541 (VTI.Vec (Inst (VTI.Vec MQPR:$val)))>;
3542 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$val), (VTI.Pred VCCR:$pred),
3543 (VTI.Vec MQPR:$inactive))),
3544 (VTI.Vec (Inst (VTI.Vec MQPR:$val), ARMVCCThen,
3545 (VTI.Pred VCCR:$pred), (VTI.Vec MQPR:$inactive)))>;
3549 multiclass MVE_VRINT_ops<MVEVectorVTInfo VTI> {
3550 defm N : MVE_VRINT_m<VTI, "n", 0b000, int_arm_mve_vrintn>;
3551 defm X : MVE_VRINT_m<VTI, "x", 0b001, frint>;
3552 defm A : MVE_VRINT_m<VTI, "a", 0b010, fround>;
3553 defm Z : MVE_VRINT_m<VTI, "z", 0b011, ftrunc>;
3554 defm M : MVE_VRINT_m<VTI, "m", 0b101, ffloor>;
3555 defm P : MVE_VRINT_m<VTI, "p", 0b111, fceil>;
3588 multiclass MVE_VMULT_fp_m<string iname, bit bit_21, MVEVectorVTInfo VTI,
3590 def "" : MVE_VMUL_fp<iname, VTI.Suffix, VTI.Size{0}>;
3594 defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? ), !cast<Instruction>(NAME)>;
3598 multiclass MVE_VMUL_fp_m<MVEVectorVTInfo VTI>
3599 : MVE_VMULT_fp_m<"vmul", 0, VTI, fmul, int_arm_mve_mul_predicated>;
3624 multiclass MVE_VCMLA_m<MVEVectorVTInfo VTI, bit size> {
3625 def "" : MVE_VCMLA<VTI.Suffix, size>;
3629 def : Pat<(VTI.Vec (int_arm_mve_vcmlaq
3630 imm:$rot, (VTI.Vec MQPR:$Qd_src),
3631 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
3632 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src),
3633 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
3636 def : Pat<(VTI.Vec (int_arm_mve_vcmlaq_predicated
3637 imm:$rot, (VTI.Vec MQPR:$Qd_src),
3638 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
3639 (VTI.Pred VCCR:$mask))),
3640 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src), (VTI.Vec MQPR:$Qn),
3641 (VTI.Vec MQPR:$Qm), imm:$rot,
3642 ARMVCCThen, (VTI.Pred VCCR:$mask)))>;
3673 multiclass MVE_VFMA_fp_multi<string iname, bit fms, MVEVectorVTInfo VTI> {
3674 def "" : MVE_VADDSUBFMA_fp<iname, VTI.Suffix, VTI.Size{0}, 0b1, 0b0, fms,
3678 defvar m1 = (VTI.Vec MQPR:$m1);
3679 defvar m2 = (VTI.Vec MQPR:$m2);
3680 defvar add = (VTI.Vec MQPR:$add);
3681 defvar pred = (VTI.Pred VCCR:$pred);
3685 def : Pat<(VTI.Vec (fma (fneg m1), m2, add)),
3687 def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$pred),
3688 (VTI.Vec (fma (fneg m1), m2, add)),
3691 def : Pat<(VTI.Vec (pred_int (fneg m1), m2, add, pred)),
3693 def : Pat<(VTI.Vec (pred_int m1, (fneg m2), add, pred)),
3696 def : Pat<(VTI.Vec (fma m1, m2, add)),
3698 def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$pred),
3699 (VTI.Vec (fma m1, m2, add)),
3702 def : Pat<(VTI.Vec (pred_int m1, m2, add, pred)),
3713 multiclass MVE_VADDSUB_fp_m<string iname, bit bit_21, MVEVectorVTInfo VTI,
3715 def "" : MVE_VADDSUBFMA_fp<iname, VTI.Suffix, VTI.Size{0}, 0, 1, bit_21> {
3721 defm : MVE_TwoOpPattern<VTI, Op, PredInt, (? ), !cast<Instruction>(NAME)>;
3725 multiclass MVE_VADD_fp_m<MVEVectorVTInfo VTI>
3726 : MVE_VADDSUB_fp_m<"vadd", 0, VTI, fadd, int_arm_mve_add_predicated>;
3727 multiclass MVE_VSUB_fp_m<MVEVectorVTInfo VTI>
3728 : MVE_VADDSUB_fp_m<"vsub", 1, VTI, fsub, int_arm_mve_sub_predicated>;
3757 multiclass MVE_VCADD_m<MVEVectorVTInfo VTI, bit size, string cstr=""> {
3758 def "" : MVE_VCADD<VTI.Suffix, size, cstr>;
3762 def : Pat<(VTI.Vec (int_arm_mve_vcaddq (i32 1),
3763 imm:$rot, (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
3764 (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
3767 def : Pat<(VTI.Vec (int_arm_mve_vcaddq_predicated (i32 1),
3768 imm:$rot, (VTI.Vec MQPR:$inactive),
3769 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
3770 (VTI.Pred VCCR:$mask))),
3771 (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
3772 imm:$rot, ARMVCCThen, (VTI.Pred VCCR:$mask),
3773 (VTI.Vec MQPR:$inactive)))>;
3801 multiclass MVE_VABDT_fp_m<MVEVectorVTInfo VTI,
3803 def "" : MVE_VABD_fp<VTI.Suffix, VTI.Size{0}>;
3807 def : Pat<(VTI.Vec (unpred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
3809 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
3810 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
3811 (i32 0), (VTI.Pred VCCR:$mask),
3812 (VTI.Vec MQPR:$inactive))),
3813 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
3814 ARMVCCThen, (VTI.Pred VCCR:$mask),
3815 (VTI.Vec MQPR:$inactive)))>;
3819 multiclass MVE_VABD_fp_m<MVEVectorVTInfo VTI>
3820 : MVE_VABDT_fp_m<VTI, int_arm_mve_vabd, int_arm_mve_abd_predicated>;
4045 MVEVectorVTInfo VTI, bit opcode> {
4046 def "" : MVE_VABSNEG_fp<iname, VTI.Suffix, VTI.Size, opcode>;
4050 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$v))),
4051 (VTI.Vec (Inst $v))>;
4052 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$v), (VTI.Pred VCCR:$mask),
4053 (VTI.Vec MQPR:$inactive))),
4054 (VTI.Vec (Inst $v, ARMVCCThen, $mask, $inactive))>;
4090 multiclass MVE_VMAXMINNMA_m<string iname, MVEVectorVTInfo VTI,
4093 def "" : MVE_VMAXMINNMA<iname, VTI.Suffix, VTI.Size{0}, bit_12>;
4098 def : Pat<(VTI.Vec (unpred_op (fabs (VTI.Vec MQPR:$Qd)),
4099 (fabs (VTI.Vec MQPR:$Qm)))),
4100 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm)))>;
4103 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm),
4104 (VTI.Pred VCCR:$mask))),
4105 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd), (VTI.Vec MQPR:$Qm),
4106 ARMVCCThen, (VTI.Pred VCCR:$mask)))>;
4110 multiclass MVE_VMAXNMA<MVEVectorVTInfo VTI, bit bit_12>
4111 : MVE_VMAXMINNMA_m<"vmaxnma", VTI, fmaxnum, int_arm_mve_vmaxnma_predicated, bit_12>;
4116 multiclass MVE_VMINNMA<MVEVectorVTInfo VTI, bit bit_12>
4117 : MVE_VMAXMINNMA_m<"vminnma", VTI, fminnum, int_arm_mve_vminnma_predicated, bit_12>;
4488 MVEVectorVTInfo VTI> {
4489 def "": MVE_VQxDMLxDH<iname, exch, round, subtract, VTI.Suffix, VTI.Size,
4490 !if(!eq(VTI.LaneBits, 32), ",@earlyclobber $Qd", "")>;
4496 def : Pat<(VTI.Vec !con((unpred_intr (VTI.Vec MQPR:$a), (VTI.Vec MQPR:$b),
4497 (VTI.Vec MQPR:$c)), ConstParams)),
4498 (VTI.Vec (Inst (VTI.Vec MQPR:$a), (VTI.Vec MQPR:$b),
4499 (VTI.Vec MQPR:$c)))>;
4500 def : Pat<(VTI.Vec !con((pred_intr (VTI.Vec MQPR:$a), (VTI.Vec MQPR:$b),
4501 (VTI.Vec MQPR:$c)), ConstParams,
4502 (? (VTI.Pred VCCR:$pred)))),
4503 (VTI.Vec (Inst (VTI.Vec MQPR:$a), (VTI.Vec MQPR:$b),
4504 (VTI.Vec MQPR:$c),
4505 ARMVCCThen, (VTI.Pred VCCR:$pred)))>;
4543 multiclass MVE_VCMUL_m<string iname, MVEVectorVTInfo VTI,
4545 def "" : MVE_VCMUL<iname, VTI.Suffix, size, cstr>;
4549 def : Pat<(VTI.Vec (int_arm_mve_vcmulq
4550 imm:$rot, (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
4551 (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
4554 def : Pat<(VTI.Vec (int_arm_mve_vcmulq_predicated
4555 imm:$rot, (VTI.Vec MQPR:$inactive),
4556 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
4557 (VTI.Pred VCCR:$mask))),
4558 (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
4559 imm:$rot, ARMVCCThen, (VTI.Pred VCCR:$mask),
4560 (VTI.Vec MQPR:$inactive)))>;
4589 multiclass MVE_VMULL_m<MVEVectorVTInfo VTI,
4592 def "" : MVE_VMULL<"vmull" # !if(Top, "t", "b"), VTI.Suffix, VTI.Unsigned,
4593 VTI.Size, Top, cstr>;
4597 defvar uflag = !if(!eq(VTI.SuffixLetter, "p"), (?), (? (i32 VTI.Unsigned)));
4600 def : Pat<(VTI.DblVec !con((unpred_op (VTI.Vec MQPR:$Qm),
4601 (VTI.Vec MQPR:$Qn)),
4603 (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
4606 def : Pat<(VTI.DblVec !con((pred_int (VTI.Vec MQPR:$Qm),
4607 (VTI.Vec MQPR:$Qn)),
4608 uflag, (? (i32 Top), (VTI.DblPred VCCR:$mask),
4609 (VTI.DblVec MQPR:$inactive)))),
4610 (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
4611 ARMVCCThen, (VTI.DblPred VCCR:$mask),
4612 (VTI.DblVec MQPR:$inactive)))>;
4719 multiclass MVE_VxMULH_m<string iname, MVEVectorVTInfo VTI, SDNode unpred_op,
4721 def "" : MVE_VxMULH<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, round>;
4726 def : Pat<(VTI.Vec (unpred_op (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
4727 (i32 VTI.Unsigned))),
4728 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
4731 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
4732 (i32 VTI.Unsigned), (VTI.Pred VCCR:$mask),
4733 (VTI.Vec MQPR:$inactive))),
4734 (VTI.Vec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
4735 ARMVCCThen, (VTI.Pred VCCR:$mask),
4736 (VTI.Vec MQPR:$inactive)))>;
4740 multiclass MVE_VMULT<string iname, MVEVectorVTInfo VTI, bit round>
4741 : MVE_VxMULH_m<iname, VTI, !if(round, int_arm_mve_vrmulh, int_arm_mve_vmulh),
4797 MVEVectorVTInfo VTI, MVEVectorVTInfo InVTI> {
4800 def : Pat<(VTI.Vec (MVEvmovn (VTI.Vec MQPR:$Qd_src),
4801 (VTI.Vec MQPR:$Qm), (i32 top))),
4802 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src), (VTI.Vec MQPR:$Qm)))>;
4811 def : Pat<(VTI.Vec (MVEvmovn (VTI.Vec MQPR:$Qm),
4812 (VTI.Vec (vrev MQPR:$Qd_src)), (i32 1))),
4813 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src), (VTI.Vec MQPR:$Qm)))>;
4819 def : Pat<(VTI.Vec (int_arm_mve_vmovn_predicated (VTI.Vec MQPR:$Qd_src),
4822 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src),
4833 MVEVectorVTInfo VTI, MVEVectorVTInfo InVTI> {
4834 def : Pat<(VTI.Vec (int_arm_mve_vqmovn (VTI.Vec MQPR:$Qd_src),
4837 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src),
4840 def : Pat<(VTI.Vec (int_arm_mve_vqmovn_predicated (VTI.Vec MQPR:$Qd_src),
4844 (VTI.Vec (Inst (VTI.Vec MQPR:$Qd_src),
4986 multiclass MVE_VxCADD_m<string iname, MVEVectorVTInfo VTI,
4988 def "" : MVE_VxCADD<iname, VTI.Suffix, VTI.Size, halve, cstr>;
4992 def : Pat<(VTI.Vec (int_arm_mve_vcaddq halve,
4993 imm:$rot, (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
4994 (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
4997 def : Pat<(VTI.Vec (int_arm_mve_vcaddq_predicated halve,
4998 imm:$rot, (VTI.Vec MQPR:$inactive),
4999 (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
5000 (VTI.Pred VCCR:$mask))),
5001 (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
5002 imm:$rot, ARMVCCThen, (VTI.Pred VCCR:$mask),
5003 (VTI.Vec MQPR:$inactive)))>;
5062 multiclass MVE_VQDMULL_m<string iname, MVEVectorVTInfo VTI, bit size, bit T,
5064 def "" : MVE_VQDMULL<iname, VTI.Suffix, size, T, cstr>;
5069 def : Pat<(VTI.DblVec (int_arm_mve_vqdmull (VTI.Vec MQPR:$Qm),
5070 (VTI.Vec MQPR:$Qn), (i32 T))),
5071 (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn)))>;
5073 def : Pat<(VTI.DblVec (int_arm_mve_vqdmull_predicated
5074 (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
5075 (i32 T), (VTI.DblPred VCCR:$mask),
5076 (VTI.DblVec MQPR:$inactive))),
5077 (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (VTI.Vec MQPR:$Qn),
5078 ARMVCCThen, (VTI.DblPred VCCR:$mask),
5079 (VTI.DblVec MQPR:$inactive)))>;
5083 multiclass MVE_VQDMULL_halves<MVEVectorVTInfo VTI, bit size, string cstr=""> {
5084 defm bh : MVE_VQDMULL_m<"vqdmullb", VTI, size, 0b0, cstr>;
5085 defm th : MVE_VQDMULL_m<"vqdmullt", VTI, size, 0b1, cstr>;
5136 multiclass MVE_vec_scalar_int_pat_m<Instruction inst, MVEVectorVTInfo VTI,
5140 defvar UnpredSign = !if(unpred_has_sign, (? (i32 VTI.Unsigned)), (?));
5141 defvar PredSign = !if(pred_has_sign, (? (i32 VTI.Unsigned)), (?));
5145 def : Pat<(VTI.Vec !con((unpred_op (VTI.Vec MQPR:$Qm),
5146 (VTI.Vec (ARMvdup rGPR:$val))),
5148 (VTI.Vec (inst (VTI.Vec MQPR:$Qm), (i32 rGPR:$val)))>;
5150 def : Pat<(VTI.Vec !con((pred_op (VTI.Vec MQPR:$Qm),
5151 (VTI.Vec (ARMvdup rGPR:$val))),
5153 (pred_op (VTI.Pred VCCR:$mask),
5154 (VTI.Vec MQPR:$inactive)))),
5155 (VTI.Vec (inst (VTI.Vec MQPR:$Qm), (i32 rGPR:$val),
5156 ARMVCCThen, (VTI.Pred VCCR:$mask),
5157 (VTI.Vec MQPR:$inactive)))>;
5175 multiclass MVE_VADDSUB_qr_m<string iname, MVEVectorVTInfo VTI, bit subtract,
5177 def "" : MVE_VADDSUB_qr<iname, VTI.Suffix, VTI.Size, 0b0, subtract, 0b1, 0b0>;
5179 defm : MVE_TwoOpPatternDup<VTI, Op, PredInt, (? ), !cast<Instruction>(NAME), ARMimmAllZerosV>;
5183 multiclass MVE_VADD_qr_m<MVEVectorVTInfo VTI>
5184 : MVE_VADDSUB_qr_m<"vadd", VTI, 0b0, add, int_arm_mve_add_predicated>;
5186 multiclass MVE_VSUB_qr_m<MVEVectorVTInfo VTI>
5187 : MVE_VADDSUB_qr_m<"vsub", VTI, 0b1, sub, int_arm_mve_sub_predicated>;
5198 multiclass MVE_VQADDSUB_qr_m<string iname, MVEVectorVTInfo VTI, bit subtract,
5200 def "" : MVE_VADDSUB_qr<iname, VTI.Suffix, VTI.Size, 0b1, subtract,
5201 0b0, VTI.Unsigned>;
5204 defm : MVE_TwoOpPatternDup<VTI, Op, PredInt, (? (i32 VTI.Unsigned)),
5209 multiclass MVE_VQADD_qr_m<MVEVectorVTInfo VTI, SDNode Op>
5210 : MVE_VQADDSUB_qr_m<"vqadd", VTI, 0b0, Op, int_arm_mve_qadd_predicated>;
5212 multiclass MVE_VQSUB_qr_m<MVEVectorVTInfo VTI, SDNode Op>
5213 : MVE_VQADDSUB_qr_m<"vqsub", VTI, 0b1, Op, int_arm_mve_qsub_predicated>;
5243 multiclass MVE_VQDMULL_qr_m<string iname, MVEVectorVTInfo VTI, bit size,
5245 def "" : MVE_VQDMULL_qr<iname, VTI.Suffix, size, T, cstr>;
5250 def : Pat<(VTI.DblVec (int_arm_mve_vqdmull (VTI.Vec MQPR:$Qm),
5251 (VTI.Vec (ARMvdup rGPR:$val)),
5253 (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (i32 rGPR:$val)))>;
5255 def : Pat<(VTI.DblVec (int_arm_mve_vqdmull_predicated
5256 (VTI.Vec MQPR:$Qm),
5257 (VTI.Vec (ARMvdup rGPR:$val)),
5259 (VTI.DblPred VCCR:$mask),
5260 (VTI.DblVec MQPR:$inactive))),
5261 (VTI.DblVec (Inst (VTI.Vec MQPR:$Qm), (i32 rGPR:$val),
5262 ARMVCCThen, (VTI.DblPred VCCR:$mask),
5263 (VTI.DblVec MQPR:$inactive)))>;
5267 multiclass MVE_VQDMULL_qr_halves<MVEVectorVTInfo VTI, bit size, string cstr=""> {
5268 defm bh : MVE_VQDMULL_qr_m<"vqdmullb", VTI, size, 0b0, cstr>;
5269 defm th : MVE_VQDMULL_qr_m<"vqdmullt", VTI, size, 0b1, cstr>;
5289 multiclass MVE_VHADDSUB_qr_m<string iname, MVEVectorVTInfo VTI, bit subtract,
5291 def "" : MVE_VxADDSUB_qr<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, subtract>;
5293 VTI, unpred_int, pred_int, 1, 1>;
5296 multiclass MVE_VHADD_qr_m<MVEVectorVTInfo VTI> :
5297 MVE_VHADDSUB_qr_m<"vhadd", VTI, 0b0, int_arm_mve_vhadd,
5300 multiclass MVE_VHSUB_qr_m<MVEVectorVTInfo VTI> :
5301 MVE_VHADDSUB_qr_m<"vhsub", VTI, 0b1, int_arm_mve_vhsub,
5318 multiclass MVE_VADDSUB_qr_f<string iname, MVEVectorVTInfo VTI, bit subtract,
5320 def "" : MVE_VxADDSUB_qr<iname, VTI.Suffix, VTI.Size{0}, 0b11, subtract>;
5321 defm : MVE_TwoOpPatternDup<VTI, Op, PredInt, (? ),
5353 multiclass MVE_VxSHL_qr_p<string iname, MVEVectorVTInfo VTI, bit q, bit r> {
5354 def "" : MVE_VxSHL_qr<iname, VTI.Suffix, VTI.Unsigned, VTI.Size, q, r>;
5357 def : Pat<(VTI.Vec (int_arm_mve_vshl_scalar
5358 (VTI.Vec MQPR:$in), (i32 rGPR:$sh),
5359 (i32 q), (i32 r), (i32 VTI.Unsigned))),
5360 (VTI.Vec (Inst (VTI.Vec MQPR:$in), (i32 rGPR:$sh)))>;
5362 def : Pat<(VTI.Vec (int_arm_mve_vshl_scalar_predicated
5363 (VTI.Vec MQPR:$in), (i32 rGPR:$sh),
5364 (i32 q), (i32 r), (i32 VTI.Unsigned),
5365 (VTI.Pred VCCR:$mask))),
5366 (VTI.Vec (Inst (VTI.Vec MQPR:$in), (i32 rGPR:$sh),
5367 ARMVCCThen, (VTI.Pred VCCR:$mask)))>;
5416 multiclass MVE_VBRSR_pat_m<MVEVectorVTInfo VTI, Instruction Inst> {
5418 def : Pat<(VTI.Vec (int_arm_mve_vbrsr (VTI.Vec MQPR:$Qn), (i32 rGPR:$Rm))),
5419 (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (i32 rGPR:$Rm)))>;
5421 def : Pat<(VTI.Vec (int_arm_mve_vbrsr_predicated
5422 (VTI.Vec MQPR:$inactive),
5423 (VTI.Vec MQPR:$Qn), (i32 rGPR:$Rm),
5424 (VTI.Pred VCCR:$mask))),
5425 (VTI.Vec (Inst (VTI.Vec MQPR:$Qn), (i32 rGPR:$Rm),
5426 ARMVCCThen, (VTI.Pred VCCR:$mask),
5427 (VTI.Vec MQPR:$inactive)))>;
5462 multiclass MVE_VMUL_qr_int_m<MVEVectorVTInfo VTI> {
5463 def "" : MVE_VMUL_qr_int<"vmul", VTI.Suffix, VTI.Size>;
5465 defm : MVE_TwoOpPatternDup<VTI, mul, int_arm_mve_mul_predicated, (? ),
5487 multiclass MVE_VxxMUL_qr_m<string iname, MVEVectorVTInfo VTI, bit bit_28,
5489 def "" : MVE_VxxMUL_qr<iname, VTI.Suffix, bit_28, VTI.Size>;
5492 defm : MVE_TwoOpPatternDup<VTI, Op, int_pred, (? ), !cast<Instruction>(NAME)>;
5494 defm : MVE_vec_scalar_int_pat_m<!cast<Instruction>(NAME), VTI, int_unpred, int_pred>;
5497 multiclass MVE_VQDMULH_qr_m<MVEVectorVTInfo VTI> :
5498 MVE_VxxMUL_qr_m<"vqdmulh", VTI, 0b0, MVEvqdmulh,
5501 multiclass MVE_VQRDMULH_qr_m<MVEVectorVTInfo VTI> :
5502 MVE_VxxMUL_qr_m<"vqrdmulh", VTI, 0b1, null_frag,
5513 multiclass MVE_VxxMUL_qr_f_m<MVEVectorVTInfo VTI> {
5515 def "" : MVE_VxxMUL_qr<"vmul", VTI.Suffix, VTI.Size{0}, 0b11>;
5516 defm : MVE_TwoOpPatternDup<VTI, fmul, int_arm_mve_mul_predicated, (? ),
5540 multiclass MVE_VMLA_qr_multi<string iname, MVEVectorVTInfo VTI,
5542 def "": MVE_VFMAMLA_qr<iname, VTI.Suffix, VTI.Unsigned, VTI.Size,
5546 defvar v1 = (VTI.Vec MQPR:$v1);
5547 defvar v2 = (VTI.Vec MQPR:$v2);
5548 defvar vs = (VTI.Vec (ARMvdup rGPR:$s));
5550 defvar pred = (VTI.Pred VCCR:$pred);
5555 if VTI.Unsigned then let Predicates = [HasMVEInt] in {
5557 def : Pat<(VTI.Vec (add (mul v1, v2), vs)),
5558 (VTI.Vec (Inst v1, v2, s))>;
5560 def : Pat<(VTI.Vec (add (mul v2, vs), v1)),
5561 (VTI.Vec (Inst v1, v2, s))>;
5564 def : Pat<(VTI.Vec (pred_int v1, v2, s, pred)),
5565 (VTI.Vec (Inst v1, v2, s, ARMVCCThen, pred))>;
5583 multiclass MVE_VFMA_qr_multi<string iname, MVEVectorVTInfo VTI,
5585 def "": MVE_VFMAMLA_qr<iname, VTI.Suffix, VTI.Size{0}, 0b11, scalar_addend>;
5588 defvar v1 = (VTI.Vec MQPR:$v1);
5589 defvar v2 = (VTI.Vec MQPR:$v2);
5590 defvar vs = (VTI.Vec (ARMvdup (i32 rGPR:$s)));
5592 defvar pred = (VTI.Pred VCCR:$pred);
5596 def : Pat<(VTI.Vec (fma v1, v2, vs)),
5597 (VTI.Vec (Inst v1, v2, is))>;
5598 def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$pred),
5599 (VTI.Vec (fma v1, v2, vs)),
5601 (VTI.Vec (Inst v1, v2, is, ARMVCCThen, $pred))>;
5602 def : Pat<(VTI.Vec (pred_int v1, v2, vs, pred)),
5603 (VTI.Vec (Inst v1, v2, is, ARMVCCThen, pred))>;
5605 def : Pat<(VTI.Vec (fma v1, vs, v2)),
5606 (VTI.Vec (Inst v2, v1, is))>;
5607 def : Pat<(VTI.Vec (fma vs, v1, v2)),
5608 (VTI.Vec (Inst v2, v1, is))>;
5609 def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$pred),
5610 (VTI.Vec (fma vs, v2, v1)),
5612 (VTI.Vec (Inst v1, v2, is, ARMVCCThen, $pred))>;
5613 def : Pat<(VTI.Vec (vselect (VTI.Pred VCCR:$pred),
5614 (VTI.Vec (fma v2, vs, v1)),
5616 (VTI.Vec (Inst v1, v2, is, ARMVCCThen, $pred))>;
5617 def : Pat<(VTI.Vec (pred_int v1, vs, v2, pred)),
5618 (VTI.Vec (Inst v2, v1, is, ARMVCCThen, pred))>;
5619 def : Pat<(VTI.Vec (pred_int vs, v1, v2, pred)),
5620 (VTI.Vec (Inst v2, v1, is, ARMVCCThen, pred))>;
5644 multiclass MVE_VQDMLAH_qr_multi<string iname, MVEVectorVTInfo VTI,
5646 def "": MVE_VQDMLAH_qr<iname, VTI.Suffix, 0b0, VTI.Size, bit_5, bit_12>;
5652 def : Pat<(VTI.Vec (unpred_int (VTI.Vec MQPR:$v1), (VTI.Vec MQPR:$v2),
5654 (VTI.Vec (Inst (VTI.Vec MQPR:$v1), (VTI.Vec MQPR:$v2),
5656 def : Pat<(VTI.Vec (pred_int (VTI.Vec MQPR:$v1), (VTI.Vec MQPR:$v2),
5657 (i32 rGPR:$s), (VTI.Pred VCCR:$pred))),
5658 (VTI.Vec (Inst (VTI.Vec MQPR:$v1), (VTI.Vec MQPR:$v2),
5660 (VTI.Pred VCCR:$pred)))>;
5764 multiclass MVE_VCTP<MVEVectorVTInfo VTI, Intrinsic intr> {
5765 def "": MVE_VCTPInst<VTI.BitsSuffix, VTI.Size>;
5770 (VTI.Pred (Inst rGPR:$Rn))>;
5771 def : Pat<(and (intr rGPR:$Rn), (VTI.Pred VCCR:$mask)),
5772 (VTI.Pred (Inst rGPR:$Rn, ARMVCCThen, VCCR:$mask))>;
6265 foreach VTI = VTIs in
6266 foreach UnsignedFlag = !if(!eq(VTI.Size, memsz.encoding),
6267 [0,1], [VTI.Unsigned]) in {
6268 …def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), memsz.T…
6269 (VTI.Vec (InstU GPR:$base, MQPR:$offsets))>;
6270 …def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), memsz.T…
6271 (VTI.Vec (Inst GPR:$base, MQPR:$offsets))>;
6272 …def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset_predicated GPR:$base, (VTIs[0].Vec MQPR:$offset…
6273 (VTI.Vec (InstU GPR:$base, MQPR:$offsets, ARMVCCThen, VCCR:$pred))>;
6274 …: Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset_predicated GPR:$base, (VTIs[0].Vec MQPR:$offsets), …
6275 (VTI.Vec (Inst GPR:$base, MQPR:$offsets, ARMVCCThen, VCCR:$pred))>;
6283 foreach VTI = VTIs in {
6284 …def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), 8, 0, V…
6285 (VTI.Vec (Inst GPR:$base, MQPR:$offsets))>;
6286 …def : Pat<(VTI.Vec (int_arm_mve_vldr_gather_offset_predicated GPR:$base, (VTIs[0].Vec MQPR:$offset…
6287 (VTI.Vec (Inst GPR:$base, MQPR:$offsets, ARMVCCThen, VCCR:$pred))>;
6296 foreach VTI = VTIs in {
6297 …def : Pat<(int_arm_mve_vstr_scatter_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$…
6299 …def : Pat<(int_arm_mve_vstr_scatter_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$…
6301 …t_predicated GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$data), memsz.TypeBits, 0, (VTI…
6303 …dicated GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$data), memsz.TypeBits, memsz.shift,…
6312 foreach VTI = VTIs in {
6313 …def : Pat<(int_arm_mve_vstr_scatter_offset GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$…
6315 …r_offset_predicated GPR:$base, (VTIs[0].Vec MQPR:$offsets), (VTI.Vec MQPR:$data), 8, 0, (VTI.Pred …