Lines Matching refs:OpNode
1311 X86VectorVTInfo _, SDPatternOperator OpNode,
1318 (_.VT (OpNode SrcRC:$src)), /*IsCommutable*/0,
1324 X86VectorVTInfo _, SDPatternOperator OpNode,
1334 def : Pat <(_.VT (OpNode SrcRC:$src)),
1339 def : Pat <(vselect _.KRCWM:$mask, (_.VT (OpNode SrcRC:$src)), _.RC:$src0),
1343 def : Pat <(vselect _.KRCWM:$mask, (_.VT (OpNode SrcRC:$src)), _.ImmAllZerosV),
1349 AVX512VLVectorVTInfo _, SDPatternOperator OpNode,
1353 OpNode, SrcRC, Subreg>, EVEX_V512;
1356 _.info256, OpNode, SrcRC, Subreg>, EVEX_V256;
1358 _.info128, OpNode, SrcRC, Subreg>, EVEX_V128;
1363 SDPatternOperator OpNode,
1366 defm Z : avx512_int_broadcast_reg<opc, WriteShuffle256, _.info512, OpNode,
1369 defm Z256 : avx512_int_broadcast_reg<opc, WriteShuffle256, _.info256, OpNode,
1371 defm Z128 : avx512_int_broadcast_reg<opc, WriteShuffle, _.info128, OpNode,
2054 multiclass avx512_cmp_scalar<X86VectorVTInfo _, SDNode OpNode, SDNode OpNodeSAE,
2062 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2), timm:$cc),
2071 (OpNode (_.VT _.RC:$src1), (_.ScalarIntMemFrags addr:$src2),
2095 [(set _.KRC:$dst, (OpNode _.FRC:$src1,
2104 [(set _.KRC:$dst, (OpNode _.FRC:$src1,
2977 RegisterClass KRC, SDPatternOperator OpNode,
2982 [(set KRC:$dst, (OpNode KRC:$src))]>,
2987 SDPatternOperator OpNode,
2989 defm B : avx512_mask_unop<opc, !strconcat(OpcodeStr, "b"), VK8, OpNode,
2991 defm W : avx512_mask_unop<opc, !strconcat(OpcodeStr, "w"), VK16, OpNode,
2993 defm D : avx512_mask_unop<opc, !strconcat(OpcodeStr, "d"), VK32, OpNode,
2995 defm Q : avx512_mask_unop<opc, !strconcat(OpcodeStr, "q"), VK64, OpNode,
3017 RegisterClass KRC, SDPatternOperator OpNode,
3024 [(set KRC:$dst, (OpNode KRC:$src1, KRC:$src2))]>,
3029 SDPatternOperator OpNode,
3032 defm B : avx512_mask_binop<opc, !strconcat(OpcodeStr, "b"), VK8, OpNode,
3034 defm W : avx512_mask_binop<opc, !strconcat(OpcodeStr, "w"), VK16, OpNode,
3036 defm D : avx512_mask_binop<opc, !strconcat(OpcodeStr, "d"), VK32, OpNode,
3038 defm Q : avx512_mask_binop<opc, !strconcat(OpcodeStr, "q"), VK64, OpNode,
3107 SDNode OpNode, X86FoldableSchedWrite sched,
3112 [(set EFLAGS, (OpNode KRC:$src1, KRC:$src2))]>,
3116 multiclass avx512_mask_testop_w<bits<8> opc, string OpcodeStr, SDNode OpNode,
3119 defm B : avx512_mask_testop<opc, OpcodeStr#"b", VK8, OpNode, sched, HasDQI>,
3121 defm W : avx512_mask_testop<opc, OpcodeStr#"w", VK16, OpNode, sched, prdW>,
3123 defm Q : avx512_mask_testop<opc, OpcodeStr#"q", VK64, OpNode, sched, HasBWI>,
3125 defm D : avx512_mask_testop<opc, OpcodeStr#"d", VK32, OpNode, sched, HasBWI>,
3135 SDNode OpNode, X86FoldableSchedWrite sched> {
3140 [(set KRC:$dst, (OpNode KRC:$src, (i8 timm:$imm)))]>,
3145 SDNode OpNode, X86FoldableSchedWrite sched> {
3146 defm W : avx512_mask_shiftop<opc1, !strconcat(OpcodeStr, "w"), VK16, OpNode,
3149 defm B : avx512_mask_shiftop<opc1, !strconcat(OpcodeStr, "b"), VK8, OpNode,
3152 defm Q : avx512_mask_shiftop<opc2, !strconcat(OpcodeStr, "q"), VK64, OpNode,
3154 defm D : avx512_mask_shiftop<opc2, !strconcat(OpcodeStr, "d"), VK32, OpNode,
3979 multiclass avx512_move_scalar<string asm, SDNode OpNode, PatFrag vzload_frag,
3985 [(set _.RC:$dst, (_.VT (OpNode _.RC:$src1, _.RC:$src2)))],
3992 (_.VT (OpNode _.RC:$src1, _.RC:$src2)),
4001 (_.VT (OpNode _.RC:$src1, _.RC:$src2)),
4048 multiclass avx512_move_scalar_lowering<string InstrStr, SDNode OpNode,
4051 def : Pat<(_.VT (OpNode _.RC:$src0,
4062 def : Pat<(_.VT (OpNode _.RC:$src0,
4641 multiclass avx512_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
4647 (_.VT (OpNode _.RC:$src1, _.RC:$src2)),
4654 (_.VT (OpNode _.RC:$src1, (_.LdFrag addr:$src2)))>,
4659 multiclass avx512_binop_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
4662 avx512_binop_rm<opc, OpcodeStr, OpNode, _, sched, IsCommutable> {
4667 (_.VT (OpNode _.RC:$src1,
4673 multiclass avx512_binop_rm_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
4678 defm Z : avx512_binop_rm<opc, OpcodeStr, OpNode, VTInfo.info512, sched.ZMM,
4682 defm Z256 : avx512_binop_rm<opc, OpcodeStr, OpNode, VTInfo.info256,
4684 defm Z128 : avx512_binop_rm<opc, OpcodeStr, OpNode, VTInfo.info128,
4689 multiclass avx512_binop_rmb_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
4694 defm Z : avx512_binop_rmb<opc, OpcodeStr, OpNode, VTInfo.info512, sched.ZMM,
4698 defm Z256 : avx512_binop_rmb<opc, OpcodeStr, OpNode, VTInfo.info256,
4700 defm Z128 : avx512_binop_rmb<opc, OpcodeStr, OpNode, VTInfo.info128,
4705 multiclass avx512_binop_rm_vl_q<bits<8> opc, string OpcodeStr, SDNode OpNode,
4708 defm NAME : avx512_binop_rmb_vl<opc, OpcodeStr, OpNode, avx512vl_i64_info,
4713 multiclass avx512_binop_rm_vl_d<bits<8> opc, string OpcodeStr, SDNode OpNode,
4716 defm NAME : avx512_binop_rmb_vl<opc, OpcodeStr, OpNode, avx512vl_i32_info,
4720 multiclass avx512_binop_rm_vl_w<bits<8> opc, string OpcodeStr, SDNode OpNode,
4723 defm NAME : avx512_binop_rm_vl<opc, OpcodeStr, OpNode, avx512vl_i16_info,
4728 multiclass avx512_binop_rm_vl_b<bits<8> opc, string OpcodeStr, SDNode OpNode,
4731 defm NAME : avx512_binop_rm_vl<opc, OpcodeStr, OpNode, avx512vl_i8_info,
4737 SDNode OpNode, X86SchedWriteWidths sched,
4739 defm Q : avx512_binop_rm_vl_q<opc_q, OpcodeStr#"q", OpNode, sched, prd,
4742 defm D : avx512_binop_rm_vl_d<opc_d, OpcodeStr#"d", OpNode, sched, prd,
4747 SDNode OpNode, X86SchedWriteWidths sched,
4749 defm W : avx512_binop_rm_vl_w<opc_w, OpcodeStr#"w", OpNode, sched, prd,
4752 defm B : avx512_binop_rm_vl_b<opc_b, OpcodeStr#"b", OpNode, sched, prd,
4758 string OpcodeStr, SDNode OpNode,
4761 defm NAME : avx512_binop_rm_vl_dq<opc_d, opc_q, OpcodeStr, OpNode,
4763 avx512_binop_rm_vl_bw<opc_b, opc_w, OpcodeStr, OpNode,
4769 SDNode OpNode,X86VectorVTInfo _Src,
4775 (_Dst.VT (OpNode
4783 (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1),
4793 (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1), (bitconvert
4835 SDNode OpNode, Predicate prd, bit IsCommutable = 0> {
4837 defm NAME#Z : avx512_binop_rm2<opc, OpcodeStr, sched.ZMM, OpNode,
4842 defm NAME#Z256 : avx512_binop_rm2<opc, OpcodeStr, sched.YMM, OpNode,
4846 defm NAME#Z128 : avx512_binop_rm2<opc, OpcodeStr, sched.XMM, OpNode,
4857 multiclass avx512_packs_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
4865 (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1), (bitconvert
4872 SDNode OpNode,X86VectorVTInfo _Src,
4878 (_Dst.VT (OpNode
4886 (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1),
4893 SDNode OpNode> {
4895 defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, v16i32_info,
4897 avx512_packs_rmb<opc, OpcodeStr, OpNode, v16i32_info,
4900 defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, v8i32x_info,
4902 avx512_packs_rmb<opc, OpcodeStr, OpNode, v8i32x_info,
4905 defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, v4i32x_info,
4907 avx512_packs_rmb<opc, OpcodeStr, OpNode, v4i32x_info,
4913 SDNode OpNode> {
4915 defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, v32i16_info, v64i8_info,
4918 defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, v16i16x_info,
4921 defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, v8i16x_info,
4928 SDNode OpNode, AVX512VLVectorVTInfo _Src,
4931 defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, _Src.info512,
4935 defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, _Src.info256,
4938 defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, _Src.info128,
5023 multiclass avx512_min_max_lowering<string Instr, SDNode OpNode> {
5024 def : Pat<(v4i64 (OpNode VR256X:$src1, VR256X:$src2)),
5030 def : Pat<(v4i64 (OpNode (v4i64 VR256X:$src1), (v4i64 (X86VBroadcastld64 addr:$src2)))),
5037 def : Pat<(v2i64 (OpNode VR128X:$src1, VR128X:$src2)),
5043 def : Pat<(v2i64 (OpNode (v2i64 VR128X:$src1), (v2i64 (X86VBroadcastld64 addr:$src2)))),
5196 multiclass avx512_logical_lowering<string InstrStr, SDNode OpNode,
5201 (bitconvert (IntInfo.VT (OpNode _.RC:$src1, _.RC:$src2))),
5207 (bitconvert (IntInfo.VT (OpNode _.RC:$src1, _.RC:$src2))),
5214 (bitconvert (IntInfo.VT (OpNode _.RC:$src1,
5220 (bitconvert (IntInfo.VT (OpNode _.RC:$src1,
5227 multiclass avx512_logical_lowering_bcast<string InstrStr, SDNode OpNode,
5233 (IntInfo.VT (OpNode _.RC:$src1,
5240 (IntInfo.VT (OpNode _.RC:$src1,
5247 multiclass avx512_logical_lowering_sizes<string InstrStr, SDNode OpNode,
5251 defm : avx512_logical_lowering<InstrStr#"Z128", OpNode, SelectInfo.info128,
5253 defm : avx512_logical_lowering<InstrStr#"Z256", OpNode, SelectInfo.info256,
5257 defm : avx512_logical_lowering<InstrStr#"Z", OpNode, SelectInfo.info512,
5262 multiclass avx512_logical_lowering_sizes_bcast<string InstrStr, SDNode OpNode,
5266 defm : avx512_logical_lowering_bcast<InstrStr#"Z128", OpNode,
5268 defm : avx512_logical_lowering_bcast<InstrStr#"Z256", OpNode,
5272 defm : avx512_logical_lowering_bcast<InstrStr#"Z", OpNode,
5277 multiclass avx512_logical_lowering_types<string InstrStr, SDNode OpNode> {
5279 defm : avx512_logical_lowering_sizes<InstrStr#"Q", OpNode, avx512vl_i64_info,
5281 defm : avx512_logical_lowering_sizes<InstrStr#"Q", OpNode, avx512vl_i64_info,
5283 defm : avx512_logical_lowering_sizes<InstrStr#"Q", OpNode, avx512vl_i64_info,
5287 defm : avx512_logical_lowering_sizes<InstrStr#"D", OpNode, avx512vl_i32_info,
5289 defm : avx512_logical_lowering_sizes<InstrStr#"D", OpNode, avx512vl_i32_info,
5291 defm : avx512_logical_lowering_sizes<InstrStr#"D", OpNode, avx512vl_i32_info,
5295 defm : avx512_logical_lowering_sizes<InstrStr#"D", OpNode, avx512vl_f32_info,
5297 defm : avx512_logical_lowering_sizes<InstrStr#"D", OpNode, avx512vl_f32_info,
5299 defm : avx512_logical_lowering_sizes<InstrStr#"D", OpNode, avx512vl_f32_info,
5301 defm : avx512_logical_lowering_sizes<InstrStr#"D", OpNode, avx512vl_f32_info,
5305 defm : avx512_logical_lowering_sizes<InstrStr#"Q", OpNode, avx512vl_f64_info,
5307 defm : avx512_logical_lowering_sizes<InstrStr#"Q", OpNode, avx512vl_f64_info,
5309 defm : avx512_logical_lowering_sizes<InstrStr#"Q", OpNode, avx512vl_f64_info,
5311 defm : avx512_logical_lowering_sizes<InstrStr#"Q", OpNode, avx512vl_f64_info,
5314 defm : avx512_logical_lowering_sizes_bcast<InstrStr#"D", OpNode,
5317 defm : avx512_logical_lowering_sizes_bcast<InstrStr#"Q", OpNode,
5332 SDNode OpNode, SDNode VecNode,
5351 [(set _.FRC:$dst, (OpNode _.FRC:$src1, _.FRC:$src2))]>,
5358 [(set _.FRC:$dst, (OpNode _.FRC:$src1,
5377 SDNode OpNode, SDNode VecNode, SDNode SaeNode,
5399 [(set _.FRC:$dst, (OpNode _.FRC:$src1, _.FRC:$src2))]>,
5407 [(set _.FRC:$dst, (OpNode _.FRC:$src1,
5422 multiclass avx512_binop_s_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
5425 defm SSZ : avx512_fp_scalar<opc, OpcodeStr#"ss", f32x_info, OpNode, VecNode,
5430 defm SDZ : avx512_fp_scalar<opc, OpcodeStr#"sd", f64x_info, OpNode, VecNode,
5437 multiclass avx512_binop_s_sae<bits<8> opc, string OpcodeStr, SDNode OpNode,
5440 defm SSZ : avx512_fp_scalar_sae<opc, OpcodeStr#"ss", f32x_info, OpNode,
5444 defm SDZ : avx512_fp_scalar_sae<opc, OpcodeStr#"sd", f64x_info, OpNode,
5465 X86VectorVTInfo _, SDNode OpNode,
5472 [(set _.FRC:$dst, (OpNode _.FRC:$src1, _.FRC:$src2))]>,
5479 [(set _.FRC:$dst, (OpNode _.FRC:$src1,
5503 multiclass avx512_fp_packed<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,
5513 (_.VT (OpNode _.RC:$src1, _.RC:$src2)),
5521 (OpNode _.RC:$src1, (_.LdFrag addr:$src2)),
5528 (OpNode _.RC:$src1, (_.VT (_.BroadcastLdFrag addr:$src2))),
5558 multiclass avx512_fp_binop_p<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,
5564 defm PSZ : avx512_fp_packed<opc, OpcodeStr, OpNode, MaskOpNode, v16f32_info,
5567 defm PDZ : avx512_fp_packed<opc, OpcodeStr, OpNode, MaskOpNode, v8f64_info,
5574 defm PSZ128 : avx512_fp_packed<opc, OpcodeStr, OpNode, MaskOpNode, v4f32x_info,
5577 defm PSZ256 : avx512_fp_packed<opc, OpcodeStr, OpNode, MaskOpNode, v8f32x_info,
5580 defm PDZ128 : avx512_fp_packed<opc, OpcodeStr, OpNode, MaskOpNode, v2f64x_info,
5584 defm PDZ256 : avx512_fp_packed<opc, OpcodeStr, OpNode, MaskOpNode, v4f64x_info,
5647 multiclass avx512_fp_scalef_p<bits<8> opc, string OpcodeStr, SDNode OpNode,
5653 (_.VT (OpNode _.RC:$src1, _.RC:$src2))>,
5658 (OpNode _.RC:$src1, (_.LdFrag addr:$src2))>,
5664 (OpNode _.RC:$src1, (_.VT (_.BroadcastLdFrag addr:$src2)))>,
5669 multiclass avx512_fp_scalef_scalar<bits<8> opc, string OpcodeStr, SDNode OpNode,
5675 (_.VT (OpNode _.RC:$src1, _.RC:$src2))>,
5680 (OpNode _.RC:$src1, (_.ScalarIntMemFrags addr:$src2))>,
5813 string OpcodeStr, SDNode OpNode,
5819 (_.VT (OpNode _.RC:$src1, (i8 timm:$src2)))>,
5824 (_.VT (OpNode (_.VT (_.LdFrag addr:$src1)),
5831 string OpcodeStr, SDNode OpNode,
5837 (_.VT (OpNode (_.BroadcastLdFrag addr:$src1), (i8 timm:$src2)))>,
5841 multiclass avx512_shift_rrm<bits<8> opc, string OpcodeStr, SDNode OpNode,
5849 (_.VT (OpNode _.RC:$src1, (SrcVT VR128X:$src2)))>,
5854 (_.VT (OpNode _.RC:$src1, (SrcVT (load addr:$src2))))>,
5860 multiclass avx512_shift_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,
5865 defm Z : avx512_shift_rrm<opc, OpcodeStr, OpNode, sched.ZMM, SrcVT,
5869 defm Z256 : avx512_shift_rrm<opc, OpcodeStr, OpNode, sched.YMM, SrcVT,
5872 defm Z128 : avx512_shift_rrm<opc, OpcodeStr, OpNode, sched.XMM, SrcVT,
5879 string OpcodeStr, SDNode OpNode,
5882 defm D : avx512_shift_sizes<opcd, OpcodeStr#"d", OpNode, sched, v4i32,
5885 defm Q : avx512_shift_sizes<opcq, OpcodeStr#"q", OpNode, sched, v2i64,
5887 defm W : avx512_shift_sizes<opcw, OpcodeStr#"w", OpNode, sched, v8i16,
5892 string OpcodeStr, SDNode OpNode,
5896 defm Z: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
5898 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode, sched.ZMM,
5901 defm Z256: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
5903 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode, sched.YMM,
5905 defm Z128: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
5907 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode, sched.XMM,
5913 string OpcodeStr, SDNode OpNode,
5916 defm WZ: avx512_shift_rmi<opcw, ImmFormR, ImmFormM, OpcodeStr, OpNode,
5919 defm WZ256: avx512_shift_rmi<opcw, ImmFormR, ImmFormM, OpcodeStr, OpNode,
5921 defm WZ128: avx512_shift_rmi<opcw, ImmFormR, ImmFormM, OpcodeStr, OpNode,
5928 string OpcodeStr, SDNode OpNode,
5931 defm D: avx512_shift_rmi_sizes<opcd, ImmFormR, ImmFormM, OpcodeStr#"d", OpNode,
5934 defm Q: avx512_shift_rmi_sizes<opcq, ImmFormR, ImmFormM, OpcodeStr#"q", OpNode,
5996 multiclass avx512_var_shift<bits<8> opc, string OpcodeStr, SDNode OpNode,
6002 (_.VT (OpNode _.RC:$src1, (_.VT _.RC:$src2)))>,
6007 (_.VT (OpNode _.RC:$src1,
6014 multiclass avx512_var_shift_mb<bits<8> opc, string OpcodeStr, SDNode OpNode,
6021 (_.VT (OpNode _.RC:$src1, (_.VT (_.BroadcastLdFrag addr:$src2))))>,
6026 multiclass avx512_var_shift_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,
6029 defm Z : avx512_var_shift<opc, OpcodeStr, OpNode, sched.ZMM, _.info512>,
6030 avx512_var_shift_mb<opc, OpcodeStr, OpNode, sched.ZMM, _.info512>, EVEX_V512;
6033 defm Z256 : avx512_var_shift<opc, OpcodeStr, OpNode, sched.YMM, _.info256>,
6034 avx512_var_shift_mb<opc, OpcodeStr, OpNode, sched.YMM, _.info256>, EVEX_V256;
6035 defm Z128 : avx512_var_shift<opc, OpcodeStr, OpNode, sched.XMM, _.info128>,
6036 avx512_var_shift_mb<opc, OpcodeStr, OpNode, sched.XMM, _.info128>, EVEX_V128;
6041 SDNode OpNode, X86SchedWriteWidths sched> {
6042 defm D : avx512_var_shift_sizes<opc, OpcodeStr#"d", OpNode, sched,
6044 defm Q : avx512_var_shift_sizes<opc, OpcodeStr#"q", OpNode, sched,
6050 SDNode OpNode, list<Predicate> p> {
6052 def : Pat<(_.info256.VT (OpNode (_.info256.VT _.info256.RC:$src1),
6060 def : Pat<(_.info128.VT (OpNode (_.info128.VT _.info128.RC:$src1),
6070 SDNode OpNode, X86SchedWriteWidths sched> {
6072 defm WZ: avx512_var_shift<opc, OpcodeStr, OpNode, sched.ZMM, v32i16_info>,
6076 defm WZ256: avx512_var_shift<opc, OpcodeStr, OpNode, sched.YMM, v16i16x_info>,
6078 defm WZ128: avx512_var_shift<opc, OpcodeStr, OpNode, sched.XMM, v8i16x_info>,
6207 multiclass avx512_vperm_dq_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,
6210 defm Z : avx512_var_shift<opc, OpcodeStr, OpNode, sched, _.info512>,
6211 avx512_var_shift_mb<opc, OpcodeStr, OpNode, sched, _.info512>, EVEX_V512;
6214 defm Z256 : avx512_var_shift<opc, OpcodeStr, OpNode, sched, _.info256>,
6215 avx512_var_shift_mb<opc, OpcodeStr, OpNode, sched, _.info256>, EVEX_V256;
6219 string OpcodeStr, SDNode OpNode,
6222 defm Z: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
6224 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode,
6227 defm Z256: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
6229 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode,
6234 Predicate prd, SDNode OpNode,
6237 defm Z: avx512_var_shift<opc, OpcodeStr, OpNode, sched, _.info512>,
6240 defm Z256: avx512_var_shift<opc, OpcodeStr, OpNode, sched, _.info256>,
6242 defm Z128: avx512_var_shift<opc, OpcodeStr, OpNode, sched, _.info128>,
6272 multiclass avx512_permil_vec<bits<8> OpcVar, string OpcodeStr, SDNode OpNode,
6278 (_.VT (OpNode _.RC:$src1,
6284 (_.VT (OpNode
6293 (_.VT (OpNode
6350 multiclass avx512_pshufb_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,
6353 defm Z: avx512_var_shift<opc, OpcodeStr, OpNode, sched.ZMM, v64i8_info>,
6357 defm Z256: avx512_var_shift<opc, OpcodeStr, OpNode, sched.YMM, v32i8x_info>,
6359 defm Z128: avx512_var_shift<opc, OpcodeStr, OpNode, sched.XMM, v16i8x_info>,
6389 SDPatternOperator OpNode,
6397 (OpNode _.RC:$src1,
6461 multiclass avx512_fma3p_213_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
6469 (_.VT (OpNode _.RC:$src2, _.RC:$src1, _.RC:$src3)),
6476 (_.VT (OpNode _.RC:$src2, _.RC:$src1, (_.LdFrag addr:$src3))),
6484 (OpNode _.RC:$src2,
6492 multiclass avx512_fma3_213_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
6500 (_.VT ( OpNode _.RC:$src2, _.RC:$src1, _.RC:$src3, (i32 timm:$rc))),
6501 (_.VT ( OpNode _.RC:$src2, _.RC:$src1, _.RC:$src3, (i32 timm:$rc))), 1, 1>,
6505 multiclass avx512_fma3p_213_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
6510 defm Z : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, MaskOpNode,
6517 defm Z256 : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, MaskOpNode,
6520 defm Z128 : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, MaskOpNode,
6526 multiclass avx512_fma3p_213_f<bits<8> opc, string OpcodeStr, SDNode OpNode,
6528 defm PS : avx512_fma3p_213_common<opc, OpcodeStr#"ps", OpNode, MaskOpNode,
6531 defm PD : avx512_fma3p_213_common<opc, OpcodeStr#"pd", OpNode, MaskOpNode,
6550 multiclass avx512_fma3p_231_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
6565 (_.VT (OpNode _.RC:$src2, (_.LdFrag addr:$src3), _.RC:$src1)),
6573 (_.VT (OpNode _.RC:$src2,
6583 multiclass avx512_fma3_231_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
6592 (_.VT (OpNode _.RC:$src2, _.RC:$src3, _.RC:$src1, (i32 timm:$rc))),
6596 multiclass avx512_fma3p_231_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
6601 defm Z : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, MaskOpNode,
6608 defm Z256 : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, MaskOpNode,
6611 defm Z128 : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, MaskOpNode,
6617 multiclass avx512_fma3p_231_f<bits<8> opc, string OpcodeStr, SDNode OpNode,
6619 defm PS : avx512_fma3p_231_common<opc, OpcodeStr#"ps", OpNode, MaskOpNode,
6622 defm PD : avx512_fma3p_231_common<opc, OpcodeStr#"pd", OpNode, MaskOpNode,
6640 multiclass avx512_fma3p_132_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
6657 (_.VT (OpNode (_.LdFrag addr:$src3), _.RC:$src1, _.RC:$src2)),
6667 (_.VT (OpNode (_.VT (_.BroadcastLdFrag addr:$src3)),
6675 multiclass avx512_fma3_132_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
6684 (_.VT (OpNode _.RC:$src1, _.RC:$src3, _.RC:$src2, (i32 timm:$rc))),
6688 multiclass avx512_fma3p_132_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
6693 defm Z : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, MaskOpNode,
6700 defm Z256 : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, MaskOpNode,
6703 defm Z128 : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, MaskOpNode,
6709 multiclass avx512_fma3p_132_f<bits<8> opc, string OpcodeStr, SDNode OpNode,
6711 defm PS : avx512_fma3p_132_common<opc, OpcodeStr#"ps", OpNode, MaskOpNode,
6714 defm PD : avx512_fma3p_132_common<opc, OpcodeStr#"pd", OpNode, MaskOpNode,
6777 string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd,
6783 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, _.FRC:$src1,
6785 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, _.FRC:$src1,
6791 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, _.FRC:$src3,
6793 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2,
6801 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src1, _.FRC:$src3,
6803 (set _.FRC:$dst, (_.EltVT (OpNode (_.ScalarLdFrag addr:$src3),
6811 string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd> {
6813 defm NAME : avx512_fma3s_all<opc213, opc231, opc132, OpcodeStr, OpNode,
6816 defm NAME : avx512_fma3s_all<opc213, opc231, opc132, OpcodeStr, OpNode,
7052 multiclass avx512_pmadd52_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
7060 (_.VT (OpNode _.RC:$src2, _.RC:$src3, _.RC:$src1)), 1, 1>,
7066 (_.VT (OpNode _.RC:$src2, (_.LdFrag addr:$src3), _.RC:$src1))>,
7073 (OpNode _.RC:$src2,
7081 multiclass avx512_pmadd52_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
7084 defm Z : avx512_pmadd52_rm<opc, OpcodeStr, OpNode, sched.ZMM, _.info512>,
7088 defm Z256 : avx512_pmadd52_rm<opc, OpcodeStr, OpNode, sched.YMM, _.info256>,
7090 defm Z128 : avx512_pmadd52_rm<opc, OpcodeStr, OpNode, sched.XMM, _.info128>,
7106 multiclass avx512_vcvtsi<bits<8> opc, SDPatternOperator OpNode, X86FoldableSchedWrite sched,
7128 (OpNode (DstVT.VT DstVT.RC:$src1), SrcRC:$src2))]>,
7135 (OpNode (DstVT.VT DstVT.RC:$src1),
7144 multiclass avx512_vcvtsi_round<bits<8> opc, SDNode OpNode,
7154 (OpNode (DstVT.VT DstVT.RC:$src1),
7163 multiclass avx512_vcvtsi_common<bits<8> opc, SDNode OpNode, SDNode OpNodeRnd,
7169 avx512_vcvtsi<opc, OpNode, sched, SrcRC, DstVT, x86memop,
7258 X86VectorVTInfo DstVT, SDNode OpNode,
7265 [(set DstVT.RC:$dst, (OpNode (SrcVT.VT SrcVT.RC:$src)))]>,
7275 [(set DstVT.RC:$dst, (OpNode
7316 X86VectorVTInfo DstVT, SDNode OpNode,
7323 [(set DstVT.RC:$dst, (OpNode SrcVT.FRC:$src))]>,
7327 [(set DstVT.RC:$dst, (OpNode (SrcVT.ScalarLdFrag addr:$src)))]>,
7440 X86VectorVTInfo _DstRC, SDNode OpNode,
7447 [(set _DstRC.RC:$dst, (OpNode _SrcRC.FRC:$src))]>,
7451 [(set _DstRC.RC:$dst, (OpNode (_SrcRC.ScalarLdFrag addr:$src)))]>,
7513 X86VectorVTInfo _Src, SDNode OpNode,
7518 (_.VT (OpNode (_.VT _.RC:$src1),
7524 (_.VT (OpNode (_.VT _.RC:$src1),
7569 SDNode OpNode, SDNode OpNodeRnd,
7573 defm Z : avx512_cvt_fp_scalar<opc, OpcodeStr, _dst, _src, OpNode, sched>,
7580 SDNode OpNode, SDNode OpNodeSAE,
7584 defm Z : avx512_cvt_fp_scalar<opc, OpcodeStr, _dst, _src, OpNode, sched>,
7627 X86VectorVTInfo _Src, SDNode OpNode, SDNode MaskOpNode,
7632 dag LdDAG = (_.VT (OpNode (_Src.VT (_Src.LdFrag addr:$src)))),
7640 (_.VT (OpNode (_Src.VT _Src.RC:$src))),
7665 (_.VT (OpNode (_Src.VT
7709 X86VectorVTInfo _Src, SDNode OpNode,
7715 : avx512_vcvt_fp<opc, OpcodeStr, _, _Src, OpNode, MaskOpNode, sched, Broadcast,
7834 multiclass avx512_cvtdq2pd<bits<8> opc, string OpcodeStr, SDNode OpNode,
7840 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f64_info, v8i32x_info, OpNode,
7854 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f64x_info, v4i32x_info, OpNode,
7860 multiclass avx512_cvtdq2ps<bits<8> opc, string OpcodeStr, SDNode OpNode,
7864 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16f32_info, v16i32_info, OpNode,
7870 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4i32x_info, OpNode,
7872 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8i32x_info, OpNode,
7878 multiclass avx512_cvttps2dq<bits<8> opc, string OpcodeStr, SDNode OpNode,
7882 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16i32_info, v16f32_info, OpNode,
7888 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f32x_info, OpNode,
7890 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f32x_info, OpNode,
7896 multiclass avx512_cvtps2dq<bits<8> opc, string OpcodeStr, SDNode OpNode,
7900 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16i32_info, v16f32_info, OpNode,
7906 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f32x_info, OpNode,
7908 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f32x_info, OpNode,
7914 multiclass avx512_cvttpd2dq<bits<8> opc, string OpcodeStr, SDNode OpNode,
7918 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f64_info, OpNode,
7931 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f64x_info, OpNode,
7979 multiclass avx512_cvtpd2dq<bits<8> opc, string OpcodeStr, SDNode OpNode,
7983 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f64_info, OpNode,
7996 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f64x_info, OpNode,
8042 multiclass avx512_cvtpd2qq<bits<8> opc, string OpcodeStr, SDNode OpNode,
8046 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f64_info, OpNode,
8052 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v2f64x_info, OpNode,
8054 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f64x_info, OpNode,
8060 multiclass avx512_cvttpd2qq<bits<8> opc, string OpcodeStr, SDNode OpNode,
8064 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f64_info, OpNode,
8070 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v2f64x_info, OpNode,
8072 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f64x_info, OpNode,
8078 multiclass avx512_cvtqq2pd<bits<8> opc, string OpcodeStr, SDNode OpNode,
8082 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f64_info, v8i64_info, OpNode,
8088 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2f64x_info, v2i64x_info, OpNode,
8090 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f64x_info, v4i64x_info, OpNode,
8096 multiclass avx512_cvtps2qq<bits<8> opc, string OpcodeStr, SDNode OpNode,
8100 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f32x_info, OpNode,
8108 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v4f32x_info, OpNode,
8110 (v2i64 (OpNode (bc_v4f32
8117 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f32x_info, OpNode,
8123 multiclass avx512_cvttps2qq<bits<8> opc, string OpcodeStr, SDNode OpNode,
8127 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f32x_info, OpNode,
8135 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v4f32x_info, OpNode,
8137 (v2i64 (OpNode (bc_v4f32
8144 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f32x_info, OpNode,
8150 multiclass avx512_cvtqq2ps<bits<8> opc, string OpcodeStr, SDNode OpNode,
8154 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8i64_info, OpNode,
8167 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4i64x_info, OpNode,
8747 multiclass avx512_fp14_s<bits<8> opc, string OpcodeStr, SDNode OpNode,
8753 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2))>,
8758 (OpNode (_.VT _.RC:$src1),
8778 multiclass avx512_fp14_p<bits<8> opc, string OpcodeStr, SDNode OpNode,
8783 (_.VT (OpNode _.RC:$src))>, EVEX, T8PD,
8787 (OpNode (_.VT
8793 (OpNode (_.VT
8800 multiclass avx512_fp14_p_vl_all<bits<8> opc, string OpcodeStr, SDNode OpNode,
8802 defm PSZ : avx512_fp14_p<opc, !strconcat(OpcodeStr, "ps"), OpNode, sched.ZMM,
8804 defm PDZ : avx512_fp14_p<opc, !strconcat(OpcodeStr, "pd"), OpNode, sched.ZMM,
8810 OpNode, sched.XMM, v4f32x_info>,
8813 OpNode, sched.YMM, v8f32x_info>,
8816 OpNode, sched.XMM, v2f64x_info>,
8819 OpNode, sched.YMM, v4f64x_info>,
8829 SDNode OpNode, SDNode OpNodeSAE,
8835 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2))>,
8847 (OpNode (_.VT _.RC:$src1), (_.ScalarIntMemFrags addr:$src2))>,
8852 multiclass avx512_eri_s<bits<8> opc, string OpcodeStr, SDNode OpNode,
8854 defm SSZ : avx512_fp28_s<opc, OpcodeStr#"ss", f32x_info, OpNode, OpNodeSAE,
8856 defm SDZ : avx512_fp28_s<opc, OpcodeStr#"sd", f64x_info, OpNode, OpNodeSAE,
8872 SDNode OpNode, X86FoldableSchedWrite sched> {
8876 (OpNode (_.VT _.RC:$src))>,
8881 (OpNode (_.VT
8888 (OpNode (_.VT
8894 SDNode OpNode, X86FoldableSchedWrite sched> {
8899 (OpNode (_.VT _.RC:$src))>,
8903 multiclass avx512_eri<bits<8> opc, string OpcodeStr, SDNode OpNode,
8905 defm PSZ : avx512_fp28_p<opc, OpcodeStr#"ps", v16f32_info, OpNode, sched.ZMM>,
8908 defm PDZ : avx512_fp28_p<opc, OpcodeStr#"pd", v8f64_info, OpNode, sched.ZMM>,
8914 SDNode OpNode, X86SchedWriteWidths sched> {
8917 defm PSZ128 : avx512_fp28_p<opc, OpcodeStr#"ps", v4f32x_info, OpNode,
8920 defm PSZ256 : avx512_fp28_p<opc, OpcodeStr#"ps", v8f32x_info, OpNode,
8923 defm PDZ128 : avx512_fp28_p<opc, OpcodeStr#"pd", v2f64x_info, OpNode,
8926 defm PDZ256 : avx512_fp28_p<opc, OpcodeStr#"pd", v4f64x_info, OpNode,
9139 multiclass avx512_masked_scalar<SDNode OpNode, string OpcPrefix, SDNode Move,
9144 (OpNode (extractelt _.VT:$src2, (iPTR 0))),
9150 (OpNode (extractelt _.VT:$src2, (iPTR 0))),
9182 multiclass avx512_trunc_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
9191 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src))))]>,
9270 multiclass avx512_trunc_qb<bits<8> opc, string OpcodeStr, SDNode OpNode,
9282 multiclass avx512_trunc_qw<bits<8> opc, string OpcodeStr, SDNode OpNode,
9287 defm NAME: avx512_trunc<opc, OpcodeStr, InVecNode, InVecNode, OpNode,
9294 multiclass avx512_trunc_qd<bits<8> opc, string OpcodeStr, SDNode OpNode,
9299 defm NAME: avx512_trunc<opc, OpcodeStr, InVecNode, OpNode, OpNode,
9306 multiclass avx512_trunc_db<bits<8> opc, string OpcodeStr, SDNode OpNode,
9311 defm NAME: avx512_trunc<opc, OpcodeStr, InVecNode, InVecNode, OpNode,
9318 multiclass avx512_trunc_dw<bits<8> opc, string OpcodeStr, SDNode OpNode,
9323 defm NAME: avx512_trunc<opc, OpcodeStr, InVecNode, OpNode, OpNode,
9330 multiclass avx512_trunc_wb<bits<8> opc, string OpcodeStr, SDNode OpNode,
9335 defm NAME: avx512_trunc<opc, OpcodeStr, InVecNode, OpNode, OpNode,
9433 multiclass mtrunc_lowering<string InstrName, SDNode OpNode,
9436 def : Pat<(DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src),
9443 def : Pat<(DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src),
9472 X86MemOperand x86memop, PatFrag LdFrag, SDNode OpNode>{
9476 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src)))>,
9487 SDNode OpNode, SDNode InVecNode, string ExtTy,
9495 v16i8x_info, i128mem, LdFrag, OpNode>,
9500 v32i8x_info, i256mem, LdFrag, OpNode>,
9506 SDNode OpNode, SDNode InVecNode, string ExtTy,
9519 v16i8x_info, i128mem, LdFrag, OpNode>,
9525 SDNode OpNode, SDNode InVecNode, string ExtTy,
9544 SDNode OpNode, SDNode InVecNode, string ExtTy,
9552 v8i16x_info, i128mem, LdFrag, OpNode>,
9557 v16i16x_info, i256mem, LdFrag, OpNode>,
9563 SDNode OpNode, SDNode InVecNode, string ExtTy,
9576 v8i16x_info, i128mem, LdFrag, OpNode>,
9582 SDNode OpNode, SDNode InVecNode, string ExtTy,
9591 v4i32x_info, i128mem, LdFrag, OpNode>,
9596 v8i32x_info, i256mem, LdFrag, OpNode>,
10126 SDNode OpNode, SDNode MaskOpNode,
10133 (OpNode (_.VT _.RC:$src1), (i32 timm:$src2)),
10139 (OpNode (_.VT (bitconvert (_.LdFrag addr:$src1))),
10148 (OpNode (_.VT (_.BroadcastLdFrag addr:$src1)),
10158 SDNode OpNode, X86FoldableSchedWrite sched,
10165 (OpNode (_.VT _.RC:$src1),
10171 AVX512VLVectorVTInfo _, bits<8> opc, SDNode OpNode,
10175 defm Z : avx512_unary_fp_packed_imm<opc, OpcodeStr, OpNode, MaskOpNode,
10181 defm Z128 : avx512_unary_fp_packed_imm<opc, OpcodeStr, OpNode, MaskOpNode,
10183 defm Z256 : avx512_unary_fp_packed_imm<opc, OpcodeStr, OpNode, MaskOpNode,
10192 multiclass avx512_fp_packed_imm<bits<8> opc, string OpcodeStr, SDNode OpNode,
10198 (OpNode (_.VT _.RC:$src1),
10205 (OpNode (_.VT _.RC:$src1),
10213 (OpNode (_.VT _.RC:$src1),
10222 multiclass avx512_3Op_rm_imm8<bits<8> opc, string OpcodeStr, SDNode OpNode,
10229 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1),
10236 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1),
10247 multiclass avx512_3Op_imm8<bits<8> opc, string OpcodeStr, SDNode OpNode,
10249 avx512_3Op_rm_imm8<opc, OpcodeStr, OpNode, sched, _, _>{
10256 (OpNode (_.VT _.RC:$src1),
10264 multiclass avx512_fp_scalar_imm<bits<8> opc, string OpcodeStr, SDNode OpNode,
10270 (OpNode (_.VT _.RC:$src1),
10277 (OpNode (_.VT _.RC:$src1),
10286 SDNode OpNode, X86FoldableSchedWrite sched,
10293 (OpNode (_.VT _.RC:$src1),
10300 multiclass avx512_fp_sae_scalar_imm<bits<8> opc, string OpcodeStr, SDNode OpNode,
10307 (OpNode (_.VT _.RC:$src1),
10314 AVX512VLVectorVTInfo _, bits<8> opc, SDNode OpNode,
10317 defm Z : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, sched.ZMM, _.info512>,
10323 defm Z128 : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, sched.XMM, _.info128>,
10325 defm Z256 : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, sched.YMM, _.info256>,
10330 multiclass avx512_common_3Op_rm_imm8<bits<8> opc, SDNode OpNode, string OpStr,
10334 defm Z : avx512_3Op_rm_imm8<opc, OpStr, OpNode, sched.ZMM, DestInfo.info512,
10338 defm Z128 : avx512_3Op_rm_imm8<opc, OpStr, OpNode, sched.XMM, DestInfo.info128,
10340 defm Z256 : avx512_3Op_rm_imm8<opc, OpStr, OpNode, sched.YMM, DestInfo.info256,
10346 bits<8> opc, SDNode OpNode, X86SchedWriteWidths sched,
10349 defm Z : avx512_3Op_imm8<opc, OpcodeStr, OpNode, sched.ZMM, _.info512>,
10353 defm Z128 : avx512_3Op_imm8<opc, OpcodeStr, OpNode, sched.XMM, _.info128>,
10355 defm Z256 : avx512_3Op_imm8<opc, OpcodeStr, OpNode, sched.YMM, _.info256>,
10361 X86VectorVTInfo _, bits<8> opc, SDNode OpNode,
10364 defm Z : avx512_fp_scalar_imm<opc, OpcodeStr, OpNode, sched.XMM, _>,
10370 bits<8> opcPs, bits<8> opcPd, SDNode OpNode,
10374 opcPs, OpNode, MaskOpNode, OpNodeSAE, sched, prd>,
10377 opcPd, OpNode, MaskOpNode, OpNodeSAE, sched, prd>,
10583 multiclass avx512_vpalign_mask_lowering<string OpcodeStr, SDNode OpNode,
10588 (From.VT (OpNode From.RC:$src1, From.RC:$src2,
10597 (From.VT (OpNode From.RC:$src1, From.RC:$src2,
10606 (From.VT (OpNode From.RC:$src1,
10616 (From.VT (OpNode From.RC:$src1,
10625 multiclass avx512_vpalign_mask_lowering_mb<string OpcodeStr, SDNode OpNode,
10629 avx512_vpalign_mask_lowering<OpcodeStr, OpNode, From, To, ImmXForm> {
10630 def : Pat<(From.VT (OpNode From.RC:$src1,
10638 (From.VT (OpNode From.RC:$src1,
10649 (From.VT (OpNode From.RC:$src1,
10689 multiclass avx512_unary_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
10695 (_.VT (OpNode (_.VT _.RC:$src1)))>, EVEX, AVX5128IBase,
10701 (_.VT (OpNode (_.VT (bitconvert (_.LdFrag addr:$src1)))))>,
10707 multiclass avx512_unary_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
10709 avx512_unary_rm<opc, OpcodeStr, OpNode, sched, _> {
10714 (_.VT (OpNode (_.VT (_.BroadcastLdFrag addr:$src1))))>,
10719 multiclass avx512_unary_rm_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
10723 defm Z : avx512_unary_rm<opc, OpcodeStr, OpNode, sched.ZMM, VTInfo.info512>,
10727 defm Z256 : avx512_unary_rm<opc, OpcodeStr, OpNode, sched.YMM, VTInfo.info256>,
10729 defm Z128 : avx512_unary_rm<opc, OpcodeStr, OpNode, sched.XMM, VTInfo.info128>,
10734 multiclass avx512_unary_rmb_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
10738 defm Z : avx512_unary_rmb<opc, OpcodeStr, OpNode, sched.ZMM, VTInfo.info512>,
10742 defm Z256 : avx512_unary_rmb<opc, OpcodeStr, OpNode, sched.YMM, VTInfo.info256>,
10744 defm Z128 : avx512_unary_rmb<opc, OpcodeStr, OpNode, sched.XMM, VTInfo.info128>,
10750 SDNode OpNode, X86SchedWriteWidths sched,
10752 defm Q : avx512_unary_rmb_vl<opc_q, OpcodeStr#"q", OpNode, sched,
10754 defm D : avx512_unary_rmb_vl<opc_d, OpcodeStr#"d", OpNode, sched,
10759 SDNode OpNode, X86SchedWriteWidths sched,
10761 defm W : avx512_unary_rm_vl<opc_w, OpcodeStr#"w", OpNode, sched,
10763 defm B : avx512_unary_rm_vl<opc_b, OpcodeStr#"b", OpNode, sched,
10769 string OpcodeStr, SDNode OpNode,
10771 defm NAME : avx512_unary_rm_vl_dq<opc_d, opc_q, OpcodeStr, OpNode, sched,
10773 avx512_unary_rm_vl_bw<opc_b, opc_w, OpcodeStr, OpNode, sched,
10795 multiclass avx512_unary_lowering<string InstrStr, SDNode OpNode,
10798 def : Pat<(_.info256.VT (OpNode (_.info256.VT _.info256.RC:$src1))),
10806 def : Pat<(_.info128.VT (OpNode (_.info128.VT _.info128.RC:$src1))),
10842 multiclass avx512_replicate<bits<8> opc, string OpcodeStr, SDNode OpNode,
10844 defm NAME: avx512_unary_rm_vl<opc, OpcodeStr, OpNode, sched,
10872 multiclass avx512_movddup_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
10885 multiclass avx512_movddup<bits<8> opc, string OpcodeStr, SDNode OpNode,
10887 defm NAME: avx512_movddup_common<opc, OpcodeStr, OpNode, sched,
10939 multiclass avx512_extract_elt_bw_m<bits<8> opc, string OpcodeStr, SDNode OpNode,
10944 [(store (_.EltVT (trunc (OpNode (_.VT _.RC:$src1), timm:$src2))),
11007 multiclass avx512_insert_elt_m<bits<8> opc, string OpcodeStr, SDNode OpNode,
11014 (_.VT (OpNode _.RC:$src1, (LdFrag addr:$src2), immoperator:$src3)))]>,
11018 multiclass avx512_insert_elt_bw<bits<8> opc, string OpcodeStr, SDNode OpNode,
11025 (OpNode _.RC:$src1, GR32orGR64:$src2, timm:$src3))]>, EVEX_4V,
11028 defm NAME : avx512_insert_elt_m<opc, OpcodeStr, OpNode, _, LdFrag, timm>;
11073 multiclass avx512_shift_packed<bits<8> opc, SDNode OpNode, Format MRMr,
11079 [(set _.RC:$dst,(_.VT (OpNode _.RC:$src1, (i8 timm:$src2))))]>,
11084 [(set _.RC:$dst,(_.VT (OpNode
11090 multiclass avx512_shift_packed_all<bits<8> opc, SDNode OpNode, Format MRMr,
11094 defm Z : avx512_shift_packed<opc, OpNode, MRMr, MRMm, OpcodeStr,
11097 defm Z256 : avx512_shift_packed<opc, OpNode, MRMr, MRMm, OpcodeStr,
11099 defm Z128 : avx512_shift_packed<opc, OpNode, MRMr, MRMm, OpcodeStr,
11110 multiclass avx512_psadbw_packed<bits<8> opc, SDNode OpNode,
11118 (OpNode (_src.VT _src.RC:$src1),
11125 (OpNode (_src.VT _src.RC:$src1),
11131 multiclass avx512_psadbw_packed_all<bits<8> opc, SDNode OpNode,
11135 defm Z : avx512_psadbw_packed<opc, OpNode, OpcodeStr, sched.ZMM,
11138 defm Z256 : avx512_psadbw_packed<opc, OpNode, OpcodeStr, sched.YMM,
11140 defm Z128 : avx512_psadbw_packed<opc, OpNode, OpcodeStr, sched.XMM,
11210 multiclass avx512_ternlog<bits<8> opc, string OpcodeStr, SDNode OpNode,
11217 (OpNode (_.VT _.RC:$src1),
11225 (OpNode (_.VT _.RC:$src1),
11235 (OpNode (_.VT _.RC:$src1),
11245 (OpNode _.RC:$src3, _.RC:$src2, _.RC:$src1, (i8 timm:$src4)),
11250 (OpNode _.RC:$src2, _.RC:$src1, _.RC:$src3, (i8 timm:$src4)),
11258 (OpNode (bitconvert (_.LdFrag addr:$src3)),
11264 (OpNode _.RC:$src1, (bitconvert (_.LdFrag addr:$src3)),
11273 (OpNode _.RC:$src1, (bitconvert (_.LdFrag addr:$src3)),
11279 (OpNode (bitconvert (_.LdFrag addr:$src3)),
11285 (OpNode _.RC:$src2, _.RC:$src1,
11291 (OpNode _.RC:$src2, (bitconvert (_.LdFrag addr:$src3)),
11297 (OpNode (bitconvert (_.LdFrag addr:$src3)),
11306 (OpNode (_.BroadcastLdFrag addr:$src3),
11313 (OpNode _.RC:$src1,
11324 (OpNode _.RC:$src1, (_.BroadcastLdFrag addr:$src3),
11330 (OpNode (_.BroadcastLdFrag addr:$src3),
11336 (OpNode _.RC:$src2, _.RC:$src1,
11342 (OpNode _.RC:$src2,
11349 (OpNode (_.BroadcastLdFrag addr:$src3),
11700 multiclass AVX512_scalar_unary_math_patterns<SDNode OpNode, string OpcPrefix,
11704 (scalar_to_vector (OpNode (extractelt _.VT:$src, 0))))),
11765 multiclass VBMI2_shift_var_rm<bits<8> Op, string OpStr, SDNode OpNode,
11772 (VTI.VT (OpNode VTI.RC:$src1, VTI.RC:$src2, VTI.RC:$src3))>,
11777 (VTI.VT (OpNode VTI.RC:$src1, VTI.RC:$src2,
11784 multiclass VBMI2_shift_var_rmb<bits<8> Op, string OpStr, SDNode OpNode,
11786 : VBMI2_shift_var_rm<Op, OpStr, OpNode, sched, VTI> {
11793 (OpNode VTI.RC:$src1, VTI.RC:$src2,
11799 multiclass VBMI2_shift_var_rm_common<bits<8> Op, string OpStr, SDNode OpNode,
11802 defm Z : VBMI2_shift_var_rm<Op, OpStr, OpNode, sched.ZMM, VTI.info512>,
11805 defm Z256 : VBMI2_shift_var_rm<Op, OpStr, OpNode, sched.YMM, VTI.info256>,
11807 defm Z128 : VBMI2_shift_var_rm<Op, OpStr, OpNode, sched.XMM, VTI.info128>,
11812 multiclass VBMI2_shift_var_rmb_common<bits<8> Op, string OpStr, SDNode OpNode,
11815 defm Z : VBMI2_shift_var_rmb<Op, OpStr, OpNode, sched.ZMM, VTI.info512>,
11818 defm Z256 : VBMI2_shift_var_rmb<Op, OpStr, OpNode, sched.YMM, VTI.info256>,
11820 defm Z128 : VBMI2_shift_var_rmb<Op, OpStr, OpNode, sched.XMM, VTI.info128>,
11825 SDNode OpNode, X86SchedWriteWidths sched> {
11826 defm W : VBMI2_shift_var_rm_common<wOp, Prefix#"w", OpNode, sched,
11828 defm D : VBMI2_shift_var_rmb_common<dqOp, Prefix#"d", OpNode, sched,
11830 defm Q : VBMI2_shift_var_rmb_common<dqOp, Prefix#"q", OpNode, sched,
11835 SDNode OpNode, X86SchedWriteWidths sched> {
11836 defm W : avx512_common_3Op_rm_imm8<wOp, OpNode, Prefix#"w", sched,
11840 OpNode, sched, HasVBMI2>, AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>;
11841 defm Q : avx512_common_3Op_imm8<Prefix#"q", avx512vl_i64_info, dqOp, OpNode,
11869 multiclass VNNI_rmb<bits<8> Op, string OpStr, SDNode OpNode,
11876 (VTI.VT (OpNode VTI.RC:$src1,
11883 (VTI.VT (OpNode VTI.RC:$src1, VTI.RC:$src2,
11891 (OpNode VTI.RC:$src1, VTI.RC:$src2,
11898 multiclass VNNI_common<bits<8> Op, string OpStr, SDNode OpNode,
11901 defm Z : VNNI_rmb<Op, OpStr, OpNode, sched.ZMM, v16i32_info,
11904 defm Z256 : VNNI_rmb<Op, OpStr, OpNode, sched.YMM, v8i32x_info,
11906 defm Z128 : VNNI_rmb<Op, OpStr, OpNode, sched.XMM, v4i32x_info,
12002 multiclass GF2P8MULB_avx512_common<bits<8> Op, string OpStr, SDNode OpNode,
12005 defm Z : avx512_binop_rm<Op, OpStr, OpNode, v64i8_info, sched.ZMM, 1>,
12008 defm Z256 : avx512_binop_rm<Op, OpStr, OpNode, v32i8x_info, sched.YMM, 1>,
12010 defm Z128 : avx512_binop_rm<Op, OpStr, OpNode, v16i8x_info, sched.XMM, 1>,
12019 multiclass GF2P8AFFINE_avx512_rmb_imm<bits<8> Op, string OpStr, SDNode OpNode,
12022 : avx512_3Op_rm_imm8<Op, OpStr, OpNode, sched, VTI, VTI> {
12028 (OpNode (VTI.VT VTI.RC:$src1),
12034 multiclass GF2P8AFFINE_avx512_common<bits<8> Op, string OpStr, SDNode OpNode,
12037 defm Z : GF2P8AFFINE_avx512_rmb_imm<Op, OpStr, OpNode, sched.ZMM,
12040 defm Z256 : GF2P8AFFINE_avx512_rmb_imm<Op, OpStr, OpNode, sched.YMM,
12042 defm Z128 : GF2P8AFFINE_avx512_rmb_imm<Op, OpStr, OpNode, sched.XMM,
12164 SDNode OpNode, Predicate prd,
12167 defm NAME#Z : avx512_binop_rm2<opc, OpcodeStr, sched.ZMM, OpNode,
12172 defm NAME#Z256 : avx512_binop_rm2<opc, OpcodeStr, sched.YMM, OpNode,
12176 defm NAME#Z128 : avx512_binop_rm2<opc, OpcodeStr, sched.XMM, OpNode,
12260 multiclass avx512_dpbf16ps_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
12266 (_.VT (OpNode _.RC:$src1, src_v.RC:$src2, src_v.RC:$src3))>,
12272 (_.VT (OpNode _.RC:$src1, src_v.RC:$src2,
12281 (_.VT (OpNode _.RC:$src1, src_v.RC:$src2,
12288 multiclass avx512_dpbf16ps_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,
12292 defm Z : avx512_dpbf16ps_rm<opc, OpcodeStr, OpNode, sched.ZMM, _.info512,
12296 defm Z256 : avx512_dpbf16ps_rm<opc, OpcodeStr, OpNode, sched.YMM, _.info256,
12298 defm Z128 : avx512_dpbf16ps_rm<opc, OpcodeStr, OpNode, sched.XMM, _.info128,