Lines Matching refs:Pat
194 // Prefer over VMOV*rrk Pat<>
204 let AddedComplexity = 30 in // Prefer over VMOV*rrkz Pat<>
376 def : Pat<(v8f64 (bitconvert (v8i64 VR512:$src))), (v8f64 VR512:$src)>;
377 def : Pat<(v8f64 (bitconvert (v16i32 VR512:$src))), (v8f64 VR512:$src)>;
378 def : Pat<(v8f64 (bitconvert (v32i16 VR512:$src))), (v8f64 VR512:$src)>;
379 def : Pat<(v8f64 (bitconvert (v64i8 VR512:$src))), (v8f64 VR512:$src)>;
380 def : Pat<(v8f64 (bitconvert (v16f32 VR512:$src))), (v8f64 VR512:$src)>;
381 def : Pat<(v16f32 (bitconvert (v8i64 VR512:$src))), (v16f32 VR512:$src)>;
382 def : Pat<(v16f32 (bitconvert (v16i32 VR512:$src))), (v16f32 VR512:$src)>;
383 def : Pat<(v16f32 (bitconvert (v32i16 VR512:$src))), (v16f32 VR512:$src)>;
384 def : Pat<(v16f32 (bitconvert (v64i8 VR512:$src))), (v16f32 VR512:$src)>;
385 def : Pat<(v16f32 (bitconvert (v8f64 VR512:$src))), (v16f32 VR512:$src)>;
386 def : Pat<(v8i64 (bitconvert (v16i32 VR512:$src))), (v8i64 VR512:$src)>;
387 def : Pat<(v8i64 (bitconvert (v32i16 VR512:$src))), (v8i64 VR512:$src)>;
388 def : Pat<(v8i64 (bitconvert (v64i8 VR512:$src))), (v8i64 VR512:$src)>;
389 def : Pat<(v8i64 (bitconvert (v8f64 VR512:$src))), (v8i64 VR512:$src)>;
390 def : Pat<(v8i64 (bitconvert (v16f32 VR512:$src))), (v8i64 VR512:$src)>;
391 def : Pat<(v16i32 (bitconvert (v8i64 VR512:$src))), (v16i32 VR512:$src)>;
392 def : Pat<(v16i32 (bitconvert (v16f32 VR512:$src))), (v16i32 VR512:$src)>;
393 def : Pat<(v16i32 (bitconvert (v32i16 VR512:$src))), (v16i32 VR512:$src)>;
394 def : Pat<(v16i32 (bitconvert (v64i8 VR512:$src))), (v16i32 VR512:$src)>;
395 def : Pat<(v16i32 (bitconvert (v8f64 VR512:$src))), (v16i32 VR512:$src)>;
396 def : Pat<(v32i16 (bitconvert (v8i64 VR512:$src))), (v32i16 VR512:$src)>;
397 def : Pat<(v32i16 (bitconvert (v16i32 VR512:$src))), (v32i16 VR512:$src)>;
398 def : Pat<(v32i16 (bitconvert (v64i8 VR512:$src))), (v32i16 VR512:$src)>;
399 def : Pat<(v32i16 (bitconvert (v8f64 VR512:$src))), (v32i16 VR512:$src)>;
400 def : Pat<(v32i16 (bitconvert (v16f32 VR512:$src))), (v32i16 VR512:$src)>;
401 def : Pat<(v32i16 (bitconvert (v16f32 VR512:$src))), (v32i16 VR512:$src)>;
402 def : Pat<(v64i8 (bitconvert (v8i64 VR512:$src))), (v64i8 VR512:$src)>;
403 def : Pat<(v64i8 (bitconvert (v16i32 VR512:$src))), (v64i8 VR512:$src)>;
404 def : Pat<(v64i8 (bitconvert (v32i16 VR512:$src))), (v64i8 VR512:$src)>;
405 def : Pat<(v64i8 (bitconvert (v8f64 VR512:$src))), (v64i8 VR512:$src)>;
406 def : Pat<(v64i8 (bitconvert (v16f32 VR512:$src))), (v64i8 VR512:$src)>;
408 def : Pat<(v2i64 (bitconvert (v4i32 VR128X:$src))), (v2i64 VR128X:$src)>;
409 def : Pat<(v2i64 (bitconvert (v8i16 VR128X:$src))), (v2i64 VR128X:$src)>;
410 def : Pat<(v2i64 (bitconvert (v16i8 VR128X:$src))), (v2i64 VR128X:$src)>;
411 def : Pat<(v2i64 (bitconvert (v2f64 VR128X:$src))), (v2i64 VR128X:$src)>;
412 def : Pat<(v2i64 (bitconvert (v4f32 VR128X:$src))), (v2i64 VR128X:$src)>;
413 def : Pat<(v4i32 (bitconvert (v2i64 VR128X:$src))), (v4i32 VR128X:$src)>;
414 def : Pat<(v4i32 (bitconvert (v8i16 VR128X:$src))), (v4i32 VR128X:$src)>;
415 def : Pat<(v4i32 (bitconvert (v16i8 VR128X:$src))), (v4i32 VR128X:$src)>;
416 def : Pat<(v4i32 (bitconvert (v2f64 VR128X:$src))), (v4i32 VR128X:$src)>;
417 def : Pat<(v4i32 (bitconvert (v4f32 VR128X:$src))), (v4i32 VR128X:$src)>;
418 def : Pat<(v8i16 (bitconvert (v2i64 VR128X:$src))), (v8i16 VR128X:$src)>;
419 def : Pat<(v8i16 (bitconvert (v4i32 VR128X:$src))), (v8i16 VR128X:$src)>;
420 def : Pat<(v8i16 (bitconvert (v16i8 VR128X:$src))), (v8i16 VR128X:$src)>;
421 def : Pat<(v8i16 (bitconvert (v2f64 VR128X:$src))), (v8i16 VR128X:$src)>;
422 def : Pat<(v8i16 (bitconvert (v4f32 VR128X:$src))), (v8i16 VR128X:$src)>;
423 def : Pat<(v16i8 (bitconvert (v2i64 VR128X:$src))), (v16i8 VR128X:$src)>;
424 def : Pat<(v16i8 (bitconvert (v4i32 VR128X:$src))), (v16i8 VR128X:$src)>;
425 def : Pat<(v16i8 (bitconvert (v8i16 VR128X:$src))), (v16i8 VR128X:$src)>;
426 def : Pat<(v16i8 (bitconvert (v2f64 VR128X:$src))), (v16i8 VR128X:$src)>;
427 def : Pat<(v16i8 (bitconvert (v4f32 VR128X:$src))), (v16i8 VR128X:$src)>;
428 def : Pat<(v4f32 (bitconvert (v2i64 VR128X:$src))), (v4f32 VR128X:$src)>;
429 def : Pat<(v4f32 (bitconvert (v4i32 VR128X:$src))), (v4f32 VR128X:$src)>;
430 def : Pat<(v4f32 (bitconvert (v8i16 VR128X:$src))), (v4f32 VR128X:$src)>;
431 def : Pat<(v4f32 (bitconvert (v16i8 VR128X:$src))), (v4f32 VR128X:$src)>;
432 def : Pat<(v4f32 (bitconvert (v2f64 VR128X:$src))), (v4f32 VR128X:$src)>;
433 def : Pat<(v2f64 (bitconvert (v2i64 VR128X:$src))), (v2f64 VR128X:$src)>;
434 def : Pat<(v2f64 (bitconvert (v4i32 VR128X:$src))), (v2f64 VR128X:$src)>;
435 def : Pat<(v2f64 (bitconvert (v8i16 VR128X:$src))), (v2f64 VR128X:$src)>;
436 def : Pat<(v2f64 (bitconvert (v16i8 VR128X:$src))), (v2f64 VR128X:$src)>;
437 def : Pat<(v2f64 (bitconvert (v4f32 VR128X:$src))), (v2f64 VR128X:$src)>;
441 def : Pat<(v4f64 (bitconvert (v8f32 VR256X:$src))), (v4f64 VR256X:$src)>;
442 def : Pat<(v4f64 (bitconvert (v8i32 VR256X:$src))), (v4f64 VR256X:$src)>;
443 def : Pat<(v4f64 (bitconvert (v4i64 VR256X:$src))), (v4f64 VR256X:$src)>;
444 def : Pat<(v4f64 (bitconvert (v16i16 VR256X:$src))), (v4f64 VR256X:$src)>;
445 def : Pat<(v4f64 (bitconvert (v32i8 VR256X:$src))), (v4f64 VR256X:$src)>;
446 def : Pat<(v8f32 (bitconvert (v8i32 VR256X:$src))), (v8f32 VR256X:$src)>;
447 def : Pat<(v8f32 (bitconvert (v4i64 VR256X:$src))), (v8f32 VR256X:$src)>;
448 def : Pat<(v8f32 (bitconvert (v4f64 VR256X:$src))), (v8f32 VR256X:$src)>;
449 def : Pat<(v8f32 (bitconvert (v32i8 VR256X:$src))), (v8f32 VR256X:$src)>;
450 def : Pat<(v8f32 (bitconvert (v16i16 VR256X:$src))), (v8f32 VR256X:$src)>;
451 def : Pat<(v4i64 (bitconvert (v8f32 VR256X:$src))), (v4i64 VR256X:$src)>;
452 def : Pat<(v4i64 (bitconvert (v8i32 VR256X:$src))), (v4i64 VR256X:$src)>;
453 def : Pat<(v4i64 (bitconvert (v4f64 VR256X:$src))), (v4i64 VR256X:$src)>;
454 def : Pat<(v4i64 (bitconvert (v32i8 VR256X:$src))), (v4i64 VR256X:$src)>;
455 def : Pat<(v4i64 (bitconvert (v16i16 VR256X:$src))), (v4i64 VR256X:$src)>;
456 def : Pat<(v32i8 (bitconvert (v4f64 VR256X:$src))), (v32i8 VR256X:$src)>;
457 def : Pat<(v32i8 (bitconvert (v4i64 VR256X:$src))), (v32i8 VR256X:$src)>;
458 def : Pat<(v32i8 (bitconvert (v8f32 VR256X:$src))), (v32i8 VR256X:$src)>;
459 def : Pat<(v32i8 (bitconvert (v8i32 VR256X:$src))), (v32i8 VR256X:$src)>;
460 def : Pat<(v32i8 (bitconvert (v16i16 VR256X:$src))), (v32i8 VR256X:$src)>;
461 def : Pat<(v8i32 (bitconvert (v32i8 VR256X:$src))), (v8i32 VR256X:$src)>;
462 def : Pat<(v8i32 (bitconvert (v16i16 VR256X:$src))), (v8i32 VR256X:$src)>;
463 def : Pat<(v8i32 (bitconvert (v8f32 VR256X:$src))), (v8i32 VR256X:$src)>;
464 def : Pat<(v8i32 (bitconvert (v4i64 VR256X:$src))), (v8i32 VR256X:$src)>;
465 def : Pat<(v8i32 (bitconvert (v4f64 VR256X:$src))), (v8i32 VR256X:$src)>;
466 def : Pat<(v16i16 (bitconvert (v8f32 VR256X:$src))), (v16i16 VR256X:$src)>;
467 def : Pat<(v16i16 (bitconvert (v8i32 VR256X:$src))), (v16i16 VR256X:$src)>;
468 def : Pat<(v16i16 (bitconvert (v4i64 VR256X:$src))), (v16i16 VR256X:$src)>;
469 def : Pat<(v16i16 (bitconvert (v4f64 VR256X:$src))), (v16i16 VR256X:$src)>;
470 def : Pat<(v16i16 (bitconvert (v32i8 VR256X:$src))), (v16i16 VR256X:$src)>;
484 def : Pat<(v8i64 immAllZerosV), (AVX512_512_SET0)>;
485 def : Pat<(v16i32 immAllZerosV), (AVX512_512_SET0)>;
486 def : Pat<(v8f64 immAllZerosV), (AVX512_512_SET0)>;
519 def : Pat<(vinsert_insert:$ins
525 def : Pat<(vinsert_insert:$ins
631 Pat<(To.VT (extract_subvector (From.VT From.RC:$src),(iPTR 0))),
669 def : Pat<(!cast<Intrinsic>("int_x86_avx512_mask_vextract" # To.EltTypeName #
679 def : Pat<(!cast<Intrinsic>("int_x86_avx512_mask_vextract" # To.EltTypeName #
688 def : Pat<(!cast<Intrinsic>("int_x86_avx512_mask_vextract" # To.EltTypeName #
703 def : Pat<(vextract_extract:$ext (From.VT From.RC:$src1), (iPTR imm)),
780 def : Pat<(insert_subvector undef, (v2i64 VR128X:$src), (iPTR 0)),
784 def : Pat<(insert_subvector undef, (v2f64 VR128X:$src), (iPTR 0)),
788 def : Pat<(insert_subvector undef, (v4i32 VR128X:$src), (iPTR 0)),
792 def : Pat<(insert_subvector undef, (v4f32 VR128X:$src), (iPTR 0)),
797 def : Pat<(insert_subvector undef, (v4i64 VR256X:$src), (iPTR 0)),
799 def : Pat<(insert_subvector undef, (v4f64 VR256X:$src), (iPTR 0)),
801 def : Pat<(insert_subvector undef, (v8i32 VR256X:$src), (iPTR 0)),
803 def : Pat<(insert_subvector undef, (v8f32 VR256X:$src), (iPTR 0)),
805 def : Pat<(insert_subvector undef, (v16i16 VR256X:$src), (iPTR 0)),
807 def : Pat<(insert_subvector undef, (v32i8 VR256X:$src), (iPTR 0)),
875 def : Pat<(_.VT (OpNode (_.EltVT SrcRC_s:$src))),
880 def : Pat<(_.VT (vselect _.KRCWM:$mask,
885 def : Pat<(_.VT(vselect _.KRCWM:$mask,
906 def : Pat<(v16f32 (X86VBroadcast (loadf32 addr:$src))),
908 def : Pat<(v8f64 (X86VBroadcast (loadf64 addr:$src))),
911 def : Pat<(int_x86_avx512_vbroadcast_ss_512 addr:$src),
913 def : Pat<(int_x86_avx512_vbroadcast_sd_512 addr:$src),
942 def : Pat <(v16i32 (X86vzext VK16WM:$mask)),
945 def : Pat <(v8i64 (X86vzext VK8WM:$mask)),
948 def : Pat<(v16i32 (X86VBroadcast (i32 GR32:$src))),
950 def : Pat<(v8i64 (X86VBroadcast (i64 GR64:$src))),
953 def : Pat<(v16i32 (int_x86_avx512_pbroadcastd_i32_512 (i32 GR32:$src))),
955 def : Pat<(v8i64 (int_x86_avx512_pbroadcastq_i64_512 (i64 GR64:$src))),
958 def : Pat<(v16i32 (int_x86_avx512_mask_pbroadcast_d_gpr_512 (i32 GR32:$src),
961 def : Pat<(v8i64 (int_x86_avx512_mask_pbroadcast_q_gpr_512 (i64 GR64:$src),
969 def : Pat<(DestInfo.VT (X86VBroadcast (SrcInfo.VT SrcInfo.RC:$src))),
1106 def : Pat<(v16f32 (X86VBroadcast (v16f32 VR512:$src))),
1108 def : Pat<(v16f32 (X86VBroadcast (v8f32 VR256X:$src))),
1111 def : Pat<(v8f64 (X86VBroadcast (v8f64 VR512:$src))),
1113 def : Pat<(v8f64 (X86VBroadcast (v4f64 VR256X:$src))),
1116 def : Pat<(v16f32 (int_x86_avx512_vbroadcast_ss_ps_512 (v4f32 VR128X:$src))),
1118 def : Pat<(v8f64 (int_x86_avx512_vbroadcast_sd_pd_512 (v2f64 VR128X:$src))),
1123 def : Pat<(v16f32 (X86VBroadcast FR32X:$src)),
1125 def : Pat<(v8f64 (X86VBroadcast FR64X:$src)),
1401 def : Pat<(v8f32 (vselect (v8i1 VK8WM:$mask), (v8f32 VR256X:$src1),
1408 def : Pat<(v8i32 (vselect (v8i1 VK8WM:$mask), (v8i32 VR256X:$src1),
1620 def : Pat<(v8i1 (X86pcmpgtm (v8i32 VR256X:$src1), (v8i32 VR256X:$src2))),
1625 def : Pat<(v8i1 (X86pcmpeqm (v8i32 VR256X:$src1), (v8i32 VR256X:$src2))),
1883 def : Pat<(v8i1 (X86cmpm (v8f32 VR256X:$src1), (v8f32 VR256X:$src2), imm:$cc)),
1888 def : Pat<(v8i1 (X86cmpm (v8i32 VR256X:$src1), (v8i32 VR256X:$src2), imm:$cc)),
1893 def : Pat<(v8i1 (X86cmpmu (v8i32 VR256X:$src1), (v8i32 VR256X:$src2), imm:$cc)),
2081 def : Pat<(v8i1 (bitconvert (i8 GR8:$src))),
2083 def : Pat<(i8 (bitconvert (v8i1 VK8:$src))),
2087 def : Pat<(v16i1 (bitconvert (i16 GR16:$src))),
2089 def : Pat<(i16 (bitconvert (v16i1 VK16:$src))),
2093 def : Pat<(v32i1 (bitconvert (i32 GR32:$src))), (KMOVDkr GR32:$src)>;
2094 def : Pat<(i32 (bitconvert (v32i1 VK32:$src))), (KMOVDrk VK32:$src)>;
2097 def : Pat<(v64i1 (bitconvert (i64 GR64:$src))), (KMOVQkr GR64:$src)>;
2098 def : Pat<(i64 (bitconvert (v64i1 VK64:$src))), (KMOVQrk VK64:$src)>;
2103 def : Pat<(store (i8 (bitconvert (v8i1 VK8:$src))), addr:$dst),
2105 def : Pat<(v8i1 (bitconvert (i8 (load addr:$src)))),
2108 def : Pat<(store VK4:$src, addr:$dst),
2110 def : Pat<(store VK2:$src, addr:$dst),
2114 def : Pat<(store (i8 (bitconvert (v8i1 VK8:$src))), addr:$dst),
2116 def : Pat<(v8i1 (bitconvert (i8 (load addr:$src)))),
2120 def : Pat<(store (i16 (bitconvert (v16i1 VK16:$src))), addr:$dst),
2122 def : Pat<(i1 (load addr:$src)),
2126 def : Pat<(v16i1 (bitconvert (i16 (load addr:$src)))),
2130 def : Pat<(store (i32 (bitconvert (v32i1 VK32:$src))), addr:$dst),
2132 def : Pat<(v32i1 (bitconvert (i32 (load addr:$src)))),
2136 def : Pat<(store (i64 (bitconvert (v64i1 VK64:$src))), addr:$dst),
2138 def : Pat<(v64i1 (bitconvert (i64 (load addr:$src)))),
2143 def : Pat<(i1 (trunc (i64 GR64:$src))),
2147 def : Pat<(i1 (trunc (i32 GR32:$src))),
2150 def : Pat<(i1 (trunc (i8 GR8:$src))),
2154 def : Pat<(i1 (trunc (i16 GR16:$src))),
2159 def : Pat<(i32 (zext VK1:$src)),
2161 def : Pat<(i32 (anyext VK1:$src)),
2164 def : Pat<(i8 (zext VK1:$src)),
2168 def : Pat<(i8 (anyext VK1:$src)),
2172 def : Pat<(i64 (zext VK1:$src)),
2175 def : Pat<(i16 (zext VK1:$src)),
2180 def : Pat<(v16i1 (scalar_to_vector VK1:$src)),
2182 def : Pat<(v8i1 (scalar_to_vector VK1:$src)),
2184 def : Pat<(v4i1 (scalar_to_vector VK1:$src)),
2186 def : Pat<(v2i1 (scalar_to_vector VK1:$src)),
2188 def : Pat<(v32i1 (scalar_to_vector VK1:$src)),
2190 def : Pat<(v64i1 (scalar_to_vector VK1:$src)),
2197 def : Pat<(v8i1 (bitconvert (i8 GR8:$src))),
2200 def : Pat<(i8 (bitconvert (v8i1 VK8:$src))),
2207 def : Pat<(i1 (X86Vextract VK16:$src, (iPTR 0))),
2209 def : Pat<(i1 (X86Vextract VK8:$src, (iPTR 0))),
2213 def : Pat<(i1 (X86Vextract VK32:$src, (iPTR 0))),
2215 def : Pat<(i1 (X86Vextract VK64:$src, (iPTR 0))),
2246 def : Pat<(!cast<Intrinsic>("int_x86_avx512_"##IntName##"_w")
2254 def : Pat<(xor VK8:$src1, (v8i1 immAllOnesV)), (KNOTBrr VK8:$src1)>;
2256 def : Pat<(xor VK16:$src1, (v16i1 immAllOnesV)), (KNOTWrr VK16:$src1)>;
2258 def : Pat<(xor VK32:$src1, (v32i1 immAllOnesV)), (KNOTDrr VK32:$src1)>;
2260 def : Pat<(xor VK64:$src1, (v64i1 immAllOnesV)), (KNOTQrr VK64:$src1)>;
2264 def : Pat<(xor VK8:$src1, (v8i1 immAllOnesV)),
2266 def : Pat<(not VK8:$src),
2270 def : Pat<(xor VK4:$src1, (v4i1 immAllOnesV)),
2272 def : Pat<(xor VK2:$src1, (v2i1 immAllOnesV)),
2312 def : Pat<(!cast<Intrinsic>("int_x86_avx512_"##IntName##"_w")
2329 def : Pat<(OpNode VK8:$src1, VK8:$src2),
2335 def : Pat<(OpNode VK1:$src1, VK1:$src2),
2339 def : Pat<(OpNode VK2:$src1, VK2:$src2),
2343 def : Pat<(OpNode VK4:$src1, VK4:$src2),
2355 def : Pat<(xor (xor VK16:$src1, VK16:$src2), (v16i1 immAllOnesV)),
2357 def : Pat<(xor (xor VK8:$src1, VK8:$src2), (v8i1 immAllOnesV)),
2359 def : Pat<(xor (xor VK32:$src1, VK32:$src2), (v32i1 immAllOnesV)),
2361 def : Pat<(xor (xor VK64:$src1, VK64:$src2), (v64i1 immAllOnesV)),
2365 def : Pat<(xor (xor VK8:$src1, VK8:$src2), (v8i1 immAllOnesV)),
2369 def : Pat<(xor (xor VK4:$src1, VK4:$src2), (v4i1 immAllOnesV)),
2373 def : Pat<(xor (xor VK2:$src1, VK2:$src2), (v2i1 immAllOnesV)),
2377 def : Pat<(xor (xor VK1:$src1, VK1:$src2), (i1 1)),
2390 def : Pat<(VT (concat_vectors KRCSrc:$src1, KRCSrc:$src2)),
2474 def : Pat<(v8i1 immAllZerosV), (COPY_TO_REGCLASS (KSET0W), VK8)>;
2475 def : Pat<(v8i1 immAllOnesV), (COPY_TO_REGCLASS (KSET1W), VK8)>;
2476 def : Pat<(v4i1 immAllOnesV), (COPY_TO_REGCLASS (KSET1W), VK4)>;
2477 def : Pat<(v2i1 immAllOnesV), (COPY_TO_REGCLASS (KSET1W), VK2)>;
2478 def : Pat<(i1 0), (COPY_TO_REGCLASS (KSET0W), VK1)>;
2479 def : Pat<(i1 1), (COPY_TO_REGCLASS (KSHIFTRWri (KSET1W), (i8 15)), VK1)>;
2480 def : Pat<(i1 -1), (COPY_TO_REGCLASS (KSHIFTRWri (KSET1W), (i8 15)), VK1)>;
2482 def : Pat<(v8i1 (extract_subvector (v16i1 VK16:$src), (iPTR 0))),
2485 def : Pat<(v16i1 (insert_subvector undef, (v8i1 VK8:$src), (iPTR 0))),
2488 def : Pat<(v8i1 (extract_subvector (v16i1 VK16:$src), (iPTR 8))),
2491 def : Pat<(v16i1 (extract_subvector (v32i1 VK32:$src), (iPTR 0))),
2494 def : Pat<(v16i1 (extract_subvector (v32i1 VK32:$src), (iPTR 16))),
2497 def : Pat<(v32i1 (extract_subvector (v64i1 VK64:$src), (iPTR 0))),
2500 def : Pat<(v32i1 (extract_subvector (v64i1 VK64:$src), (iPTR 32))),
2503 def : Pat<(v4i1 (extract_subvector (v8i1 VK8:$src), (iPTR 0))),
2506 def : Pat<(v2i1 (extract_subvector (v8i1 VK8:$src), (iPTR 0))),
2509 def : Pat<(v4i1 (insert_subvector undef, (v2i1 VK2:$src), (iPTR 0))),
2512 def : Pat<(v8i1 (insert_subvector undef, (v4i1 VK4:$src), (iPTR 0))),
2514 def : Pat<(v8i1 (insert_subvector undef, (v2i1 VK2:$src), (iPTR 0))),
2517 def : Pat<(v32i1 (insert_subvector undef, VK2:$src, (iPTR 0))),
2519 def : Pat<(v32i1 (insert_subvector undef, VK4:$src, (iPTR 0))),
2521 def : Pat<(v32i1 (insert_subvector undef, VK8:$src, (iPTR 0))),
2523 def : Pat<(v32i1 (insert_subvector undef, VK16:$src, (iPTR 0))),
2526 def : Pat<(v64i1 (insert_subvector undef, VK2:$src, (iPTR 0))),
2528 def : Pat<(v64i1 (insert_subvector undef, VK4:$src, (iPTR 0))),
2530 def : Pat<(v64i1 (insert_subvector undef, VK8:$src, (iPTR 0))),
2532 def : Pat<(v64i1 (insert_subvector undef, VK16:$src, (iPTR 0))),
2534 def : Pat<(v64i1 (insert_subvector undef, VK32:$src, (iPTR 0))),
2538 def : Pat<(v8i1 (X86vshli VK8:$src, (i8 imm:$imm))),
2543 def : Pat<(v8i1 (X86vsrli VK8:$src, (i8 imm:$imm))),
2548 def : Pat<(v4i1 (X86vshli VK4:$src, (i8 imm:$imm))),
2553 def : Pat<(v4i1 (X86vsrli VK4:$src, (i8 imm:$imm))),
2611 def : Pat<(_.VT (mload addr:$ptr, _.KRCWM:$mask, undef)),
2614 def : Pat<(_.VT (mload addr:$ptr, _.KRCWM:$mask, _.ImmAllZerosV)),
2617 def : Pat<(_.VT (mload addr:$ptr, _.KRCWM:$mask, (_.VT _.RC:$src0))),
2681 def: Pat<(mstore addr:$ptr, _.KRCWM:$mask, (_.VT _.RC:$src)),
2733 def: Pat<(v8f64 (int_x86_avx512_mask_loadu_pd_512 addr:$ptr,
2737 def: Pat<(v16f32 (int_x86_avx512_mask_loadu_ps_512 addr:$ptr,
2741 def: Pat<(v8f64 (int_x86_avx512_mask_load_pd_512 addr:$ptr,
2745 def: Pat<(v16f32 (int_x86_avx512_mask_load_ps_512 addr:$ptr,
2749 def: Pat<(v8f64 (int_x86_avx512_mask_load_pd_512 addr:$ptr,
2753 def: Pat<(v16f32 (int_x86_avx512_mask_load_ps_512 addr:$ptr,
2757 def: Pat<(int_x86_avx512_mask_storeu_ps_512 addr:$ptr, (v16f32 VR512:$src),
2761 def: Pat<(int_x86_avx512_mask_storeu_pd_512 addr:$ptr, (v8f64 VR512:$src),
2766 def: Pat<(int_x86_avx512_mask_store_ps_512 addr:$ptr, (v16f32 VR512:$src),
2770 def: Pat<(int_x86_avx512_mask_store_pd_512 addr:$ptr, (v8f64 VR512:$src),
2801 def: Pat<(v16i32 (int_x86_avx512_mask_loadu_d_512 addr:$ptr,
2805 def: Pat<(v8i64 (int_x86_avx512_mask_loadu_q_512 addr:$ptr,
2809 def: Pat<(int_x86_avx512_mask_storeu_d_512 addr:$ptr, (v16i32 VR512:$src),
2813 def: Pat<(int_x86_avx512_mask_storeu_q_512 addr:$ptr, (v8i64 VR512:$src),
2819 def : Pat<(v8i64 (vselect VK8WM:$mask, (v8i64 VR512:$src),
2823 def : Pat<(v8i64 (vselect VK8WM:$mask, (bc_v8i64 (v16i32 immAllZerosV)),
2828 def : Pat<(v16i32 (vselect VK16WM:$mask, (v16i32 VR512:$src),
2832 def : Pat<(v16i32 (vselect VK16WM:$mask, (v16i32 immAllZerosV),
2995 def : Pat<(f32 (X86select VK1WM:$mask, (f32 FR32X:$src1), (f32 FR32X:$src2))),
2999 def : Pat<(f64 (X86select VK1WM:$mask, (f64 FR64X:$src1), (f64 FR64X:$src2))),
3003 def : Pat<(int_x86_avx512_mask_store_ss addr:$dst, VR128X:$src, GR8:$mask),
3021 def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector FR32X:$src)))),
3023 def : Pat<(v4f32 (X86vzmovl (v4f32 VR128X:$src))),
3025 def : Pat<(v4i32 (X86vzmovl (v4i32 VR128X:$src))),
3027 def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector FR64X:$src)))),
3031 def : Pat<(v8f32 (X86vzmovl (v8f32 VR256X:$src))),
3035 def : Pat<(v8i32 (X86vzmovl (v8i32 VR256X:$src))),
3044 def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector (loadf32 addr:$src))))),
3046 def : Pat<(v4f32 (scalar_to_vector (loadf32 addr:$src))),
3048 def : Pat<(v4f32 (X86vzmovl (loadv4f32 addr:$src))),
3053 def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector (loadf64 addr:$src))))),
3055 def : Pat<(v2f64 (scalar_to_vector (loadf64 addr:$src))),
3057 def : Pat<(v2f64 (X86vzmovl (loadv2f64 addr:$src))),
3059 def : Pat<(v2f64 (X86vzmovl (bc_v2f64 (loadv4f32 addr:$src)))),
3061 def : Pat<(v2f64 (X86vzload addr:$src)),
3066 def : Pat<(v8i32 (X86vzmovl (insert_subvector undef,
3069 def : Pat<(v8f32 (X86vzmovl (insert_subvector undef,
3072 def : Pat<(v4f64 (X86vzmovl (insert_subvector undef,
3076 def : Pat<(v8f32 (X86vzmovl (insert_subvector undef,
3080 def : Pat<(v4f64 (X86vzmovl (insert_subvector undef,
3084 def : Pat<(v4i64 (X86vzmovl (insert_subvector undef,
3089 def : Pat<(v4f64 (X86vzmovl (v4f64 VR256X:$src))),
3094 def : Pat<(v4i64 (X86vzmovl (v4i64 VR256X:$src))),
3099 def : Pat<(store (f32 (extractelt (v4f32 VR128X:$src), (iPTR 0))),
3102 def : Pat<(store (f64 (extractelt (v2f64 VR128X:$src), (iPTR 0))),
3107 def : Pat<(v4i32 (X86Movss VR128X:$src1, VR128X:$src2)),
3110 def : Pat<(v4f32 (X86Movss VR128X:$src1, VR128X:$src2)),
3115 def : Pat<(v8i32 (X86Movss VR256X:$src1, VR256X:$src2)),
3120 def : Pat<(v8f32 (X86Movss VR256X:$src1, VR256X:$src2)),
3127 def : Pat<(v2i64 (X86Movsd VR128X:$src1, VR128X:$src2)),
3129 def : Pat<(v2f64 (X86Movsd VR128X:$src1, VR128X:$src2)),
3131 def : Pat<(v4f32 (X86Movsd VR128X:$src1, VR128X:$src2)),
3133 def : Pat<(v4i32 (X86Movsd VR128X:$src1, VR128X:$src2)),
3137 def : Pat<(v4i64 (X86Movsd VR256X:$src1, VR256X:$src2)),
3142 def : Pat<(v4f64 (X86Movsd VR256X:$src1, VR256X:$src2)),
3148 def : Pat<(v2f64 (X86Movlpd VR128X:$src1, VR128X:$src2)),
3150 def : Pat<(v2i64 (X86Movlpd VR128X:$src1, VR128X:$src2)),
3152 def : Pat<(v4f32 (X86Movlps VR128X:$src1, VR128X:$src2)),
3154 def : Pat<(v4i32 (X86Movlps VR128X:$src1, VR128X:$src2)),
3178 def : Pat<(v4i32 (X86vzmovl (v4i32 (scalar_to_vector (loadi32 addr:$src))))),
3180 def : Pat<(v2i64 (X86vzmovl (v2i64 (scalar_to_vector GR64:$src)))),
3182 def : Pat<(v4i32 (X86vzmovl (v4i32 (scalar_to_vector GR32:$src)))),
3185 def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv4f32 addr:$src)))),
3187 def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv2i64 addr:$src)))),
3189 def : Pat<(v2i64 (X86vzmovl (loadv2i64 addr:$src))),
3191 def : Pat<(v2f64 (X86vzmovl (v2f64 VR128X:$src))),
3193 def : Pat<(v2i64 (X86vzload addr:$src)),
3198 def : Pat<(v8i32 (X86vzmovl (insert_subvector undef,
3201 def : Pat<(v4i64 (X86vzmovl (insert_subvector undef,
3206 def : Pat<(v16i32 (X86Vinsert (v16i32 immAllZerosV), GR32:$src2, (iPTR 0))),
3209 def : Pat<(v8i64 (X86Vinsert (bc_v8i64 (v16i32 immAllZerosV)), GR64:$src2, (iPTR 0))),
3212 def : Pat<(v16i32 (X86Vinsert undef, GR32:$src2, (iPTR 0))),
3215 def : Pat<(v8i64 (X86Vinsert undef, GR64:$src2, (iPTR 0))),
3962 def : Pat <(i16 (int_x86_avx512_mask_ptestm_d_512 (v16i32 VR512:$src1),
3966 def : Pat <(i8 (int_x86_avx512_mask_ptestm_q_512 (v8i64 VR512:$src1),
4332 def : Pat<(v4i32 (X86Movlhps VR128X:$src1, VR128X:$src2)),
4334 def : Pat<(v2i64 (X86Movlhps VR128X:$src1, VR128X:$src2)),
4338 def : Pat<(v4i32 (X86Movhlps VR128X:$src1, VR128X:$src2)),
4371 def : Pat<(X86Movlhps VR128X:$src1,
4374 def : Pat<(X86Movlhps VR128X:$src1,
4378 def : Pat<(v2f64 (X86Unpckl VR128X:$src1,
4381 def : Pat<(v2f64 (X86Unpckl VR128X:$src1,
4385 def : Pat<(v4f32 (X86Movlps VR128X:$src1, (load addr:$src2))),
4387 def : Pat<(v4i32 (X86Movlps VR128X:$src1, (load addr:$src2))),
4390 def : Pat<(v2f64 (X86Movlpd VR128X:$src1, (load addr:$src2))),
4392 def : Pat<(v2i64 (X86Movlpd VR128X:$src1, (load addr:$src2))),
4394 def : Pat<(v2f64 (X86Movsd VR128X:$src1,
4432 def : Pat<(store (f64 (vector_extract
4437 def : Pat<(store (v4f32 (X86Movlps (load addr:$src1), VR128X:$src2)),
4440 def : Pat<(store (v4i32 (X86Movlps
4444 def : Pat<(store (v2f64 (X86Movlpd (load addr:$src1), VR128X:$src2)),
4447 def : Pat<(store (v2i64 (X86Movlpd (load addr:$src1), VR128X:$src2)),
4817 def : Pat<(f32 (sint_to_fp (loadi32 addr:$src))),
4819 def : Pat<(f32 (sint_to_fp (loadi64 addr:$src))),
4821 def : Pat<(f64 (sint_to_fp (loadi32 addr:$src))),
4823 def : Pat<(f64 (sint_to_fp (loadi64 addr:$src))),
4826 def : Pat<(f32 (sint_to_fp GR32:$src)),
4828 def : Pat<(f32 (sint_to_fp GR64:$src)),
4830 def : Pat<(f64 (sint_to_fp GR32:$src)),
4832 def : Pat<(f64 (sint_to_fp GR64:$src)),
4848 def : Pat<(f32 (uint_to_fp (loadi32 addr:$src))),
4850 def : Pat<(f32 (uint_to_fp (loadi64 addr:$src))),
4852 def : Pat<(f64 (uint_to_fp (loadi32 addr:$src))),
4854 def : Pat<(f64 (uint_to_fp (loadi64 addr:$src))),
4857 def : Pat<(f32 (uint_to_fp GR32:$src)),
4859 def : Pat<(f32 (uint_to_fp GR64:$src)),
4861 def : Pat<(f64 (uint_to_fp GR32:$src)),
4863 def : Pat<(f64 (uint_to_fp GR64:$src)),
5000 def : Pat<(i32 (int_x86_sse_cvttss2si (v4f32 VR128X:$src))),
5002 def : Pat<(i64 (int_x86_sse_cvttss2si64 (v4f32 VR128X:$src))),
5004 def : Pat<(i32 (int_x86_sse2_cvttsd2si (v2f64 VR128X:$src))),
5006 def : Pat<(i64 (int_x86_sse2_cvttsd2si64 (v2f64 VR128X:$src))),
5078 def : Pat<(f64 (fextend FR32X:$src)),
5082 def : Pat<(f64 (fextend (loadf32 addr:$src))),
5086 def : Pat<(f64 (extloadf32 addr:$src)),
5090 def : Pat<(f64 (extloadf32 addr:$src)),
5095 def : Pat<(f32 (fround FR64X:$src)),
5181 def : Pat<(v8f64 (extloadv8f32 addr:$src)),
5185 def : Pat<(v4f64 (extloadv4f32 addr:$src)),
5482 def : Pat<(v8i32 (fp_to_uint (v8f32 VR256X:$src1))),
5486 def : Pat<(v4i32 (fp_to_uint (v4f32 VR128X:$src1))),
5490 def : Pat<(v8f32 (uint_to_fp (v8i32 VR256X:$src1))),
5494 def : Pat<(v4f32 (uint_to_fp (v4i32 VR128X:$src1))),
5498 def : Pat<(v4f64 (uint_to_fp (v4i32 VR128X:$src1))),
5504 def : Pat<(v8f32 (fround (loadv8f64 addr:$src))),
5506 def : Pat<(v8f64 (extloadv8f32 addr:$src)),
5715 def : Pat <(v16f32 (int_x86_avx512_rsqrt14_ps_512 (v16f32 VR512:$src),
5718 def : Pat <(v8f64 (int_x86_avx512_rsqrt14_pd_512 (v8f64 VR512:$src),
5722 def : Pat <(v16f32 (int_x86_avx512_rcp14_ps_512 (v16f32 VR512:$src),
5725 def : Pat <(v8f64 (int_x86_avx512_rcp14_pd_512 (v8f64 VR512:$src),
5926 def : Pat<(_.EltVT (OpNode _.FRC:$src)),
5930 def : Pat<(_.EltVT (OpNode (load addr:$src))),
5948 def : Pat<(f32 (X86frsqrt FR32X:$src)),
5950 def : Pat<(f32 (X86frsqrt (load addr:$src))),
5953 def : Pat<(f32 (X86frcp FR32X:$src)),
5955 def : Pat<(f32 (X86frcp (load addr:$src))),
5985 def : Pat<(ffloor _.FRC:$src), (COPY_TO_REGCLASS
5988 def : Pat<(fceil _.FRC:$src), (COPY_TO_REGCLASS
5991 def : Pat<(ftrunc _.FRC:$src), (COPY_TO_REGCLASS
5994 def : Pat<(frint _.FRC:$src), (COPY_TO_REGCLASS
5997 def : Pat<(fnearbyint _.FRC:$src), (COPY_TO_REGCLASS
6001 def : Pat<(ffloor (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS
6004 def : Pat<(fceil (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS
6007 def : Pat<(ftrunc (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS
6010 def : Pat<(frint (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS
6013 def : Pat<(fnearbyint (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS
6039 def : Pat<(DestInfo.VT (X86select DestInfo.KRCWM:$mask,
6045 def : Pat<(DestInfo.VT (X86select DestInfo.KRCWM:$mask,
6051 def : Pat<(DestInfo.VT (X86select DestInfo.KRCWM:$mask,
6075 def : Pat<(truncFrag (SrcInfo.VT SrcInfo.RC:$src), addr:$dst),
6079 def : Pat<(mtruncFrag addr:$dst, SrcInfo.KRCWM:$mask,
6088 def: Pat<(!cast<Intrinsic>("int_x86_avx512_mask_pmov"#sat#"_"#SrcInfo.Suffix#
6095 def: Pat<(!cast<Intrinsic>("int_x86_avx512_mask_pmov"#sat#"_"#SrcInfo.Suffix#
6242 def: Pat<(v8i16 (X86vtrunc (v8i32 VR256X:$src))),
6246 def: Pat<(v4i32 (X86vtrunc (v4i64 VR256X:$src))),
6253 def: Pat<(v16i8 (X86vtrunc (v16i16 VR256X:$src))),
6576 def : Pat<(store (i1 -1), addr:$dst), (MOV8mi addr:$dst, (i8 1))>;
6577 def : Pat<(store (i1 1), addr:$dst), (MOV8mi addr:$dst, (i8 1))>;
6578 def : Pat<(store (i1 0), addr:$dst), (MOV8mi addr:$dst, (i8 0))>;
6580 def : Pat<(store VK1:$src, addr:$dst),
6585 def : Pat<(store VK8:$src, addr:$dst),
6595 def : Pat<(truncstorei1 GR8:$src, addr:$dst),
7037 def : Pat<(v16f32 (ffloor VR512:$src)),
7039 def : Pat<(v16f32 (fnearbyint VR512:$src)),
7041 def : Pat<(v16f32 (fceil VR512:$src)),
7043 def : Pat<(v16f32 (frint VR512:$src)),
7045 def : Pat<(v16f32 (ftrunc VR512:$src)),
7048 def : Pat<(v8f64 (ffloor VR512:$src)),
7050 def : Pat<(v8f64 (fnearbyint VR512:$src)),
7052 def : Pat<(v8f64 (fceil VR512:$src)),
7054 def : Pat<(v8f64 (frint VR512:$src)),
7056 def : Pat<(v8f64 (ftrunc VR512:$src)),
7087 Pat<(_.VT (X86PAlignr _.RC:$src1, _.RC:$src2, (i8 imm:$imm))),
7189 def : Pat<(xor
7193 def : Pat<(xor
7263 def : Pat<(X86Movddup (loadv2f64 addr:$src)),
7265 def : Pat<(v2f64 (X86VBroadcast (loadf64 addr:$src))),