1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\ 2|* *| 3|* "Fast" Instruction Selector for the X86 target *| 4|* *| 5|* Automatically generated file, do not edit! *| 6|* *| 7\*===----------------------------------------------------------------------===*/ 8 9 10// FastEmit Immediate Predicate functions. 11static bool Predicate_i64immSExt32(int64_t Imm) { 12 return isInt<32>(Imm); 13} 14static bool Predicate_AndMask64(int64_t Imm) { 15 16 return isMask_64(Imm) && !isUInt<32>(Imm); 17 18} 19static bool Predicate_BTRMask64(int64_t Imm) { 20 21 return !isUInt<32>(Imm) && !isInt<32>(Imm) && isPowerOf2_64(~Imm); 22 23} 24static bool Predicate_BTCBTSMask64(int64_t Imm) { 25 26 return !isInt<32>(Imm) && isPowerOf2_64(Imm); 27 28} 29static bool Predicate_i16immSExt8(int64_t Imm) { 30 return isInt<8>(Imm); 31} 32static bool Predicate_i32immSExt8(int64_t Imm) { 33 return isInt<8>(Imm); 34} 35static bool Predicate_i64immSExt8(int64_t Imm) { 36 return isInt<8>(Imm); 37} 38 39 40// FastEmit functions for ISD::ABS. 41 42unsigned fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 43 if (RetVT.SimpleTy != MVT::v16i8) 44 return 0; 45 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 46 return fastEmitInst_r(X86::VPABSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 47 } 48 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) { 49 return fastEmitInst_r(X86::PABSBrr, &X86::VR128RegClass, Op0, Op0IsKill); 50 } 51 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 52 return fastEmitInst_r(X86::VPABSBrr, &X86::VR128RegClass, Op0, Op0IsKill); 53 } 54 return 0; 55} 56 57unsigned fastEmit_ISD_ABS_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 58 if (RetVT.SimpleTy != MVT::v32i8) 59 return 0; 60 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 61 return fastEmitInst_r(X86::VPABSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 62 } 63 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 64 return fastEmitInst_r(X86::VPABSBYrr, &X86::VR256RegClass, Op0, Op0IsKill); 65 } 66 return 0; 67} 68 69unsigned fastEmit_ISD_ABS_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 70 if (RetVT.SimpleTy != MVT::v64i8) 71 return 0; 72 if ((Subtarget->hasBWI())) { 73 return fastEmitInst_r(X86::VPABSBZrr, &X86::VR512RegClass, Op0, Op0IsKill); 74 } 75 return 0; 76} 77 78unsigned fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 79 if (RetVT.SimpleTy != MVT::v8i16) 80 return 0; 81 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 82 return fastEmitInst_r(X86::VPABSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 83 } 84 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) { 85 return fastEmitInst_r(X86::PABSWrr, &X86::VR128RegClass, Op0, Op0IsKill); 86 } 87 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 88 return fastEmitInst_r(X86::VPABSWrr, &X86::VR128RegClass, Op0, Op0IsKill); 89 } 90 return 0; 91} 92 93unsigned fastEmit_ISD_ABS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 94 if (RetVT.SimpleTy != MVT::v16i16) 95 return 0; 96 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 97 return fastEmitInst_r(X86::VPABSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 98 } 99 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 100 return fastEmitInst_r(X86::VPABSWYrr, &X86::VR256RegClass, Op0, Op0IsKill); 101 } 102 return 0; 103} 104 105unsigned fastEmit_ISD_ABS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 106 if (RetVT.SimpleTy != MVT::v32i16) 107 return 0; 108 if ((Subtarget->hasBWI())) { 109 return fastEmitInst_r(X86::VPABSWZrr, &X86::VR512RegClass, Op0, Op0IsKill); 110 } 111 return 0; 112} 113 114unsigned fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 115 if (RetVT.SimpleTy != MVT::v4i32) 116 return 0; 117 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 118 return fastEmitInst_r(X86::VPABSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 119 } 120 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) { 121 return fastEmitInst_r(X86::PABSDrr, &X86::VR128RegClass, Op0, Op0IsKill); 122 } 123 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 124 return fastEmitInst_r(X86::VPABSDrr, &X86::VR128RegClass, Op0, Op0IsKill); 125 } 126 return 0; 127} 128 129unsigned fastEmit_ISD_ABS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 130 if (RetVT.SimpleTy != MVT::v8i32) 131 return 0; 132 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 133 return fastEmitInst_r(X86::VPABSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 134 } 135 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 136 return fastEmitInst_r(X86::VPABSDYrr, &X86::VR256RegClass, Op0, Op0IsKill); 137 } 138 return 0; 139} 140 141unsigned fastEmit_ISD_ABS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 142 if (RetVT.SimpleTy != MVT::v16i32) 143 return 0; 144 if ((Subtarget->hasAVX512())) { 145 return fastEmitInst_r(X86::VPABSDZrr, &X86::VR512RegClass, Op0, Op0IsKill); 146 } 147 return 0; 148} 149 150unsigned fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 151 if (RetVT.SimpleTy != MVT::v2i64) 152 return 0; 153 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 154 return fastEmitInst_r(X86::VPABSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 155 } 156 return 0; 157} 158 159unsigned fastEmit_ISD_ABS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 160 if (RetVT.SimpleTy != MVT::v4i64) 161 return 0; 162 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 163 return fastEmitInst_r(X86::VPABSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 164 } 165 return 0; 166} 167 168unsigned fastEmit_ISD_ABS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 169 if (RetVT.SimpleTy != MVT::v8i64) 170 return 0; 171 if ((Subtarget->hasAVX512())) { 172 return fastEmitInst_r(X86::VPABSQZrr, &X86::VR512RegClass, Op0, Op0IsKill); 173 } 174 return 0; 175} 176 177unsigned fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 178 switch (VT.SimpleTy) { 179 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0, Op0IsKill); 180 case MVT::v32i8: return fastEmit_ISD_ABS_MVT_v32i8_r(RetVT, Op0, Op0IsKill); 181 case MVT::v64i8: return fastEmit_ISD_ABS_MVT_v64i8_r(RetVT, Op0, Op0IsKill); 182 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0, Op0IsKill); 183 case MVT::v16i16: return fastEmit_ISD_ABS_MVT_v16i16_r(RetVT, Op0, Op0IsKill); 184 case MVT::v32i16: return fastEmit_ISD_ABS_MVT_v32i16_r(RetVT, Op0, Op0IsKill); 185 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0, Op0IsKill); 186 case MVT::v8i32: return fastEmit_ISD_ABS_MVT_v8i32_r(RetVT, Op0, Op0IsKill); 187 case MVT::v16i32: return fastEmit_ISD_ABS_MVT_v16i32_r(RetVT, Op0, Op0IsKill); 188 case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0, Op0IsKill); 189 case MVT::v4i64: return fastEmit_ISD_ABS_MVT_v4i64_r(RetVT, Op0, Op0IsKill); 190 case MVT::v8i64: return fastEmit_ISD_ABS_MVT_v8i64_r(RetVT, Op0, Op0IsKill); 191 default: return 0; 192 } 193} 194 195// FastEmit functions for ISD::ANY_EXTEND. 196 197unsigned fastEmit_ISD_ANY_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 198 if (RetVT.SimpleTy != MVT::i32) 199 return 0; 200 return fastEmitInst_r(X86::MOVZX32rr8, &X86::GR32RegClass, Op0, Op0IsKill); 201} 202 203unsigned fastEmit_ISD_ANY_EXTEND_MVT_v2i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 204 if (RetVT.SimpleTy != MVT::v2i64) 205 return 0; 206 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 207 return fastEmitInst_r(X86::VPMOVM2QZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 208 } 209 return 0; 210} 211 212unsigned fastEmit_ISD_ANY_EXTEND_MVT_v4i1_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 213 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 214 return fastEmitInst_r(X86::VPMOVM2DZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 215 } 216 return 0; 217} 218 219unsigned fastEmit_ISD_ANY_EXTEND_MVT_v4i1_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) { 220 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 221 return fastEmitInst_r(X86::VPMOVM2QZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 222 } 223 return 0; 224} 225 226unsigned fastEmit_ISD_ANY_EXTEND_MVT_v4i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 227switch (RetVT.SimpleTy) { 228 case MVT::v4i32: return fastEmit_ISD_ANY_EXTEND_MVT_v4i1_MVT_v4i32_r(Op0, Op0IsKill); 229 case MVT::v4i64: return fastEmit_ISD_ANY_EXTEND_MVT_v4i1_MVT_v4i64_r(Op0, Op0IsKill); 230 default: return 0; 231} 232} 233 234unsigned fastEmit_ISD_ANY_EXTEND_MVT_v8i1_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) { 235 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 236 return fastEmitInst_r(X86::VPMOVM2WZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 237 } 238 return 0; 239} 240 241unsigned fastEmit_ISD_ANY_EXTEND_MVT_v8i1_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) { 242 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 243 return fastEmitInst_r(X86::VPMOVM2DZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 244 } 245 return 0; 246} 247 248unsigned fastEmit_ISD_ANY_EXTEND_MVT_v8i1_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) { 249 if ((Subtarget->hasDQI())) { 250 return fastEmitInst_r(X86::VPMOVM2QZrr, &X86::VR512RegClass, Op0, Op0IsKill); 251 } 252 return 0; 253} 254 255unsigned fastEmit_ISD_ANY_EXTEND_MVT_v8i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 256switch (RetVT.SimpleTy) { 257 case MVT::v8i16: return fastEmit_ISD_ANY_EXTEND_MVT_v8i1_MVT_v8i16_r(Op0, Op0IsKill); 258 case MVT::v8i32: return fastEmit_ISD_ANY_EXTEND_MVT_v8i1_MVT_v8i32_r(Op0, Op0IsKill); 259 case MVT::v8i64: return fastEmit_ISD_ANY_EXTEND_MVT_v8i1_MVT_v8i64_r(Op0, Op0IsKill); 260 default: return 0; 261} 262} 263 264unsigned fastEmit_ISD_ANY_EXTEND_MVT_v16i1_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) { 265 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 266 return fastEmitInst_r(X86::VPMOVM2BZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 267 } 268 return 0; 269} 270 271unsigned fastEmit_ISD_ANY_EXTEND_MVT_v16i1_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) { 272 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 273 return fastEmitInst_r(X86::VPMOVM2WZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 274 } 275 return 0; 276} 277 278unsigned fastEmit_ISD_ANY_EXTEND_MVT_v16i1_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) { 279 if ((Subtarget->hasDQI())) { 280 return fastEmitInst_r(X86::VPMOVM2DZrr, &X86::VR512RegClass, Op0, Op0IsKill); 281 } 282 return 0; 283} 284 285unsigned fastEmit_ISD_ANY_EXTEND_MVT_v16i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 286switch (RetVT.SimpleTy) { 287 case MVT::v16i8: return fastEmit_ISD_ANY_EXTEND_MVT_v16i1_MVT_v16i8_r(Op0, Op0IsKill); 288 case MVT::v16i16: return fastEmit_ISD_ANY_EXTEND_MVT_v16i1_MVT_v16i16_r(Op0, Op0IsKill); 289 case MVT::v16i32: return fastEmit_ISD_ANY_EXTEND_MVT_v16i1_MVT_v16i32_r(Op0, Op0IsKill); 290 default: return 0; 291} 292} 293 294unsigned fastEmit_ISD_ANY_EXTEND_MVT_v32i1_MVT_v32i8_r(unsigned Op0, bool Op0IsKill) { 295 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 296 return fastEmitInst_r(X86::VPMOVM2BZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 297 } 298 return 0; 299} 300 301unsigned fastEmit_ISD_ANY_EXTEND_MVT_v32i1_MVT_v32i16_r(unsigned Op0, bool Op0IsKill) { 302 if ((Subtarget->hasBWI())) { 303 return fastEmitInst_r(X86::VPMOVM2WZrr, &X86::VR512RegClass, Op0, Op0IsKill); 304 } 305 return 0; 306} 307 308unsigned fastEmit_ISD_ANY_EXTEND_MVT_v32i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 309switch (RetVT.SimpleTy) { 310 case MVT::v32i8: return fastEmit_ISD_ANY_EXTEND_MVT_v32i1_MVT_v32i8_r(Op0, Op0IsKill); 311 case MVT::v32i16: return fastEmit_ISD_ANY_EXTEND_MVT_v32i1_MVT_v32i16_r(Op0, Op0IsKill); 312 default: return 0; 313} 314} 315 316unsigned fastEmit_ISD_ANY_EXTEND_MVT_v64i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 317 if (RetVT.SimpleTy != MVT::v64i8) 318 return 0; 319 if ((Subtarget->hasBWI())) { 320 return fastEmitInst_r(X86::VPMOVM2BZrr, &X86::VR512RegClass, Op0, Op0IsKill); 321 } 322 return 0; 323} 324 325unsigned fastEmit_ISD_ANY_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 326 switch (VT.SimpleTy) { 327 case MVT::i8: return fastEmit_ISD_ANY_EXTEND_MVT_i8_r(RetVT, Op0, Op0IsKill); 328 case MVT::v2i1: return fastEmit_ISD_ANY_EXTEND_MVT_v2i1_r(RetVT, Op0, Op0IsKill); 329 case MVT::v4i1: return fastEmit_ISD_ANY_EXTEND_MVT_v4i1_r(RetVT, Op0, Op0IsKill); 330 case MVT::v8i1: return fastEmit_ISD_ANY_EXTEND_MVT_v8i1_r(RetVT, Op0, Op0IsKill); 331 case MVT::v16i1: return fastEmit_ISD_ANY_EXTEND_MVT_v16i1_r(RetVT, Op0, Op0IsKill); 332 case MVT::v32i1: return fastEmit_ISD_ANY_EXTEND_MVT_v32i1_r(RetVT, Op0, Op0IsKill); 333 case MVT::v64i1: return fastEmit_ISD_ANY_EXTEND_MVT_v64i1_r(RetVT, Op0, Op0IsKill); 334 default: return 0; 335 } 336} 337 338// FastEmit functions for ISD::BITCAST. 339 340unsigned fastEmit_ISD_BITCAST_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 341 if (RetVT.SimpleTy != MVT::f32) 342 return 0; 343 if ((Subtarget->hasAVX512())) { 344 return fastEmitInst_r(X86::VMOVDI2SSZrr, &X86::FR32XRegClass, Op0, Op0IsKill); 345 } 346 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 347 return fastEmitInst_r(X86::MOVDI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill); 348 } 349 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 350 return fastEmitInst_r(X86::VMOVDI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill); 351 } 352 return 0; 353} 354 355unsigned fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(unsigned Op0, bool Op0IsKill) { 356 if ((Subtarget->hasAVX512())) { 357 return fastEmitInst_r(X86::VMOV64toSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill); 358 } 359 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 360 return fastEmitInst_r(X86::MOV64toSDrr, &X86::FR64RegClass, Op0, Op0IsKill); 361 } 362 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 363 return fastEmitInst_r(X86::VMOV64toSDrr, &X86::FR64RegClass, Op0, Op0IsKill); 364 } 365 return 0; 366} 367 368unsigned fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(unsigned Op0, bool Op0IsKill) { 369 if ((Subtarget->hasMMX())) { 370 return fastEmitInst_r(X86::MMX_MOVD64to64rr, &X86::VR64RegClass, Op0, Op0IsKill); 371 } 372 return 0; 373} 374 375unsigned fastEmit_ISD_BITCAST_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 376switch (RetVT.SimpleTy) { 377 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(Op0, Op0IsKill); 378 case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(Op0, Op0IsKill); 379 default: return 0; 380} 381} 382 383unsigned fastEmit_ISD_BITCAST_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 384 if (RetVT.SimpleTy != MVT::i32) 385 return 0; 386 if ((Subtarget->hasAVX512())) { 387 return fastEmitInst_r(X86::VMOVSS2DIZrr, &X86::GR32RegClass, Op0, Op0IsKill); 388 } 389 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 390 return fastEmitInst_r(X86::MOVSS2DIrr, &X86::GR32RegClass, Op0, Op0IsKill); 391 } 392 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 393 return fastEmitInst_r(X86::VMOVSS2DIrr, &X86::GR32RegClass, Op0, Op0IsKill); 394 } 395 return 0; 396} 397 398unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) { 399 if ((Subtarget->hasAVX512())) { 400 return fastEmitInst_r(X86::VMOVSDto64Zrr, &X86::GR64RegClass, Op0, Op0IsKill); 401 } 402 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 403 return fastEmitInst_r(X86::MOVSDto64rr, &X86::GR64RegClass, Op0, Op0IsKill); 404 } 405 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 406 return fastEmitInst_r(X86::VMOVSDto64rr, &X86::GR64RegClass, Op0, Op0IsKill); 407 } 408 return 0; 409} 410 411unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(unsigned Op0, bool Op0IsKill) { 412 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) { 413 return fastEmitInst_r(X86::MMX_MOVFR642Qrr, &X86::VR64RegClass, Op0, Op0IsKill); 414 } 415 return 0; 416} 417 418unsigned fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 419switch (RetVT.SimpleTy) { 420 case MVT::i64: return fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(Op0, Op0IsKill); 421 case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(Op0, Op0IsKill); 422 default: return 0; 423} 424} 425 426unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(unsigned Op0, bool Op0IsKill) { 427 if ((Subtarget->hasMMX())) { 428 return fastEmitInst_r(X86::MMX_MOVD64from64rr, &X86::GR64RegClass, Op0, Op0IsKill); 429 } 430 return 0; 431} 432 433unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(unsigned Op0, bool Op0IsKill) { 434 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) { 435 return fastEmitInst_r(X86::MMX_MOVQ2FR64rr, &X86::FR64RegClass, Op0, Op0IsKill); 436 } 437 return 0; 438} 439 440unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 441switch (RetVT.SimpleTy) { 442 case MVT::i64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(Op0, Op0IsKill); 443 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(Op0, Op0IsKill); 444 default: return 0; 445} 446} 447 448unsigned fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 449 switch (VT.SimpleTy) { 450 case MVT::i32: return fastEmit_ISD_BITCAST_MVT_i32_r(RetVT, Op0, Op0IsKill); 451 case MVT::i64: return fastEmit_ISD_BITCAST_MVT_i64_r(RetVT, Op0, Op0IsKill); 452 case MVT::f32: return fastEmit_ISD_BITCAST_MVT_f32_r(RetVT, Op0, Op0IsKill); 453 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0, Op0IsKill); 454 case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_x86mmx_r(RetVT, Op0, Op0IsKill); 455 default: return 0; 456 } 457} 458 459// FastEmit functions for ISD::BRIND. 460 461unsigned fastEmit_ISD_BRIND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 462 if (RetVT.SimpleTy != MVT::isVoid) 463 return 0; 464 if ((!Subtarget->is64Bit())) { 465 return fastEmitInst_r(X86::JMP16r, &X86::GR16RegClass, Op0, Op0IsKill); 466 } 467 return 0; 468} 469 470unsigned fastEmit_ISD_BRIND_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 471 if (RetVT.SimpleTy != MVT::isVoid) 472 return 0; 473 if ((!Subtarget->is64Bit())) { 474 return fastEmitInst_r(X86::JMP32r, &X86::GR32RegClass, Op0, Op0IsKill); 475 } 476 return 0; 477} 478 479unsigned fastEmit_ISD_BRIND_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 480 if (RetVT.SimpleTy != MVT::isVoid) 481 return 0; 482 if ((Subtarget->is64Bit())) { 483 return fastEmitInst_r(X86::JMP64r, &X86::GR64RegClass, Op0, Op0IsKill); 484 } 485 return 0; 486} 487 488unsigned fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 489 switch (VT.SimpleTy) { 490 case MVT::i16: return fastEmit_ISD_BRIND_MVT_i16_r(RetVT, Op0, Op0IsKill); 491 case MVT::i32: return fastEmit_ISD_BRIND_MVT_i32_r(RetVT, Op0, Op0IsKill); 492 case MVT::i64: return fastEmit_ISD_BRIND_MVT_i64_r(RetVT, Op0, Op0IsKill); 493 default: return 0; 494 } 495} 496 497// FastEmit functions for ISD::BSWAP. 498 499unsigned fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 500 if (RetVT.SimpleTy != MVT::i32) 501 return 0; 502 return fastEmitInst_r(X86::BSWAP32r, &X86::GR32RegClass, Op0, Op0IsKill); 503} 504 505unsigned fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 506 if (RetVT.SimpleTy != MVT::i64) 507 return 0; 508 return fastEmitInst_r(X86::BSWAP64r, &X86::GR64RegClass, Op0, Op0IsKill); 509} 510 511unsigned fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 512 switch (VT.SimpleTy) { 513 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0, Op0IsKill); 514 case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0, Op0IsKill); 515 default: return 0; 516 } 517} 518 519// FastEmit functions for ISD::CTLZ. 520 521unsigned fastEmit_ISD_CTLZ_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 522 if (RetVT.SimpleTy != MVT::i16) 523 return 0; 524 if ((Subtarget->hasLZCNT())) { 525 return fastEmitInst_r(X86::LZCNT16rr, &X86::GR16RegClass, Op0, Op0IsKill); 526 } 527 return 0; 528} 529 530unsigned fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 531 if (RetVT.SimpleTy != MVT::i32) 532 return 0; 533 if ((Subtarget->hasLZCNT())) { 534 return fastEmitInst_r(X86::LZCNT32rr, &X86::GR32RegClass, Op0, Op0IsKill); 535 } 536 return 0; 537} 538 539unsigned fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 540 if (RetVT.SimpleTy != MVT::i64) 541 return 0; 542 if ((Subtarget->hasLZCNT())) { 543 return fastEmitInst_r(X86::LZCNT64rr, &X86::GR64RegClass, Op0, Op0IsKill); 544 } 545 return 0; 546} 547 548unsigned fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 549 if (RetVT.SimpleTy != MVT::v4i32) 550 return 0; 551 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) { 552 return fastEmitInst_r(X86::VPLZCNTDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 553 } 554 return 0; 555} 556 557unsigned fastEmit_ISD_CTLZ_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 558 if (RetVT.SimpleTy != MVT::v8i32) 559 return 0; 560 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) { 561 return fastEmitInst_r(X86::VPLZCNTDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 562 } 563 return 0; 564} 565 566unsigned fastEmit_ISD_CTLZ_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 567 if (RetVT.SimpleTy != MVT::v16i32) 568 return 0; 569 if ((Subtarget->hasCDI())) { 570 return fastEmitInst_r(X86::VPLZCNTDZrr, &X86::VR512RegClass, Op0, Op0IsKill); 571 } 572 return 0; 573} 574 575unsigned fastEmit_ISD_CTLZ_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 576 if (RetVT.SimpleTy != MVT::v2i64) 577 return 0; 578 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) { 579 return fastEmitInst_r(X86::VPLZCNTQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 580 } 581 return 0; 582} 583 584unsigned fastEmit_ISD_CTLZ_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 585 if (RetVT.SimpleTy != MVT::v4i64) 586 return 0; 587 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) { 588 return fastEmitInst_r(X86::VPLZCNTQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 589 } 590 return 0; 591} 592 593unsigned fastEmit_ISD_CTLZ_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 594 if (RetVT.SimpleTy != MVT::v8i64) 595 return 0; 596 if ((Subtarget->hasCDI())) { 597 return fastEmitInst_r(X86::VPLZCNTQZrr, &X86::VR512RegClass, Op0, Op0IsKill); 598 } 599 return 0; 600} 601 602unsigned fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 603 switch (VT.SimpleTy) { 604 case MVT::i16: return fastEmit_ISD_CTLZ_MVT_i16_r(RetVT, Op0, Op0IsKill); 605 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0, Op0IsKill); 606 case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0, Op0IsKill); 607 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0, Op0IsKill); 608 case MVT::v8i32: return fastEmit_ISD_CTLZ_MVT_v8i32_r(RetVT, Op0, Op0IsKill); 609 case MVT::v16i32: return fastEmit_ISD_CTLZ_MVT_v16i32_r(RetVT, Op0, Op0IsKill); 610 case MVT::v2i64: return fastEmit_ISD_CTLZ_MVT_v2i64_r(RetVT, Op0, Op0IsKill); 611 case MVT::v4i64: return fastEmit_ISD_CTLZ_MVT_v4i64_r(RetVT, Op0, Op0IsKill); 612 case MVT::v8i64: return fastEmit_ISD_CTLZ_MVT_v8i64_r(RetVT, Op0, Op0IsKill); 613 default: return 0; 614 } 615} 616 617// FastEmit functions for ISD::CTPOP. 618 619unsigned fastEmit_ISD_CTPOP_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 620 if (RetVT.SimpleTy != MVT::i16) 621 return 0; 622 if ((Subtarget->hasPOPCNT())) { 623 return fastEmitInst_r(X86::POPCNT16rr, &X86::GR16RegClass, Op0, Op0IsKill); 624 } 625 return 0; 626} 627 628unsigned fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 629 if (RetVT.SimpleTy != MVT::i32) 630 return 0; 631 if ((Subtarget->hasPOPCNT())) { 632 return fastEmitInst_r(X86::POPCNT32rr, &X86::GR32RegClass, Op0, Op0IsKill); 633 } 634 return 0; 635} 636 637unsigned fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 638 if (RetVT.SimpleTy != MVT::i64) 639 return 0; 640 if ((Subtarget->hasPOPCNT())) { 641 return fastEmitInst_r(X86::POPCNT64rr, &X86::GR64RegClass, Op0, Op0IsKill); 642 } 643 return 0; 644} 645 646unsigned fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 647 if (RetVT.SimpleTy != MVT::v16i8) 648 return 0; 649 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) { 650 return fastEmitInst_r(X86::VPOPCNTBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 651 } 652 return 0; 653} 654 655unsigned fastEmit_ISD_CTPOP_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 656 if (RetVT.SimpleTy != MVT::v32i8) 657 return 0; 658 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) { 659 return fastEmitInst_r(X86::VPOPCNTBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 660 } 661 return 0; 662} 663 664unsigned fastEmit_ISD_CTPOP_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 665 if (RetVT.SimpleTy != MVT::v64i8) 666 return 0; 667 if ((Subtarget->hasBITALG())) { 668 return fastEmitInst_r(X86::VPOPCNTBZrr, &X86::VR512RegClass, Op0, Op0IsKill); 669 } 670 return 0; 671} 672 673unsigned fastEmit_ISD_CTPOP_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 674 if (RetVT.SimpleTy != MVT::v8i16) 675 return 0; 676 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) { 677 return fastEmitInst_r(X86::VPOPCNTWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 678 } 679 return 0; 680} 681 682unsigned fastEmit_ISD_CTPOP_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 683 if (RetVT.SimpleTy != MVT::v16i16) 684 return 0; 685 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) { 686 return fastEmitInst_r(X86::VPOPCNTWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 687 } 688 return 0; 689} 690 691unsigned fastEmit_ISD_CTPOP_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 692 if (RetVT.SimpleTy != MVT::v32i16) 693 return 0; 694 if ((Subtarget->hasBITALG())) { 695 return fastEmitInst_r(X86::VPOPCNTWZrr, &X86::VR512RegClass, Op0, Op0IsKill); 696 } 697 return 0; 698} 699 700unsigned fastEmit_ISD_CTPOP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 701 if (RetVT.SimpleTy != MVT::v4i32) 702 return 0; 703 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) { 704 return fastEmitInst_r(X86::VPOPCNTDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 705 } 706 return 0; 707} 708 709unsigned fastEmit_ISD_CTPOP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 710 if (RetVT.SimpleTy != MVT::v8i32) 711 return 0; 712 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) { 713 return fastEmitInst_r(X86::VPOPCNTDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 714 } 715 return 0; 716} 717 718unsigned fastEmit_ISD_CTPOP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 719 if (RetVT.SimpleTy != MVT::v16i32) 720 return 0; 721 if ((Subtarget->hasVPOPCNTDQ())) { 722 return fastEmitInst_r(X86::VPOPCNTDZrr, &X86::VR512RegClass, Op0, Op0IsKill); 723 } 724 return 0; 725} 726 727unsigned fastEmit_ISD_CTPOP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 728 if (RetVT.SimpleTy != MVT::v2i64) 729 return 0; 730 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) { 731 return fastEmitInst_r(X86::VPOPCNTQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 732 } 733 return 0; 734} 735 736unsigned fastEmit_ISD_CTPOP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 737 if (RetVT.SimpleTy != MVT::v4i64) 738 return 0; 739 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) { 740 return fastEmitInst_r(X86::VPOPCNTQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 741 } 742 return 0; 743} 744 745unsigned fastEmit_ISD_CTPOP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 746 if (RetVT.SimpleTy != MVT::v8i64) 747 return 0; 748 if ((Subtarget->hasVPOPCNTDQ())) { 749 return fastEmitInst_r(X86::VPOPCNTQZrr, &X86::VR512RegClass, Op0, Op0IsKill); 750 } 751 return 0; 752} 753 754unsigned fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 755 switch (VT.SimpleTy) { 756 case MVT::i16: return fastEmit_ISD_CTPOP_MVT_i16_r(RetVT, Op0, Op0IsKill); 757 case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0, Op0IsKill); 758 case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0, Op0IsKill); 759 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0, Op0IsKill); 760 case MVT::v32i8: return fastEmit_ISD_CTPOP_MVT_v32i8_r(RetVT, Op0, Op0IsKill); 761 case MVT::v64i8: return fastEmit_ISD_CTPOP_MVT_v64i8_r(RetVT, Op0, Op0IsKill); 762 case MVT::v8i16: return fastEmit_ISD_CTPOP_MVT_v8i16_r(RetVT, Op0, Op0IsKill); 763 case MVT::v16i16: return fastEmit_ISD_CTPOP_MVT_v16i16_r(RetVT, Op0, Op0IsKill); 764 case MVT::v32i16: return fastEmit_ISD_CTPOP_MVT_v32i16_r(RetVT, Op0, Op0IsKill); 765 case MVT::v4i32: return fastEmit_ISD_CTPOP_MVT_v4i32_r(RetVT, Op0, Op0IsKill); 766 case MVT::v8i32: return fastEmit_ISD_CTPOP_MVT_v8i32_r(RetVT, Op0, Op0IsKill); 767 case MVT::v16i32: return fastEmit_ISD_CTPOP_MVT_v16i32_r(RetVT, Op0, Op0IsKill); 768 case MVT::v2i64: return fastEmit_ISD_CTPOP_MVT_v2i64_r(RetVT, Op0, Op0IsKill); 769 case MVT::v4i64: return fastEmit_ISD_CTPOP_MVT_v4i64_r(RetVT, Op0, Op0IsKill); 770 case MVT::v8i64: return fastEmit_ISD_CTPOP_MVT_v8i64_r(RetVT, Op0, Op0IsKill); 771 default: return 0; 772 } 773} 774 775// FastEmit functions for ISD::CTTZ. 776 777unsigned fastEmit_ISD_CTTZ_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 778 if (RetVT.SimpleTy != MVT::i16) 779 return 0; 780 if ((Subtarget->hasBMI())) { 781 return fastEmitInst_r(X86::TZCNT16rr, &X86::GR16RegClass, Op0, Op0IsKill); 782 } 783 return 0; 784} 785 786unsigned fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 787 if (RetVT.SimpleTy != MVT::i32) 788 return 0; 789 if ((Subtarget->hasBMI())) { 790 return fastEmitInst_r(X86::TZCNT32rr, &X86::GR32RegClass, Op0, Op0IsKill); 791 } 792 return 0; 793} 794 795unsigned fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 796 if (RetVT.SimpleTy != MVT::i64) 797 return 0; 798 if ((Subtarget->hasBMI())) { 799 return fastEmitInst_r(X86::TZCNT64rr, &X86::GR64RegClass, Op0, Op0IsKill); 800 } 801 return 0; 802} 803 804unsigned fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 805 switch (VT.SimpleTy) { 806 case MVT::i16: return fastEmit_ISD_CTTZ_MVT_i16_r(RetVT, Op0, Op0IsKill); 807 case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0, Op0IsKill); 808 case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0, Op0IsKill); 809 default: return 0; 810 } 811} 812 813// FastEmit functions for ISD::CTTZ_ZERO_UNDEF. 814 815unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 816 if (RetVT.SimpleTy != MVT::i16) 817 return 0; 818 return fastEmitInst_r(X86::BSF16rr, &X86::GR16RegClass, Op0, Op0IsKill); 819} 820 821unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 822 if (RetVT.SimpleTy != MVT::i32) 823 return 0; 824 return fastEmitInst_r(X86::BSF32rr, &X86::GR32RegClass, Op0, Op0IsKill); 825} 826 827unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 828 if (RetVT.SimpleTy != MVT::i64) 829 return 0; 830 return fastEmitInst_r(X86::BSF64rr, &X86::GR64RegClass, Op0, Op0IsKill); 831} 832 833unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 834 switch (VT.SimpleTy) { 835 case MVT::i16: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i16_r(RetVT, Op0, Op0IsKill); 836 case MVT::i32: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i32_r(RetVT, Op0, Op0IsKill); 837 case MVT::i64: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i64_r(RetVT, Op0, Op0IsKill); 838 default: return 0; 839 } 840} 841 842// FastEmit functions for ISD::FABS. 843 844unsigned fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 845 if (RetVT.SimpleTy != MVT::f32) 846 return 0; 847 if ((!Subtarget->hasSSE1())) { 848 return fastEmitInst_r(X86::ABS_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill); 849 } 850 return 0; 851} 852 853unsigned fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 854 if (RetVT.SimpleTy != MVT::f64) 855 return 0; 856 if ((!Subtarget->hasSSE2())) { 857 return fastEmitInst_r(X86::ABS_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill); 858 } 859 return 0; 860} 861 862unsigned fastEmit_ISD_FABS_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 863 if (RetVT.SimpleTy != MVT::f80) 864 return 0; 865 return fastEmitInst_r(X86::ABS_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill); 866} 867 868unsigned fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 869 switch (VT.SimpleTy) { 870 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0, Op0IsKill); 871 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0, Op0IsKill); 872 case MVT::f80: return fastEmit_ISD_FABS_MVT_f80_r(RetVT, Op0, Op0IsKill); 873 default: return 0; 874 } 875} 876 877// FastEmit functions for ISD::FNEG. 878 879unsigned fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 880 if (RetVT.SimpleTy != MVT::f32) 881 return 0; 882 if ((!Subtarget->hasSSE1())) { 883 return fastEmitInst_r(X86::CHS_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill); 884 } 885 return 0; 886} 887 888unsigned fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 889 if (RetVT.SimpleTy != MVT::f64) 890 return 0; 891 if ((!Subtarget->hasSSE2())) { 892 return fastEmitInst_r(X86::CHS_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill); 893 } 894 return 0; 895} 896 897unsigned fastEmit_ISD_FNEG_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 898 if (RetVT.SimpleTy != MVT::f80) 899 return 0; 900 return fastEmitInst_r(X86::CHS_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill); 901} 902 903unsigned fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 904 switch (VT.SimpleTy) { 905 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0, Op0IsKill); 906 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0, Op0IsKill); 907 case MVT::f80: return fastEmit_ISD_FNEG_MVT_f80_r(RetVT, Op0, Op0IsKill); 908 default: return 0; 909 } 910} 911 912// FastEmit functions for ISD::FP_EXTEND. 913 914unsigned fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 915 if (RetVT.SimpleTy != MVT::f64) 916 return 0; 917 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 918 return fastEmitInst_r(X86::CVTSS2SDrr, &X86::FR64RegClass, Op0, Op0IsKill); 919 } 920 return 0; 921} 922 923unsigned fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 924 if (RetVT.SimpleTy != MVT::v4f64) 925 return 0; 926 if ((Subtarget->hasVLX())) { 927 return fastEmitInst_r(X86::VCVTPS2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 928 } 929 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 930 return fastEmitInst_r(X86::VCVTPS2PDYrr, &X86::VR256RegClass, Op0, Op0IsKill); 931 } 932 return 0; 933} 934 935unsigned fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 936 if (RetVT.SimpleTy != MVT::v8f64) 937 return 0; 938 if ((Subtarget->hasAVX512())) { 939 return fastEmitInst_r(X86::VCVTPS2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill); 940 } 941 return 0; 942} 943 944unsigned fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 945 switch (VT.SimpleTy) { 946 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0, Op0IsKill); 947 case MVT::v4f32: return fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(RetVT, Op0, Op0IsKill); 948 case MVT::v8f32: return fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(RetVT, Op0, Op0IsKill); 949 default: return 0; 950 } 951} 952 953// FastEmit functions for ISD::FP_ROUND. 954 955unsigned fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 956 if (RetVT.SimpleTy != MVT::f32) 957 return 0; 958 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 959 return fastEmitInst_r(X86::CVTSD2SSrr, &X86::FR32RegClass, Op0, Op0IsKill); 960 } 961 return 0; 962} 963 964unsigned fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 965 switch (VT.SimpleTy) { 966 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0, Op0IsKill); 967 default: return 0; 968 } 969} 970 971// FastEmit functions for ISD::FP_TO_SINT. 972 973unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) { 974 if ((Subtarget->hasAVX512())) { 975 return fastEmitInst_r(X86::VCVTTSS2SIZrr, &X86::GR32RegClass, Op0, Op0IsKill); 976 } 977 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 978 return fastEmitInst_r(X86::CVTTSS2SIrr, &X86::GR32RegClass, Op0, Op0IsKill); 979 } 980 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 981 return fastEmitInst_r(X86::VCVTTSS2SIrr, &X86::GR32RegClass, Op0, Op0IsKill); 982 } 983 return 0; 984} 985 986unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) { 987 if ((Subtarget->hasAVX512())) { 988 return fastEmitInst_r(X86::VCVTTSS2SI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill); 989 } 990 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 991 return fastEmitInst_r(X86::CVTTSS2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill); 992 } 993 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 994 return fastEmitInst_r(X86::VCVTTSS2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill); 995 } 996 return 0; 997} 998 999unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1000switch (RetVT.SimpleTy) { 1001 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill); 1002 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill); 1003 default: return 0; 1004} 1005} 1006 1007unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) { 1008 if ((Subtarget->hasAVX512())) { 1009 return fastEmitInst_r(X86::VCVTTSD2SIZrr, &X86::GR32RegClass, Op0, Op0IsKill); 1010 } 1011 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 1012 return fastEmitInst_r(X86::CVTTSD2SIrr, &X86::GR32RegClass, Op0, Op0IsKill); 1013 } 1014 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 1015 return fastEmitInst_r(X86::VCVTTSD2SIrr, &X86::GR32RegClass, Op0, Op0IsKill); 1016 } 1017 return 0; 1018} 1019 1020unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) { 1021 if ((Subtarget->hasAVX512())) { 1022 return fastEmitInst_r(X86::VCVTTSD2SI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill); 1023 } 1024 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 1025 return fastEmitInst_r(X86::CVTTSD2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill); 1026 } 1027 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 1028 return fastEmitInst_r(X86::VCVTTSD2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill); 1029 } 1030 return 0; 1031} 1032 1033unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1034switch (RetVT.SimpleTy) { 1035 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill); 1036 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill); 1037 default: return 0; 1038} 1039} 1040 1041unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1042 if (RetVT.SimpleTy != MVT::v4i32) 1043 return 0; 1044 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 1045 return fastEmitInst_r(X86::VCVTTPD2DQYrr, &X86::VR128RegClass, Op0, Op0IsKill); 1046 } 1047 return 0; 1048} 1049 1050unsigned fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 1051 switch (VT.SimpleTy) { 1052 case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0, Op0IsKill); 1053 case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0, Op0IsKill); 1054 case MVT::v4f64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(RetVT, Op0, Op0IsKill); 1055 default: return 0; 1056 } 1057} 1058 1059// FastEmit functions for ISD::FP_TO_UINT. 1060 1061unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) { 1062 if ((Subtarget->hasAVX512())) { 1063 return fastEmitInst_r(X86::VCVTTSS2USIZrr, &X86::GR32RegClass, Op0, Op0IsKill); 1064 } 1065 return 0; 1066} 1067 1068unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) { 1069 if ((Subtarget->hasAVX512())) { 1070 return fastEmitInst_r(X86::VCVTTSS2USI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill); 1071 } 1072 return 0; 1073} 1074 1075unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1076switch (RetVT.SimpleTy) { 1077 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill); 1078 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill); 1079 default: return 0; 1080} 1081} 1082 1083unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) { 1084 if ((Subtarget->hasAVX512())) { 1085 return fastEmitInst_r(X86::VCVTTSD2USIZrr, &X86::GR32RegClass, Op0, Op0IsKill); 1086 } 1087 return 0; 1088} 1089 1090unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) { 1091 if ((Subtarget->hasAVX512())) { 1092 return fastEmitInst_r(X86::VCVTTSD2USI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill); 1093 } 1094 return 0; 1095} 1096 1097unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1098switch (RetVT.SimpleTy) { 1099 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill); 1100 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill); 1101 default: return 0; 1102} 1103} 1104 1105unsigned fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 1106 switch (VT.SimpleTy) { 1107 case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0, Op0IsKill); 1108 case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0, Op0IsKill); 1109 default: return 0; 1110 } 1111} 1112 1113// FastEmit functions for ISD::FSQRT. 1114 1115unsigned fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1116 if (RetVT.SimpleTy != MVT::f32) 1117 return 0; 1118 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 1119 return fastEmitInst_r(X86::SQRTSSr, &X86::FR32RegClass, Op0, Op0IsKill); 1120 } 1121 if ((!Subtarget->hasSSE1())) { 1122 return fastEmitInst_r(X86::SQRT_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill); 1123 } 1124 return 0; 1125} 1126 1127unsigned fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1128 if (RetVT.SimpleTy != MVT::f64) 1129 return 0; 1130 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 1131 return fastEmitInst_r(X86::SQRTSDr, &X86::FR64RegClass, Op0, Op0IsKill); 1132 } 1133 if ((!Subtarget->hasSSE2())) { 1134 return fastEmitInst_r(X86::SQRT_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill); 1135 } 1136 return 0; 1137} 1138 1139unsigned fastEmit_ISD_FSQRT_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1140 if (RetVT.SimpleTy != MVT::f80) 1141 return 0; 1142 return fastEmitInst_r(X86::SQRT_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill); 1143} 1144 1145unsigned fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1146 if (RetVT.SimpleTy != MVT::v4f32) 1147 return 0; 1148 if ((Subtarget->hasVLX())) { 1149 return fastEmitInst_r(X86::VSQRTPSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill); 1150 } 1151 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 1152 return fastEmitInst_r(X86::SQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill); 1153 } 1154 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 1155 return fastEmitInst_r(X86::VSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill); 1156 } 1157 return 0; 1158} 1159 1160unsigned fastEmit_ISD_FSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1161 if (RetVT.SimpleTy != MVT::v8f32) 1162 return 0; 1163 if ((Subtarget->hasVLX())) { 1164 return fastEmitInst_r(X86::VSQRTPSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill); 1165 } 1166 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 1167 return fastEmitInst_r(X86::VSQRTPSYr, &X86::VR256RegClass, Op0, Op0IsKill); 1168 } 1169 return 0; 1170} 1171 1172unsigned fastEmit_ISD_FSQRT_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1173 if (RetVT.SimpleTy != MVT::v16f32) 1174 return 0; 1175 if ((Subtarget->hasAVX512())) { 1176 return fastEmitInst_r(X86::VSQRTPSZr, &X86::VR512RegClass, Op0, Op0IsKill); 1177 } 1178 return 0; 1179} 1180 1181unsigned fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1182 if (RetVT.SimpleTy != MVT::v2f64) 1183 return 0; 1184 if ((Subtarget->hasVLX())) { 1185 return fastEmitInst_r(X86::VSQRTPDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill); 1186 } 1187 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 1188 return fastEmitInst_r(X86::SQRTPDr, &X86::VR128RegClass, Op0, Op0IsKill); 1189 } 1190 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 1191 return fastEmitInst_r(X86::VSQRTPDr, &X86::VR128RegClass, Op0, Op0IsKill); 1192 } 1193 return 0; 1194} 1195 1196unsigned fastEmit_ISD_FSQRT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1197 if (RetVT.SimpleTy != MVT::v4f64) 1198 return 0; 1199 if ((Subtarget->hasVLX())) { 1200 return fastEmitInst_r(X86::VSQRTPDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill); 1201 } 1202 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 1203 return fastEmitInst_r(X86::VSQRTPDYr, &X86::VR256RegClass, Op0, Op0IsKill); 1204 } 1205 return 0; 1206} 1207 1208unsigned fastEmit_ISD_FSQRT_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1209 if (RetVT.SimpleTy != MVT::v8f64) 1210 return 0; 1211 if ((Subtarget->hasAVX512())) { 1212 return fastEmitInst_r(X86::VSQRTPDZr, &X86::VR512RegClass, Op0, Op0IsKill); 1213 } 1214 return 0; 1215} 1216 1217unsigned fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 1218 switch (VT.SimpleTy) { 1219 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0, Op0IsKill); 1220 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0, Op0IsKill); 1221 case MVT::f80: return fastEmit_ISD_FSQRT_MVT_f80_r(RetVT, Op0, Op0IsKill); 1222 case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0, Op0IsKill); 1223 case MVT::v8f32: return fastEmit_ISD_FSQRT_MVT_v8f32_r(RetVT, Op0, Op0IsKill); 1224 case MVT::v16f32: return fastEmit_ISD_FSQRT_MVT_v16f32_r(RetVT, Op0, Op0IsKill); 1225 case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0, Op0IsKill); 1226 case MVT::v4f64: return fastEmit_ISD_FSQRT_MVT_v4f64_r(RetVT, Op0, Op0IsKill); 1227 case MVT::v8f64: return fastEmit_ISD_FSQRT_MVT_v8f64_r(RetVT, Op0, Op0IsKill); 1228 default: return 0; 1229 } 1230} 1231 1232// FastEmit functions for ISD::SCALAR_TO_VECTOR. 1233 1234unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 1235 if ((Subtarget->hasAVX512())) { 1236 return fastEmitInst_r(X86::VMOVDI2PDIZrr, &X86::VR128XRegClass, Op0, Op0IsKill); 1237 } 1238 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 1239 return fastEmitInst_r(X86::MOVDI2PDIrr, &X86::VR128RegClass, Op0, Op0IsKill); 1240 } 1241 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 1242 return fastEmitInst_r(X86::VMOVDI2PDIrr, &X86::VR128RegClass, Op0, Op0IsKill); 1243 } 1244 return 0; 1245} 1246 1247unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_x86mmx_r(unsigned Op0, bool Op0IsKill) { 1248 if ((Subtarget->hasMMX())) { 1249 return fastEmitInst_r(X86::MMX_MOVD64rr, &X86::VR64RegClass, Op0, Op0IsKill); 1250 } 1251 return 0; 1252} 1253 1254unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1255switch (RetVT.SimpleTy) { 1256 case MVT::v4i32: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_v4i32_r(Op0, Op0IsKill); 1257 case MVT::x86mmx: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_x86mmx_r(Op0, Op0IsKill); 1258 default: return 0; 1259} 1260} 1261 1262unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1263 if (RetVT.SimpleTy != MVT::v2i64) 1264 return 0; 1265 if ((Subtarget->hasAVX512())) { 1266 return fastEmitInst_r(X86::VMOV64toPQIZrr, &X86::VR128XRegClass, Op0, Op0IsKill); 1267 } 1268 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 1269 return fastEmitInst_r(X86::MOV64toPQIrr, &X86::VR128RegClass, Op0, Op0IsKill); 1270 } 1271 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 1272 return fastEmitInst_r(X86::VMOV64toPQIrr, &X86::VR128RegClass, Op0, Op0IsKill); 1273 } 1274 return 0; 1275} 1276 1277unsigned fastEmit_ISD_SCALAR_TO_VECTOR_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 1278 switch (VT.SimpleTy) { 1279 case MVT::i32: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(RetVT, Op0, Op0IsKill); 1280 case MVT::i64: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(RetVT, Op0, Op0IsKill); 1281 default: return 0; 1282 } 1283} 1284 1285// FastEmit functions for ISD::SIGN_EXTEND. 1286 1287unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(unsigned Op0, bool Op0IsKill) { 1288 return fastEmitInst_r(X86::MOVSX32rr8, &X86::GR32RegClass, Op0, Op0IsKill); 1289} 1290 1291unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(unsigned Op0, bool Op0IsKill) { 1292 return fastEmitInst_r(X86::MOVSX64rr8, &X86::GR64RegClass, Op0, Op0IsKill); 1293} 1294 1295unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1296switch (RetVT.SimpleTy) { 1297 case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(Op0, Op0IsKill); 1298 case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(Op0, Op0IsKill); 1299 default: return 0; 1300} 1301} 1302 1303unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(unsigned Op0, bool Op0IsKill) { 1304 return fastEmitInst_r(X86::MOVSX32rr16, &X86::GR32RegClass, Op0, Op0IsKill); 1305} 1306 1307unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(unsigned Op0, bool Op0IsKill) { 1308 return fastEmitInst_r(X86::MOVSX64rr16, &X86::GR64RegClass, Op0, Op0IsKill); 1309} 1310 1311unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1312switch (RetVT.SimpleTy) { 1313 case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(Op0, Op0IsKill); 1314 case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(Op0, Op0IsKill); 1315 default: return 0; 1316} 1317} 1318 1319unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1320 if (RetVT.SimpleTy != MVT::i64) 1321 return 0; 1322 if ((Subtarget->is64Bit())) { 1323 return fastEmitInst_r(X86::MOVSX64rr32, &X86::GR64RegClass, Op0, Op0IsKill); 1324 } 1325 return 0; 1326} 1327 1328unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1329 if (RetVT.SimpleTy != MVT::v2i64) 1330 return 0; 1331 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 1332 return fastEmitInst_r(X86::VPMOVM2QZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 1333 } 1334 return 0; 1335} 1336 1337unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 1338 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 1339 return fastEmitInst_r(X86::VPMOVM2DZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 1340 } 1341 return 0; 1342} 1343 1344unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) { 1345 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 1346 return fastEmitInst_r(X86::VPMOVM2QZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 1347 } 1348 return 0; 1349} 1350 1351unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1352switch (RetVT.SimpleTy) { 1353 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(Op0, Op0IsKill); 1354 case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(Op0, Op0IsKill); 1355 default: return 0; 1356} 1357} 1358 1359unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) { 1360 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 1361 return fastEmitInst_r(X86::VPMOVM2WZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 1362 } 1363 return 0; 1364} 1365 1366unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) { 1367 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 1368 return fastEmitInst_r(X86::VPMOVM2DZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 1369 } 1370 return 0; 1371} 1372 1373unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) { 1374 if ((Subtarget->hasDQI())) { 1375 return fastEmitInst_r(X86::VPMOVM2QZrr, &X86::VR512RegClass, Op0, Op0IsKill); 1376 } 1377 return 0; 1378} 1379 1380unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1381switch (RetVT.SimpleTy) { 1382 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(Op0, Op0IsKill); 1383 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(Op0, Op0IsKill); 1384 case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(Op0, Op0IsKill); 1385 default: return 0; 1386} 1387} 1388 1389unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) { 1390 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 1391 return fastEmitInst_r(X86::VPMOVM2BZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 1392 } 1393 return 0; 1394} 1395 1396unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) { 1397 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 1398 return fastEmitInst_r(X86::VPMOVM2WZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 1399 } 1400 return 0; 1401} 1402 1403unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) { 1404 if ((Subtarget->hasDQI())) { 1405 return fastEmitInst_r(X86::VPMOVM2DZrr, &X86::VR512RegClass, Op0, Op0IsKill); 1406 } 1407 return 0; 1408} 1409 1410unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1411switch (RetVT.SimpleTy) { 1412 case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(Op0, Op0IsKill); 1413 case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(Op0, Op0IsKill); 1414 case MVT::v16i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(Op0, Op0IsKill); 1415 default: return 0; 1416} 1417} 1418 1419unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(unsigned Op0, bool Op0IsKill) { 1420 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 1421 return fastEmitInst_r(X86::VPMOVM2BZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 1422 } 1423 return 0; 1424} 1425 1426unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(unsigned Op0, bool Op0IsKill) { 1427 if ((Subtarget->hasBWI())) { 1428 return fastEmitInst_r(X86::VPMOVM2WZrr, &X86::VR512RegClass, Op0, Op0IsKill); 1429 } 1430 return 0; 1431} 1432 1433unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1434switch (RetVT.SimpleTy) { 1435 case MVT::v32i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(Op0, Op0IsKill); 1436 case MVT::v32i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(Op0, Op0IsKill); 1437 default: return 0; 1438} 1439} 1440 1441unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1442 if (RetVT.SimpleTy != MVT::v64i8) 1443 return 0; 1444 if ((Subtarget->hasBWI())) { 1445 return fastEmitInst_r(X86::VPMOVM2BZrr, &X86::VR512RegClass, Op0, Op0IsKill); 1446 } 1447 return 0; 1448} 1449 1450unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) { 1451 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 1452 return fastEmitInst_r(X86::VPMOVSXBWYrr, &X86::VR256RegClass, Op0, Op0IsKill); 1453 } 1454 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 1455 return fastEmitInst_r(X86::VPMOVSXBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 1456 } 1457 return 0; 1458} 1459 1460unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) { 1461 if ((Subtarget->hasAVX512())) { 1462 return fastEmitInst_r(X86::VPMOVSXBDZrr, &X86::VR512RegClass, Op0, Op0IsKill); 1463 } 1464 return 0; 1465} 1466 1467unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1468switch (RetVT.SimpleTy) { 1469 case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i16_r(Op0, Op0IsKill); 1470 case MVT::v16i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i32_r(Op0, Op0IsKill); 1471 default: return 0; 1472} 1473} 1474 1475unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1476 if (RetVT.SimpleTy != MVT::v32i16) 1477 return 0; 1478 if ((Subtarget->hasBWI())) { 1479 return fastEmitInst_r(X86::VPMOVSXBWZrr, &X86::VR512RegClass, Op0, Op0IsKill); 1480 } 1481 return 0; 1482} 1483 1484unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) { 1485 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 1486 return fastEmitInst_r(X86::VPMOVSXWDYrr, &X86::VR256RegClass, Op0, Op0IsKill); 1487 } 1488 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 1489 return fastEmitInst_r(X86::VPMOVSXWDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 1490 } 1491 return 0; 1492} 1493 1494unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) { 1495 if ((Subtarget->hasAVX512())) { 1496 return fastEmitInst_r(X86::VPMOVSXWQZrr, &X86::VR512RegClass, Op0, Op0IsKill); 1497 } 1498 return 0; 1499} 1500 1501unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1502switch (RetVT.SimpleTy) { 1503 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i32_r(Op0, Op0IsKill); 1504 case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i64_r(Op0, Op0IsKill); 1505 default: return 0; 1506} 1507} 1508 1509unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1510 if (RetVT.SimpleTy != MVT::v16i32) 1511 return 0; 1512 if ((Subtarget->hasAVX512())) { 1513 return fastEmitInst_r(X86::VPMOVSXWDZrr, &X86::VR512RegClass, Op0, Op0IsKill); 1514 } 1515 return 0; 1516} 1517 1518unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1519 if (RetVT.SimpleTy != MVT::v4i64) 1520 return 0; 1521 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 1522 return fastEmitInst_r(X86::VPMOVSXDQYrr, &X86::VR256RegClass, Op0, Op0IsKill); 1523 } 1524 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 1525 return fastEmitInst_r(X86::VPMOVSXDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 1526 } 1527 return 0; 1528} 1529 1530unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1531 if (RetVT.SimpleTy != MVT::v8i64) 1532 return 0; 1533 if ((Subtarget->hasAVX512())) { 1534 return fastEmitInst_r(X86::VPMOVSXDQZrr, &X86::VR512RegClass, Op0, Op0IsKill); 1535 } 1536 return 0; 1537} 1538 1539unsigned fastEmit_ISD_SIGN_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 1540 switch (VT.SimpleTy) { 1541 case MVT::i8: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(RetVT, Op0, Op0IsKill); 1542 case MVT::i16: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(RetVT, Op0, Op0IsKill); 1543 case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(RetVT, Op0, Op0IsKill); 1544 case MVT::v2i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(RetVT, Op0, Op0IsKill); 1545 case MVT::v4i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(RetVT, Op0, Op0IsKill); 1546 case MVT::v8i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(RetVT, Op0, Op0IsKill); 1547 case MVT::v16i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(RetVT, Op0, Op0IsKill); 1548 case MVT::v32i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(RetVT, Op0, Op0IsKill); 1549 case MVT::v64i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(RetVT, Op0, Op0IsKill); 1550 case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_r(RetVT, Op0, Op0IsKill); 1551 case MVT::v32i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i8_r(RetVT, Op0, Op0IsKill); 1552 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_r(RetVT, Op0, Op0IsKill); 1553 case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i16_r(RetVT, Op0, Op0IsKill); 1554 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i32_r(RetVT, Op0, Op0IsKill); 1555 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i32_r(RetVT, Op0, Op0IsKill); 1556 default: return 0; 1557 } 1558} 1559 1560// FastEmit functions for ISD::SIGN_EXTEND_VECTOR_INREG. 1561 1562unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) { 1563 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 1564 return fastEmitInst_r(X86::PMOVSXBWrr, &X86::VR128RegClass, Op0, Op0IsKill); 1565 } 1566 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 1567 return fastEmitInst_r(X86::VPMOVSXBWrr, &X86::VR128RegClass, Op0, Op0IsKill); 1568 } 1569 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 1570 return fastEmitInst_r(X86::VPMOVSXBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 1571 } 1572 return 0; 1573} 1574 1575unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 1576 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 1577 return fastEmitInst_r(X86::PMOVSXBDrr, &X86::VR128RegClass, Op0, Op0IsKill); 1578 } 1579 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 1580 return fastEmitInst_r(X86::VPMOVSXBDrr, &X86::VR128RegClass, Op0, Op0IsKill); 1581 } 1582 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 1583 return fastEmitInst_r(X86::VPMOVSXBDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 1584 } 1585 return 0; 1586} 1587 1588unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) { 1589 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 1590 return fastEmitInst_r(X86::VPMOVSXBDYrr, &X86::VR256RegClass, Op0, Op0IsKill); 1591 } 1592 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 1593 return fastEmitInst_r(X86::VPMOVSXBDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 1594 } 1595 return 0; 1596} 1597 1598unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) { 1599 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 1600 return fastEmitInst_r(X86::PMOVSXBQrr, &X86::VR128RegClass, Op0, Op0IsKill); 1601 } 1602 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 1603 return fastEmitInst_r(X86::VPMOVSXBQrr, &X86::VR128RegClass, Op0, Op0IsKill); 1604 } 1605 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 1606 return fastEmitInst_r(X86::VPMOVSXBQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 1607 } 1608 return 0; 1609} 1610 1611unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) { 1612 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 1613 return fastEmitInst_r(X86::VPMOVSXBQYrr, &X86::VR256RegClass, Op0, Op0IsKill); 1614 } 1615 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 1616 return fastEmitInst_r(X86::VPMOVSXBQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 1617 } 1618 return 0; 1619} 1620 1621unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) { 1622 if ((Subtarget->hasAVX512())) { 1623 return fastEmitInst_r(X86::VPMOVSXBQZrr, &X86::VR512RegClass, Op0, Op0IsKill); 1624 } 1625 return 0; 1626} 1627 1628unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1629switch (RetVT.SimpleTy) { 1630 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0, Op0IsKill); 1631 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0, Op0IsKill); 1632 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Op0, Op0IsKill); 1633 case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0, Op0IsKill); 1634 case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Op0, Op0IsKill); 1635 case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Op0, Op0IsKill); 1636 default: return 0; 1637} 1638} 1639 1640unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 1641 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 1642 return fastEmitInst_r(X86::PMOVSXWDrr, &X86::VR128RegClass, Op0, Op0IsKill); 1643 } 1644 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 1645 return fastEmitInst_r(X86::VPMOVSXWDrr, &X86::VR128RegClass, Op0, Op0IsKill); 1646 } 1647 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 1648 return fastEmitInst_r(X86::VPMOVSXWDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 1649 } 1650 return 0; 1651} 1652 1653unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) { 1654 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 1655 return fastEmitInst_r(X86::PMOVSXWQrr, &X86::VR128RegClass, Op0, Op0IsKill); 1656 } 1657 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 1658 return fastEmitInst_r(X86::VPMOVSXWQrr, &X86::VR128RegClass, Op0, Op0IsKill); 1659 } 1660 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 1661 return fastEmitInst_r(X86::VPMOVSXWQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 1662 } 1663 return 0; 1664} 1665 1666unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) { 1667 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 1668 return fastEmitInst_r(X86::VPMOVSXWQYrr, &X86::VR256RegClass, Op0, Op0IsKill); 1669 } 1670 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 1671 return fastEmitInst_r(X86::VPMOVSXWQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 1672 } 1673 return 0; 1674} 1675 1676unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1677switch (RetVT.SimpleTy) { 1678 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0, Op0IsKill); 1679 case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0, Op0IsKill); 1680 case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Op0, Op0IsKill); 1681 default: return 0; 1682} 1683} 1684 1685unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1686 if (RetVT.SimpleTy != MVT::v2i64) 1687 return 0; 1688 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 1689 return fastEmitInst_r(X86::PMOVSXDQrr, &X86::VR128RegClass, Op0, Op0IsKill); 1690 } 1691 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 1692 return fastEmitInst_r(X86::VPMOVSXDQrr, &X86::VR128RegClass, Op0, Op0IsKill); 1693 } 1694 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 1695 return fastEmitInst_r(X86::VPMOVSXDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 1696 } 1697 return 0; 1698} 1699 1700unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 1701 switch (VT.SimpleTy) { 1702 case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0, Op0IsKill); 1703 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0, Op0IsKill); 1704 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0, Op0IsKill); 1705 default: return 0; 1706 } 1707} 1708 1709// FastEmit functions for ISD::SINT_TO_FP. 1710 1711unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0, bool Op0IsKill) { 1712 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 1713 return fastEmitInst_r(X86::CVTSI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill); 1714 } 1715 return 0; 1716} 1717 1718unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0, bool Op0IsKill) { 1719 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 1720 return fastEmitInst_r(X86::CVTSI2SDrr, &X86::FR64RegClass, Op0, Op0IsKill); 1721 } 1722 return 0; 1723} 1724 1725unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1726switch (RetVT.SimpleTy) { 1727 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0, Op0IsKill); 1728 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0, Op0IsKill); 1729 default: return 0; 1730} 1731} 1732 1733unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0, bool Op0IsKill) { 1734 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 1735 return fastEmitInst_r(X86::CVTSI642SSrr, &X86::FR32RegClass, Op0, Op0IsKill); 1736 } 1737 return 0; 1738} 1739 1740unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0, bool Op0IsKill) { 1741 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 1742 return fastEmitInst_r(X86::CVTSI642SDrr, &X86::FR64RegClass, Op0, Op0IsKill); 1743 } 1744 return 0; 1745} 1746 1747unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1748switch (RetVT.SimpleTy) { 1749 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0, Op0IsKill); 1750 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0, Op0IsKill); 1751 default: return 0; 1752} 1753} 1754 1755unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) { 1756 if ((Subtarget->hasVLX())) { 1757 return fastEmitInst_r(X86::VCVTDQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 1758 } 1759 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 1760 return fastEmitInst_r(X86::CVTDQ2PSrr, &X86::VR128RegClass, Op0, Op0IsKill); 1761 } 1762 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 1763 return fastEmitInst_r(X86::VCVTDQ2PSrr, &X86::VR128RegClass, Op0, Op0IsKill); 1764 } 1765 return 0; 1766} 1767 1768unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) { 1769 if ((Subtarget->hasVLX())) { 1770 return fastEmitInst_r(X86::VCVTDQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 1771 } 1772 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 1773 return fastEmitInst_r(X86::VCVTDQ2PDYrr, &X86::VR256RegClass, Op0, Op0IsKill); 1774 } 1775 return 0; 1776} 1777 1778unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1779switch (RetVT.SimpleTy) { 1780 case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0, Op0IsKill); 1781 case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0, Op0IsKill); 1782 default: return 0; 1783} 1784} 1785 1786unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) { 1787 if ((Subtarget->hasVLX())) { 1788 return fastEmitInst_r(X86::VCVTDQ2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 1789 } 1790 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 1791 return fastEmitInst_r(X86::VCVTDQ2PSYrr, &X86::VR256RegClass, Op0, Op0IsKill); 1792 } 1793 return 0; 1794} 1795 1796unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) { 1797 if ((Subtarget->hasAVX512())) { 1798 return fastEmitInst_r(X86::VCVTDQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill); 1799 } 1800 return 0; 1801} 1802 1803unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1804switch (RetVT.SimpleTy) { 1805 case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0, Op0IsKill); 1806 case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0, Op0IsKill); 1807 default: return 0; 1808} 1809} 1810 1811unsigned fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1812 if (RetVT.SimpleTy != MVT::v16f32) 1813 return 0; 1814 if ((Subtarget->hasAVX512())) { 1815 return fastEmitInst_r(X86::VCVTDQ2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill); 1816 } 1817 return 0; 1818} 1819 1820unsigned fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1821 if (RetVT.SimpleTy != MVT::v2f64) 1822 return 0; 1823 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 1824 return fastEmitInst_r(X86::VCVTQQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 1825 } 1826 return 0; 1827} 1828 1829unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) { 1830 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 1831 return fastEmitInst_r(X86::VCVTQQ2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 1832 } 1833 return 0; 1834} 1835 1836unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) { 1837 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 1838 return fastEmitInst_r(X86::VCVTQQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 1839 } 1840 return 0; 1841} 1842 1843unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1844switch (RetVT.SimpleTy) { 1845 case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0, Op0IsKill); 1846 case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0, Op0IsKill); 1847 default: return 0; 1848} 1849} 1850 1851unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) { 1852 if ((Subtarget->hasDQI())) { 1853 return fastEmitInst_r(X86::VCVTQQ2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill); 1854 } 1855 return 0; 1856} 1857 1858unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) { 1859 if ((Subtarget->hasDQI())) { 1860 return fastEmitInst_r(X86::VCVTQQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill); 1861 } 1862 return 0; 1863} 1864 1865unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1866switch (RetVT.SimpleTy) { 1867 case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0, Op0IsKill); 1868 case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0, Op0IsKill); 1869 default: return 0; 1870} 1871} 1872 1873unsigned fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 1874 switch (VT.SimpleTy) { 1875 case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0, Op0IsKill); 1876 case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0, Op0IsKill); 1877 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill); 1878 case MVT::v8i32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(RetVT, Op0, Op0IsKill); 1879 case MVT::v16i32: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(RetVT, Op0, Op0IsKill); 1880 case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0, Op0IsKill); 1881 case MVT::v4i64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(RetVT, Op0, Op0IsKill); 1882 case MVT::v8i64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(RetVT, Op0, Op0IsKill); 1883 default: return 0; 1884 } 1885} 1886 1887// FastEmit functions for ISD::STRICT_FP_EXTEND. 1888 1889unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1890 if (RetVT.SimpleTy != MVT::f64) 1891 return 0; 1892 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 1893 return fastEmitInst_r(X86::CVTSS2SDrr, &X86::FR64RegClass, Op0, Op0IsKill); 1894 } 1895 return 0; 1896} 1897 1898unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1899 if (RetVT.SimpleTy != MVT::v4f64) 1900 return 0; 1901 if ((Subtarget->hasVLX())) { 1902 return fastEmitInst_r(X86::VCVTPS2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 1903 } 1904 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 1905 return fastEmitInst_r(X86::VCVTPS2PDYrr, &X86::VR256RegClass, Op0, Op0IsKill); 1906 } 1907 return 0; 1908} 1909 1910unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1911 if (RetVT.SimpleTy != MVT::v8f64) 1912 return 0; 1913 if ((Subtarget->hasAVX512())) { 1914 return fastEmitInst_r(X86::VCVTPS2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill); 1915 } 1916 return 0; 1917} 1918 1919unsigned fastEmit_ISD_STRICT_FP_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 1920 switch (VT.SimpleTy) { 1921 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(RetVT, Op0, Op0IsKill); 1922 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f32_r(RetVT, Op0, Op0IsKill); 1923 case MVT::v8f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f32_r(RetVT, Op0, Op0IsKill); 1924 default: return 0; 1925 } 1926} 1927 1928// FastEmit functions for ISD::STRICT_FP_ROUND. 1929 1930unsigned fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1931 if (RetVT.SimpleTy != MVT::f32) 1932 return 0; 1933 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 1934 return fastEmitInst_r(X86::CVTSD2SSrr, &X86::FR32RegClass, Op0, Op0IsKill); 1935 } 1936 return 0; 1937} 1938 1939unsigned fastEmit_ISD_STRICT_FP_ROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 1940 switch (VT.SimpleTy) { 1941 case MVT::f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(RetVT, Op0, Op0IsKill); 1942 default: return 0; 1943 } 1944} 1945 1946// FastEmit functions for ISD::STRICT_FP_TO_SINT. 1947 1948unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) { 1949 if ((Subtarget->hasAVX512())) { 1950 return fastEmitInst_r(X86::VCVTTSS2SIZrr, &X86::GR32RegClass, Op0, Op0IsKill); 1951 } 1952 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 1953 return fastEmitInst_r(X86::CVTTSS2SIrr, &X86::GR32RegClass, Op0, Op0IsKill); 1954 } 1955 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 1956 return fastEmitInst_r(X86::VCVTTSS2SIrr, &X86::GR32RegClass, Op0, Op0IsKill); 1957 } 1958 return 0; 1959} 1960 1961unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) { 1962 if ((Subtarget->hasAVX512())) { 1963 return fastEmitInst_r(X86::VCVTTSS2SI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill); 1964 } 1965 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 1966 return fastEmitInst_r(X86::CVTTSS2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill); 1967 } 1968 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 1969 return fastEmitInst_r(X86::VCVTTSS2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill); 1970 } 1971 return 0; 1972} 1973 1974unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 1975switch (RetVT.SimpleTy) { 1976 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill); 1977 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill); 1978 default: return 0; 1979} 1980} 1981 1982unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) { 1983 if ((Subtarget->hasAVX512())) { 1984 return fastEmitInst_r(X86::VCVTTSD2SIZrr, &X86::GR32RegClass, Op0, Op0IsKill); 1985 } 1986 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 1987 return fastEmitInst_r(X86::CVTTSD2SIrr, &X86::GR32RegClass, Op0, Op0IsKill); 1988 } 1989 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 1990 return fastEmitInst_r(X86::VCVTTSD2SIrr, &X86::GR32RegClass, Op0, Op0IsKill); 1991 } 1992 return 0; 1993} 1994 1995unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) { 1996 if ((Subtarget->hasAVX512())) { 1997 return fastEmitInst_r(X86::VCVTTSD2SI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill); 1998 } 1999 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 2000 return fastEmitInst_r(X86::CVTTSD2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill); 2001 } 2002 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 2003 return fastEmitInst_r(X86::VCVTTSD2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill); 2004 } 2005 return 0; 2006} 2007 2008unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2009switch (RetVT.SimpleTy) { 2010 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill); 2011 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill); 2012 default: return 0; 2013} 2014} 2015 2016unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2017 if (RetVT.SimpleTy != MVT::v4i32) 2018 return 0; 2019 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 2020 return fastEmitInst_r(X86::VCVTTPD2DQYrr, &X86::VR128RegClass, Op0, Op0IsKill); 2021 } 2022 return 0; 2023} 2024 2025unsigned fastEmit_ISD_STRICT_FP_TO_SINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 2026 switch (VT.SimpleTy) { 2027 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(RetVT, Op0, Op0IsKill); 2028 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(RetVT, Op0, Op0IsKill); 2029 case MVT::v4f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f64_r(RetVT, Op0, Op0IsKill); 2030 default: return 0; 2031 } 2032} 2033 2034// FastEmit functions for ISD::STRICT_FP_TO_UINT. 2035 2036unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) { 2037 if ((Subtarget->hasAVX512())) { 2038 return fastEmitInst_r(X86::VCVTTSS2USIZrr, &X86::GR32RegClass, Op0, Op0IsKill); 2039 } 2040 return 0; 2041} 2042 2043unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) { 2044 if ((Subtarget->hasAVX512())) { 2045 return fastEmitInst_r(X86::VCVTTSS2USI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill); 2046 } 2047 return 0; 2048} 2049 2050unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2051switch (RetVT.SimpleTy) { 2052 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill); 2053 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill); 2054 default: return 0; 2055} 2056} 2057 2058unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) { 2059 if ((Subtarget->hasAVX512())) { 2060 return fastEmitInst_r(X86::VCVTTSD2USIZrr, &X86::GR32RegClass, Op0, Op0IsKill); 2061 } 2062 return 0; 2063} 2064 2065unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) { 2066 if ((Subtarget->hasAVX512())) { 2067 return fastEmitInst_r(X86::VCVTTSD2USI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill); 2068 } 2069 return 0; 2070} 2071 2072unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2073switch (RetVT.SimpleTy) { 2074 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill); 2075 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill); 2076 default: return 0; 2077} 2078} 2079 2080unsigned fastEmit_ISD_STRICT_FP_TO_UINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 2081 switch (VT.SimpleTy) { 2082 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(RetVT, Op0, Op0IsKill); 2083 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(RetVT, Op0, Op0IsKill); 2084 default: return 0; 2085 } 2086} 2087 2088// FastEmit functions for ISD::STRICT_FSQRT. 2089 2090unsigned fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2091 if (RetVT.SimpleTy != MVT::f32) 2092 return 0; 2093 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 2094 return fastEmitInst_r(X86::SQRTSSr, &X86::FR32RegClass, Op0, Op0IsKill); 2095 } 2096 if ((!Subtarget->hasSSE1())) { 2097 return fastEmitInst_r(X86::SQRT_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill); 2098 } 2099 return 0; 2100} 2101 2102unsigned fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2103 if (RetVT.SimpleTy != MVT::f64) 2104 return 0; 2105 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 2106 return fastEmitInst_r(X86::SQRTSDr, &X86::FR64RegClass, Op0, Op0IsKill); 2107 } 2108 if ((!Subtarget->hasSSE2())) { 2109 return fastEmitInst_r(X86::SQRT_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill); 2110 } 2111 return 0; 2112} 2113 2114unsigned fastEmit_ISD_STRICT_FSQRT_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2115 if (RetVT.SimpleTy != MVT::f80) 2116 return 0; 2117 return fastEmitInst_r(X86::SQRT_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill); 2118} 2119 2120unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2121 if (RetVT.SimpleTy != MVT::v4f32) 2122 return 0; 2123 if ((Subtarget->hasVLX())) { 2124 return fastEmitInst_r(X86::VSQRTPSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill); 2125 } 2126 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 2127 return fastEmitInst_r(X86::SQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill); 2128 } 2129 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 2130 return fastEmitInst_r(X86::VSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill); 2131 } 2132 return 0; 2133} 2134 2135unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2136 if (RetVT.SimpleTy != MVT::v8f32) 2137 return 0; 2138 if ((Subtarget->hasVLX())) { 2139 return fastEmitInst_r(X86::VSQRTPSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill); 2140 } 2141 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 2142 return fastEmitInst_r(X86::VSQRTPSYr, &X86::VR256RegClass, Op0, Op0IsKill); 2143 } 2144 return 0; 2145} 2146 2147unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2148 if (RetVT.SimpleTy != MVT::v16f32) 2149 return 0; 2150 if ((Subtarget->hasAVX512())) { 2151 return fastEmitInst_r(X86::VSQRTPSZr, &X86::VR512RegClass, Op0, Op0IsKill); 2152 } 2153 return 0; 2154} 2155 2156unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2157 if (RetVT.SimpleTy != MVT::v2f64) 2158 return 0; 2159 if ((Subtarget->hasVLX())) { 2160 return fastEmitInst_r(X86::VSQRTPDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill); 2161 } 2162 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 2163 return fastEmitInst_r(X86::SQRTPDr, &X86::VR128RegClass, Op0, Op0IsKill); 2164 } 2165 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 2166 return fastEmitInst_r(X86::VSQRTPDr, &X86::VR128RegClass, Op0, Op0IsKill); 2167 } 2168 return 0; 2169} 2170 2171unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2172 if (RetVT.SimpleTy != MVT::v4f64) 2173 return 0; 2174 if ((Subtarget->hasVLX())) { 2175 return fastEmitInst_r(X86::VSQRTPDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill); 2176 } 2177 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 2178 return fastEmitInst_r(X86::VSQRTPDYr, &X86::VR256RegClass, Op0, Op0IsKill); 2179 } 2180 return 0; 2181} 2182 2183unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2184 if (RetVT.SimpleTy != MVT::v8f64) 2185 return 0; 2186 if ((Subtarget->hasAVX512())) { 2187 return fastEmitInst_r(X86::VSQRTPDZr, &X86::VR512RegClass, Op0, Op0IsKill); 2188 } 2189 return 0; 2190} 2191 2192unsigned fastEmit_ISD_STRICT_FSQRT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 2193 switch (VT.SimpleTy) { 2194 case MVT::f32: return fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(RetVT, Op0, Op0IsKill); 2195 case MVT::f64: return fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(RetVT, Op0, Op0IsKill); 2196 case MVT::f80: return fastEmit_ISD_STRICT_FSQRT_MVT_f80_r(RetVT, Op0, Op0IsKill); 2197 case MVT::v4f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(RetVT, Op0, Op0IsKill); 2198 case MVT::v8f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f32_r(RetVT, Op0, Op0IsKill); 2199 case MVT::v16f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v16f32_r(RetVT, Op0, Op0IsKill); 2200 case MVT::v2f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(RetVT, Op0, Op0IsKill); 2201 case MVT::v4f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f64_r(RetVT, Op0, Op0IsKill); 2202 case MVT::v8f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f64_r(RetVT, Op0, Op0IsKill); 2203 default: return 0; 2204 } 2205} 2206 2207// FastEmit functions for ISD::STRICT_SINT_TO_FP. 2208 2209unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0, bool Op0IsKill) { 2210 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 2211 return fastEmitInst_r(X86::CVTSI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill); 2212 } 2213 return 0; 2214} 2215 2216unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0, bool Op0IsKill) { 2217 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 2218 return fastEmitInst_r(X86::CVTSI2SDrr, &X86::FR64RegClass, Op0, Op0IsKill); 2219 } 2220 return 0; 2221} 2222 2223unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2224switch (RetVT.SimpleTy) { 2225 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0, Op0IsKill); 2226 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0, Op0IsKill); 2227 default: return 0; 2228} 2229} 2230 2231unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0, bool Op0IsKill) { 2232 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 2233 return fastEmitInst_r(X86::CVTSI642SSrr, &X86::FR32RegClass, Op0, Op0IsKill); 2234 } 2235 return 0; 2236} 2237 2238unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0, bool Op0IsKill) { 2239 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 2240 return fastEmitInst_r(X86::CVTSI642SDrr, &X86::FR64RegClass, Op0, Op0IsKill); 2241 } 2242 return 0; 2243} 2244 2245unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2246switch (RetVT.SimpleTy) { 2247 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0, Op0IsKill); 2248 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0, Op0IsKill); 2249 default: return 0; 2250} 2251} 2252 2253unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) { 2254 if ((Subtarget->hasVLX())) { 2255 return fastEmitInst_r(X86::VCVTDQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 2256 } 2257 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 2258 return fastEmitInst_r(X86::CVTDQ2PSrr, &X86::VR128RegClass, Op0, Op0IsKill); 2259 } 2260 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 2261 return fastEmitInst_r(X86::VCVTDQ2PSrr, &X86::VR128RegClass, Op0, Op0IsKill); 2262 } 2263 return 0; 2264} 2265 2266unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) { 2267 if ((Subtarget->hasVLX())) { 2268 return fastEmitInst_r(X86::VCVTDQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 2269 } 2270 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 2271 return fastEmitInst_r(X86::VCVTDQ2PDYrr, &X86::VR256RegClass, Op0, Op0IsKill); 2272 } 2273 return 0; 2274} 2275 2276unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2277switch (RetVT.SimpleTy) { 2278 case MVT::v4f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0, Op0IsKill); 2279 case MVT::v4f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0, Op0IsKill); 2280 default: return 0; 2281} 2282} 2283 2284unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) { 2285 if ((Subtarget->hasVLX())) { 2286 return fastEmitInst_r(X86::VCVTDQ2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 2287 } 2288 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 2289 return fastEmitInst_r(X86::VCVTDQ2PSYrr, &X86::VR256RegClass, Op0, Op0IsKill); 2290 } 2291 return 0; 2292} 2293 2294unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) { 2295 if ((Subtarget->hasAVX512())) { 2296 return fastEmitInst_r(X86::VCVTDQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill); 2297 } 2298 return 0; 2299} 2300 2301unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2302switch (RetVT.SimpleTy) { 2303 case MVT::v8f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0, Op0IsKill); 2304 case MVT::v8f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0, Op0IsKill); 2305 default: return 0; 2306} 2307} 2308 2309unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2310 if (RetVT.SimpleTy != MVT::v16f32) 2311 return 0; 2312 if ((Subtarget->hasAVX512())) { 2313 return fastEmitInst_r(X86::VCVTDQ2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill); 2314 } 2315 return 0; 2316} 2317 2318unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2319 if (RetVT.SimpleTy != MVT::v2f64) 2320 return 0; 2321 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 2322 return fastEmitInst_r(X86::VCVTQQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 2323 } 2324 return 0; 2325} 2326 2327unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) { 2328 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 2329 return fastEmitInst_r(X86::VCVTQQ2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 2330 } 2331 return 0; 2332} 2333 2334unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) { 2335 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 2336 return fastEmitInst_r(X86::VCVTQQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 2337 } 2338 return 0; 2339} 2340 2341unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2342switch (RetVT.SimpleTy) { 2343 case MVT::v4f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0, Op0IsKill); 2344 case MVT::v4f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0, Op0IsKill); 2345 default: return 0; 2346} 2347} 2348 2349unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) { 2350 if ((Subtarget->hasDQI())) { 2351 return fastEmitInst_r(X86::VCVTQQ2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill); 2352 } 2353 return 0; 2354} 2355 2356unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) { 2357 if ((Subtarget->hasDQI())) { 2358 return fastEmitInst_r(X86::VCVTQQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill); 2359 } 2360 return 0; 2361} 2362 2363unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2364switch (RetVT.SimpleTy) { 2365 case MVT::v8f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0, Op0IsKill); 2366 case MVT::v8f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0, Op0IsKill); 2367 default: return 0; 2368} 2369} 2370 2371unsigned fastEmit_ISD_STRICT_SINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 2372 switch (VT.SimpleTy) { 2373 case MVT::i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(RetVT, Op0, Op0IsKill); 2374 case MVT::i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(RetVT, Op0, Op0IsKill); 2375 case MVT::v4i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill); 2376 case MVT::v8i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_r(RetVT, Op0, Op0IsKill); 2377 case MVT::v16i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_r(RetVT, Op0, Op0IsKill); 2378 case MVT::v2i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0, Op0IsKill); 2379 case MVT::v4i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_r(RetVT, Op0, Op0IsKill); 2380 case MVT::v8i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_r(RetVT, Op0, Op0IsKill); 2381 default: return 0; 2382 } 2383} 2384 2385// FastEmit functions for ISD::STRICT_UINT_TO_FP. 2386 2387unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) { 2388 if ((Subtarget->hasVLX())) { 2389 return fastEmitInst_r(X86::VCVTUDQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 2390 } 2391 return 0; 2392} 2393 2394unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) { 2395 if ((Subtarget->hasVLX())) { 2396 return fastEmitInst_r(X86::VCVTUDQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 2397 } 2398 return 0; 2399} 2400 2401unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2402switch (RetVT.SimpleTy) { 2403 case MVT::v4f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0, Op0IsKill); 2404 case MVT::v4f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0, Op0IsKill); 2405 default: return 0; 2406} 2407} 2408 2409unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) { 2410 if ((Subtarget->hasVLX())) { 2411 return fastEmitInst_r(X86::VCVTUDQ2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 2412 } 2413 return 0; 2414} 2415 2416unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) { 2417 if ((Subtarget->hasAVX512())) { 2418 return fastEmitInst_r(X86::VCVTUDQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill); 2419 } 2420 return 0; 2421} 2422 2423unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2424switch (RetVT.SimpleTy) { 2425 case MVT::v8f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0, Op0IsKill); 2426 case MVT::v8f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0, Op0IsKill); 2427 default: return 0; 2428} 2429} 2430 2431unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2432 if (RetVT.SimpleTy != MVT::v16f32) 2433 return 0; 2434 if ((Subtarget->hasAVX512())) { 2435 return fastEmitInst_r(X86::VCVTUDQ2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill); 2436 } 2437 return 0; 2438} 2439 2440unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2441 if (RetVT.SimpleTy != MVT::v2f64) 2442 return 0; 2443 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 2444 return fastEmitInst_r(X86::VCVTUQQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 2445 } 2446 return 0; 2447} 2448 2449unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) { 2450 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 2451 return fastEmitInst_r(X86::VCVTUQQ2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 2452 } 2453 return 0; 2454} 2455 2456unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) { 2457 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 2458 return fastEmitInst_r(X86::VCVTUQQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 2459 } 2460 return 0; 2461} 2462 2463unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2464switch (RetVT.SimpleTy) { 2465 case MVT::v4f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0, Op0IsKill); 2466 case MVT::v4f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0, Op0IsKill); 2467 default: return 0; 2468} 2469} 2470 2471unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) { 2472 if ((Subtarget->hasDQI())) { 2473 return fastEmitInst_r(X86::VCVTUQQ2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill); 2474 } 2475 return 0; 2476} 2477 2478unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) { 2479 if ((Subtarget->hasDQI())) { 2480 return fastEmitInst_r(X86::VCVTUQQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill); 2481 } 2482 return 0; 2483} 2484 2485unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2486switch (RetVT.SimpleTy) { 2487 case MVT::v8f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0, Op0IsKill); 2488 case MVT::v8f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0, Op0IsKill); 2489 default: return 0; 2490} 2491} 2492 2493unsigned fastEmit_ISD_STRICT_UINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 2494 switch (VT.SimpleTy) { 2495 case MVT::v4i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill); 2496 case MVT::v8i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_r(RetVT, Op0, Op0IsKill); 2497 case MVT::v16i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_r(RetVT, Op0, Op0IsKill); 2498 case MVT::v2i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0, Op0IsKill); 2499 case MVT::v4i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_r(RetVT, Op0, Op0IsKill); 2500 case MVT::v8i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_r(RetVT, Op0, Op0IsKill); 2501 default: return 0; 2502 } 2503} 2504 2505// FastEmit functions for ISD::TRUNCATE. 2506 2507unsigned fastEmit_ISD_TRUNCATE_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2508 if (RetVT.SimpleTy != MVT::i8) 2509 return 0; 2510 if ((Subtarget->is64Bit())) { 2511 return fastEmitInst_extractsubreg(RetVT, Op0, Op0IsKill, X86::sub_8bit); 2512 } 2513 return 0; 2514} 2515 2516unsigned fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(unsigned Op0, bool Op0IsKill) { 2517 if ((Subtarget->is64Bit())) { 2518 return fastEmitInst_extractsubreg(MVT::i8, Op0, Op0IsKill, X86::sub_8bit); 2519 } 2520 return 0; 2521} 2522 2523unsigned fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(unsigned Op0, bool Op0IsKill) { 2524 return fastEmitInst_extractsubreg(MVT::i16, Op0, Op0IsKill, X86::sub_16bit); 2525} 2526 2527unsigned fastEmit_ISD_TRUNCATE_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2528switch (RetVT.SimpleTy) { 2529 case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(Op0, Op0IsKill); 2530 case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(Op0, Op0IsKill); 2531 default: return 0; 2532} 2533} 2534 2535unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(unsigned Op0, bool Op0IsKill) { 2536 return fastEmitInst_extractsubreg(MVT::i8, Op0, Op0IsKill, X86::sub_8bit); 2537} 2538 2539unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(unsigned Op0, bool Op0IsKill) { 2540 return fastEmitInst_extractsubreg(MVT::i16, Op0, Op0IsKill, X86::sub_16bit); 2541} 2542 2543unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(unsigned Op0, bool Op0IsKill) { 2544 return fastEmitInst_extractsubreg(MVT::i32, Op0, Op0IsKill, X86::sub_32bit); 2545} 2546 2547unsigned fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2548switch (RetVT.SimpleTy) { 2549 case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(Op0, Op0IsKill); 2550 case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(Op0, Op0IsKill); 2551 case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(Op0, Op0IsKill); 2552 default: return 0; 2553} 2554} 2555 2556unsigned fastEmit_ISD_TRUNCATE_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2557 if (RetVT.SimpleTy != MVT::v16i8) 2558 return 0; 2559 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 2560 return fastEmitInst_r(X86::VPMOVWBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 2561 } 2562 return 0; 2563} 2564 2565unsigned fastEmit_ISD_TRUNCATE_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2566 if (RetVT.SimpleTy != MVT::v32i8) 2567 return 0; 2568 if ((Subtarget->hasBWI())) { 2569 return fastEmitInst_r(X86::VPMOVWBZrr, &X86::VR256XRegClass, Op0, Op0IsKill); 2570 } 2571 return 0; 2572} 2573 2574unsigned fastEmit_ISD_TRUNCATE_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2575 if (RetVT.SimpleTy != MVT::v8i16) 2576 return 0; 2577 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 2578 return fastEmitInst_r(X86::VPMOVDWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 2579 } 2580 return 0; 2581} 2582 2583unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) { 2584 if ((Subtarget->hasAVX512())) { 2585 return fastEmitInst_r(X86::VPMOVDBZrr, &X86::VR128XRegClass, Op0, Op0IsKill); 2586 } 2587 return 0; 2588} 2589 2590unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) { 2591 if ((Subtarget->hasAVX512())) { 2592 return fastEmitInst_r(X86::VPMOVDWZrr, &X86::VR256XRegClass, Op0, Op0IsKill); 2593 } 2594 return 0; 2595} 2596 2597unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2598switch (RetVT.SimpleTy) { 2599 case MVT::v16i8: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(Op0, Op0IsKill); 2600 case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(Op0, Op0IsKill); 2601 default: return 0; 2602} 2603} 2604 2605unsigned fastEmit_ISD_TRUNCATE_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2606 if (RetVT.SimpleTy != MVT::v4i32) 2607 return 0; 2608 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 2609 return fastEmitInst_r(X86::VPMOVQDZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 2610 } 2611 return 0; 2612} 2613 2614unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) { 2615 if ((Subtarget->hasAVX512())) { 2616 return fastEmitInst_r(X86::VPMOVQWZrr, &X86::VR128XRegClass, Op0, Op0IsKill); 2617 } 2618 return 0; 2619} 2620 2621unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) { 2622 if ((Subtarget->hasAVX512())) { 2623 return fastEmitInst_r(X86::VPMOVQDZrr, &X86::VR256XRegClass, Op0, Op0IsKill); 2624 } 2625 return 0; 2626} 2627 2628unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2629switch (RetVT.SimpleTy) { 2630 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(Op0, Op0IsKill); 2631 case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(Op0, Op0IsKill); 2632 default: return 0; 2633} 2634} 2635 2636unsigned fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 2637 switch (VT.SimpleTy) { 2638 case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i16_r(RetVT, Op0, Op0IsKill); 2639 case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i32_r(RetVT, Op0, Op0IsKill); 2640 case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0, Op0IsKill); 2641 case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i16_r(RetVT, Op0, Op0IsKill); 2642 case MVT::v32i16: return fastEmit_ISD_TRUNCATE_MVT_v32i16_r(RetVT, Op0, Op0IsKill); 2643 case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i32_r(RetVT, Op0, Op0IsKill); 2644 case MVT::v16i32: return fastEmit_ISD_TRUNCATE_MVT_v16i32_r(RetVT, Op0, Op0IsKill); 2645 case MVT::v4i64: return fastEmit_ISD_TRUNCATE_MVT_v4i64_r(RetVT, Op0, Op0IsKill); 2646 case MVT::v8i64: return fastEmit_ISD_TRUNCATE_MVT_v8i64_r(RetVT, Op0, Op0IsKill); 2647 default: return 0; 2648 } 2649} 2650 2651// FastEmit functions for ISD::UINT_TO_FP. 2652 2653unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) { 2654 if ((Subtarget->hasVLX())) { 2655 return fastEmitInst_r(X86::VCVTUDQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 2656 } 2657 return 0; 2658} 2659 2660unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) { 2661 if ((Subtarget->hasVLX())) { 2662 return fastEmitInst_r(X86::VCVTUDQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 2663 } 2664 return 0; 2665} 2666 2667unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2668switch (RetVT.SimpleTy) { 2669 case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0, Op0IsKill); 2670 case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0, Op0IsKill); 2671 default: return 0; 2672} 2673} 2674 2675unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) { 2676 if ((Subtarget->hasVLX())) { 2677 return fastEmitInst_r(X86::VCVTUDQ2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 2678 } 2679 return 0; 2680} 2681 2682unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) { 2683 if ((Subtarget->hasAVX512())) { 2684 return fastEmitInst_r(X86::VCVTUDQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill); 2685 } 2686 return 0; 2687} 2688 2689unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2690switch (RetVT.SimpleTy) { 2691 case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0, Op0IsKill); 2692 case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0, Op0IsKill); 2693 default: return 0; 2694} 2695} 2696 2697unsigned fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2698 if (RetVT.SimpleTy != MVT::v16f32) 2699 return 0; 2700 if ((Subtarget->hasAVX512())) { 2701 return fastEmitInst_r(X86::VCVTUDQ2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill); 2702 } 2703 return 0; 2704} 2705 2706unsigned fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2707 if (RetVT.SimpleTy != MVT::v2f64) 2708 return 0; 2709 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 2710 return fastEmitInst_r(X86::VCVTUQQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 2711 } 2712 return 0; 2713} 2714 2715unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) { 2716 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 2717 return fastEmitInst_r(X86::VCVTUQQ2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 2718 } 2719 return 0; 2720} 2721 2722unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) { 2723 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 2724 return fastEmitInst_r(X86::VCVTUQQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 2725 } 2726 return 0; 2727} 2728 2729unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2730switch (RetVT.SimpleTy) { 2731 case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0, Op0IsKill); 2732 case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0, Op0IsKill); 2733 default: return 0; 2734} 2735} 2736 2737unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) { 2738 if ((Subtarget->hasDQI())) { 2739 return fastEmitInst_r(X86::VCVTUQQ2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill); 2740 } 2741 return 0; 2742} 2743 2744unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) { 2745 if ((Subtarget->hasDQI())) { 2746 return fastEmitInst_r(X86::VCVTUQQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill); 2747 } 2748 return 0; 2749} 2750 2751unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2752switch (RetVT.SimpleTy) { 2753 case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0, Op0IsKill); 2754 case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0, Op0IsKill); 2755 default: return 0; 2756} 2757} 2758 2759unsigned fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 2760 switch (VT.SimpleTy) { 2761 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill); 2762 case MVT::v8i32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(RetVT, Op0, Op0IsKill); 2763 case MVT::v16i32: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(RetVT, Op0, Op0IsKill); 2764 case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0, Op0IsKill); 2765 case MVT::v4i64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(RetVT, Op0, Op0IsKill); 2766 case MVT::v8i64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(RetVT, Op0, Op0IsKill); 2767 default: return 0; 2768 } 2769} 2770 2771// FastEmit functions for ISD::ZERO_EXTEND. 2772 2773unsigned fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2774 if (RetVT.SimpleTy != MVT::i32) 2775 return 0; 2776 return fastEmitInst_r(X86::MOVZX32rr8, &X86::GR32RegClass, Op0, Op0IsKill); 2777} 2778 2779unsigned fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2780 if (RetVT.SimpleTy != MVT::i32) 2781 return 0; 2782 return fastEmitInst_r(X86::MOVZX32rr16, &X86::GR32RegClass, Op0, Op0IsKill); 2783} 2784 2785unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) { 2786 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 2787 return fastEmitInst_r(X86::VPMOVZXBWYrr, &X86::VR256RegClass, Op0, Op0IsKill); 2788 } 2789 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 2790 return fastEmitInst_r(X86::VPMOVZXBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 2791 } 2792 return 0; 2793} 2794 2795unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) { 2796 if ((Subtarget->hasAVX512())) { 2797 return fastEmitInst_r(X86::VPMOVZXBDZrr, &X86::VR512RegClass, Op0, Op0IsKill); 2798 } 2799 return 0; 2800} 2801 2802unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2803switch (RetVT.SimpleTy) { 2804 case MVT::v16i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i16_r(Op0, Op0IsKill); 2805 case MVT::v16i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i32_r(Op0, Op0IsKill); 2806 default: return 0; 2807} 2808} 2809 2810unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2811 if (RetVT.SimpleTy != MVT::v32i16) 2812 return 0; 2813 if ((Subtarget->hasBWI())) { 2814 return fastEmitInst_r(X86::VPMOVZXBWZrr, &X86::VR512RegClass, Op0, Op0IsKill); 2815 } 2816 return 0; 2817} 2818 2819unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) { 2820 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 2821 return fastEmitInst_r(X86::VPMOVZXWDYrr, &X86::VR256RegClass, Op0, Op0IsKill); 2822 } 2823 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 2824 return fastEmitInst_r(X86::VPMOVZXWDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 2825 } 2826 return 0; 2827} 2828 2829unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) { 2830 if ((Subtarget->hasAVX512())) { 2831 return fastEmitInst_r(X86::VPMOVZXWQZrr, &X86::VR512RegClass, Op0, Op0IsKill); 2832 } 2833 return 0; 2834} 2835 2836unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2837switch (RetVT.SimpleTy) { 2838 case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i32_r(Op0, Op0IsKill); 2839 case MVT::v8i64: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i64_r(Op0, Op0IsKill); 2840 default: return 0; 2841} 2842} 2843 2844unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2845 if (RetVT.SimpleTy != MVT::v16i32) 2846 return 0; 2847 if ((Subtarget->hasAVX512())) { 2848 return fastEmitInst_r(X86::VPMOVZXWDZrr, &X86::VR512RegClass, Op0, Op0IsKill); 2849 } 2850 return 0; 2851} 2852 2853unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2854 if (RetVT.SimpleTy != MVT::v4i64) 2855 return 0; 2856 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 2857 return fastEmitInst_r(X86::VPMOVZXDQYrr, &X86::VR256RegClass, Op0, Op0IsKill); 2858 } 2859 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 2860 return fastEmitInst_r(X86::VPMOVZXDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 2861 } 2862 return 0; 2863} 2864 2865unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2866 if (RetVT.SimpleTy != MVT::v8i64) 2867 return 0; 2868 if ((Subtarget->hasAVX512())) { 2869 return fastEmitInst_r(X86::VPMOVZXDQZrr, &X86::VR512RegClass, Op0, Op0IsKill); 2870 } 2871 return 0; 2872} 2873 2874unsigned fastEmit_ISD_ZERO_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 2875 switch (VT.SimpleTy) { 2876 case MVT::i8: return fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(RetVT, Op0, Op0IsKill); 2877 case MVT::i16: return fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(RetVT, Op0, Op0IsKill); 2878 case MVT::v16i8: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_r(RetVT, Op0, Op0IsKill); 2879 case MVT::v32i8: return fastEmit_ISD_ZERO_EXTEND_MVT_v32i8_r(RetVT, Op0, Op0IsKill); 2880 case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_r(RetVT, Op0, Op0IsKill); 2881 case MVT::v16i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i16_r(RetVT, Op0, Op0IsKill); 2882 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v4i32_r(RetVT, Op0, Op0IsKill); 2883 case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i32_r(RetVT, Op0, Op0IsKill); 2884 default: return 0; 2885 } 2886} 2887 2888// FastEmit functions for ISD::ZERO_EXTEND_VECTOR_INREG. 2889 2890unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) { 2891 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 2892 return fastEmitInst_r(X86::PMOVZXBWrr, &X86::VR128RegClass, Op0, Op0IsKill); 2893 } 2894 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 2895 return fastEmitInst_r(X86::VPMOVZXBWrr, &X86::VR128RegClass, Op0, Op0IsKill); 2896 } 2897 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 2898 return fastEmitInst_r(X86::VPMOVZXBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 2899 } 2900 return 0; 2901} 2902 2903unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 2904 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 2905 return fastEmitInst_r(X86::PMOVZXBDrr, &X86::VR128RegClass, Op0, Op0IsKill); 2906 } 2907 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 2908 return fastEmitInst_r(X86::VPMOVZXBDrr, &X86::VR128RegClass, Op0, Op0IsKill); 2909 } 2910 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 2911 return fastEmitInst_r(X86::VPMOVZXBDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 2912 } 2913 return 0; 2914} 2915 2916unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) { 2917 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 2918 return fastEmitInst_r(X86::VPMOVZXBDYrr, &X86::VR256RegClass, Op0, Op0IsKill); 2919 } 2920 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 2921 return fastEmitInst_r(X86::VPMOVZXBDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 2922 } 2923 return 0; 2924} 2925 2926unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) { 2927 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 2928 return fastEmitInst_r(X86::PMOVZXBQrr, &X86::VR128RegClass, Op0, Op0IsKill); 2929 } 2930 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 2931 return fastEmitInst_r(X86::VPMOVZXBQrr, &X86::VR128RegClass, Op0, Op0IsKill); 2932 } 2933 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 2934 return fastEmitInst_r(X86::VPMOVZXBQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 2935 } 2936 return 0; 2937} 2938 2939unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) { 2940 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 2941 return fastEmitInst_r(X86::VPMOVZXBQYrr, &X86::VR256RegClass, Op0, Op0IsKill); 2942 } 2943 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 2944 return fastEmitInst_r(X86::VPMOVZXBQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 2945 } 2946 return 0; 2947} 2948 2949unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) { 2950 if ((Subtarget->hasAVX512())) { 2951 return fastEmitInst_r(X86::VPMOVZXBQZrr, &X86::VR512RegClass, Op0, Op0IsKill); 2952 } 2953 return 0; 2954} 2955 2956unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 2957switch (RetVT.SimpleTy) { 2958 case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0, Op0IsKill); 2959 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0, Op0IsKill); 2960 case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Op0, Op0IsKill); 2961 case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0, Op0IsKill); 2962 case MVT::v4i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Op0, Op0IsKill); 2963 case MVT::v8i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Op0, Op0IsKill); 2964 default: return 0; 2965} 2966} 2967 2968unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 2969 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 2970 return fastEmitInst_r(X86::PMOVZXWDrr, &X86::VR128RegClass, Op0, Op0IsKill); 2971 } 2972 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 2973 return fastEmitInst_r(X86::VPMOVZXWDrr, &X86::VR128RegClass, Op0, Op0IsKill); 2974 } 2975 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 2976 return fastEmitInst_r(X86::VPMOVZXWDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 2977 } 2978 return 0; 2979} 2980 2981unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) { 2982 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 2983 return fastEmitInst_r(X86::PMOVZXWQrr, &X86::VR128RegClass, Op0, Op0IsKill); 2984 } 2985 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 2986 return fastEmitInst_r(X86::VPMOVZXWQrr, &X86::VR128RegClass, Op0, Op0IsKill); 2987 } 2988 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 2989 return fastEmitInst_r(X86::VPMOVZXWQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 2990 } 2991 return 0; 2992} 2993 2994unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) { 2995 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 2996 return fastEmitInst_r(X86::VPMOVZXWQYrr, &X86::VR256RegClass, Op0, Op0IsKill); 2997 } 2998 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 2999 return fastEmitInst_r(X86::VPMOVZXWQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 3000 } 3001 return 0; 3002} 3003 3004unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3005switch (RetVT.SimpleTy) { 3006 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0, Op0IsKill); 3007 case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0, Op0IsKill); 3008 case MVT::v4i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Op0, Op0IsKill); 3009 default: return 0; 3010} 3011} 3012 3013unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3014 if (RetVT.SimpleTy != MVT::v2i64) 3015 return 0; 3016 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 3017 return fastEmitInst_r(X86::PMOVZXDQrr, &X86::VR128RegClass, Op0, Op0IsKill); 3018 } 3019 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 3020 return fastEmitInst_r(X86::VPMOVZXDQrr, &X86::VR128RegClass, Op0, Op0IsKill); 3021 } 3022 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 3023 return fastEmitInst_r(X86::VPMOVZXDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 3024 } 3025 return 0; 3026} 3027 3028unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 3029 switch (VT.SimpleTy) { 3030 case MVT::v16i8: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0, Op0IsKill); 3031 case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0, Op0IsKill); 3032 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0, Op0IsKill); 3033 default: return 0; 3034 } 3035} 3036 3037// FastEmit functions for X86ISD::CALL. 3038 3039unsigned fastEmit_X86ISD_CALL_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3040 if (RetVT.SimpleTy != MVT::isVoid) 3041 return 0; 3042 if ((!Subtarget->is64Bit())) { 3043 return fastEmitInst_r(X86::CALL16r, &X86::GR16RegClass, Op0, Op0IsKill); 3044 } 3045 return 0; 3046} 3047 3048unsigned fastEmit_X86ISD_CALL_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3049 if (RetVT.SimpleTy != MVT::isVoid) 3050 return 0; 3051 if ((!Subtarget->is64Bit()) && (Subtarget->useRetpolineIndirectCalls())) { 3052 return fastEmitInst_r(X86::RETPOLINE_CALL32, &X86::GR32RegClass, Op0, Op0IsKill); 3053 } 3054 if ((!Subtarget->is64Bit()) && (!Subtarget->useRetpolineIndirectCalls())) { 3055 return fastEmitInst_r(X86::CALL32r, &X86::GR32RegClass, Op0, Op0IsKill); 3056 } 3057 return 0; 3058} 3059 3060unsigned fastEmit_X86ISD_CALL_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3061 if (RetVT.SimpleTy != MVT::isVoid) 3062 return 0; 3063 if ((Subtarget->is64Bit()) && (Subtarget->useRetpolineIndirectCalls())) { 3064 return fastEmitInst_r(X86::RETPOLINE_CALL64, &X86::GR64RegClass, Op0, Op0IsKill); 3065 } 3066 if ((Subtarget->is64Bit()) && (!Subtarget->useRetpolineIndirectCalls())) { 3067 return fastEmitInst_r(X86::CALL64r, &X86::GR64RegClass, Op0, Op0IsKill); 3068 } 3069 return 0; 3070} 3071 3072unsigned fastEmit_X86ISD_CALL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 3073 switch (VT.SimpleTy) { 3074 case MVT::i16: return fastEmit_X86ISD_CALL_MVT_i16_r(RetVT, Op0, Op0IsKill); 3075 case MVT::i32: return fastEmit_X86ISD_CALL_MVT_i32_r(RetVT, Op0, Op0IsKill); 3076 case MVT::i64: return fastEmit_X86ISD_CALL_MVT_i64_r(RetVT, Op0, Op0IsKill); 3077 default: return 0; 3078 } 3079} 3080 3081// FastEmit functions for X86ISD::CONFLICT. 3082 3083unsigned fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3084 if (RetVT.SimpleTy != MVT::v4i32) 3085 return 0; 3086 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) { 3087 return fastEmitInst_r(X86::VPCONFLICTDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 3088 } 3089 return 0; 3090} 3091 3092unsigned fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3093 if (RetVT.SimpleTy != MVT::v8i32) 3094 return 0; 3095 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) { 3096 return fastEmitInst_r(X86::VPCONFLICTDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 3097 } 3098 return 0; 3099} 3100 3101unsigned fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3102 if (RetVT.SimpleTy != MVT::v16i32) 3103 return 0; 3104 if ((Subtarget->hasCDI())) { 3105 return fastEmitInst_r(X86::VPCONFLICTDZrr, &X86::VR512RegClass, Op0, Op0IsKill); 3106 } 3107 return 0; 3108} 3109 3110unsigned fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3111 if (RetVT.SimpleTy != MVT::v2i64) 3112 return 0; 3113 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) { 3114 return fastEmitInst_r(X86::VPCONFLICTQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 3115 } 3116 return 0; 3117} 3118 3119unsigned fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3120 if (RetVT.SimpleTy != MVT::v4i64) 3121 return 0; 3122 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) { 3123 return fastEmitInst_r(X86::VPCONFLICTQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 3124 } 3125 return 0; 3126} 3127 3128unsigned fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3129 if (RetVT.SimpleTy != MVT::v8i64) 3130 return 0; 3131 if ((Subtarget->hasCDI())) { 3132 return fastEmitInst_r(X86::VPCONFLICTQZrr, &X86::VR512RegClass, Op0, Op0IsKill); 3133 } 3134 return 0; 3135} 3136 3137unsigned fastEmit_X86ISD_CONFLICT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 3138 switch (VT.SimpleTy) { 3139 case MVT::v4i32: return fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(RetVT, Op0, Op0IsKill); 3140 case MVT::v8i32: return fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(RetVT, Op0, Op0IsKill); 3141 case MVT::v16i32: return fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(RetVT, Op0, Op0IsKill); 3142 case MVT::v2i64: return fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(RetVT, Op0, Op0IsKill); 3143 case MVT::v4i64: return fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(RetVT, Op0, Op0IsKill); 3144 case MVT::v8i64: return fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(RetVT, Op0, Op0IsKill); 3145 default: return 0; 3146 } 3147} 3148 3149// FastEmit functions for X86ISD::CVTNEPS2BF16. 3150 3151unsigned fastEmit_X86ISD_CVTNEPS2BF16_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3152 if (RetVT.SimpleTy != MVT::v8i16) 3153 return 0; 3154 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) { 3155 return fastEmitInst_r(X86::VCVTNEPS2BF16Z128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 3156 } 3157 return 0; 3158} 3159 3160unsigned fastEmit_X86ISD_CVTNEPS2BF16_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3161 if (RetVT.SimpleTy != MVT::v8i16) 3162 return 0; 3163 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) { 3164 return fastEmitInst_r(X86::VCVTNEPS2BF16Z256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 3165 } 3166 return 0; 3167} 3168 3169unsigned fastEmit_X86ISD_CVTNEPS2BF16_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3170 if (RetVT.SimpleTy != MVT::v16i16) 3171 return 0; 3172 if ((Subtarget->hasBF16())) { 3173 return fastEmitInst_r(X86::VCVTNEPS2BF16Zrr, &X86::VR256XRegClass, Op0, Op0IsKill); 3174 } 3175 return 0; 3176} 3177 3178unsigned fastEmit_X86ISD_CVTNEPS2BF16_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 3179 switch (VT.SimpleTy) { 3180 case MVT::v4f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v4f32_r(RetVT, Op0, Op0IsKill); 3181 case MVT::v8f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v8f32_r(RetVT, Op0, Op0IsKill); 3182 case MVT::v16f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v16f32_r(RetVT, Op0, Op0IsKill); 3183 default: return 0; 3184 } 3185} 3186 3187// FastEmit functions for X86ISD::CVTP2SI. 3188 3189unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 3190 if ((Subtarget->hasVLX())) { 3191 return fastEmitInst_r(X86::VCVTPS2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 3192 } 3193 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 3194 return fastEmitInst_r(X86::CVTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill); 3195 } 3196 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 3197 return fastEmitInst_r(X86::VCVTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill); 3198 } 3199 return 0; 3200} 3201 3202unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) { 3203 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 3204 return fastEmitInst_r(X86::VCVTPS2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 3205 } 3206 return 0; 3207} 3208 3209unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) { 3210 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 3211 return fastEmitInst_r(X86::VCVTPS2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 3212 } 3213 return 0; 3214} 3215 3216unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3217switch (RetVT.SimpleTy) { 3218 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill); 3219 case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill); 3220 case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill); 3221 default: return 0; 3222} 3223} 3224 3225unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) { 3226 if ((Subtarget->hasVLX())) { 3227 return fastEmitInst_r(X86::VCVTPS2DQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 3228 } 3229 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 3230 return fastEmitInst_r(X86::VCVTPS2DQYrr, &X86::VR256RegClass, Op0, Op0IsKill); 3231 } 3232 return 0; 3233} 3234 3235unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) { 3236 if ((Subtarget->hasDQI())) { 3237 return fastEmitInst_r(X86::VCVTPS2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill); 3238 } 3239 return 0; 3240} 3241 3242unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3243switch (RetVT.SimpleTy) { 3244 case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill); 3245 case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill); 3246 default: return 0; 3247} 3248} 3249 3250unsigned fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3251 if (RetVT.SimpleTy != MVT::v16i32) 3252 return 0; 3253 if ((Subtarget->hasAVX512())) { 3254 return fastEmitInst_r(X86::VCVTPS2DQZrr, &X86::VR512RegClass, Op0, Op0IsKill); 3255 } 3256 return 0; 3257} 3258 3259unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 3260 if ((Subtarget->hasVLX())) { 3261 return fastEmitInst_r(X86::VCVTPD2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 3262 } 3263 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 3264 return fastEmitInst_r(X86::CVTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill); 3265 } 3266 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 3267 return fastEmitInst_r(X86::VCVTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill); 3268 } 3269 return 0; 3270} 3271 3272unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) { 3273 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 3274 return fastEmitInst_r(X86::VCVTPD2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 3275 } 3276 return 0; 3277} 3278 3279unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3280switch (RetVT.SimpleTy) { 3281 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill); 3282 case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill); 3283 default: return 0; 3284} 3285} 3286 3287unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 3288 if ((Subtarget->hasVLX())) { 3289 return fastEmitInst_r(X86::VCVTPD2DQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 3290 } 3291 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 3292 return fastEmitInst_r(X86::VCVTPD2DQYrr, &X86::VR128RegClass, Op0, Op0IsKill); 3293 } 3294 return 0; 3295} 3296 3297unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) { 3298 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 3299 return fastEmitInst_r(X86::VCVTPD2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 3300 } 3301 return 0; 3302} 3303 3304unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3305switch (RetVT.SimpleTy) { 3306 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill); 3307 case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill); 3308 default: return 0; 3309} 3310} 3311 3312unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) { 3313 if ((Subtarget->hasAVX512())) { 3314 return fastEmitInst_r(X86::VCVTPD2DQZrr, &X86::VR256XRegClass, Op0, Op0IsKill); 3315 } 3316 return 0; 3317} 3318 3319unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) { 3320 if ((Subtarget->hasDQI())) { 3321 return fastEmitInst_r(X86::VCVTPD2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill); 3322 } 3323 return 0; 3324} 3325 3326unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3327switch (RetVT.SimpleTy) { 3328 case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill); 3329 case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill); 3330 default: return 0; 3331} 3332} 3333 3334unsigned fastEmit_X86ISD_CVTP2SI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 3335 switch (VT.SimpleTy) { 3336 case MVT::v4f32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(RetVT, Op0, Op0IsKill); 3337 case MVT::v8f32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(RetVT, Op0, Op0IsKill); 3338 case MVT::v16f32: return fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(RetVT, Op0, Op0IsKill); 3339 case MVT::v2f64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(RetVT, Op0, Op0IsKill); 3340 case MVT::v4f64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(RetVT, Op0, Op0IsKill); 3341 case MVT::v8f64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(RetVT, Op0, Op0IsKill); 3342 default: return 0; 3343 } 3344} 3345 3346// FastEmit functions for X86ISD::CVTP2UI. 3347 3348unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 3349 if ((Subtarget->hasVLX())) { 3350 return fastEmitInst_r(X86::VCVTPS2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 3351 } 3352 return 0; 3353} 3354 3355unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) { 3356 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 3357 return fastEmitInst_r(X86::VCVTPS2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 3358 } 3359 return 0; 3360} 3361 3362unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) { 3363 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 3364 return fastEmitInst_r(X86::VCVTPS2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 3365 } 3366 return 0; 3367} 3368 3369unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3370switch (RetVT.SimpleTy) { 3371 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill); 3372 case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill); 3373 case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill); 3374 default: return 0; 3375} 3376} 3377 3378unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) { 3379 if ((Subtarget->hasVLX())) { 3380 return fastEmitInst_r(X86::VCVTPS2UDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 3381 } 3382 return 0; 3383} 3384 3385unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) { 3386 if ((Subtarget->hasDQI())) { 3387 return fastEmitInst_r(X86::VCVTPS2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill); 3388 } 3389 return 0; 3390} 3391 3392unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3393switch (RetVT.SimpleTy) { 3394 case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill); 3395 case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill); 3396 default: return 0; 3397} 3398} 3399 3400unsigned fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3401 if (RetVT.SimpleTy != MVT::v16i32) 3402 return 0; 3403 if ((Subtarget->hasAVX512())) { 3404 return fastEmitInst_r(X86::VCVTPS2UDQZrr, &X86::VR512RegClass, Op0, Op0IsKill); 3405 } 3406 return 0; 3407} 3408 3409unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 3410 if ((Subtarget->hasVLX())) { 3411 return fastEmitInst_r(X86::VCVTPD2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 3412 } 3413 return 0; 3414} 3415 3416unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) { 3417 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 3418 return fastEmitInst_r(X86::VCVTPD2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 3419 } 3420 return 0; 3421} 3422 3423unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3424switch (RetVT.SimpleTy) { 3425 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill); 3426 case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill); 3427 default: return 0; 3428} 3429} 3430 3431unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 3432 if ((Subtarget->hasVLX())) { 3433 return fastEmitInst_r(X86::VCVTPD2UDQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 3434 } 3435 return 0; 3436} 3437 3438unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) { 3439 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 3440 return fastEmitInst_r(X86::VCVTPD2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 3441 } 3442 return 0; 3443} 3444 3445unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3446switch (RetVT.SimpleTy) { 3447 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill); 3448 case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill); 3449 default: return 0; 3450} 3451} 3452 3453unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) { 3454 if ((Subtarget->hasAVX512())) { 3455 return fastEmitInst_r(X86::VCVTPD2UDQZrr, &X86::VR256XRegClass, Op0, Op0IsKill); 3456 } 3457 return 0; 3458} 3459 3460unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) { 3461 if ((Subtarget->hasDQI())) { 3462 return fastEmitInst_r(X86::VCVTPD2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill); 3463 } 3464 return 0; 3465} 3466 3467unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3468switch (RetVT.SimpleTy) { 3469 case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill); 3470 case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill); 3471 default: return 0; 3472} 3473} 3474 3475unsigned fastEmit_X86ISD_CVTP2UI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 3476 switch (VT.SimpleTy) { 3477 case MVT::v4f32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(RetVT, Op0, Op0IsKill); 3478 case MVT::v8f32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(RetVT, Op0, Op0IsKill); 3479 case MVT::v16f32: return fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(RetVT, Op0, Op0IsKill); 3480 case MVT::v2f64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(RetVT, Op0, Op0IsKill); 3481 case MVT::v4f64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(RetVT, Op0, Op0IsKill); 3482 case MVT::v8f64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(RetVT, Op0, Op0IsKill); 3483 default: return 0; 3484 } 3485} 3486 3487// FastEmit functions for X86ISD::CVTPH2PS. 3488 3489unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) { 3490 if ((Subtarget->hasVLX())) { 3491 return fastEmitInst_r(X86::VCVTPH2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 3492 } 3493 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) { 3494 return fastEmitInst_r(X86::VCVTPH2PSrr, &X86::VR128RegClass, Op0, Op0IsKill); 3495 } 3496 return 0; 3497} 3498 3499unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) { 3500 if ((Subtarget->hasVLX())) { 3501 return fastEmitInst_r(X86::VCVTPH2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 3502 } 3503 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) { 3504 return fastEmitInst_r(X86::VCVTPH2PSYrr, &X86::VR256RegClass, Op0, Op0IsKill); 3505 } 3506 return 0; 3507} 3508 3509unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3510switch (RetVT.SimpleTy) { 3511 case MVT::v4f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Op0, Op0IsKill); 3512 case MVT::v8f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Op0, Op0IsKill); 3513 default: return 0; 3514} 3515} 3516 3517unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3518 if (RetVT.SimpleTy != MVT::v16f32) 3519 return 0; 3520 if ((Subtarget->hasAVX512())) { 3521 return fastEmitInst_r(X86::VCVTPH2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill); 3522 } 3523 return 0; 3524} 3525 3526unsigned fastEmit_X86ISD_CVTPH2PS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 3527 switch (VT.SimpleTy) { 3528 case MVT::v8i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(RetVT, Op0, Op0IsKill); 3529 case MVT::v16i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(RetVT, Op0, Op0IsKill); 3530 default: return 0; 3531 } 3532} 3533 3534// FastEmit functions for X86ISD::CVTPH2PS_SAE. 3535 3536unsigned fastEmit_X86ISD_CVTPH2PS_SAE_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3537 if (RetVT.SimpleTy != MVT::v16f32) 3538 return 0; 3539 if ((Subtarget->hasAVX512())) { 3540 return fastEmitInst_r(X86::VCVTPH2PSZrrb, &X86::VR512RegClass, Op0, Op0IsKill); 3541 } 3542 return 0; 3543} 3544 3545unsigned fastEmit_X86ISD_CVTPH2PS_SAE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 3546 switch (VT.SimpleTy) { 3547 case MVT::v16i16: return fastEmit_X86ISD_CVTPH2PS_SAE_MVT_v16i16_r(RetVT, Op0, Op0IsKill); 3548 default: return 0; 3549 } 3550} 3551 3552// FastEmit functions for X86ISD::CVTS2SI. 3553 3554unsigned fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) { 3555 if ((Subtarget->hasAVX512())) { 3556 return fastEmitInst_r(X86::VCVTSS2SIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill); 3557 } 3558 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 3559 return fastEmitInst_r(X86::CVTSS2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill); 3560 } 3561 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 3562 return fastEmitInst_r(X86::VCVTSS2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill); 3563 } 3564 return 0; 3565} 3566 3567unsigned fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) { 3568 if ((Subtarget->hasAVX512())) { 3569 return fastEmitInst_r(X86::VCVTSS2SI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill); 3570 } 3571 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 3572 return fastEmitInst_r(X86::CVTSS2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill); 3573 } 3574 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 3575 return fastEmitInst_r(X86::VCVTSS2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill); 3576 } 3577 return 0; 3578} 3579 3580unsigned fastEmit_X86ISD_CVTS2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3581switch (RetVT.SimpleTy) { 3582 case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i32_r(Op0, Op0IsKill); 3583 case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i64_r(Op0, Op0IsKill); 3584 default: return 0; 3585} 3586} 3587 3588unsigned fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) { 3589 if ((Subtarget->hasAVX512())) { 3590 return fastEmitInst_r(X86::VCVTSD2SIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill); 3591 } 3592 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 3593 return fastEmitInst_r(X86::CVTSD2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill); 3594 } 3595 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 3596 return fastEmitInst_r(X86::VCVTSD2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill); 3597 } 3598 return 0; 3599} 3600 3601unsigned fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) { 3602 if ((Subtarget->hasAVX512())) { 3603 return fastEmitInst_r(X86::VCVTSD2SI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill); 3604 } 3605 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 3606 return fastEmitInst_r(X86::CVTSD2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill); 3607 } 3608 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 3609 return fastEmitInst_r(X86::VCVTSD2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill); 3610 } 3611 return 0; 3612} 3613 3614unsigned fastEmit_X86ISD_CVTS2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3615switch (RetVT.SimpleTy) { 3616 case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i32_r(Op0, Op0IsKill); 3617 case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i64_r(Op0, Op0IsKill); 3618 default: return 0; 3619} 3620} 3621 3622unsigned fastEmit_X86ISD_CVTS2SI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 3623 switch (VT.SimpleTy) { 3624 case MVT::v4f32: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_r(RetVT, Op0, Op0IsKill); 3625 case MVT::v2f64: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_r(RetVT, Op0, Op0IsKill); 3626 default: return 0; 3627 } 3628} 3629 3630// FastEmit functions for X86ISD::CVTS2UI. 3631 3632unsigned fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) { 3633 if ((Subtarget->hasAVX512())) { 3634 return fastEmitInst_r(X86::VCVTSS2USIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill); 3635 } 3636 return 0; 3637} 3638 3639unsigned fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) { 3640 if ((Subtarget->hasAVX512())) { 3641 return fastEmitInst_r(X86::VCVTSS2USI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill); 3642 } 3643 return 0; 3644} 3645 3646unsigned fastEmit_X86ISD_CVTS2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3647switch (RetVT.SimpleTy) { 3648 case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i32_r(Op0, Op0IsKill); 3649 case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i64_r(Op0, Op0IsKill); 3650 default: return 0; 3651} 3652} 3653 3654unsigned fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) { 3655 if ((Subtarget->hasAVX512())) { 3656 return fastEmitInst_r(X86::VCVTSD2USIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill); 3657 } 3658 return 0; 3659} 3660 3661unsigned fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) { 3662 if ((Subtarget->hasAVX512())) { 3663 return fastEmitInst_r(X86::VCVTSD2USI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill); 3664 } 3665 return 0; 3666} 3667 3668unsigned fastEmit_X86ISD_CVTS2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3669switch (RetVT.SimpleTy) { 3670 case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i32_r(Op0, Op0IsKill); 3671 case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i64_r(Op0, Op0IsKill); 3672 default: return 0; 3673} 3674} 3675 3676unsigned fastEmit_X86ISD_CVTS2UI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 3677 switch (VT.SimpleTy) { 3678 case MVT::v4f32: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_r(RetVT, Op0, Op0IsKill); 3679 case MVT::v2f64: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_r(RetVT, Op0, Op0IsKill); 3680 default: return 0; 3681 } 3682} 3683 3684// FastEmit functions for X86ISD::CVTSI2P. 3685 3686unsigned fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3687 if (RetVT.SimpleTy != MVT::v2f64) 3688 return 0; 3689 if ((Subtarget->hasVLX())) { 3690 return fastEmitInst_r(X86::VCVTDQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 3691 } 3692 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 3693 return fastEmitInst_r(X86::CVTDQ2PDrr, &X86::VR128RegClass, Op0, Op0IsKill); 3694 } 3695 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 3696 return fastEmitInst_r(X86::VCVTDQ2PDrr, &X86::VR128RegClass, Op0, Op0IsKill); 3697 } 3698 return 0; 3699} 3700 3701unsigned fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3702 if (RetVT.SimpleTy != MVT::v4f32) 3703 return 0; 3704 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 3705 return fastEmitInst_r(X86::VCVTQQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 3706 } 3707 return 0; 3708} 3709 3710unsigned fastEmit_X86ISD_CVTSI2P_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 3711 switch (VT.SimpleTy) { 3712 case MVT::v4i32: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(RetVT, Op0, Op0IsKill); 3713 case MVT::v2i64: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(RetVT, Op0, Op0IsKill); 3714 default: return 0; 3715 } 3716} 3717 3718// FastEmit functions for X86ISD::CVTTP2SI. 3719 3720unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 3721 if ((Subtarget->hasVLX())) { 3722 return fastEmitInst_r(X86::VCVTTPS2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 3723 } 3724 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 3725 return fastEmitInst_r(X86::CVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill); 3726 } 3727 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 3728 return fastEmitInst_r(X86::VCVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill); 3729 } 3730 return 0; 3731} 3732 3733unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) { 3734 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 3735 return fastEmitInst_r(X86::VCVTTPS2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 3736 } 3737 return 0; 3738} 3739 3740unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) { 3741 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 3742 return fastEmitInst_r(X86::VCVTTPS2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 3743 } 3744 return 0; 3745} 3746 3747unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3748switch (RetVT.SimpleTy) { 3749 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill); 3750 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill); 3751 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill); 3752 default: return 0; 3753} 3754} 3755 3756unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) { 3757 if ((Subtarget->hasVLX())) { 3758 return fastEmitInst_r(X86::VCVTTPS2DQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 3759 } 3760 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 3761 return fastEmitInst_r(X86::VCVTTPS2DQYrr, &X86::VR256RegClass, Op0, Op0IsKill); 3762 } 3763 return 0; 3764} 3765 3766unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) { 3767 if ((Subtarget->hasDQI())) { 3768 return fastEmitInst_r(X86::VCVTTPS2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill); 3769 } 3770 return 0; 3771} 3772 3773unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3774switch (RetVT.SimpleTy) { 3775 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill); 3776 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill); 3777 default: return 0; 3778} 3779} 3780 3781unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3782 if (RetVT.SimpleTy != MVT::v16i32) 3783 return 0; 3784 if ((Subtarget->hasAVX512())) { 3785 return fastEmitInst_r(X86::VCVTTPS2DQZrr, &X86::VR512RegClass, Op0, Op0IsKill); 3786 } 3787 return 0; 3788} 3789 3790unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 3791 if ((Subtarget->hasVLX())) { 3792 return fastEmitInst_r(X86::VCVTTPD2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 3793 } 3794 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 3795 return fastEmitInst_r(X86::CVTTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill); 3796 } 3797 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 3798 return fastEmitInst_r(X86::VCVTTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill); 3799 } 3800 return 0; 3801} 3802 3803unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) { 3804 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 3805 return fastEmitInst_r(X86::VCVTTPD2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 3806 } 3807 return 0; 3808} 3809 3810unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3811switch (RetVT.SimpleTy) { 3812 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill); 3813 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill); 3814 default: return 0; 3815} 3816} 3817 3818unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 3819 if ((Subtarget->hasVLX())) { 3820 return fastEmitInst_r(X86::VCVTTPD2DQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 3821 } 3822 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 3823 return fastEmitInst_r(X86::VCVTTPD2DQYrr, &X86::VR128RegClass, Op0, Op0IsKill); 3824 } 3825 return 0; 3826} 3827 3828unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) { 3829 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 3830 return fastEmitInst_r(X86::VCVTTPD2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 3831 } 3832 return 0; 3833} 3834 3835unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3836switch (RetVT.SimpleTy) { 3837 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill); 3838 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill); 3839 default: return 0; 3840} 3841} 3842 3843unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) { 3844 if ((Subtarget->hasAVX512())) { 3845 return fastEmitInst_r(X86::VCVTTPD2DQZrr, &X86::VR256XRegClass, Op0, Op0IsKill); 3846 } 3847 return 0; 3848} 3849 3850unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) { 3851 if ((Subtarget->hasDQI())) { 3852 return fastEmitInst_r(X86::VCVTTPD2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill); 3853 } 3854 return 0; 3855} 3856 3857unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3858switch (RetVT.SimpleTy) { 3859 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill); 3860 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill); 3861 default: return 0; 3862} 3863} 3864 3865unsigned fastEmit_X86ISD_CVTTP2SI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 3866 switch (VT.SimpleTy) { 3867 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(RetVT, Op0, Op0IsKill); 3868 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_r(RetVT, Op0, Op0IsKill); 3869 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f32_r(RetVT, Op0, Op0IsKill); 3870 case MVT::v2f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(RetVT, Op0, Op0IsKill); 3871 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_r(RetVT, Op0, Op0IsKill); 3872 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_r(RetVT, Op0, Op0IsKill); 3873 default: return 0; 3874 } 3875} 3876 3877// FastEmit functions for X86ISD::CVTTP2SI_SAE. 3878 3879unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3880 if (RetVT.SimpleTy != MVT::v8i64) 3881 return 0; 3882 if ((Subtarget->hasDQI())) { 3883 return fastEmitInst_r(X86::VCVTTPS2QQZrrb, &X86::VR512RegClass, Op0, Op0IsKill); 3884 } 3885 return 0; 3886} 3887 3888unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3889 if (RetVT.SimpleTy != MVT::v16i32) 3890 return 0; 3891 if ((Subtarget->hasAVX512())) { 3892 return fastEmitInst_r(X86::VCVTTPS2DQZrrb, &X86::VR512RegClass, Op0, Op0IsKill); 3893 } 3894 return 0; 3895} 3896 3897unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) { 3898 if ((Subtarget->hasAVX512())) { 3899 return fastEmitInst_r(X86::VCVTTPD2DQZrrb, &X86::VR256XRegClass, Op0, Op0IsKill); 3900 } 3901 return 0; 3902} 3903 3904unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) { 3905 if ((Subtarget->hasDQI())) { 3906 return fastEmitInst_r(X86::VCVTTPD2QQZrrb, &X86::VR512RegClass, Op0, Op0IsKill); 3907 } 3908 return 0; 3909} 3910 3911unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3912switch (RetVT.SimpleTy) { 3913 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill); 3914 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill); 3915 default: return 0; 3916} 3917} 3918 3919unsigned fastEmit_X86ISD_CVTTP2SI_SAE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 3920 switch (VT.SimpleTy) { 3921 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f32_r(RetVT, Op0, Op0IsKill); 3922 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f32_r(RetVT, Op0, Op0IsKill); 3923 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_r(RetVT, Op0, Op0IsKill); 3924 default: return 0; 3925 } 3926} 3927 3928// FastEmit functions for X86ISD::CVTTP2UI. 3929 3930unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 3931 if ((Subtarget->hasVLX())) { 3932 return fastEmitInst_r(X86::VCVTTPS2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 3933 } 3934 return 0; 3935} 3936 3937unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) { 3938 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 3939 return fastEmitInst_r(X86::VCVTTPS2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 3940 } 3941 return 0; 3942} 3943 3944unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) { 3945 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 3946 return fastEmitInst_r(X86::VCVTTPS2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 3947 } 3948 return 0; 3949} 3950 3951unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3952switch (RetVT.SimpleTy) { 3953 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill); 3954 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill); 3955 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill); 3956 default: return 0; 3957} 3958} 3959 3960unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) { 3961 if ((Subtarget->hasVLX())) { 3962 return fastEmitInst_r(X86::VCVTTPS2UDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 3963 } 3964 return 0; 3965} 3966 3967unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) { 3968 if ((Subtarget->hasDQI())) { 3969 return fastEmitInst_r(X86::VCVTTPS2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill); 3970 } 3971 return 0; 3972} 3973 3974unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3975switch (RetVT.SimpleTy) { 3976 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill); 3977 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill); 3978 default: return 0; 3979} 3980} 3981 3982unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 3983 if (RetVT.SimpleTy != MVT::v16i32) 3984 return 0; 3985 if ((Subtarget->hasAVX512())) { 3986 return fastEmitInst_r(X86::VCVTTPS2UDQZrr, &X86::VR512RegClass, Op0, Op0IsKill); 3987 } 3988 return 0; 3989} 3990 3991unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 3992 if ((Subtarget->hasVLX())) { 3993 return fastEmitInst_r(X86::VCVTTPD2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 3994 } 3995 return 0; 3996} 3997 3998unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) { 3999 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 4000 return fastEmitInst_r(X86::VCVTTPD2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 4001 } 4002 return 0; 4003} 4004 4005unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4006switch (RetVT.SimpleTy) { 4007 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill); 4008 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill); 4009 default: return 0; 4010} 4011} 4012 4013unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 4014 if ((Subtarget->hasVLX())) { 4015 return fastEmitInst_r(X86::VCVTTPD2UDQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 4016 } 4017 return 0; 4018} 4019 4020unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) { 4021 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 4022 return fastEmitInst_r(X86::VCVTTPD2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 4023 } 4024 return 0; 4025} 4026 4027unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4028switch (RetVT.SimpleTy) { 4029 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill); 4030 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill); 4031 default: return 0; 4032} 4033} 4034 4035unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) { 4036 if ((Subtarget->hasAVX512())) { 4037 return fastEmitInst_r(X86::VCVTTPD2UDQZrr, &X86::VR256XRegClass, Op0, Op0IsKill); 4038 } 4039 return 0; 4040} 4041 4042unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) { 4043 if ((Subtarget->hasDQI())) { 4044 return fastEmitInst_r(X86::VCVTTPD2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill); 4045 } 4046 return 0; 4047} 4048 4049unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4050switch (RetVT.SimpleTy) { 4051 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill); 4052 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill); 4053 default: return 0; 4054} 4055} 4056 4057unsigned fastEmit_X86ISD_CVTTP2UI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 4058 switch (VT.SimpleTy) { 4059 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(RetVT, Op0, Op0IsKill); 4060 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_r(RetVT, Op0, Op0IsKill); 4061 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f32_r(RetVT, Op0, Op0IsKill); 4062 case MVT::v2f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(RetVT, Op0, Op0IsKill); 4063 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_r(RetVT, Op0, Op0IsKill); 4064 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_r(RetVT, Op0, Op0IsKill); 4065 default: return 0; 4066 } 4067} 4068 4069// FastEmit functions for X86ISD::CVTTP2UI_SAE. 4070 4071unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4072 if (RetVT.SimpleTy != MVT::v8i64) 4073 return 0; 4074 if ((Subtarget->hasDQI())) { 4075 return fastEmitInst_r(X86::VCVTTPS2UQQZrrb, &X86::VR512RegClass, Op0, Op0IsKill); 4076 } 4077 return 0; 4078} 4079 4080unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4081 if (RetVT.SimpleTy != MVT::v16i32) 4082 return 0; 4083 if ((Subtarget->hasAVX512())) { 4084 return fastEmitInst_r(X86::VCVTTPS2UDQZrrb, &X86::VR512RegClass, Op0, Op0IsKill); 4085 } 4086 return 0; 4087} 4088 4089unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) { 4090 if ((Subtarget->hasAVX512())) { 4091 return fastEmitInst_r(X86::VCVTTPD2UDQZrrb, &X86::VR256XRegClass, Op0, Op0IsKill); 4092 } 4093 return 0; 4094} 4095 4096unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) { 4097 if ((Subtarget->hasDQI())) { 4098 return fastEmitInst_r(X86::VCVTTPD2UQQZrrb, &X86::VR512RegClass, Op0, Op0IsKill); 4099 } 4100 return 0; 4101} 4102 4103unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4104switch (RetVT.SimpleTy) { 4105 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill); 4106 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill); 4107 default: return 0; 4108} 4109} 4110 4111unsigned fastEmit_X86ISD_CVTTP2UI_SAE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 4112 switch (VT.SimpleTy) { 4113 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f32_r(RetVT, Op0, Op0IsKill); 4114 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f32_r(RetVT, Op0, Op0IsKill); 4115 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_r(RetVT, Op0, Op0IsKill); 4116 default: return 0; 4117 } 4118} 4119 4120// FastEmit functions for X86ISD::CVTTS2SI. 4121 4122unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) { 4123 if ((Subtarget->hasAVX512())) { 4124 return fastEmitInst_r(X86::VCVTTSS2SIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill); 4125 } 4126 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 4127 return fastEmitInst_r(X86::CVTTSS2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill); 4128 } 4129 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 4130 return fastEmitInst_r(X86::VCVTTSS2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill); 4131 } 4132 return 0; 4133} 4134 4135unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) { 4136 if ((Subtarget->hasAVX512())) { 4137 return fastEmitInst_r(X86::VCVTTSS2SI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill); 4138 } 4139 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 4140 return fastEmitInst_r(X86::CVTTSS2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill); 4141 } 4142 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 4143 return fastEmitInst_r(X86::VCVTTSS2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill); 4144 } 4145 return 0; 4146} 4147 4148unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4149switch (RetVT.SimpleTy) { 4150 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i32_r(Op0, Op0IsKill); 4151 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i64_r(Op0, Op0IsKill); 4152 default: return 0; 4153} 4154} 4155 4156unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) { 4157 if ((Subtarget->hasAVX512())) { 4158 return fastEmitInst_r(X86::VCVTTSD2SIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill); 4159 } 4160 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 4161 return fastEmitInst_r(X86::CVTTSD2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill); 4162 } 4163 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 4164 return fastEmitInst_r(X86::VCVTTSD2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill); 4165 } 4166 return 0; 4167} 4168 4169unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) { 4170 if ((Subtarget->hasAVX512())) { 4171 return fastEmitInst_r(X86::VCVTTSD2SI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill); 4172 } 4173 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 4174 return fastEmitInst_r(X86::CVTTSD2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill); 4175 } 4176 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 4177 return fastEmitInst_r(X86::VCVTTSD2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill); 4178 } 4179 return 0; 4180} 4181 4182unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4183switch (RetVT.SimpleTy) { 4184 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i32_r(Op0, Op0IsKill); 4185 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i64_r(Op0, Op0IsKill); 4186 default: return 0; 4187} 4188} 4189 4190unsigned fastEmit_X86ISD_CVTTS2SI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 4191 switch (VT.SimpleTy) { 4192 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_r(RetVT, Op0, Op0IsKill); 4193 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_r(RetVT, Op0, Op0IsKill); 4194 default: return 0; 4195 } 4196} 4197 4198// FastEmit functions for X86ISD::CVTTS2SI_SAE. 4199 4200unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) { 4201 if ((Subtarget->hasAVX512())) { 4202 return fastEmitInst_r(X86::VCVTTSS2SIZrrb_Int, &X86::GR32RegClass, Op0, Op0IsKill); 4203 } 4204 return 0; 4205} 4206 4207unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) { 4208 if ((Subtarget->hasAVX512())) { 4209 return fastEmitInst_r(X86::VCVTTSS2SI64Zrrb_Int, &X86::GR64RegClass, Op0, Op0IsKill); 4210 } 4211 return 0; 4212} 4213 4214unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4215switch (RetVT.SimpleTy) { 4216 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i32_r(Op0, Op0IsKill); 4217 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i64_r(Op0, Op0IsKill); 4218 default: return 0; 4219} 4220} 4221 4222unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) { 4223 if ((Subtarget->hasAVX512())) { 4224 return fastEmitInst_r(X86::VCVTTSD2SIZrrb_Int, &X86::GR32RegClass, Op0, Op0IsKill); 4225 } 4226 return 0; 4227} 4228 4229unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) { 4230 if ((Subtarget->hasAVX512())) { 4231 return fastEmitInst_r(X86::VCVTTSD2SI64Zrrb_Int, &X86::GR64RegClass, Op0, Op0IsKill); 4232 } 4233 return 0; 4234} 4235 4236unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4237switch (RetVT.SimpleTy) { 4238 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i32_r(Op0, Op0IsKill); 4239 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i64_r(Op0, Op0IsKill); 4240 default: return 0; 4241} 4242} 4243 4244unsigned fastEmit_X86ISD_CVTTS2SI_SAE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 4245 switch (VT.SimpleTy) { 4246 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_r(RetVT, Op0, Op0IsKill); 4247 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_r(RetVT, Op0, Op0IsKill); 4248 default: return 0; 4249 } 4250} 4251 4252// FastEmit functions for X86ISD::CVTTS2UI. 4253 4254unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) { 4255 if ((Subtarget->hasAVX512())) { 4256 return fastEmitInst_r(X86::VCVTTSS2USIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill); 4257 } 4258 return 0; 4259} 4260 4261unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) { 4262 if ((Subtarget->hasAVX512())) { 4263 return fastEmitInst_r(X86::VCVTTSS2USI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill); 4264 } 4265 return 0; 4266} 4267 4268unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4269switch (RetVT.SimpleTy) { 4270 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i32_r(Op0, Op0IsKill); 4271 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i64_r(Op0, Op0IsKill); 4272 default: return 0; 4273} 4274} 4275 4276unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) { 4277 if ((Subtarget->hasAVX512())) { 4278 return fastEmitInst_r(X86::VCVTTSD2USIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill); 4279 } 4280 return 0; 4281} 4282 4283unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) { 4284 if ((Subtarget->hasAVX512())) { 4285 return fastEmitInst_r(X86::VCVTTSD2USI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill); 4286 } 4287 return 0; 4288} 4289 4290unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4291switch (RetVT.SimpleTy) { 4292 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i32_r(Op0, Op0IsKill); 4293 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i64_r(Op0, Op0IsKill); 4294 default: return 0; 4295} 4296} 4297 4298unsigned fastEmit_X86ISD_CVTTS2UI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 4299 switch (VT.SimpleTy) { 4300 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_r(RetVT, Op0, Op0IsKill); 4301 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_r(RetVT, Op0, Op0IsKill); 4302 default: return 0; 4303 } 4304} 4305 4306// FastEmit functions for X86ISD::CVTTS2UI_SAE. 4307 4308unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) { 4309 if ((Subtarget->hasAVX512())) { 4310 return fastEmitInst_r(X86::VCVTTSS2USIZrrb_Int, &X86::GR32RegClass, Op0, Op0IsKill); 4311 } 4312 return 0; 4313} 4314 4315unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) { 4316 if ((Subtarget->hasAVX512())) { 4317 return fastEmitInst_r(X86::VCVTTSS2USI64Zrrb_Int, &X86::GR64RegClass, Op0, Op0IsKill); 4318 } 4319 return 0; 4320} 4321 4322unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4323switch (RetVT.SimpleTy) { 4324 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i32_r(Op0, Op0IsKill); 4325 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i64_r(Op0, Op0IsKill); 4326 default: return 0; 4327} 4328} 4329 4330unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) { 4331 if ((Subtarget->hasAVX512())) { 4332 return fastEmitInst_r(X86::VCVTTSD2USIZrrb_Int, &X86::GR32RegClass, Op0, Op0IsKill); 4333 } 4334 return 0; 4335} 4336 4337unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) { 4338 if ((Subtarget->hasAVX512())) { 4339 return fastEmitInst_r(X86::VCVTTSD2USI64Zrrb_Int, &X86::GR64RegClass, Op0, Op0IsKill); 4340 } 4341 return 0; 4342} 4343 4344unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4345switch (RetVT.SimpleTy) { 4346 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i32_r(Op0, Op0IsKill); 4347 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i64_r(Op0, Op0IsKill); 4348 default: return 0; 4349} 4350} 4351 4352unsigned fastEmit_X86ISD_CVTTS2UI_SAE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 4353 switch (VT.SimpleTy) { 4354 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_r(RetVT, Op0, Op0IsKill); 4355 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_r(RetVT, Op0, Op0IsKill); 4356 default: return 0; 4357 } 4358} 4359 4360// FastEmit functions for X86ISD::CVTUI2P. 4361 4362unsigned fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4363 if (RetVT.SimpleTy != MVT::v2f64) 4364 return 0; 4365 if ((Subtarget->hasVLX())) { 4366 return fastEmitInst_r(X86::VCVTUDQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 4367 } 4368 return 0; 4369} 4370 4371unsigned fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4372 if (RetVT.SimpleTy != MVT::v4f32) 4373 return 0; 4374 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 4375 return fastEmitInst_r(X86::VCVTUQQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 4376 } 4377 return 0; 4378} 4379 4380unsigned fastEmit_X86ISD_CVTUI2P_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 4381 switch (VT.SimpleTy) { 4382 case MVT::v4i32: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(RetVT, Op0, Op0IsKill); 4383 case MVT::v2i64: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(RetVT, Op0, Op0IsKill); 4384 default: return 0; 4385 } 4386} 4387 4388// FastEmit functions for X86ISD::EH_RETURN. 4389 4390unsigned fastEmit_X86ISD_EH_RETURN_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4391 if (RetVT.SimpleTy != MVT::isVoid) 4392 return 0; 4393 return fastEmitInst_r(X86::EH_RETURN, &X86::GR32RegClass, Op0, Op0IsKill); 4394} 4395 4396unsigned fastEmit_X86ISD_EH_RETURN_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4397 if (RetVT.SimpleTy != MVT::isVoid) 4398 return 0; 4399 return fastEmitInst_r(X86::EH_RETURN64, &X86::GR64RegClass, Op0, Op0IsKill); 4400} 4401 4402unsigned fastEmit_X86ISD_EH_RETURN_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 4403 switch (VT.SimpleTy) { 4404 case MVT::i32: return fastEmit_X86ISD_EH_RETURN_MVT_i32_r(RetVT, Op0, Op0IsKill); 4405 case MVT::i64: return fastEmit_X86ISD_EH_RETURN_MVT_i64_r(RetVT, Op0, Op0IsKill); 4406 default: return 0; 4407 } 4408} 4409 4410// FastEmit functions for X86ISD::EXP2. 4411 4412unsigned fastEmit_X86ISD_EXP2_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4413 if (RetVT.SimpleTy != MVT::v16f32) 4414 return 0; 4415 if ((Subtarget->hasERI())) { 4416 return fastEmitInst_r(X86::VEXP2PSZr, &X86::VR512RegClass, Op0, Op0IsKill); 4417 } 4418 return 0; 4419} 4420 4421unsigned fastEmit_X86ISD_EXP2_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4422 if (RetVT.SimpleTy != MVT::v8f64) 4423 return 0; 4424 if ((Subtarget->hasERI())) { 4425 return fastEmitInst_r(X86::VEXP2PDZr, &X86::VR512RegClass, Op0, Op0IsKill); 4426 } 4427 return 0; 4428} 4429 4430unsigned fastEmit_X86ISD_EXP2_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 4431 switch (VT.SimpleTy) { 4432 case MVT::v16f32: return fastEmit_X86ISD_EXP2_MVT_v16f32_r(RetVT, Op0, Op0IsKill); 4433 case MVT::v8f64: return fastEmit_X86ISD_EXP2_MVT_v8f64_r(RetVT, Op0, Op0IsKill); 4434 default: return 0; 4435 } 4436} 4437 4438// FastEmit functions for X86ISD::EXP2_SAE. 4439 4440unsigned fastEmit_X86ISD_EXP2_SAE_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4441 if (RetVT.SimpleTy != MVT::v16f32) 4442 return 0; 4443 if ((Subtarget->hasERI())) { 4444 return fastEmitInst_r(X86::VEXP2PSZrb, &X86::VR512RegClass, Op0, Op0IsKill); 4445 } 4446 return 0; 4447} 4448 4449unsigned fastEmit_X86ISD_EXP2_SAE_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4450 if (RetVT.SimpleTy != MVT::v8f64) 4451 return 0; 4452 if ((Subtarget->hasERI())) { 4453 return fastEmitInst_r(X86::VEXP2PDZrb, &X86::VR512RegClass, Op0, Op0IsKill); 4454 } 4455 return 0; 4456} 4457 4458unsigned fastEmit_X86ISD_EXP2_SAE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 4459 switch (VT.SimpleTy) { 4460 case MVT::v16f32: return fastEmit_X86ISD_EXP2_SAE_MVT_v16f32_r(RetVT, Op0, Op0IsKill); 4461 case MVT::v8f64: return fastEmit_X86ISD_EXP2_SAE_MVT_v8f64_r(RetVT, Op0, Op0IsKill); 4462 default: return 0; 4463 } 4464} 4465 4466// FastEmit functions for X86ISD::FGETEXP. 4467 4468unsigned fastEmit_X86ISD_FGETEXP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4469 if (RetVT.SimpleTy != MVT::v4f32) 4470 return 0; 4471 if ((Subtarget->hasVLX())) { 4472 return fastEmitInst_r(X86::VGETEXPPSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill); 4473 } 4474 return 0; 4475} 4476 4477unsigned fastEmit_X86ISD_FGETEXP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4478 if (RetVT.SimpleTy != MVT::v8f32) 4479 return 0; 4480 if ((Subtarget->hasVLX())) { 4481 return fastEmitInst_r(X86::VGETEXPPSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill); 4482 } 4483 return 0; 4484} 4485 4486unsigned fastEmit_X86ISD_FGETEXP_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4487 if (RetVT.SimpleTy != MVT::v16f32) 4488 return 0; 4489 if ((Subtarget->hasAVX512())) { 4490 return fastEmitInst_r(X86::VGETEXPPSZr, &X86::VR512RegClass, Op0, Op0IsKill); 4491 } 4492 return 0; 4493} 4494 4495unsigned fastEmit_X86ISD_FGETEXP_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4496 if (RetVT.SimpleTy != MVT::v2f64) 4497 return 0; 4498 if ((Subtarget->hasVLX())) { 4499 return fastEmitInst_r(X86::VGETEXPPDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill); 4500 } 4501 return 0; 4502} 4503 4504unsigned fastEmit_X86ISD_FGETEXP_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4505 if (RetVT.SimpleTy != MVT::v4f64) 4506 return 0; 4507 if ((Subtarget->hasVLX())) { 4508 return fastEmitInst_r(X86::VGETEXPPDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill); 4509 } 4510 return 0; 4511} 4512 4513unsigned fastEmit_X86ISD_FGETEXP_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4514 if (RetVT.SimpleTy != MVT::v8f64) 4515 return 0; 4516 if ((Subtarget->hasAVX512())) { 4517 return fastEmitInst_r(X86::VGETEXPPDZr, &X86::VR512RegClass, Op0, Op0IsKill); 4518 } 4519 return 0; 4520} 4521 4522unsigned fastEmit_X86ISD_FGETEXP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 4523 switch (VT.SimpleTy) { 4524 case MVT::v4f32: return fastEmit_X86ISD_FGETEXP_MVT_v4f32_r(RetVT, Op0, Op0IsKill); 4525 case MVT::v8f32: return fastEmit_X86ISD_FGETEXP_MVT_v8f32_r(RetVT, Op0, Op0IsKill); 4526 case MVT::v16f32: return fastEmit_X86ISD_FGETEXP_MVT_v16f32_r(RetVT, Op0, Op0IsKill); 4527 case MVT::v2f64: return fastEmit_X86ISD_FGETEXP_MVT_v2f64_r(RetVT, Op0, Op0IsKill); 4528 case MVT::v4f64: return fastEmit_X86ISD_FGETEXP_MVT_v4f64_r(RetVT, Op0, Op0IsKill); 4529 case MVT::v8f64: return fastEmit_X86ISD_FGETEXP_MVT_v8f64_r(RetVT, Op0, Op0IsKill); 4530 default: return 0; 4531 } 4532} 4533 4534// FastEmit functions for X86ISD::FGETEXP_SAE. 4535 4536unsigned fastEmit_X86ISD_FGETEXP_SAE_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4537 if (RetVT.SimpleTy != MVT::v16f32) 4538 return 0; 4539 if ((Subtarget->hasAVX512())) { 4540 return fastEmitInst_r(X86::VGETEXPPSZrb, &X86::VR512RegClass, Op0, Op0IsKill); 4541 } 4542 return 0; 4543} 4544 4545unsigned fastEmit_X86ISD_FGETEXP_SAE_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4546 if (RetVT.SimpleTy != MVT::v8f64) 4547 return 0; 4548 if ((Subtarget->hasAVX512())) { 4549 return fastEmitInst_r(X86::VGETEXPPDZrb, &X86::VR512RegClass, Op0, Op0IsKill); 4550 } 4551 return 0; 4552} 4553 4554unsigned fastEmit_X86ISD_FGETEXP_SAE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 4555 switch (VT.SimpleTy) { 4556 case MVT::v16f32: return fastEmit_X86ISD_FGETEXP_SAE_MVT_v16f32_r(RetVT, Op0, Op0IsKill); 4557 case MVT::v8f64: return fastEmit_X86ISD_FGETEXP_SAE_MVT_v8f64_r(RetVT, Op0, Op0IsKill); 4558 default: return 0; 4559 } 4560} 4561 4562// FastEmit functions for X86ISD::FRCP. 4563 4564unsigned fastEmit_X86ISD_FRCP_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4565 if (RetVT.SimpleTy != MVT::f32) 4566 return 0; 4567 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 4568 return fastEmitInst_r(X86::RCPSSr, &X86::FR32RegClass, Op0, Op0IsKill); 4569 } 4570 return 0; 4571} 4572 4573unsigned fastEmit_X86ISD_FRCP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4574 if (RetVT.SimpleTy != MVT::v4f32) 4575 return 0; 4576 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 4577 return fastEmitInst_r(X86::RCPPSr, &X86::VR128RegClass, Op0, Op0IsKill); 4578 } 4579 if ((Subtarget->hasAVX())) { 4580 return fastEmitInst_r(X86::VRCPPSr, &X86::VR128RegClass, Op0, Op0IsKill); 4581 } 4582 return 0; 4583} 4584 4585unsigned fastEmit_X86ISD_FRCP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4586 if (RetVT.SimpleTy != MVT::v8f32) 4587 return 0; 4588 if ((Subtarget->hasAVX())) { 4589 return fastEmitInst_r(X86::VRCPPSYr, &X86::VR256RegClass, Op0, Op0IsKill); 4590 } 4591 return 0; 4592} 4593 4594unsigned fastEmit_X86ISD_FRCP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 4595 switch (VT.SimpleTy) { 4596 case MVT::f32: return fastEmit_X86ISD_FRCP_MVT_f32_r(RetVT, Op0, Op0IsKill); 4597 case MVT::v4f32: return fastEmit_X86ISD_FRCP_MVT_v4f32_r(RetVT, Op0, Op0IsKill); 4598 case MVT::v8f32: return fastEmit_X86ISD_FRCP_MVT_v8f32_r(RetVT, Op0, Op0IsKill); 4599 default: return 0; 4600 } 4601} 4602 4603// FastEmit functions for X86ISD::FRSQRT. 4604 4605unsigned fastEmit_X86ISD_FRSQRT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4606 if (RetVT.SimpleTy != MVT::f32) 4607 return 0; 4608 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 4609 return fastEmitInst_r(X86::RSQRTSSr, &X86::FR32RegClass, Op0, Op0IsKill); 4610 } 4611 return 0; 4612} 4613 4614unsigned fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4615 if (RetVT.SimpleTy != MVT::v4f32) 4616 return 0; 4617 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 4618 return fastEmitInst_r(X86::RSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill); 4619 } 4620 if ((Subtarget->hasAVX())) { 4621 return fastEmitInst_r(X86::VRSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill); 4622 } 4623 return 0; 4624} 4625 4626unsigned fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4627 if (RetVT.SimpleTy != MVT::v8f32) 4628 return 0; 4629 if ((Subtarget->hasAVX())) { 4630 return fastEmitInst_r(X86::VRSQRTPSYr, &X86::VR256RegClass, Op0, Op0IsKill); 4631 } 4632 return 0; 4633} 4634 4635unsigned fastEmit_X86ISD_FRSQRT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 4636 switch (VT.SimpleTy) { 4637 case MVT::f32: return fastEmit_X86ISD_FRSQRT_MVT_f32_r(RetVT, Op0, Op0IsKill); 4638 case MVT::v4f32: return fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(RetVT, Op0, Op0IsKill); 4639 case MVT::v8f32: return fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(RetVT, Op0, Op0IsKill); 4640 default: return 0; 4641 } 4642} 4643 4644// FastEmit functions for X86ISD::MMX_MOVD2W. 4645 4646unsigned fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4647 if (RetVT.SimpleTy != MVT::i32) 4648 return 0; 4649 if ((Subtarget->hasMMX())) { 4650 return fastEmitInst_r(X86::MMX_MOVD64grr, &X86::GR32RegClass, Op0, Op0IsKill); 4651 } 4652 return 0; 4653} 4654 4655unsigned fastEmit_X86ISD_MMX_MOVD2W_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 4656 switch (VT.SimpleTy) { 4657 case MVT::x86mmx: return fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(RetVT, Op0, Op0IsKill); 4658 default: return 0; 4659 } 4660} 4661 4662// FastEmit functions for X86ISD::MMX_MOVW2D. 4663 4664unsigned fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4665 if (RetVT.SimpleTy != MVT::x86mmx) 4666 return 0; 4667 if ((Subtarget->hasMMX())) { 4668 return fastEmitInst_r(X86::MMX_MOVD64rr, &X86::VR64RegClass, Op0, Op0IsKill); 4669 } 4670 return 0; 4671} 4672 4673unsigned fastEmit_X86ISD_MMX_MOVW2D_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 4674 switch (VT.SimpleTy) { 4675 case MVT::i32: return fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(RetVT, Op0, Op0IsKill); 4676 default: return 0; 4677 } 4678} 4679 4680// FastEmit functions for X86ISD::MOVDDUP. 4681 4682unsigned fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4683 if (RetVT.SimpleTy != MVT::v2f64) 4684 return 0; 4685 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) { 4686 return fastEmitInst_r(X86::MOVDDUPrr, &X86::VR128RegClass, Op0, Op0IsKill); 4687 } 4688 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 4689 return fastEmitInst_r(X86::VMOVDDUPrr, &X86::VR128RegClass, Op0, Op0IsKill); 4690 } 4691 return 0; 4692} 4693 4694unsigned fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4695 if (RetVT.SimpleTy != MVT::v4f64) 4696 return 0; 4697 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 4698 return fastEmitInst_r(X86::VMOVDDUPZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 4699 } 4700 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 4701 return fastEmitInst_r(X86::VMOVDDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill); 4702 } 4703 return 0; 4704} 4705 4706unsigned fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4707 if (RetVT.SimpleTy != MVT::v8f64) 4708 return 0; 4709 if ((Subtarget->hasAVX512())) { 4710 return fastEmitInst_r(X86::VMOVDDUPZrr, &X86::VR512RegClass, Op0, Op0IsKill); 4711 } 4712 return 0; 4713} 4714 4715unsigned fastEmit_X86ISD_MOVDDUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 4716 switch (VT.SimpleTy) { 4717 case MVT::v2f64: return fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(RetVT, Op0, Op0IsKill); 4718 case MVT::v4f64: return fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(RetVT, Op0, Op0IsKill); 4719 case MVT::v8f64: return fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(RetVT, Op0, Op0IsKill); 4720 default: return 0; 4721 } 4722} 4723 4724// FastEmit functions for X86ISD::MOVDQ2Q. 4725 4726unsigned fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4727 if (RetVT.SimpleTy != MVT::x86mmx) 4728 return 0; 4729 return fastEmitInst_r(X86::MMX_MOVDQ2Qrr, &X86::VR64RegClass, Op0, Op0IsKill); 4730} 4731 4732unsigned fastEmit_X86ISD_MOVDQ2Q_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 4733 switch (VT.SimpleTy) { 4734 case MVT::v2i64: return fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(RetVT, Op0, Op0IsKill); 4735 default: return 0; 4736 } 4737} 4738 4739// FastEmit functions for X86ISD::MOVMSK. 4740 4741unsigned fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4742 if (RetVT.SimpleTy != MVT::i32) 4743 return 0; 4744 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 4745 return fastEmitInst_r(X86::PMOVMSKBrr, &X86::GR32RegClass, Op0, Op0IsKill); 4746 } 4747 if ((Subtarget->hasAVX())) { 4748 return fastEmitInst_r(X86::VPMOVMSKBrr, &X86::GR32RegClass, Op0, Op0IsKill); 4749 } 4750 return 0; 4751} 4752 4753unsigned fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4754 if (RetVT.SimpleTy != MVT::i32) 4755 return 0; 4756 if ((Subtarget->hasAVX2())) { 4757 return fastEmitInst_r(X86::VPMOVMSKBYrr, &X86::GR32RegClass, Op0, Op0IsKill); 4758 } 4759 return 0; 4760} 4761 4762unsigned fastEmit_X86ISD_MOVMSK_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4763 if (RetVT.SimpleTy != MVT::i32) 4764 return 0; 4765 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 4766 return fastEmitInst_r(X86::MOVMSKPSrr, &X86::GR32RegClass, Op0, Op0IsKill); 4767 } 4768 if ((Subtarget->hasAVX())) { 4769 return fastEmitInst_r(X86::VMOVMSKPSrr, &X86::GR32RegClass, Op0, Op0IsKill); 4770 } 4771 return 0; 4772} 4773 4774unsigned fastEmit_X86ISD_MOVMSK_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4775 if (RetVT.SimpleTy != MVT::i32) 4776 return 0; 4777 if ((Subtarget->hasAVX())) { 4778 return fastEmitInst_r(X86::VMOVMSKPSYrr, &X86::GR32RegClass, Op0, Op0IsKill); 4779 } 4780 return 0; 4781} 4782 4783unsigned fastEmit_X86ISD_MOVMSK_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4784 if (RetVT.SimpleTy != MVT::i32) 4785 return 0; 4786 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 4787 return fastEmitInst_r(X86::MOVMSKPDrr, &X86::GR32RegClass, Op0, Op0IsKill); 4788 } 4789 if ((Subtarget->hasAVX())) { 4790 return fastEmitInst_r(X86::VMOVMSKPDrr, &X86::GR32RegClass, Op0, Op0IsKill); 4791 } 4792 return 0; 4793} 4794 4795unsigned fastEmit_X86ISD_MOVMSK_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4796 if (RetVT.SimpleTy != MVT::i32) 4797 return 0; 4798 if ((Subtarget->hasAVX())) { 4799 return fastEmitInst_r(X86::VMOVMSKPDYrr, &X86::GR32RegClass, Op0, Op0IsKill); 4800 } 4801 return 0; 4802} 4803 4804unsigned fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4805 if (RetVT.SimpleTy != MVT::i32) 4806 return 0; 4807 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 4808 return fastEmitInst_r(X86::MOVMSKPSrr, &X86::GR32RegClass, Op0, Op0IsKill); 4809 } 4810 if ((Subtarget->hasAVX())) { 4811 return fastEmitInst_r(X86::VMOVMSKPSrr, &X86::GR32RegClass, Op0, Op0IsKill); 4812 } 4813 return 0; 4814} 4815 4816unsigned fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4817 if (RetVT.SimpleTy != MVT::i32) 4818 return 0; 4819 if ((Subtarget->hasAVX())) { 4820 return fastEmitInst_r(X86::VMOVMSKPSYrr, &X86::GR32RegClass, Op0, Op0IsKill); 4821 } 4822 return 0; 4823} 4824 4825unsigned fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4826 if (RetVT.SimpleTy != MVT::i32) 4827 return 0; 4828 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 4829 return fastEmitInst_r(X86::MOVMSKPDrr, &X86::GR32RegClass, Op0, Op0IsKill); 4830 } 4831 if ((Subtarget->hasAVX())) { 4832 return fastEmitInst_r(X86::VMOVMSKPDrr, &X86::GR32RegClass, Op0, Op0IsKill); 4833 } 4834 return 0; 4835} 4836 4837unsigned fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4838 if (RetVT.SimpleTy != MVT::i32) 4839 return 0; 4840 if ((Subtarget->hasAVX())) { 4841 return fastEmitInst_r(X86::VMOVMSKPDYrr, &X86::GR32RegClass, Op0, Op0IsKill); 4842 } 4843 return 0; 4844} 4845 4846unsigned fastEmit_X86ISD_MOVMSK_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 4847 switch (VT.SimpleTy) { 4848 case MVT::v16i8: return fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(RetVT, Op0, Op0IsKill); 4849 case MVT::v32i8: return fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(RetVT, Op0, Op0IsKill); 4850 case MVT::v4i32: return fastEmit_X86ISD_MOVMSK_MVT_v4i32_r(RetVT, Op0, Op0IsKill); 4851 case MVT::v8i32: return fastEmit_X86ISD_MOVMSK_MVT_v8i32_r(RetVT, Op0, Op0IsKill); 4852 case MVT::v2i64: return fastEmit_X86ISD_MOVMSK_MVT_v2i64_r(RetVT, Op0, Op0IsKill); 4853 case MVT::v4i64: return fastEmit_X86ISD_MOVMSK_MVT_v4i64_r(RetVT, Op0, Op0IsKill); 4854 case MVT::v4f32: return fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(RetVT, Op0, Op0IsKill); 4855 case MVT::v8f32: return fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(RetVT, Op0, Op0IsKill); 4856 case MVT::v2f64: return fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(RetVT, Op0, Op0IsKill); 4857 case MVT::v4f64: return fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(RetVT, Op0, Op0IsKill); 4858 default: return 0; 4859 } 4860} 4861 4862// FastEmit functions for X86ISD::MOVQ2DQ. 4863 4864unsigned fastEmit_X86ISD_MOVQ2DQ_MVT_x86mmx_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4865 if (RetVT.SimpleTy != MVT::v2i64) 4866 return 0; 4867 return fastEmitInst_r(X86::MMX_MOVQ2DQrr, &X86::VR128RegClass, Op0, Op0IsKill); 4868} 4869 4870unsigned fastEmit_X86ISD_MOVQ2DQ_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 4871 switch (VT.SimpleTy) { 4872 case MVT::x86mmx: return fastEmit_X86ISD_MOVQ2DQ_MVT_x86mmx_r(RetVT, Op0, Op0IsKill); 4873 default: return 0; 4874 } 4875} 4876 4877// FastEmit functions for X86ISD::MOVSHDUP. 4878 4879unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4880 if (RetVT.SimpleTy != MVT::v4i32) 4881 return 0; 4882 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) { 4883 return fastEmitInst_r(X86::MOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill); 4884 } 4885 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 4886 return fastEmitInst_r(X86::VMOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill); 4887 } 4888 return 0; 4889} 4890 4891unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4892 if (RetVT.SimpleTy != MVT::v8i32) 4893 return 0; 4894 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 4895 return fastEmitInst_r(X86::VMOVSHDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill); 4896 } 4897 return 0; 4898} 4899 4900unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4901 if (RetVT.SimpleTy != MVT::v4f32) 4902 return 0; 4903 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 4904 return fastEmitInst_r(X86::VMOVSHDUPZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 4905 } 4906 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) { 4907 return fastEmitInst_r(X86::MOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill); 4908 } 4909 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 4910 return fastEmitInst_r(X86::VMOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill); 4911 } 4912 return 0; 4913} 4914 4915unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4916 if (RetVT.SimpleTy != MVT::v8f32) 4917 return 0; 4918 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 4919 return fastEmitInst_r(X86::VMOVSHDUPZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 4920 } 4921 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 4922 return fastEmitInst_r(X86::VMOVSHDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill); 4923 } 4924 return 0; 4925} 4926 4927unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4928 if (RetVT.SimpleTy != MVT::v16f32) 4929 return 0; 4930 if ((Subtarget->hasAVX512())) { 4931 return fastEmitInst_r(X86::VMOVSHDUPZrr, &X86::VR512RegClass, Op0, Op0IsKill); 4932 } 4933 return 0; 4934} 4935 4936unsigned fastEmit_X86ISD_MOVSHDUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 4937 switch (VT.SimpleTy) { 4938 case MVT::v4i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(RetVT, Op0, Op0IsKill); 4939 case MVT::v8i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(RetVT, Op0, Op0IsKill); 4940 case MVT::v4f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(RetVT, Op0, Op0IsKill); 4941 case MVT::v8f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(RetVT, Op0, Op0IsKill); 4942 case MVT::v16f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(RetVT, Op0, Op0IsKill); 4943 default: return 0; 4944 } 4945} 4946 4947// FastEmit functions for X86ISD::MOVSLDUP. 4948 4949unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4950 if (RetVT.SimpleTy != MVT::v4i32) 4951 return 0; 4952 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) { 4953 return fastEmitInst_r(X86::MOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill); 4954 } 4955 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 4956 return fastEmitInst_r(X86::VMOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill); 4957 } 4958 return 0; 4959} 4960 4961unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4962 if (RetVT.SimpleTy != MVT::v8i32) 4963 return 0; 4964 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 4965 return fastEmitInst_r(X86::VMOVSLDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill); 4966 } 4967 return 0; 4968} 4969 4970unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4971 if (RetVT.SimpleTy != MVT::v4f32) 4972 return 0; 4973 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 4974 return fastEmitInst_r(X86::VMOVSLDUPZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 4975 } 4976 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) { 4977 return fastEmitInst_r(X86::MOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill); 4978 } 4979 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 4980 return fastEmitInst_r(X86::VMOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill); 4981 } 4982 return 0; 4983} 4984 4985unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4986 if (RetVT.SimpleTy != MVT::v8f32) 4987 return 0; 4988 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 4989 return fastEmitInst_r(X86::VMOVSLDUPZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 4990 } 4991 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 4992 return fastEmitInst_r(X86::VMOVSLDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill); 4993 } 4994 return 0; 4995} 4996 4997unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 4998 if (RetVT.SimpleTy != MVT::v16f32) 4999 return 0; 5000 if ((Subtarget->hasAVX512())) { 5001 return fastEmitInst_r(X86::VMOVSLDUPZrr, &X86::VR512RegClass, Op0, Op0IsKill); 5002 } 5003 return 0; 5004} 5005 5006unsigned fastEmit_X86ISD_MOVSLDUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 5007 switch (VT.SimpleTy) { 5008 case MVT::v4i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(RetVT, Op0, Op0IsKill); 5009 case MVT::v8i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(RetVT, Op0, Op0IsKill); 5010 case MVT::v4f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(RetVT, Op0, Op0IsKill); 5011 case MVT::v8f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(RetVT, Op0, Op0IsKill); 5012 case MVT::v16f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(RetVT, Op0, Op0IsKill); 5013 default: return 0; 5014 } 5015} 5016 5017// FastEmit functions for X86ISD::NT_BRIND. 5018 5019unsigned fastEmit_X86ISD_NT_BRIND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5020 if (RetVT.SimpleTy != MVT::isVoid) 5021 return 0; 5022 if ((!Subtarget->is64Bit())) { 5023 return fastEmitInst_r(X86::JMP16r_NT, &X86::GR16RegClass, Op0, Op0IsKill); 5024 } 5025 return 0; 5026} 5027 5028unsigned fastEmit_X86ISD_NT_BRIND_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5029 if (RetVT.SimpleTy != MVT::isVoid) 5030 return 0; 5031 if ((!Subtarget->is64Bit())) { 5032 return fastEmitInst_r(X86::JMP32r_NT, &X86::GR32RegClass, Op0, Op0IsKill); 5033 } 5034 return 0; 5035} 5036 5037unsigned fastEmit_X86ISD_NT_BRIND_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5038 if (RetVT.SimpleTy != MVT::isVoid) 5039 return 0; 5040 if ((Subtarget->is64Bit())) { 5041 return fastEmitInst_r(X86::JMP64r_NT, &X86::GR64RegClass, Op0, Op0IsKill); 5042 } 5043 return 0; 5044} 5045 5046unsigned fastEmit_X86ISD_NT_BRIND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 5047 switch (VT.SimpleTy) { 5048 case MVT::i16: return fastEmit_X86ISD_NT_BRIND_MVT_i16_r(RetVT, Op0, Op0IsKill); 5049 case MVT::i32: return fastEmit_X86ISD_NT_BRIND_MVT_i32_r(RetVT, Op0, Op0IsKill); 5050 case MVT::i64: return fastEmit_X86ISD_NT_BRIND_MVT_i64_r(RetVT, Op0, Op0IsKill); 5051 default: return 0; 5052 } 5053} 5054 5055// FastEmit functions for X86ISD::NT_CALL. 5056 5057unsigned fastEmit_X86ISD_NT_CALL_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5058 if (RetVT.SimpleTy != MVT::isVoid) 5059 return 0; 5060 if ((!Subtarget->is64Bit())) { 5061 return fastEmitInst_r(X86::CALL16r_NT, &X86::GR16RegClass, Op0, Op0IsKill); 5062 } 5063 return 0; 5064} 5065 5066unsigned fastEmit_X86ISD_NT_CALL_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5067 if (RetVT.SimpleTy != MVT::isVoid) 5068 return 0; 5069 if ((!Subtarget->is64Bit())) { 5070 return fastEmitInst_r(X86::CALL32r_NT, &X86::GR32RegClass, Op0, Op0IsKill); 5071 } 5072 return 0; 5073} 5074 5075unsigned fastEmit_X86ISD_NT_CALL_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5076 if (RetVT.SimpleTy != MVT::isVoid) 5077 return 0; 5078 if ((Subtarget->is64Bit())) { 5079 return fastEmitInst_r(X86::CALL64r_NT, &X86::GR64RegClass, Op0, Op0IsKill); 5080 } 5081 return 0; 5082} 5083 5084unsigned fastEmit_X86ISD_NT_CALL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 5085 switch (VT.SimpleTy) { 5086 case MVT::i16: return fastEmit_X86ISD_NT_CALL_MVT_i16_r(RetVT, Op0, Op0IsKill); 5087 case MVT::i32: return fastEmit_X86ISD_NT_CALL_MVT_i32_r(RetVT, Op0, Op0IsKill); 5088 case MVT::i64: return fastEmit_X86ISD_NT_CALL_MVT_i64_r(RetVT, Op0, Op0IsKill); 5089 default: return 0; 5090 } 5091} 5092 5093// FastEmit functions for X86ISD::PHMINPOS. 5094 5095unsigned fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5096 if (RetVT.SimpleTy != MVT::v8i16) 5097 return 0; 5098 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 5099 return fastEmitInst_r(X86::PHMINPOSUWrr, &X86::VR128RegClass, Op0, Op0IsKill); 5100 } 5101 if ((Subtarget->hasAVX())) { 5102 return fastEmitInst_r(X86::VPHMINPOSUWrr, &X86::VR128RegClass, Op0, Op0IsKill); 5103 } 5104 return 0; 5105} 5106 5107unsigned fastEmit_X86ISD_PHMINPOS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 5108 switch (VT.SimpleTy) { 5109 case MVT::v8i16: return fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(RetVT, Op0, Op0IsKill); 5110 default: return 0; 5111 } 5112} 5113 5114// FastEmit functions for X86ISD::RCP14. 5115 5116unsigned fastEmit_X86ISD_RCP14_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5117 if (RetVT.SimpleTy != MVT::v4f32) 5118 return 0; 5119 if ((Subtarget->hasVLX())) { 5120 return fastEmitInst_r(X86::VRCP14PSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill); 5121 } 5122 return 0; 5123} 5124 5125unsigned fastEmit_X86ISD_RCP14_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5126 if (RetVT.SimpleTy != MVT::v8f32) 5127 return 0; 5128 if ((Subtarget->hasVLX())) { 5129 return fastEmitInst_r(X86::VRCP14PSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill); 5130 } 5131 return 0; 5132} 5133 5134unsigned fastEmit_X86ISD_RCP14_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5135 if (RetVT.SimpleTy != MVT::v16f32) 5136 return 0; 5137 if ((Subtarget->hasAVX512())) { 5138 return fastEmitInst_r(X86::VRCP14PSZr, &X86::VR512RegClass, Op0, Op0IsKill); 5139 } 5140 return 0; 5141} 5142 5143unsigned fastEmit_X86ISD_RCP14_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5144 if (RetVT.SimpleTy != MVT::v2f64) 5145 return 0; 5146 if ((Subtarget->hasVLX())) { 5147 return fastEmitInst_r(X86::VRCP14PDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill); 5148 } 5149 return 0; 5150} 5151 5152unsigned fastEmit_X86ISD_RCP14_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5153 if (RetVT.SimpleTy != MVT::v4f64) 5154 return 0; 5155 if ((Subtarget->hasVLX())) { 5156 return fastEmitInst_r(X86::VRCP14PDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill); 5157 } 5158 return 0; 5159} 5160 5161unsigned fastEmit_X86ISD_RCP14_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5162 if (RetVT.SimpleTy != MVT::v8f64) 5163 return 0; 5164 if ((Subtarget->hasAVX512())) { 5165 return fastEmitInst_r(X86::VRCP14PDZr, &X86::VR512RegClass, Op0, Op0IsKill); 5166 } 5167 return 0; 5168} 5169 5170unsigned fastEmit_X86ISD_RCP14_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 5171 switch (VT.SimpleTy) { 5172 case MVT::v4f32: return fastEmit_X86ISD_RCP14_MVT_v4f32_r(RetVT, Op0, Op0IsKill); 5173 case MVT::v8f32: return fastEmit_X86ISD_RCP14_MVT_v8f32_r(RetVT, Op0, Op0IsKill); 5174 case MVT::v16f32: return fastEmit_X86ISD_RCP14_MVT_v16f32_r(RetVT, Op0, Op0IsKill); 5175 case MVT::v2f64: return fastEmit_X86ISD_RCP14_MVT_v2f64_r(RetVT, Op0, Op0IsKill); 5176 case MVT::v4f64: return fastEmit_X86ISD_RCP14_MVT_v4f64_r(RetVT, Op0, Op0IsKill); 5177 case MVT::v8f64: return fastEmit_X86ISD_RCP14_MVT_v8f64_r(RetVT, Op0, Op0IsKill); 5178 default: return 0; 5179 } 5180} 5181 5182// FastEmit functions for X86ISD::RCP28. 5183 5184unsigned fastEmit_X86ISD_RCP28_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5185 if (RetVT.SimpleTy != MVT::v16f32) 5186 return 0; 5187 if ((Subtarget->hasERI())) { 5188 return fastEmitInst_r(X86::VRCP28PSZr, &X86::VR512RegClass, Op0, Op0IsKill); 5189 } 5190 return 0; 5191} 5192 5193unsigned fastEmit_X86ISD_RCP28_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5194 if (RetVT.SimpleTy != MVT::v8f64) 5195 return 0; 5196 if ((Subtarget->hasERI())) { 5197 return fastEmitInst_r(X86::VRCP28PDZr, &X86::VR512RegClass, Op0, Op0IsKill); 5198 } 5199 return 0; 5200} 5201 5202unsigned fastEmit_X86ISD_RCP28_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 5203 switch (VT.SimpleTy) { 5204 case MVT::v16f32: return fastEmit_X86ISD_RCP28_MVT_v16f32_r(RetVT, Op0, Op0IsKill); 5205 case MVT::v8f64: return fastEmit_X86ISD_RCP28_MVT_v8f64_r(RetVT, Op0, Op0IsKill); 5206 default: return 0; 5207 } 5208} 5209 5210// FastEmit functions for X86ISD::RCP28_SAE. 5211 5212unsigned fastEmit_X86ISD_RCP28_SAE_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5213 if (RetVT.SimpleTy != MVT::v16f32) 5214 return 0; 5215 if ((Subtarget->hasERI())) { 5216 return fastEmitInst_r(X86::VRCP28PSZrb, &X86::VR512RegClass, Op0, Op0IsKill); 5217 } 5218 return 0; 5219} 5220 5221unsigned fastEmit_X86ISD_RCP28_SAE_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5222 if (RetVT.SimpleTy != MVT::v8f64) 5223 return 0; 5224 if ((Subtarget->hasERI())) { 5225 return fastEmitInst_r(X86::VRCP28PDZrb, &X86::VR512RegClass, Op0, Op0IsKill); 5226 } 5227 return 0; 5228} 5229 5230unsigned fastEmit_X86ISD_RCP28_SAE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 5231 switch (VT.SimpleTy) { 5232 case MVT::v16f32: return fastEmit_X86ISD_RCP28_SAE_MVT_v16f32_r(RetVT, Op0, Op0IsKill); 5233 case MVT::v8f64: return fastEmit_X86ISD_RCP28_SAE_MVT_v8f64_r(RetVT, Op0, Op0IsKill); 5234 default: return 0; 5235 } 5236} 5237 5238// FastEmit functions for X86ISD::RSQRT14. 5239 5240unsigned fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5241 if (RetVT.SimpleTy != MVT::v4f32) 5242 return 0; 5243 if ((Subtarget->hasVLX())) { 5244 return fastEmitInst_r(X86::VRSQRT14PSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill); 5245 } 5246 return 0; 5247} 5248 5249unsigned fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5250 if (RetVT.SimpleTy != MVT::v8f32) 5251 return 0; 5252 if ((Subtarget->hasVLX())) { 5253 return fastEmitInst_r(X86::VRSQRT14PSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill); 5254 } 5255 return 0; 5256} 5257 5258unsigned fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5259 if (RetVT.SimpleTy != MVT::v16f32) 5260 return 0; 5261 if ((Subtarget->hasAVX512())) { 5262 return fastEmitInst_r(X86::VRSQRT14PSZr, &X86::VR512RegClass, Op0, Op0IsKill); 5263 } 5264 return 0; 5265} 5266 5267unsigned fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5268 if (RetVT.SimpleTy != MVT::v2f64) 5269 return 0; 5270 if ((Subtarget->hasVLX())) { 5271 return fastEmitInst_r(X86::VRSQRT14PDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill); 5272 } 5273 return 0; 5274} 5275 5276unsigned fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5277 if (RetVT.SimpleTy != MVT::v4f64) 5278 return 0; 5279 if ((Subtarget->hasVLX())) { 5280 return fastEmitInst_r(X86::VRSQRT14PDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill); 5281 } 5282 return 0; 5283} 5284 5285unsigned fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5286 if (RetVT.SimpleTy != MVT::v8f64) 5287 return 0; 5288 if ((Subtarget->hasAVX512())) { 5289 return fastEmitInst_r(X86::VRSQRT14PDZr, &X86::VR512RegClass, Op0, Op0IsKill); 5290 } 5291 return 0; 5292} 5293 5294unsigned fastEmit_X86ISD_RSQRT14_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 5295 switch (VT.SimpleTy) { 5296 case MVT::v4f32: return fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(RetVT, Op0, Op0IsKill); 5297 case MVT::v8f32: return fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(RetVT, Op0, Op0IsKill); 5298 case MVT::v16f32: return fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(RetVT, Op0, Op0IsKill); 5299 case MVT::v2f64: return fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(RetVT, Op0, Op0IsKill); 5300 case MVT::v4f64: return fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(RetVT, Op0, Op0IsKill); 5301 case MVT::v8f64: return fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(RetVT, Op0, Op0IsKill); 5302 default: return 0; 5303 } 5304} 5305 5306// FastEmit functions for X86ISD::RSQRT28. 5307 5308unsigned fastEmit_X86ISD_RSQRT28_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5309 if (RetVT.SimpleTy != MVT::v16f32) 5310 return 0; 5311 if ((Subtarget->hasERI())) { 5312 return fastEmitInst_r(X86::VRSQRT28PSZr, &X86::VR512RegClass, Op0, Op0IsKill); 5313 } 5314 return 0; 5315} 5316 5317unsigned fastEmit_X86ISD_RSQRT28_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5318 if (RetVT.SimpleTy != MVT::v8f64) 5319 return 0; 5320 if ((Subtarget->hasERI())) { 5321 return fastEmitInst_r(X86::VRSQRT28PDZr, &X86::VR512RegClass, Op0, Op0IsKill); 5322 } 5323 return 0; 5324} 5325 5326unsigned fastEmit_X86ISD_RSQRT28_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 5327 switch (VT.SimpleTy) { 5328 case MVT::v16f32: return fastEmit_X86ISD_RSQRT28_MVT_v16f32_r(RetVT, Op0, Op0IsKill); 5329 case MVT::v8f64: return fastEmit_X86ISD_RSQRT28_MVT_v8f64_r(RetVT, Op0, Op0IsKill); 5330 default: return 0; 5331 } 5332} 5333 5334// FastEmit functions for X86ISD::RSQRT28_SAE. 5335 5336unsigned fastEmit_X86ISD_RSQRT28_SAE_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5337 if (RetVT.SimpleTy != MVT::v16f32) 5338 return 0; 5339 if ((Subtarget->hasERI())) { 5340 return fastEmitInst_r(X86::VRSQRT28PSZrb, &X86::VR512RegClass, Op0, Op0IsKill); 5341 } 5342 return 0; 5343} 5344 5345unsigned fastEmit_X86ISD_RSQRT28_SAE_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5346 if (RetVT.SimpleTy != MVT::v8f64) 5347 return 0; 5348 if ((Subtarget->hasERI())) { 5349 return fastEmitInst_r(X86::VRSQRT28PDZrb, &X86::VR512RegClass, Op0, Op0IsKill); 5350 } 5351 return 0; 5352} 5353 5354unsigned fastEmit_X86ISD_RSQRT28_SAE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 5355 switch (VT.SimpleTy) { 5356 case MVT::v16f32: return fastEmit_X86ISD_RSQRT28_SAE_MVT_v16f32_r(RetVT, Op0, Op0IsKill); 5357 case MVT::v8f64: return fastEmit_X86ISD_RSQRT28_SAE_MVT_v8f64_r(RetVT, Op0, Op0IsKill); 5358 default: return 0; 5359 } 5360} 5361 5362// FastEmit functions for X86ISD::SEG_ALLOCA. 5363 5364unsigned fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5365 if (RetVT.SimpleTy != MVT::i32) 5366 return 0; 5367 if ((!Subtarget->isTarget64BitLP64())) { 5368 return fastEmitInst_r(X86::SEG_ALLOCA_32, &X86::GR32RegClass, Op0, Op0IsKill); 5369 } 5370 return 0; 5371} 5372 5373unsigned fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5374 if (RetVT.SimpleTy != MVT::i64) 5375 return 0; 5376 if ((Subtarget->is64Bit())) { 5377 return fastEmitInst_r(X86::SEG_ALLOCA_64, &X86::GR64RegClass, Op0, Op0IsKill); 5378 } 5379 return 0; 5380} 5381 5382unsigned fastEmit_X86ISD_SEG_ALLOCA_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 5383 switch (VT.SimpleTy) { 5384 case MVT::i32: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(RetVT, Op0, Op0IsKill); 5385 case MVT::i64: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(RetVT, Op0, Op0IsKill); 5386 default: return 0; 5387 } 5388} 5389 5390// FastEmit functions for X86ISD::STRICT_CVTSI2P. 5391 5392unsigned fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5393 if (RetVT.SimpleTy != MVT::v2f64) 5394 return 0; 5395 if ((Subtarget->hasVLX())) { 5396 return fastEmitInst_r(X86::VCVTDQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 5397 } 5398 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 5399 return fastEmitInst_r(X86::CVTDQ2PDrr, &X86::VR128RegClass, Op0, Op0IsKill); 5400 } 5401 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 5402 return fastEmitInst_r(X86::VCVTDQ2PDrr, &X86::VR128RegClass, Op0, Op0IsKill); 5403 } 5404 return 0; 5405} 5406 5407unsigned fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5408 if (RetVT.SimpleTy != MVT::v4f32) 5409 return 0; 5410 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 5411 return fastEmitInst_r(X86::VCVTQQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 5412 } 5413 return 0; 5414} 5415 5416unsigned fastEmit_X86ISD_STRICT_CVTSI2P_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 5417 switch (VT.SimpleTy) { 5418 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_r(RetVT, Op0, Op0IsKill); 5419 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_r(RetVT, Op0, Op0IsKill); 5420 default: return 0; 5421 } 5422} 5423 5424// FastEmit functions for X86ISD::STRICT_CVTTP2SI. 5425 5426unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 5427 if ((Subtarget->hasVLX())) { 5428 return fastEmitInst_r(X86::VCVTTPS2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 5429 } 5430 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 5431 return fastEmitInst_r(X86::CVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill); 5432 } 5433 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 5434 return fastEmitInst_r(X86::VCVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill); 5435 } 5436 return 0; 5437} 5438 5439unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) { 5440 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 5441 return fastEmitInst_r(X86::VCVTTPS2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 5442 } 5443 return 0; 5444} 5445 5446unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) { 5447 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 5448 return fastEmitInst_r(X86::VCVTTPS2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 5449 } 5450 return 0; 5451} 5452 5453unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5454switch (RetVT.SimpleTy) { 5455 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill); 5456 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill); 5457 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill); 5458 default: return 0; 5459} 5460} 5461 5462unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) { 5463 if ((Subtarget->hasVLX())) { 5464 return fastEmitInst_r(X86::VCVTTPS2DQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 5465 } 5466 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 5467 return fastEmitInst_r(X86::VCVTTPS2DQYrr, &X86::VR256RegClass, Op0, Op0IsKill); 5468 } 5469 return 0; 5470} 5471 5472unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) { 5473 if ((Subtarget->hasDQI())) { 5474 return fastEmitInst_r(X86::VCVTTPS2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill); 5475 } 5476 return 0; 5477} 5478 5479unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5480switch (RetVT.SimpleTy) { 5481 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill); 5482 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill); 5483 default: return 0; 5484} 5485} 5486 5487unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5488 if (RetVT.SimpleTy != MVT::v16i32) 5489 return 0; 5490 if ((Subtarget->hasAVX512())) { 5491 return fastEmitInst_r(X86::VCVTTPS2DQZrr, &X86::VR512RegClass, Op0, Op0IsKill); 5492 } 5493 return 0; 5494} 5495 5496unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 5497 if ((Subtarget->hasVLX())) { 5498 return fastEmitInst_r(X86::VCVTTPD2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 5499 } 5500 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 5501 return fastEmitInst_r(X86::CVTTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill); 5502 } 5503 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 5504 return fastEmitInst_r(X86::VCVTTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill); 5505 } 5506 return 0; 5507} 5508 5509unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) { 5510 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 5511 return fastEmitInst_r(X86::VCVTTPD2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 5512 } 5513 return 0; 5514} 5515 5516unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5517switch (RetVT.SimpleTy) { 5518 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill); 5519 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill); 5520 default: return 0; 5521} 5522} 5523 5524unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 5525 if ((Subtarget->hasVLX())) { 5526 return fastEmitInst_r(X86::VCVTTPD2DQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 5527 } 5528 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 5529 return fastEmitInst_r(X86::VCVTTPD2DQYrr, &X86::VR128RegClass, Op0, Op0IsKill); 5530 } 5531 return 0; 5532} 5533 5534unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) { 5535 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 5536 return fastEmitInst_r(X86::VCVTTPD2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 5537 } 5538 return 0; 5539} 5540 5541unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5542switch (RetVT.SimpleTy) { 5543 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill); 5544 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill); 5545 default: return 0; 5546} 5547} 5548 5549unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) { 5550 if ((Subtarget->hasAVX512())) { 5551 return fastEmitInst_r(X86::VCVTTPD2DQZrr, &X86::VR256XRegClass, Op0, Op0IsKill); 5552 } 5553 return 0; 5554} 5555 5556unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) { 5557 if ((Subtarget->hasDQI())) { 5558 return fastEmitInst_r(X86::VCVTTPD2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill); 5559 } 5560 return 0; 5561} 5562 5563unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5564switch (RetVT.SimpleTy) { 5565 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill); 5566 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill); 5567 default: return 0; 5568} 5569} 5570 5571unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 5572 switch (VT.SimpleTy) { 5573 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_r(RetVT, Op0, Op0IsKill); 5574 case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_r(RetVT, Op0, Op0IsKill); 5575 case MVT::v16f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f32_r(RetVT, Op0, Op0IsKill); 5576 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_r(RetVT, Op0, Op0IsKill); 5577 case MVT::v4f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_r(RetVT, Op0, Op0IsKill); 5578 case MVT::v8f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_r(RetVT, Op0, Op0IsKill); 5579 default: return 0; 5580 } 5581} 5582 5583// FastEmit functions for X86ISD::STRICT_CVTTP2UI. 5584 5585unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 5586 if ((Subtarget->hasVLX())) { 5587 return fastEmitInst_r(X86::VCVTTPS2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 5588 } 5589 return 0; 5590} 5591 5592unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) { 5593 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 5594 return fastEmitInst_r(X86::VCVTTPS2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 5595 } 5596 return 0; 5597} 5598 5599unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) { 5600 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 5601 return fastEmitInst_r(X86::VCVTTPS2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 5602 } 5603 return 0; 5604} 5605 5606unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5607switch (RetVT.SimpleTy) { 5608 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill); 5609 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill); 5610 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill); 5611 default: return 0; 5612} 5613} 5614 5615unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) { 5616 if ((Subtarget->hasVLX())) { 5617 return fastEmitInst_r(X86::VCVTTPS2UDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 5618 } 5619 return 0; 5620} 5621 5622unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) { 5623 if ((Subtarget->hasDQI())) { 5624 return fastEmitInst_r(X86::VCVTTPS2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill); 5625 } 5626 return 0; 5627} 5628 5629unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5630switch (RetVT.SimpleTy) { 5631 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill); 5632 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill); 5633 default: return 0; 5634} 5635} 5636 5637unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5638 if (RetVT.SimpleTy != MVT::v16i32) 5639 return 0; 5640 if ((Subtarget->hasAVX512())) { 5641 return fastEmitInst_r(X86::VCVTTPS2UDQZrr, &X86::VR512RegClass, Op0, Op0IsKill); 5642 } 5643 return 0; 5644} 5645 5646unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 5647 if ((Subtarget->hasVLX())) { 5648 return fastEmitInst_r(X86::VCVTTPD2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 5649 } 5650 return 0; 5651} 5652 5653unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) { 5654 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 5655 return fastEmitInst_r(X86::VCVTTPD2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 5656 } 5657 return 0; 5658} 5659 5660unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5661switch (RetVT.SimpleTy) { 5662 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill); 5663 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill); 5664 default: return 0; 5665} 5666} 5667 5668unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 5669 if ((Subtarget->hasVLX())) { 5670 return fastEmitInst_r(X86::VCVTTPD2UDQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 5671 } 5672 return 0; 5673} 5674 5675unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) { 5676 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 5677 return fastEmitInst_r(X86::VCVTTPD2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 5678 } 5679 return 0; 5680} 5681 5682unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5683switch (RetVT.SimpleTy) { 5684 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill); 5685 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill); 5686 default: return 0; 5687} 5688} 5689 5690unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) { 5691 if ((Subtarget->hasAVX512())) { 5692 return fastEmitInst_r(X86::VCVTTPD2UDQZrr, &X86::VR256XRegClass, Op0, Op0IsKill); 5693 } 5694 return 0; 5695} 5696 5697unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) { 5698 if ((Subtarget->hasDQI())) { 5699 return fastEmitInst_r(X86::VCVTTPD2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill); 5700 } 5701 return 0; 5702} 5703 5704unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5705switch (RetVT.SimpleTy) { 5706 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill); 5707 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill); 5708 default: return 0; 5709} 5710} 5711 5712unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 5713 switch (VT.SimpleTy) { 5714 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_r(RetVT, Op0, Op0IsKill); 5715 case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_r(RetVT, Op0, Op0IsKill); 5716 case MVT::v16f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f32_r(RetVT, Op0, Op0IsKill); 5717 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_r(RetVT, Op0, Op0IsKill); 5718 case MVT::v4f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_r(RetVT, Op0, Op0IsKill); 5719 case MVT::v8f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_r(RetVT, Op0, Op0IsKill); 5720 default: return 0; 5721 } 5722} 5723 5724// FastEmit functions for X86ISD::STRICT_CVTUI2P. 5725 5726unsigned fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5727 if (RetVT.SimpleTy != MVT::v2f64) 5728 return 0; 5729 if ((Subtarget->hasVLX())) { 5730 return fastEmitInst_r(X86::VCVTUDQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 5731 } 5732 return 0; 5733} 5734 5735unsigned fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5736 if (RetVT.SimpleTy != MVT::v4f32) 5737 return 0; 5738 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 5739 return fastEmitInst_r(X86::VCVTUQQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 5740 } 5741 return 0; 5742} 5743 5744unsigned fastEmit_X86ISD_STRICT_CVTUI2P_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 5745 switch (VT.SimpleTy) { 5746 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_r(RetVT, Op0, Op0IsKill); 5747 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_r(RetVT, Op0, Op0IsKill); 5748 default: return 0; 5749 } 5750} 5751 5752// FastEmit functions for X86ISD::STRICT_VFPEXT. 5753 5754unsigned fastEmit_X86ISD_STRICT_VFPEXT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5755 if (RetVT.SimpleTy != MVT::v2f64) 5756 return 0; 5757 if ((Subtarget->hasVLX())) { 5758 return fastEmitInst_r(X86::VCVTPS2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 5759 } 5760 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 5761 return fastEmitInst_r(X86::CVTPS2PDrr, &X86::VR128RegClass, Op0, Op0IsKill); 5762 } 5763 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 5764 return fastEmitInst_r(X86::VCVTPS2PDrr, &X86::VR128RegClass, Op0, Op0IsKill); 5765 } 5766 return 0; 5767} 5768 5769unsigned fastEmit_X86ISD_STRICT_VFPEXT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 5770 switch (VT.SimpleTy) { 5771 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v4f32_r(RetVT, Op0, Op0IsKill); 5772 default: return 0; 5773 } 5774} 5775 5776// FastEmit functions for X86ISD::STRICT_VFPROUND. 5777 5778unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5779 if (RetVT.SimpleTy != MVT::v4f32) 5780 return 0; 5781 if ((Subtarget->hasVLX())) { 5782 return fastEmitInst_r(X86::VCVTPD2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 5783 } 5784 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 5785 return fastEmitInst_r(X86::CVTPD2PSrr, &X86::VR128RegClass, Op0, Op0IsKill); 5786 } 5787 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 5788 return fastEmitInst_r(X86::VCVTPD2PSrr, &X86::VR128RegClass, Op0, Op0IsKill); 5789 } 5790 return 0; 5791} 5792 5793unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5794 if (RetVT.SimpleTy != MVT::v4f32) 5795 return 0; 5796 if ((Subtarget->hasVLX())) { 5797 return fastEmitInst_r(X86::VCVTPD2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 5798 } 5799 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 5800 return fastEmitInst_r(X86::VCVTPD2PSYrr, &X86::VR128RegClass, Op0, Op0IsKill); 5801 } 5802 return 0; 5803} 5804 5805unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5806 if (RetVT.SimpleTy != MVT::v8f32) 5807 return 0; 5808 if ((Subtarget->hasAVX512())) { 5809 return fastEmitInst_r(X86::VCVTPD2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill); 5810 } 5811 return 0; 5812} 5813 5814unsigned fastEmit_X86ISD_STRICT_VFPROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 5815 switch (VT.SimpleTy) { 5816 case MVT::v2f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_r(RetVT, Op0, Op0IsKill); 5817 case MVT::v4f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_r(RetVT, Op0, Op0IsKill); 5818 case MVT::v8f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_r(RetVT, Op0, Op0IsKill); 5819 default: return 0; 5820 } 5821} 5822 5823// FastEmit functions for X86ISD::VBROADCAST. 5824 5825unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 5826 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 5827 return fastEmitInst_r(X86::VPBROADCASTDrZ128r, &X86::VR128XRegClass, Op0, Op0IsKill); 5828 } 5829 return 0; 5830} 5831 5832unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) { 5833 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 5834 return fastEmitInst_r(X86::VPBROADCASTDrZ256r, &X86::VR256XRegClass, Op0, Op0IsKill); 5835 } 5836 return 0; 5837} 5838 5839unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) { 5840 if ((Subtarget->hasAVX512())) { 5841 return fastEmitInst_r(X86::VPBROADCASTDrZr, &X86::VR512RegClass, Op0, Op0IsKill); 5842 } 5843 return 0; 5844} 5845 5846unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5847switch (RetVT.SimpleTy) { 5848 case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(Op0, Op0IsKill); 5849 case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(Op0, Op0IsKill); 5850 case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(Op0, Op0IsKill); 5851 default: return 0; 5852} 5853} 5854 5855unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) { 5856 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 5857 return fastEmitInst_r(X86::VPBROADCASTQrZ128r, &X86::VR128XRegClass, Op0, Op0IsKill); 5858 } 5859 return 0; 5860} 5861 5862unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) { 5863 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 5864 return fastEmitInst_r(X86::VPBROADCASTQrZ256r, &X86::VR256XRegClass, Op0, Op0IsKill); 5865 } 5866 return 0; 5867} 5868 5869unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) { 5870 if ((Subtarget->hasAVX512())) { 5871 return fastEmitInst_r(X86::VPBROADCASTQrZr, &X86::VR512RegClass, Op0, Op0IsKill); 5872 } 5873 return 0; 5874} 5875 5876unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5877switch (RetVT.SimpleTy) { 5878 case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(Op0, Op0IsKill); 5879 case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(Op0, Op0IsKill); 5880 case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(Op0, Op0IsKill); 5881 default: return 0; 5882} 5883} 5884 5885unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) { 5886 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 5887 return fastEmitInst_r(X86::VPBROADCASTBZ128r, &X86::VR128XRegClass, Op0, Op0IsKill); 5888 } 5889 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 5890 return fastEmitInst_r(X86::VPBROADCASTBrr, &X86::VR128RegClass, Op0, Op0IsKill); 5891 } 5892 return 0; 5893} 5894 5895unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(unsigned Op0, bool Op0IsKill) { 5896 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 5897 return fastEmitInst_r(X86::VPBROADCASTBZ256r, &X86::VR256XRegClass, Op0, Op0IsKill); 5898 } 5899 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 5900 return fastEmitInst_r(X86::VPBROADCASTBYrr, &X86::VR256RegClass, Op0, Op0IsKill); 5901 } 5902 return 0; 5903} 5904 5905unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(unsigned Op0, bool Op0IsKill) { 5906 if ((Subtarget->hasBWI())) { 5907 return fastEmitInst_r(X86::VPBROADCASTBZr, &X86::VR512RegClass, Op0, Op0IsKill); 5908 } 5909 return 0; 5910} 5911 5912unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5913switch (RetVT.SimpleTy) { 5914 case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(Op0, Op0IsKill); 5915 case MVT::v32i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(Op0, Op0IsKill); 5916 case MVT::v64i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(Op0, Op0IsKill); 5917 default: return 0; 5918} 5919} 5920 5921unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) { 5922 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 5923 return fastEmitInst_r(X86::VPBROADCASTWZ128r, &X86::VR128XRegClass, Op0, Op0IsKill); 5924 } 5925 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 5926 return fastEmitInst_r(X86::VPBROADCASTWrr, &X86::VR128RegClass, Op0, Op0IsKill); 5927 } 5928 return 0; 5929} 5930 5931unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) { 5932 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 5933 return fastEmitInst_r(X86::VPBROADCASTWZ256r, &X86::VR256XRegClass, Op0, Op0IsKill); 5934 } 5935 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 5936 return fastEmitInst_r(X86::VPBROADCASTWYrr, &X86::VR256RegClass, Op0, Op0IsKill); 5937 } 5938 return 0; 5939} 5940 5941unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(unsigned Op0, bool Op0IsKill) { 5942 if ((Subtarget->hasBWI())) { 5943 return fastEmitInst_r(X86::VPBROADCASTWZr, &X86::VR512RegClass, Op0, Op0IsKill); 5944 } 5945 return 0; 5946} 5947 5948unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5949switch (RetVT.SimpleTy) { 5950 case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(Op0, Op0IsKill); 5951 case MVT::v16i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(Op0, Op0IsKill); 5952 case MVT::v32i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(Op0, Op0IsKill); 5953 default: return 0; 5954} 5955} 5956 5957unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 5958 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 5959 return fastEmitInst_r(X86::VPBROADCASTDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill); 5960 } 5961 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 5962 return fastEmitInst_r(X86::VPBROADCASTDrr, &X86::VR128RegClass, Op0, Op0IsKill); 5963 } 5964 return 0; 5965} 5966 5967unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) { 5968 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 5969 return fastEmitInst_r(X86::VPBROADCASTDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill); 5970 } 5971 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 5972 return fastEmitInst_r(X86::VPBROADCASTDYrr, &X86::VR256RegClass, Op0, Op0IsKill); 5973 } 5974 return 0; 5975} 5976 5977unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) { 5978 if ((Subtarget->hasAVX512())) { 5979 return fastEmitInst_r(X86::VPBROADCASTDZr, &X86::VR512RegClass, Op0, Op0IsKill); 5980 } 5981 return 0; 5982} 5983 5984unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 5985switch (RetVT.SimpleTy) { 5986 case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(Op0, Op0IsKill); 5987 case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(Op0, Op0IsKill); 5988 case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(Op0, Op0IsKill); 5989 default: return 0; 5990} 5991} 5992 5993unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) { 5994 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 5995 return fastEmitInst_r(X86::VPBROADCASTQZ128r, &X86::VR128XRegClass, Op0, Op0IsKill); 5996 } 5997 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 5998 return fastEmitInst_r(X86::VPBROADCASTQrr, &X86::VR128RegClass, Op0, Op0IsKill); 5999 } 6000 return 0; 6001} 6002 6003unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) { 6004 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6005 return fastEmitInst_r(X86::VPBROADCASTQZ256r, &X86::VR256XRegClass, Op0, Op0IsKill); 6006 } 6007 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 6008 return fastEmitInst_r(X86::VPBROADCASTQYrr, &X86::VR256RegClass, Op0, Op0IsKill); 6009 } 6010 return 0; 6011} 6012 6013unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) { 6014 if ((Subtarget->hasAVX512())) { 6015 return fastEmitInst_r(X86::VPBROADCASTQZr, &X86::VR512RegClass, Op0, Op0IsKill); 6016 } 6017 return 0; 6018} 6019 6020unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6021switch (RetVT.SimpleTy) { 6022 case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(Op0, Op0IsKill); 6023 case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(Op0, Op0IsKill); 6024 case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(Op0, Op0IsKill); 6025 default: return 0; 6026} 6027} 6028 6029unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) { 6030 if ((Subtarget->hasVLX())) { 6031 return fastEmitInst_r(X86::VBROADCASTSSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill); 6032 } 6033 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 6034 return fastEmitInst_r(X86::VBROADCASTSSrr, &X86::VR128RegClass, Op0, Op0IsKill); 6035 } 6036 return 0; 6037} 6038 6039unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) { 6040 if ((Subtarget->hasVLX())) { 6041 return fastEmitInst_r(X86::VBROADCASTSSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill); 6042 } 6043 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 6044 return fastEmitInst_r(X86::VBROADCASTSSYrr, &X86::VR256RegClass, Op0, Op0IsKill); 6045 } 6046 return 0; 6047} 6048 6049unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(unsigned Op0, bool Op0IsKill) { 6050 if ((Subtarget->hasAVX512())) { 6051 return fastEmitInst_r(X86::VBROADCASTSSZr, &X86::VR512RegClass, Op0, Op0IsKill); 6052 } 6053 return 0; 6054} 6055 6056unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6057switch (RetVT.SimpleTy) { 6058 case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(Op0, Op0IsKill); 6059 case MVT::v8f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(Op0, Op0IsKill); 6060 case MVT::v16f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(Op0, Op0IsKill); 6061 default: return 0; 6062} 6063} 6064 6065unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) { 6066 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 6067 return fastEmitInst_r(X86::VMOVDDUPrr, &X86::VR128RegClass, Op0, Op0IsKill); 6068 } 6069 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6070 return fastEmitInst_r(X86::VMOVDDUPZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6071 } 6072 return 0; 6073} 6074 6075unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) { 6076 if ((Subtarget->hasVLX())) { 6077 return fastEmitInst_r(X86::VBROADCASTSDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill); 6078 } 6079 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 6080 return fastEmitInst_r(X86::VBROADCASTSDYrr, &X86::VR256RegClass, Op0, Op0IsKill); 6081 } 6082 return 0; 6083} 6084 6085unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) { 6086 if ((Subtarget->hasAVX512())) { 6087 return fastEmitInst_r(X86::VBROADCASTSDZr, &X86::VR512RegClass, Op0, Op0IsKill); 6088 } 6089 return 0; 6090} 6091 6092unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6093switch (RetVT.SimpleTy) { 6094 case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(Op0, Op0IsKill); 6095 case MVT::v4f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(Op0, Op0IsKill); 6096 case MVT::v8f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(Op0, Op0IsKill); 6097 default: return 0; 6098} 6099} 6100 6101unsigned fastEmit_X86ISD_VBROADCAST_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 6102 switch (VT.SimpleTy) { 6103 case MVT::i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_r(RetVT, Op0, Op0IsKill); 6104 case MVT::i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_r(RetVT, Op0, Op0IsKill); 6105 case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(RetVT, Op0, Op0IsKill); 6106 case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(RetVT, Op0, Op0IsKill); 6107 case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(RetVT, Op0, Op0IsKill); 6108 case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(RetVT, Op0, Op0IsKill); 6109 case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(RetVT, Op0, Op0IsKill); 6110 case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(RetVT, Op0, Op0IsKill); 6111 default: return 0; 6112 } 6113} 6114 6115// FastEmit functions for X86ISD::VBROADCASTM. 6116 6117unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) { 6118 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) { 6119 return fastEmitInst_r(X86::VPBROADCASTMB2QZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6120 } 6121 return 0; 6122} 6123 6124unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) { 6125 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) { 6126 return fastEmitInst_r(X86::VPBROADCASTMB2QZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 6127 } 6128 return 0; 6129} 6130 6131unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) { 6132 if ((Subtarget->hasCDI())) { 6133 return fastEmitInst_r(X86::VPBROADCASTMB2QZrr, &X86::VR512RegClass, Op0, Op0IsKill); 6134 } 6135 return 0; 6136} 6137 6138unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6139switch (RetVT.SimpleTy) { 6140 case MVT::v2i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(Op0, Op0IsKill); 6141 case MVT::v4i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(Op0, Op0IsKill); 6142 case MVT::v8i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(Op0, Op0IsKill); 6143 default: return 0; 6144} 6145} 6146 6147unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 6148 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) { 6149 return fastEmitInst_r(X86::VPBROADCASTMW2DZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6150 } 6151 return 0; 6152} 6153 6154unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) { 6155 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) { 6156 return fastEmitInst_r(X86::VPBROADCASTMW2DZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill); 6157 } 6158 return 0; 6159} 6160 6161unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) { 6162 if ((Subtarget->hasCDI())) { 6163 return fastEmitInst_r(X86::VPBROADCASTMW2DZrr, &X86::VR512RegClass, Op0, Op0IsKill); 6164 } 6165 return 0; 6166} 6167 6168unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6169switch (RetVT.SimpleTy) { 6170 case MVT::v4i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(Op0, Op0IsKill); 6171 case MVT::v8i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(Op0, Op0IsKill); 6172 case MVT::v16i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(Op0, Op0IsKill); 6173 default: return 0; 6174} 6175} 6176 6177unsigned fastEmit_X86ISD_VBROADCASTM_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 6178 switch (VT.SimpleTy) { 6179 case MVT::v8i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(RetVT, Op0, Op0IsKill); 6180 case MVT::v16i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(RetVT, Op0, Op0IsKill); 6181 default: return 0; 6182 } 6183} 6184 6185// FastEmit functions for X86ISD::VFPEXT. 6186 6187unsigned fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6188 if (RetVT.SimpleTy != MVT::v2f64) 6189 return 0; 6190 if ((Subtarget->hasVLX())) { 6191 return fastEmitInst_r(X86::VCVTPS2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6192 } 6193 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 6194 return fastEmitInst_r(X86::CVTPS2PDrr, &X86::VR128RegClass, Op0, Op0IsKill); 6195 } 6196 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 6197 return fastEmitInst_r(X86::VCVTPS2PDrr, &X86::VR128RegClass, Op0, Op0IsKill); 6198 } 6199 return 0; 6200} 6201 6202unsigned fastEmit_X86ISD_VFPEXT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 6203 switch (VT.SimpleTy) { 6204 case MVT::v4f32: return fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(RetVT, Op0, Op0IsKill); 6205 default: return 0; 6206 } 6207} 6208 6209// FastEmit functions for X86ISD::VFPEXT_SAE. 6210 6211unsigned fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6212 if (RetVT.SimpleTy != MVT::v8f64) 6213 return 0; 6214 if ((Subtarget->hasAVX512())) { 6215 return fastEmitInst_r(X86::VCVTPS2PDZrrb, &X86::VR512RegClass, Op0, Op0IsKill); 6216 } 6217 return 0; 6218} 6219 6220unsigned fastEmit_X86ISD_VFPEXT_SAE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 6221 switch (VT.SimpleTy) { 6222 case MVT::v8f32: return fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f32_r(RetVT, Op0, Op0IsKill); 6223 default: return 0; 6224 } 6225} 6226 6227// FastEmit functions for X86ISD::VFPROUND. 6228 6229unsigned fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6230 if (RetVT.SimpleTy != MVT::v4f32) 6231 return 0; 6232 if ((Subtarget->hasVLX())) { 6233 return fastEmitInst_r(X86::VCVTPD2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6234 } 6235 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 6236 return fastEmitInst_r(X86::CVTPD2PSrr, &X86::VR128RegClass, Op0, Op0IsKill); 6237 } 6238 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 6239 return fastEmitInst_r(X86::VCVTPD2PSrr, &X86::VR128RegClass, Op0, Op0IsKill); 6240 } 6241 return 0; 6242} 6243 6244unsigned fastEmit_X86ISD_VFPROUND_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6245 if (RetVT.SimpleTy != MVT::v4f32) 6246 return 0; 6247 if ((Subtarget->hasVLX())) { 6248 return fastEmitInst_r(X86::VCVTPD2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6249 } 6250 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 6251 return fastEmitInst_r(X86::VCVTPD2PSYrr, &X86::VR128RegClass, Op0, Op0IsKill); 6252 } 6253 return 0; 6254} 6255 6256unsigned fastEmit_X86ISD_VFPROUND_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6257 if (RetVT.SimpleTy != MVT::v8f32) 6258 return 0; 6259 if ((Subtarget->hasAVX512())) { 6260 return fastEmitInst_r(X86::VCVTPD2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill); 6261 } 6262 return 0; 6263} 6264 6265unsigned fastEmit_X86ISD_VFPROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 6266 switch (VT.SimpleTy) { 6267 case MVT::v2f64: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(RetVT, Op0, Op0IsKill); 6268 case MVT::v4f64: return fastEmit_X86ISD_VFPROUND_MVT_v4f64_r(RetVT, Op0, Op0IsKill); 6269 case MVT::v8f64: return fastEmit_X86ISD_VFPROUND_MVT_v8f64_r(RetVT, Op0, Op0IsKill); 6270 default: return 0; 6271 } 6272} 6273 6274// FastEmit functions for X86ISD::VTRUNC. 6275 6276unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6277 if (RetVT.SimpleTy != MVT::v16i8) 6278 return 0; 6279 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 6280 return fastEmitInst_r(X86::VPMOVWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6281 } 6282 return 0; 6283} 6284 6285unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) { 6286 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6287 return fastEmitInst_r(X86::VPMOVDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6288 } 6289 return 0; 6290} 6291 6292unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) { 6293 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6294 return fastEmitInst_r(X86::VPMOVDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6295 } 6296 return 0; 6297} 6298 6299unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6300switch (RetVT.SimpleTy) { 6301 case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(Op0, Op0IsKill); 6302 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(Op0, Op0IsKill); 6303 default: return 0; 6304} 6305} 6306 6307unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6308 if (RetVT.SimpleTy != MVT::v16i8) 6309 return 0; 6310 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6311 return fastEmitInst_r(X86::VPMOVDBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6312 } 6313 return 0; 6314} 6315 6316unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) { 6317 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6318 return fastEmitInst_r(X86::VPMOVQBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6319 } 6320 return 0; 6321} 6322 6323unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) { 6324 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6325 return fastEmitInst_r(X86::VPMOVQWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6326 } 6327 return 0; 6328} 6329 6330unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 6331 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6332 return fastEmitInst_r(X86::VPMOVQDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6333 } 6334 return 0; 6335} 6336 6337unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6338switch (RetVT.SimpleTy) { 6339 case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(Op0, Op0IsKill); 6340 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(Op0, Op0IsKill); 6341 case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(Op0, Op0IsKill); 6342 default: return 0; 6343} 6344} 6345 6346unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) { 6347 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6348 return fastEmitInst_r(X86::VPMOVQBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6349 } 6350 return 0; 6351} 6352 6353unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) { 6354 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6355 return fastEmitInst_r(X86::VPMOVQWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6356 } 6357 return 0; 6358} 6359 6360unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6361switch (RetVT.SimpleTy) { 6362 case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(Op0, Op0IsKill); 6363 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(Op0, Op0IsKill); 6364 default: return 0; 6365} 6366} 6367 6368unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6369 if (RetVT.SimpleTy != MVT::v16i8) 6370 return 0; 6371 if ((Subtarget->hasAVX512())) { 6372 return fastEmitInst_r(X86::VPMOVQBZrr, &X86::VR128XRegClass, Op0, Op0IsKill); 6373 } 6374 return 0; 6375} 6376 6377unsigned fastEmit_X86ISD_VTRUNC_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 6378 switch (VT.SimpleTy) { 6379 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(RetVT, Op0, Op0IsKill); 6380 case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(RetVT, Op0, Op0IsKill); 6381 case MVT::v8i32: return fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(RetVT, Op0, Op0IsKill); 6382 case MVT::v2i64: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(RetVT, Op0, Op0IsKill); 6383 case MVT::v4i64: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(RetVT, Op0, Op0IsKill); 6384 case MVT::v8i64: return fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(RetVT, Op0, Op0IsKill); 6385 default: return 0; 6386 } 6387} 6388 6389// FastEmit functions for X86ISD::VTRUNCS. 6390 6391unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6392 if (RetVT.SimpleTy != MVT::v16i8) 6393 return 0; 6394 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 6395 return fastEmitInst_r(X86::VPMOVSWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6396 } 6397 return 0; 6398} 6399 6400unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6401 if (RetVT.SimpleTy != MVT::v16i8) 6402 return 0; 6403 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 6404 return fastEmitInst_r(X86::VPMOVSWBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6405 } 6406 return 0; 6407} 6408 6409unsigned fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6410 if (RetVT.SimpleTy != MVT::v32i8) 6411 return 0; 6412 if ((Subtarget->hasBWI())) { 6413 return fastEmitInst_r(X86::VPMOVSWBZrr, &X86::VR256XRegClass, Op0, Op0IsKill); 6414 } 6415 return 0; 6416} 6417 6418unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) { 6419 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6420 return fastEmitInst_r(X86::VPMOVSDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6421 } 6422 return 0; 6423} 6424 6425unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) { 6426 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6427 return fastEmitInst_r(X86::VPMOVSDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6428 } 6429 return 0; 6430} 6431 6432unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6433switch (RetVT.SimpleTy) { 6434 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(Op0, Op0IsKill); 6435 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(Op0, Op0IsKill); 6436 default: return 0; 6437} 6438} 6439 6440unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) { 6441 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6442 return fastEmitInst_r(X86::VPMOVSDBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6443 } 6444 return 0; 6445} 6446 6447unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) { 6448 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6449 return fastEmitInst_r(X86::VPMOVSDWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6450 } 6451 return 0; 6452} 6453 6454unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6455switch (RetVT.SimpleTy) { 6456 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(Op0, Op0IsKill); 6457 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(Op0, Op0IsKill); 6458 default: return 0; 6459} 6460} 6461 6462unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) { 6463 if ((Subtarget->hasAVX512())) { 6464 return fastEmitInst_r(X86::VPMOVSDBZrr, &X86::VR128XRegClass, Op0, Op0IsKill); 6465 } 6466 return 0; 6467} 6468 6469unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) { 6470 if ((Subtarget->hasAVX512())) { 6471 return fastEmitInst_r(X86::VPMOVSDWZrr, &X86::VR256XRegClass, Op0, Op0IsKill); 6472 } 6473 return 0; 6474} 6475 6476unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6477switch (RetVT.SimpleTy) { 6478 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(Op0, Op0IsKill); 6479 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(Op0, Op0IsKill); 6480 default: return 0; 6481} 6482} 6483 6484unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) { 6485 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6486 return fastEmitInst_r(X86::VPMOVSQBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6487 } 6488 return 0; 6489} 6490 6491unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) { 6492 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6493 return fastEmitInst_r(X86::VPMOVSQWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6494 } 6495 return 0; 6496} 6497 6498unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 6499 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6500 return fastEmitInst_r(X86::VPMOVSQDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6501 } 6502 return 0; 6503} 6504 6505unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6506switch (RetVT.SimpleTy) { 6507 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(Op0, Op0IsKill); 6508 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(Op0, Op0IsKill); 6509 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(Op0, Op0IsKill); 6510 default: return 0; 6511} 6512} 6513 6514unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) { 6515 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6516 return fastEmitInst_r(X86::VPMOVSQBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6517 } 6518 return 0; 6519} 6520 6521unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) { 6522 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6523 return fastEmitInst_r(X86::VPMOVSQWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6524 } 6525 return 0; 6526} 6527 6528unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 6529 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6530 return fastEmitInst_r(X86::VPMOVSQDZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6531 } 6532 return 0; 6533} 6534 6535unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6536switch (RetVT.SimpleTy) { 6537 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(Op0, Op0IsKill); 6538 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(Op0, Op0IsKill); 6539 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(Op0, Op0IsKill); 6540 default: return 0; 6541} 6542} 6543 6544unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) { 6545 if ((Subtarget->hasAVX512())) { 6546 return fastEmitInst_r(X86::VPMOVSQBZrr, &X86::VR128XRegClass, Op0, Op0IsKill); 6547 } 6548 return 0; 6549} 6550 6551unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) { 6552 if ((Subtarget->hasAVX512())) { 6553 return fastEmitInst_r(X86::VPMOVSQWZrr, &X86::VR128XRegClass, Op0, Op0IsKill); 6554 } 6555 return 0; 6556} 6557 6558unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) { 6559 if ((Subtarget->hasAVX512())) { 6560 return fastEmitInst_r(X86::VPMOVSQDZrr, &X86::VR256XRegClass, Op0, Op0IsKill); 6561 } 6562 return 0; 6563} 6564 6565unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6566switch (RetVT.SimpleTy) { 6567 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(Op0, Op0IsKill); 6568 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(Op0, Op0IsKill); 6569 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(Op0, Op0IsKill); 6570 default: return 0; 6571} 6572} 6573 6574unsigned fastEmit_X86ISD_VTRUNCS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 6575 switch (VT.SimpleTy) { 6576 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(RetVT, Op0, Op0IsKill); 6577 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(RetVT, Op0, Op0IsKill); 6578 case MVT::v32i16: return fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(RetVT, Op0, Op0IsKill); 6579 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(RetVT, Op0, Op0IsKill); 6580 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(RetVT, Op0, Op0IsKill); 6581 case MVT::v16i32: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(RetVT, Op0, Op0IsKill); 6582 case MVT::v2i64: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(RetVT, Op0, Op0IsKill); 6583 case MVT::v4i64: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(RetVT, Op0, Op0IsKill); 6584 case MVT::v8i64: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(RetVT, Op0, Op0IsKill); 6585 default: return 0; 6586 } 6587} 6588 6589// FastEmit functions for X86ISD::VTRUNCUS. 6590 6591unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6592 if (RetVT.SimpleTy != MVT::v16i8) 6593 return 0; 6594 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 6595 return fastEmitInst_r(X86::VPMOVUSWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6596 } 6597 return 0; 6598} 6599 6600unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6601 if (RetVT.SimpleTy != MVT::v16i8) 6602 return 0; 6603 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 6604 return fastEmitInst_r(X86::VPMOVUSWBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6605 } 6606 return 0; 6607} 6608 6609unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6610 if (RetVT.SimpleTy != MVT::v32i8) 6611 return 0; 6612 if ((Subtarget->hasBWI())) { 6613 return fastEmitInst_r(X86::VPMOVUSWBZrr, &X86::VR256XRegClass, Op0, Op0IsKill); 6614 } 6615 return 0; 6616} 6617 6618unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) { 6619 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6620 return fastEmitInst_r(X86::VPMOVUSDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6621 } 6622 return 0; 6623} 6624 6625unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) { 6626 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6627 return fastEmitInst_r(X86::VPMOVUSDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6628 } 6629 return 0; 6630} 6631 6632unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6633switch (RetVT.SimpleTy) { 6634 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(Op0, Op0IsKill); 6635 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(Op0, Op0IsKill); 6636 default: return 0; 6637} 6638} 6639 6640unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) { 6641 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6642 return fastEmitInst_r(X86::VPMOVUSDBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6643 } 6644 return 0; 6645} 6646 6647unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) { 6648 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6649 return fastEmitInst_r(X86::VPMOVUSDWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6650 } 6651 return 0; 6652} 6653 6654unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6655switch (RetVT.SimpleTy) { 6656 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(Op0, Op0IsKill); 6657 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(Op0, Op0IsKill); 6658 default: return 0; 6659} 6660} 6661 6662unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) { 6663 if ((Subtarget->hasAVX512())) { 6664 return fastEmitInst_r(X86::VPMOVUSDBZrr, &X86::VR128XRegClass, Op0, Op0IsKill); 6665 } 6666 return 0; 6667} 6668 6669unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) { 6670 if ((Subtarget->hasAVX512())) { 6671 return fastEmitInst_r(X86::VPMOVUSDWZrr, &X86::VR256XRegClass, Op0, Op0IsKill); 6672 } 6673 return 0; 6674} 6675 6676unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6677switch (RetVT.SimpleTy) { 6678 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(Op0, Op0IsKill); 6679 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(Op0, Op0IsKill); 6680 default: return 0; 6681} 6682} 6683 6684unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) { 6685 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6686 return fastEmitInst_r(X86::VPMOVUSQBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6687 } 6688 return 0; 6689} 6690 6691unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) { 6692 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6693 return fastEmitInst_r(X86::VPMOVUSQWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6694 } 6695 return 0; 6696} 6697 6698unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 6699 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6700 return fastEmitInst_r(X86::VPMOVUSQDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6701 } 6702 return 0; 6703} 6704 6705unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6706switch (RetVT.SimpleTy) { 6707 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(Op0, Op0IsKill); 6708 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(Op0, Op0IsKill); 6709 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(Op0, Op0IsKill); 6710 default: return 0; 6711} 6712} 6713 6714unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) { 6715 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6716 return fastEmitInst_r(X86::VPMOVUSQBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6717 } 6718 return 0; 6719} 6720 6721unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) { 6722 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6723 return fastEmitInst_r(X86::VPMOVUSQWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6724 } 6725 return 0; 6726} 6727 6728unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) { 6729 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 6730 return fastEmitInst_r(X86::VPMOVUSQDZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill); 6731 } 6732 return 0; 6733} 6734 6735unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6736switch (RetVT.SimpleTy) { 6737 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(Op0, Op0IsKill); 6738 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(Op0, Op0IsKill); 6739 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(Op0, Op0IsKill); 6740 default: return 0; 6741} 6742} 6743 6744unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) { 6745 if ((Subtarget->hasAVX512())) { 6746 return fastEmitInst_r(X86::VPMOVUSQBZrr, &X86::VR128XRegClass, Op0, Op0IsKill); 6747 } 6748 return 0; 6749} 6750 6751unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) { 6752 if ((Subtarget->hasAVX512())) { 6753 return fastEmitInst_r(X86::VPMOVUSQWZrr, &X86::VR128XRegClass, Op0, Op0IsKill); 6754 } 6755 return 0; 6756} 6757 6758unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) { 6759 if ((Subtarget->hasAVX512())) { 6760 return fastEmitInst_r(X86::VPMOVUSQDZrr, &X86::VR256XRegClass, Op0, Op0IsKill); 6761 } 6762 return 0; 6763} 6764 6765unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6766switch (RetVT.SimpleTy) { 6767 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(Op0, Op0IsKill); 6768 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(Op0, Op0IsKill); 6769 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(Op0, Op0IsKill); 6770 default: return 0; 6771} 6772} 6773 6774unsigned fastEmit_X86ISD_VTRUNCUS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 6775 switch (VT.SimpleTy) { 6776 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(RetVT, Op0, Op0IsKill); 6777 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(RetVT, Op0, Op0IsKill); 6778 case MVT::v32i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(RetVT, Op0, Op0IsKill); 6779 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(RetVT, Op0, Op0IsKill); 6780 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(RetVT, Op0, Op0IsKill); 6781 case MVT::v16i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(RetVT, Op0, Op0IsKill); 6782 case MVT::v2i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(RetVT, Op0, Op0IsKill); 6783 case MVT::v4i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(RetVT, Op0, Op0IsKill); 6784 case MVT::v8i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(RetVT, Op0, Op0IsKill); 6785 default: return 0; 6786 } 6787} 6788 6789// FastEmit functions for X86ISD::VZEXT_MOVL. 6790 6791unsigned fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6792 if (RetVT.SimpleTy != MVT::v2i64) 6793 return 0; 6794 if ((Subtarget->hasAVX512())) { 6795 return fastEmitInst_r(X86::VMOVZPQILo2PQIZrr, &X86::VR128XRegClass, Op0, Op0IsKill); 6796 } 6797 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 6798 return fastEmitInst_r(X86::MOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill); 6799 } 6800 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 6801 return fastEmitInst_r(X86::VMOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill); 6802 } 6803 return 0; 6804} 6805 6806unsigned fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6807 if (RetVT.SimpleTy != MVT::v2f64) 6808 return 0; 6809 if ((Subtarget->hasAVX512())) { 6810 return fastEmitInst_r(X86::VMOVZPQILo2PQIZrr, &X86::VR128XRegClass, Op0, Op0IsKill); 6811 } 6812 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 6813 return fastEmitInst_r(X86::MOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill); 6814 } 6815 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 6816 return fastEmitInst_r(X86::VMOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill); 6817 } 6818 return 0; 6819} 6820 6821unsigned fastEmit_X86ISD_VZEXT_MOVL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 6822 switch (VT.SimpleTy) { 6823 case MVT::v2i64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(RetVT, Op0, Op0IsKill); 6824 case MVT::v2f64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(RetVT, Op0, Op0IsKill); 6825 default: return 0; 6826 } 6827} 6828 6829// FastEmit functions for X86ISD::WIN_ALLOCA. 6830 6831unsigned fastEmit_X86ISD_WIN_ALLOCA_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6832 if (RetVT.SimpleTy != MVT::isVoid) 6833 return 0; 6834 if ((!Subtarget->isTarget64BitLP64())) { 6835 return fastEmitInst_r(X86::WIN_ALLOCA_32, &X86::GR32RegClass, Op0, Op0IsKill); 6836 } 6837 return 0; 6838} 6839 6840unsigned fastEmit_X86ISD_WIN_ALLOCA_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) { 6841 if (RetVT.SimpleTy != MVT::isVoid) 6842 return 0; 6843 if ((Subtarget->is64Bit())) { 6844 return fastEmitInst_r(X86::WIN_ALLOCA_64, &X86::GR64RegClass, Op0, Op0IsKill); 6845 } 6846 return 0; 6847} 6848 6849unsigned fastEmit_X86ISD_WIN_ALLOCA_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) { 6850 switch (VT.SimpleTy) { 6851 case MVT::i32: return fastEmit_X86ISD_WIN_ALLOCA_MVT_i32_r(RetVT, Op0, Op0IsKill); 6852 case MVT::i64: return fastEmit_X86ISD_WIN_ALLOCA_MVT_i64_r(RetVT, Op0, Op0IsKill); 6853 default: return 0; 6854 } 6855} 6856 6857// Top-level FastEmit function. 6858 6859unsigned fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill) override { 6860 switch (Opcode) { 6861 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0, Op0IsKill); 6862 case ISD::ANY_EXTEND: return fastEmit_ISD_ANY_EXTEND_r(VT, RetVT, Op0, Op0IsKill); 6863 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0, Op0IsKill); 6864 case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0, Op0IsKill); 6865 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0, Op0IsKill); 6866 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0, Op0IsKill); 6867 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0, Op0IsKill); 6868 case ISD::CTTZ: return fastEmit_ISD_CTTZ_r(VT, RetVT, Op0, Op0IsKill); 6869 case ISD::CTTZ_ZERO_UNDEF: return fastEmit_ISD_CTTZ_ZERO_UNDEF_r(VT, RetVT, Op0, Op0IsKill); 6870 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0, Op0IsKill); 6871 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0, Op0IsKill); 6872 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0, Op0IsKill); 6873 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0, Op0IsKill); 6874 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0, Op0IsKill); 6875 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0, Op0IsKill); 6876 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0, Op0IsKill); 6877 case ISD::SCALAR_TO_VECTOR: return fastEmit_ISD_SCALAR_TO_VECTOR_r(VT, RetVT, Op0, Op0IsKill); 6878 case ISD::SIGN_EXTEND: return fastEmit_ISD_SIGN_EXTEND_r(VT, RetVT, Op0, Op0IsKill); 6879 case ISD::SIGN_EXTEND_VECTOR_INREG: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0, Op0IsKill); 6880 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill); 6881 case ISD::STRICT_FP_EXTEND: return fastEmit_ISD_STRICT_FP_EXTEND_r(VT, RetVT, Op0, Op0IsKill); 6882 case ISD::STRICT_FP_ROUND: return fastEmit_ISD_STRICT_FP_ROUND_r(VT, RetVT, Op0, Op0IsKill); 6883 case ISD::STRICT_FP_TO_SINT: return fastEmit_ISD_STRICT_FP_TO_SINT_r(VT, RetVT, Op0, Op0IsKill); 6884 case ISD::STRICT_FP_TO_UINT: return fastEmit_ISD_STRICT_FP_TO_UINT_r(VT, RetVT, Op0, Op0IsKill); 6885 case ISD::STRICT_FSQRT: return fastEmit_ISD_STRICT_FSQRT_r(VT, RetVT, Op0, Op0IsKill); 6886 case ISD::STRICT_SINT_TO_FP: return fastEmit_ISD_STRICT_SINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill); 6887 case ISD::STRICT_UINT_TO_FP: return fastEmit_ISD_STRICT_UINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill); 6888 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0, Op0IsKill); 6889 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill); 6890 case ISD::ZERO_EXTEND: return fastEmit_ISD_ZERO_EXTEND_r(VT, RetVT, Op0, Op0IsKill); 6891 case ISD::ZERO_EXTEND_VECTOR_INREG: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0, Op0IsKill); 6892 case X86ISD::CALL: return fastEmit_X86ISD_CALL_r(VT, RetVT, Op0, Op0IsKill); 6893 case X86ISD::CONFLICT: return fastEmit_X86ISD_CONFLICT_r(VT, RetVT, Op0, Op0IsKill); 6894 case X86ISD::CVTNEPS2BF16: return fastEmit_X86ISD_CVTNEPS2BF16_r(VT, RetVT, Op0, Op0IsKill); 6895 case X86ISD::CVTP2SI: return fastEmit_X86ISD_CVTP2SI_r(VT, RetVT, Op0, Op0IsKill); 6896 case X86ISD::CVTP2UI: return fastEmit_X86ISD_CVTP2UI_r(VT, RetVT, Op0, Op0IsKill); 6897 case X86ISD::CVTPH2PS: return fastEmit_X86ISD_CVTPH2PS_r(VT, RetVT, Op0, Op0IsKill); 6898 case X86ISD::CVTPH2PS_SAE: return fastEmit_X86ISD_CVTPH2PS_SAE_r(VT, RetVT, Op0, Op0IsKill); 6899 case X86ISD::CVTS2SI: return fastEmit_X86ISD_CVTS2SI_r(VT, RetVT, Op0, Op0IsKill); 6900 case X86ISD::CVTS2UI: return fastEmit_X86ISD_CVTS2UI_r(VT, RetVT, Op0, Op0IsKill); 6901 case X86ISD::CVTSI2P: return fastEmit_X86ISD_CVTSI2P_r(VT, RetVT, Op0, Op0IsKill); 6902 case X86ISD::CVTTP2SI: return fastEmit_X86ISD_CVTTP2SI_r(VT, RetVT, Op0, Op0IsKill); 6903 case X86ISD::CVTTP2SI_SAE: return fastEmit_X86ISD_CVTTP2SI_SAE_r(VT, RetVT, Op0, Op0IsKill); 6904 case X86ISD::CVTTP2UI: return fastEmit_X86ISD_CVTTP2UI_r(VT, RetVT, Op0, Op0IsKill); 6905 case X86ISD::CVTTP2UI_SAE: return fastEmit_X86ISD_CVTTP2UI_SAE_r(VT, RetVT, Op0, Op0IsKill); 6906 case X86ISD::CVTTS2SI: return fastEmit_X86ISD_CVTTS2SI_r(VT, RetVT, Op0, Op0IsKill); 6907 case X86ISD::CVTTS2SI_SAE: return fastEmit_X86ISD_CVTTS2SI_SAE_r(VT, RetVT, Op0, Op0IsKill); 6908 case X86ISD::CVTTS2UI: return fastEmit_X86ISD_CVTTS2UI_r(VT, RetVT, Op0, Op0IsKill); 6909 case X86ISD::CVTTS2UI_SAE: return fastEmit_X86ISD_CVTTS2UI_SAE_r(VT, RetVT, Op0, Op0IsKill); 6910 case X86ISD::CVTUI2P: return fastEmit_X86ISD_CVTUI2P_r(VT, RetVT, Op0, Op0IsKill); 6911 case X86ISD::EH_RETURN: return fastEmit_X86ISD_EH_RETURN_r(VT, RetVT, Op0, Op0IsKill); 6912 case X86ISD::EXP2: return fastEmit_X86ISD_EXP2_r(VT, RetVT, Op0, Op0IsKill); 6913 case X86ISD::EXP2_SAE: return fastEmit_X86ISD_EXP2_SAE_r(VT, RetVT, Op0, Op0IsKill); 6914 case X86ISD::FGETEXP: return fastEmit_X86ISD_FGETEXP_r(VT, RetVT, Op0, Op0IsKill); 6915 case X86ISD::FGETEXP_SAE: return fastEmit_X86ISD_FGETEXP_SAE_r(VT, RetVT, Op0, Op0IsKill); 6916 case X86ISD::FRCP: return fastEmit_X86ISD_FRCP_r(VT, RetVT, Op0, Op0IsKill); 6917 case X86ISD::FRSQRT: return fastEmit_X86ISD_FRSQRT_r(VT, RetVT, Op0, Op0IsKill); 6918 case X86ISD::MMX_MOVD2W: return fastEmit_X86ISD_MMX_MOVD2W_r(VT, RetVT, Op0, Op0IsKill); 6919 case X86ISD::MMX_MOVW2D: return fastEmit_X86ISD_MMX_MOVW2D_r(VT, RetVT, Op0, Op0IsKill); 6920 case X86ISD::MOVDDUP: return fastEmit_X86ISD_MOVDDUP_r(VT, RetVT, Op0, Op0IsKill); 6921 case X86ISD::MOVDQ2Q: return fastEmit_X86ISD_MOVDQ2Q_r(VT, RetVT, Op0, Op0IsKill); 6922 case X86ISD::MOVMSK: return fastEmit_X86ISD_MOVMSK_r(VT, RetVT, Op0, Op0IsKill); 6923 case X86ISD::MOVQ2DQ: return fastEmit_X86ISD_MOVQ2DQ_r(VT, RetVT, Op0, Op0IsKill); 6924 case X86ISD::MOVSHDUP: return fastEmit_X86ISD_MOVSHDUP_r(VT, RetVT, Op0, Op0IsKill); 6925 case X86ISD::MOVSLDUP: return fastEmit_X86ISD_MOVSLDUP_r(VT, RetVT, Op0, Op0IsKill); 6926 case X86ISD::NT_BRIND: return fastEmit_X86ISD_NT_BRIND_r(VT, RetVT, Op0, Op0IsKill); 6927 case X86ISD::NT_CALL: return fastEmit_X86ISD_NT_CALL_r(VT, RetVT, Op0, Op0IsKill); 6928 case X86ISD::PHMINPOS: return fastEmit_X86ISD_PHMINPOS_r(VT, RetVT, Op0, Op0IsKill); 6929 case X86ISD::RCP14: return fastEmit_X86ISD_RCP14_r(VT, RetVT, Op0, Op0IsKill); 6930 case X86ISD::RCP28: return fastEmit_X86ISD_RCP28_r(VT, RetVT, Op0, Op0IsKill); 6931 case X86ISD::RCP28_SAE: return fastEmit_X86ISD_RCP28_SAE_r(VT, RetVT, Op0, Op0IsKill); 6932 case X86ISD::RSQRT14: return fastEmit_X86ISD_RSQRT14_r(VT, RetVT, Op0, Op0IsKill); 6933 case X86ISD::RSQRT28: return fastEmit_X86ISD_RSQRT28_r(VT, RetVT, Op0, Op0IsKill); 6934 case X86ISD::RSQRT28_SAE: return fastEmit_X86ISD_RSQRT28_SAE_r(VT, RetVT, Op0, Op0IsKill); 6935 case X86ISD::SEG_ALLOCA: return fastEmit_X86ISD_SEG_ALLOCA_r(VT, RetVT, Op0, Op0IsKill); 6936 case X86ISD::STRICT_CVTSI2P: return fastEmit_X86ISD_STRICT_CVTSI2P_r(VT, RetVT, Op0, Op0IsKill); 6937 case X86ISD::STRICT_CVTTP2SI: return fastEmit_X86ISD_STRICT_CVTTP2SI_r(VT, RetVT, Op0, Op0IsKill); 6938 case X86ISD::STRICT_CVTTP2UI: return fastEmit_X86ISD_STRICT_CVTTP2UI_r(VT, RetVT, Op0, Op0IsKill); 6939 case X86ISD::STRICT_CVTUI2P: return fastEmit_X86ISD_STRICT_CVTUI2P_r(VT, RetVT, Op0, Op0IsKill); 6940 case X86ISD::STRICT_VFPEXT: return fastEmit_X86ISD_STRICT_VFPEXT_r(VT, RetVT, Op0, Op0IsKill); 6941 case X86ISD::STRICT_VFPROUND: return fastEmit_X86ISD_STRICT_VFPROUND_r(VT, RetVT, Op0, Op0IsKill); 6942 case X86ISD::VBROADCAST: return fastEmit_X86ISD_VBROADCAST_r(VT, RetVT, Op0, Op0IsKill); 6943 case X86ISD::VBROADCASTM: return fastEmit_X86ISD_VBROADCASTM_r(VT, RetVT, Op0, Op0IsKill); 6944 case X86ISD::VFPEXT: return fastEmit_X86ISD_VFPEXT_r(VT, RetVT, Op0, Op0IsKill); 6945 case X86ISD::VFPEXT_SAE: return fastEmit_X86ISD_VFPEXT_SAE_r(VT, RetVT, Op0, Op0IsKill); 6946 case X86ISD::VFPROUND: return fastEmit_X86ISD_VFPROUND_r(VT, RetVT, Op0, Op0IsKill); 6947 case X86ISD::VTRUNC: return fastEmit_X86ISD_VTRUNC_r(VT, RetVT, Op0, Op0IsKill); 6948 case X86ISD::VTRUNCS: return fastEmit_X86ISD_VTRUNCS_r(VT, RetVT, Op0, Op0IsKill); 6949 case X86ISD::VTRUNCUS: return fastEmit_X86ISD_VTRUNCUS_r(VT, RetVT, Op0, Op0IsKill); 6950 case X86ISD::VZEXT_MOVL: return fastEmit_X86ISD_VZEXT_MOVL_r(VT, RetVT, Op0, Op0IsKill); 6951 case X86ISD::WIN_ALLOCA: return fastEmit_X86ISD_WIN_ALLOCA_r(VT, RetVT, Op0, Op0IsKill); 6952 default: return 0; 6953 } 6954} 6955 6956// FastEmit functions for ISD::ADD. 6957 6958unsigned fastEmit_ISD_ADD_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 6959 if (RetVT.SimpleTy != MVT::i8) 6960 return 0; 6961 return fastEmitInst_rr(X86::ADD8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 6962} 6963 6964unsigned fastEmit_ISD_ADD_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 6965 if (RetVT.SimpleTy != MVT::i16) 6966 return 0; 6967 return fastEmitInst_rr(X86::ADD16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 6968} 6969 6970unsigned fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 6971 if (RetVT.SimpleTy != MVT::i32) 6972 return 0; 6973 return fastEmitInst_rr(X86::ADD32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 6974} 6975 6976unsigned fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 6977 if (RetVT.SimpleTy != MVT::i64) 6978 return 0; 6979 return fastEmitInst_rr(X86::ADD64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 6980} 6981 6982unsigned fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 6983 if (RetVT.SimpleTy != MVT::v16i8) 6984 return 0; 6985 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 6986 return fastEmitInst_rr(X86::VPADDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 6987 } 6988 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 6989 return fastEmitInst_rr(X86::PADDBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 6990 } 6991 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 6992 return fastEmitInst_rr(X86::VPADDBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 6993 } 6994 return 0; 6995} 6996 6997unsigned fastEmit_ISD_ADD_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 6998 if (RetVT.SimpleTy != MVT::v32i8) 6999 return 0; 7000 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 7001 return fastEmitInst_rr(X86::VPADDBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7002 } 7003 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 7004 return fastEmitInst_rr(X86::VPADDBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7005 } 7006 return 0; 7007} 7008 7009unsigned fastEmit_ISD_ADD_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7010 if (RetVT.SimpleTy != MVT::v64i8) 7011 return 0; 7012 if ((Subtarget->hasBWI())) { 7013 return fastEmitInst_rr(X86::VPADDBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7014 } 7015 return 0; 7016} 7017 7018unsigned fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7019 if (RetVT.SimpleTy != MVT::v8i16) 7020 return 0; 7021 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 7022 return fastEmitInst_rr(X86::VPADDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7023 } 7024 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 7025 return fastEmitInst_rr(X86::PADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7026 } 7027 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 7028 return fastEmitInst_rr(X86::VPADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7029 } 7030 return 0; 7031} 7032 7033unsigned fastEmit_ISD_ADD_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7034 if (RetVT.SimpleTy != MVT::v16i16) 7035 return 0; 7036 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 7037 return fastEmitInst_rr(X86::VPADDWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7038 } 7039 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 7040 return fastEmitInst_rr(X86::VPADDWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7041 } 7042 return 0; 7043} 7044 7045unsigned fastEmit_ISD_ADD_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7046 if (RetVT.SimpleTy != MVT::v32i16) 7047 return 0; 7048 if ((Subtarget->hasBWI())) { 7049 return fastEmitInst_rr(X86::VPADDWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7050 } 7051 return 0; 7052} 7053 7054unsigned fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7055 if (RetVT.SimpleTy != MVT::v4i32) 7056 return 0; 7057 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7058 return fastEmitInst_rr(X86::VPADDDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7059 } 7060 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 7061 return fastEmitInst_rr(X86::PADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7062 } 7063 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7064 return fastEmitInst_rr(X86::VPADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7065 } 7066 return 0; 7067} 7068 7069unsigned fastEmit_ISD_ADD_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7070 if (RetVT.SimpleTy != MVT::v8i32) 7071 return 0; 7072 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7073 return fastEmitInst_rr(X86::VPADDDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7074 } 7075 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 7076 return fastEmitInst_rr(X86::VPADDDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7077 } 7078 return 0; 7079} 7080 7081unsigned fastEmit_ISD_ADD_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7082 if (RetVT.SimpleTy != MVT::v16i32) 7083 return 0; 7084 if ((Subtarget->hasAVX512())) { 7085 return fastEmitInst_rr(X86::VPADDDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7086 } 7087 return 0; 7088} 7089 7090unsigned fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7091 if (RetVT.SimpleTy != MVT::v2i64) 7092 return 0; 7093 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7094 return fastEmitInst_rr(X86::VPADDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7095 } 7096 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 7097 return fastEmitInst_rr(X86::PADDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7098 } 7099 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7100 return fastEmitInst_rr(X86::VPADDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7101 } 7102 return 0; 7103} 7104 7105unsigned fastEmit_ISD_ADD_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7106 if (RetVT.SimpleTy != MVT::v4i64) 7107 return 0; 7108 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7109 return fastEmitInst_rr(X86::VPADDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7110 } 7111 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 7112 return fastEmitInst_rr(X86::VPADDQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7113 } 7114 return 0; 7115} 7116 7117unsigned fastEmit_ISD_ADD_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7118 if (RetVT.SimpleTy != MVT::v8i64) 7119 return 0; 7120 if ((Subtarget->hasAVX512())) { 7121 return fastEmitInst_rr(X86::VPADDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7122 } 7123 return 0; 7124} 7125 7126unsigned fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7127 switch (VT.SimpleTy) { 7128 case MVT::i8: return fastEmit_ISD_ADD_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7129 case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7130 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7131 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7132 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7133 case MVT::v32i8: return fastEmit_ISD_ADD_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7134 case MVT::v64i8: return fastEmit_ISD_ADD_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7135 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7136 case MVT::v16i16: return fastEmit_ISD_ADD_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7137 case MVT::v32i16: return fastEmit_ISD_ADD_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7138 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7139 case MVT::v8i32: return fastEmit_ISD_ADD_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7140 case MVT::v16i32: return fastEmit_ISD_ADD_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7141 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7142 case MVT::v4i64: return fastEmit_ISD_ADD_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7143 case MVT::v8i64: return fastEmit_ISD_ADD_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7144 default: return 0; 7145 } 7146} 7147 7148// FastEmit functions for ISD::AND. 7149 7150unsigned fastEmit_ISD_AND_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7151 if (RetVT.SimpleTy != MVT::i8) 7152 return 0; 7153 return fastEmitInst_rr(X86::AND8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7154} 7155 7156unsigned fastEmit_ISD_AND_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7157 if (RetVT.SimpleTy != MVT::i16) 7158 return 0; 7159 return fastEmitInst_rr(X86::AND16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7160} 7161 7162unsigned fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7163 if (RetVT.SimpleTy != MVT::i32) 7164 return 0; 7165 return fastEmitInst_rr(X86::AND32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7166} 7167 7168unsigned fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7169 if (RetVT.SimpleTy != MVT::i64) 7170 return 0; 7171 return fastEmitInst_rr(X86::AND64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7172} 7173 7174unsigned fastEmit_ISD_AND_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7175 if (RetVT.SimpleTy != MVT::v8i1) 7176 return 0; 7177 if ((Subtarget->hasDQI())) { 7178 return fastEmitInst_rr(X86::KANDBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7179 } 7180 return 0; 7181} 7182 7183unsigned fastEmit_ISD_AND_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7184 if (RetVT.SimpleTy != MVT::v16i1) 7185 return 0; 7186 if ((Subtarget->hasAVX512())) { 7187 return fastEmitInst_rr(X86::KANDWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7188 } 7189 return 0; 7190} 7191 7192unsigned fastEmit_ISD_AND_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7193 if (RetVT.SimpleTy != MVT::v32i1) 7194 return 0; 7195 if ((Subtarget->hasBWI())) { 7196 return fastEmitInst_rr(X86::KANDDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7197 } 7198 return 0; 7199} 7200 7201unsigned fastEmit_ISD_AND_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7202 if (RetVT.SimpleTy != MVT::v64i1) 7203 return 0; 7204 if ((Subtarget->hasBWI())) { 7205 return fastEmitInst_rr(X86::KANDQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7206 } 7207 return 0; 7208} 7209 7210unsigned fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7211 if (RetVT.SimpleTy != MVT::v16i8) 7212 return 0; 7213 if ((Subtarget->hasVLX())) { 7214 return fastEmitInst_rr(X86::VPANDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7215 } 7216 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 7217 return fastEmitInst_rr(X86::PANDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7218 } 7219 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7220 return fastEmitInst_rr(X86::VPANDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7221 } 7222 return 0; 7223} 7224 7225unsigned fastEmit_ISD_AND_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7226 if (RetVT.SimpleTy != MVT::v32i8) 7227 return 0; 7228 if ((Subtarget->hasVLX())) { 7229 return fastEmitInst_rr(X86::VPANDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7230 } 7231 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 7232 return fastEmitInst_rr(X86::VANDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7233 } 7234 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 7235 return fastEmitInst_rr(X86::VPANDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7236 } 7237 return 0; 7238} 7239 7240unsigned fastEmit_ISD_AND_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7241 if (RetVT.SimpleTy != MVT::v64i8) 7242 return 0; 7243 if ((Subtarget->hasAVX512())) { 7244 return fastEmitInst_rr(X86::VPANDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7245 } 7246 return 0; 7247} 7248 7249unsigned fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7250 if (RetVT.SimpleTy != MVT::v8i16) 7251 return 0; 7252 if ((Subtarget->hasVLX())) { 7253 return fastEmitInst_rr(X86::VPANDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7254 } 7255 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 7256 return fastEmitInst_rr(X86::PANDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7257 } 7258 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7259 return fastEmitInst_rr(X86::VPANDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7260 } 7261 return 0; 7262} 7263 7264unsigned fastEmit_ISD_AND_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7265 if (RetVT.SimpleTy != MVT::v16i16) 7266 return 0; 7267 if ((Subtarget->hasVLX())) { 7268 return fastEmitInst_rr(X86::VPANDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7269 } 7270 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 7271 return fastEmitInst_rr(X86::VANDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7272 } 7273 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 7274 return fastEmitInst_rr(X86::VPANDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7275 } 7276 return 0; 7277} 7278 7279unsigned fastEmit_ISD_AND_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7280 if (RetVT.SimpleTy != MVT::v32i16) 7281 return 0; 7282 if ((Subtarget->hasAVX512())) { 7283 return fastEmitInst_rr(X86::VPANDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7284 } 7285 return 0; 7286} 7287 7288unsigned fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7289 if (RetVT.SimpleTy != MVT::v4i32) 7290 return 0; 7291 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 7292 return fastEmitInst_rr(X86::PANDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7293 } 7294 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7295 return fastEmitInst_rr(X86::VPANDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7296 } 7297 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7298 return fastEmitInst_rr(X86::VPANDDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7299 } 7300 return 0; 7301} 7302 7303unsigned fastEmit_ISD_AND_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7304 if (RetVT.SimpleTy != MVT::v8i32) 7305 return 0; 7306 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 7307 return fastEmitInst_rr(X86::VANDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7308 } 7309 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 7310 return fastEmitInst_rr(X86::VPANDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7311 } 7312 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7313 return fastEmitInst_rr(X86::VPANDDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7314 } 7315 return 0; 7316} 7317 7318unsigned fastEmit_ISD_AND_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7319 if (RetVT.SimpleTy != MVT::v16i32) 7320 return 0; 7321 if ((Subtarget->hasAVX512())) { 7322 return fastEmitInst_rr(X86::VPANDDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7323 } 7324 return 0; 7325} 7326 7327unsigned fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7328 if (RetVT.SimpleTy != MVT::v2i64) 7329 return 0; 7330 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7331 return fastEmitInst_rr(X86::VPANDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7332 } 7333 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 7334 return fastEmitInst_rr(X86::PANDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7335 } 7336 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7337 return fastEmitInst_rr(X86::VPANDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7338 } 7339 return 0; 7340} 7341 7342unsigned fastEmit_ISD_AND_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7343 if (RetVT.SimpleTy != MVT::v4i64) 7344 return 0; 7345 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 7346 return fastEmitInst_rr(X86::VANDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7347 } 7348 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7349 return fastEmitInst_rr(X86::VPANDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7350 } 7351 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 7352 return fastEmitInst_rr(X86::VPANDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7353 } 7354 return 0; 7355} 7356 7357unsigned fastEmit_ISD_AND_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7358 if (RetVT.SimpleTy != MVT::v8i64) 7359 return 0; 7360 if ((Subtarget->hasAVX512())) { 7361 return fastEmitInst_rr(X86::VPANDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7362 } 7363 return 0; 7364} 7365 7366unsigned fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7367 switch (VT.SimpleTy) { 7368 case MVT::i8: return fastEmit_ISD_AND_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7369 case MVT::i16: return fastEmit_ISD_AND_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7370 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7371 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7372 case MVT::v8i1: return fastEmit_ISD_AND_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7373 case MVT::v16i1: return fastEmit_ISD_AND_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7374 case MVT::v32i1: return fastEmit_ISD_AND_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7375 case MVT::v64i1: return fastEmit_ISD_AND_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7376 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7377 case MVT::v32i8: return fastEmit_ISD_AND_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7378 case MVT::v64i8: return fastEmit_ISD_AND_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7379 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7380 case MVT::v16i16: return fastEmit_ISD_AND_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7381 case MVT::v32i16: return fastEmit_ISD_AND_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7382 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7383 case MVT::v8i32: return fastEmit_ISD_AND_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7384 case MVT::v16i32: return fastEmit_ISD_AND_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7385 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7386 case MVT::v4i64: return fastEmit_ISD_AND_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7387 case MVT::v8i64: return fastEmit_ISD_AND_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7388 default: return 0; 7389 } 7390} 7391 7392// FastEmit functions for ISD::FADD. 7393 7394unsigned fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7395 if (RetVT.SimpleTy != MVT::f32) 7396 return 0; 7397 if ((Subtarget->hasAVX512())) { 7398 return fastEmitInst_rr(X86::VADDSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7399 } 7400 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 7401 return fastEmitInst_rr(X86::ADDSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7402 } 7403 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 7404 return fastEmitInst_rr(X86::VADDSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7405 } 7406 if ((!Subtarget->hasSSE1())) { 7407 return fastEmitInst_rr(X86::ADD_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7408 } 7409 return 0; 7410} 7411 7412unsigned fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7413 if (RetVT.SimpleTy != MVT::f64) 7414 return 0; 7415 if ((Subtarget->hasAVX512())) { 7416 return fastEmitInst_rr(X86::VADDSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7417 } 7418 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 7419 return fastEmitInst_rr(X86::ADDSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7420 } 7421 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 7422 return fastEmitInst_rr(X86::VADDSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7423 } 7424 if ((!Subtarget->hasSSE2())) { 7425 return fastEmitInst_rr(X86::ADD_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7426 } 7427 return 0; 7428} 7429 7430unsigned fastEmit_ISD_FADD_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7431 if (RetVT.SimpleTy != MVT::f80) 7432 return 0; 7433 return fastEmitInst_rr(X86::ADD_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7434} 7435 7436unsigned fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7437 if (RetVT.SimpleTy != MVT::v4f32) 7438 return 0; 7439 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7440 return fastEmitInst_rr(X86::VADDPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7441 } 7442 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 7443 return fastEmitInst_rr(X86::ADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7444 } 7445 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7446 return fastEmitInst_rr(X86::VADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7447 } 7448 return 0; 7449} 7450 7451unsigned fastEmit_ISD_FADD_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7452 if (RetVT.SimpleTy != MVT::v8f32) 7453 return 0; 7454 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7455 return fastEmitInst_rr(X86::VADDPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7456 } 7457 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7458 return fastEmitInst_rr(X86::VADDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7459 } 7460 return 0; 7461} 7462 7463unsigned fastEmit_ISD_FADD_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7464 if (RetVT.SimpleTy != MVT::v16f32) 7465 return 0; 7466 if ((Subtarget->hasAVX512())) { 7467 return fastEmitInst_rr(X86::VADDPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7468 } 7469 return 0; 7470} 7471 7472unsigned fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7473 if (RetVT.SimpleTy != MVT::v2f64) 7474 return 0; 7475 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7476 return fastEmitInst_rr(X86::VADDPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7477 } 7478 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 7479 return fastEmitInst_rr(X86::ADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7480 } 7481 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7482 return fastEmitInst_rr(X86::VADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7483 } 7484 return 0; 7485} 7486 7487unsigned fastEmit_ISD_FADD_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7488 if (RetVT.SimpleTy != MVT::v4f64) 7489 return 0; 7490 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7491 return fastEmitInst_rr(X86::VADDPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7492 } 7493 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7494 return fastEmitInst_rr(X86::VADDPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7495 } 7496 return 0; 7497} 7498 7499unsigned fastEmit_ISD_FADD_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7500 if (RetVT.SimpleTy != MVT::v8f64) 7501 return 0; 7502 if ((Subtarget->hasAVX512())) { 7503 return fastEmitInst_rr(X86::VADDPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7504 } 7505 return 0; 7506} 7507 7508unsigned fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7509 switch (VT.SimpleTy) { 7510 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7511 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7512 case MVT::f80: return fastEmit_ISD_FADD_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7513 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7514 case MVT::v8f32: return fastEmit_ISD_FADD_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7515 case MVT::v16f32: return fastEmit_ISD_FADD_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7516 case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7517 case MVT::v4f64: return fastEmit_ISD_FADD_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7518 case MVT::v8f64: return fastEmit_ISD_FADD_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7519 default: return 0; 7520 } 7521} 7522 7523// FastEmit functions for ISD::FDIV. 7524 7525unsigned fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7526 if (RetVT.SimpleTy != MVT::f32) 7527 return 0; 7528 if ((Subtarget->hasAVX512())) { 7529 return fastEmitInst_rr(X86::VDIVSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7530 } 7531 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 7532 return fastEmitInst_rr(X86::DIVSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7533 } 7534 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 7535 return fastEmitInst_rr(X86::VDIVSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7536 } 7537 if ((!Subtarget->hasSSE1())) { 7538 return fastEmitInst_rr(X86::DIV_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7539 } 7540 return 0; 7541} 7542 7543unsigned fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7544 if (RetVT.SimpleTy != MVT::f64) 7545 return 0; 7546 if ((Subtarget->hasAVX512())) { 7547 return fastEmitInst_rr(X86::VDIVSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7548 } 7549 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 7550 return fastEmitInst_rr(X86::DIVSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7551 } 7552 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 7553 return fastEmitInst_rr(X86::VDIVSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7554 } 7555 if ((!Subtarget->hasSSE2())) { 7556 return fastEmitInst_rr(X86::DIV_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7557 } 7558 return 0; 7559} 7560 7561unsigned fastEmit_ISD_FDIV_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7562 if (RetVT.SimpleTy != MVT::f80) 7563 return 0; 7564 return fastEmitInst_rr(X86::DIV_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7565} 7566 7567unsigned fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7568 if (RetVT.SimpleTy != MVT::v4f32) 7569 return 0; 7570 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7571 return fastEmitInst_rr(X86::VDIVPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7572 } 7573 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 7574 return fastEmitInst_rr(X86::DIVPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7575 } 7576 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7577 return fastEmitInst_rr(X86::VDIVPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7578 } 7579 return 0; 7580} 7581 7582unsigned fastEmit_ISD_FDIV_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7583 if (RetVT.SimpleTy != MVT::v8f32) 7584 return 0; 7585 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7586 return fastEmitInst_rr(X86::VDIVPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7587 } 7588 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7589 return fastEmitInst_rr(X86::VDIVPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7590 } 7591 return 0; 7592} 7593 7594unsigned fastEmit_ISD_FDIV_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7595 if (RetVT.SimpleTy != MVT::v16f32) 7596 return 0; 7597 if ((Subtarget->hasAVX512())) { 7598 return fastEmitInst_rr(X86::VDIVPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7599 } 7600 return 0; 7601} 7602 7603unsigned fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7604 if (RetVT.SimpleTy != MVT::v2f64) 7605 return 0; 7606 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7607 return fastEmitInst_rr(X86::VDIVPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7608 } 7609 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 7610 return fastEmitInst_rr(X86::DIVPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7611 } 7612 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7613 return fastEmitInst_rr(X86::VDIVPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7614 } 7615 return 0; 7616} 7617 7618unsigned fastEmit_ISD_FDIV_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7619 if (RetVT.SimpleTy != MVT::v4f64) 7620 return 0; 7621 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7622 return fastEmitInst_rr(X86::VDIVPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7623 } 7624 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7625 return fastEmitInst_rr(X86::VDIVPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7626 } 7627 return 0; 7628} 7629 7630unsigned fastEmit_ISD_FDIV_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7631 if (RetVT.SimpleTy != MVT::v8f64) 7632 return 0; 7633 if ((Subtarget->hasAVX512())) { 7634 return fastEmitInst_rr(X86::VDIVPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7635 } 7636 return 0; 7637} 7638 7639unsigned fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7640 switch (VT.SimpleTy) { 7641 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7642 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7643 case MVT::f80: return fastEmit_ISD_FDIV_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7644 case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7645 case MVT::v8f32: return fastEmit_ISD_FDIV_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7646 case MVT::v16f32: return fastEmit_ISD_FDIV_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7647 case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7648 case MVT::v4f64: return fastEmit_ISD_FDIV_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7649 case MVT::v8f64: return fastEmit_ISD_FDIV_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7650 default: return 0; 7651 } 7652} 7653 7654// FastEmit functions for ISD::FMUL. 7655 7656unsigned fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7657 if (RetVT.SimpleTy != MVT::f32) 7658 return 0; 7659 if ((Subtarget->hasAVX512())) { 7660 return fastEmitInst_rr(X86::VMULSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7661 } 7662 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 7663 return fastEmitInst_rr(X86::MULSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7664 } 7665 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 7666 return fastEmitInst_rr(X86::VMULSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7667 } 7668 if ((!Subtarget->hasSSE1())) { 7669 return fastEmitInst_rr(X86::MUL_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7670 } 7671 return 0; 7672} 7673 7674unsigned fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7675 if (RetVT.SimpleTy != MVT::f64) 7676 return 0; 7677 if ((Subtarget->hasAVX512())) { 7678 return fastEmitInst_rr(X86::VMULSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7679 } 7680 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 7681 return fastEmitInst_rr(X86::MULSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7682 } 7683 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 7684 return fastEmitInst_rr(X86::VMULSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7685 } 7686 if ((!Subtarget->hasSSE2())) { 7687 return fastEmitInst_rr(X86::MUL_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7688 } 7689 return 0; 7690} 7691 7692unsigned fastEmit_ISD_FMUL_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7693 if (RetVT.SimpleTy != MVT::f80) 7694 return 0; 7695 return fastEmitInst_rr(X86::MUL_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7696} 7697 7698unsigned fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7699 if (RetVT.SimpleTy != MVT::v4f32) 7700 return 0; 7701 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7702 return fastEmitInst_rr(X86::VMULPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7703 } 7704 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 7705 return fastEmitInst_rr(X86::MULPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7706 } 7707 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7708 return fastEmitInst_rr(X86::VMULPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7709 } 7710 return 0; 7711} 7712 7713unsigned fastEmit_ISD_FMUL_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7714 if (RetVT.SimpleTy != MVT::v8f32) 7715 return 0; 7716 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7717 return fastEmitInst_rr(X86::VMULPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7718 } 7719 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7720 return fastEmitInst_rr(X86::VMULPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7721 } 7722 return 0; 7723} 7724 7725unsigned fastEmit_ISD_FMUL_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7726 if (RetVT.SimpleTy != MVT::v16f32) 7727 return 0; 7728 if ((Subtarget->hasAVX512())) { 7729 return fastEmitInst_rr(X86::VMULPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7730 } 7731 return 0; 7732} 7733 7734unsigned fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7735 if (RetVT.SimpleTy != MVT::v2f64) 7736 return 0; 7737 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7738 return fastEmitInst_rr(X86::VMULPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7739 } 7740 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 7741 return fastEmitInst_rr(X86::MULPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7742 } 7743 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7744 return fastEmitInst_rr(X86::VMULPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7745 } 7746 return 0; 7747} 7748 7749unsigned fastEmit_ISD_FMUL_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7750 if (RetVT.SimpleTy != MVT::v4f64) 7751 return 0; 7752 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7753 return fastEmitInst_rr(X86::VMULPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7754 } 7755 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7756 return fastEmitInst_rr(X86::VMULPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7757 } 7758 return 0; 7759} 7760 7761unsigned fastEmit_ISD_FMUL_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7762 if (RetVT.SimpleTy != MVT::v8f64) 7763 return 0; 7764 if ((Subtarget->hasAVX512())) { 7765 return fastEmitInst_rr(X86::VMULPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7766 } 7767 return 0; 7768} 7769 7770unsigned fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7771 switch (VT.SimpleTy) { 7772 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7773 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7774 case MVT::f80: return fastEmit_ISD_FMUL_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7775 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7776 case MVT::v8f32: return fastEmit_ISD_FMUL_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7777 case MVT::v16f32: return fastEmit_ISD_FMUL_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7778 case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7779 case MVT::v4f64: return fastEmit_ISD_FMUL_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7780 case MVT::v8f64: return fastEmit_ISD_FMUL_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7781 default: return 0; 7782 } 7783} 7784 7785// FastEmit functions for ISD::FSUB. 7786 7787unsigned fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7788 if (RetVT.SimpleTy != MVT::f32) 7789 return 0; 7790 if ((Subtarget->hasAVX512())) { 7791 return fastEmitInst_rr(X86::VSUBSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7792 } 7793 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 7794 return fastEmitInst_rr(X86::SUBSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7795 } 7796 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 7797 return fastEmitInst_rr(X86::VSUBSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7798 } 7799 if ((!Subtarget->hasSSE1())) { 7800 return fastEmitInst_rr(X86::SUB_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7801 } 7802 return 0; 7803} 7804 7805unsigned fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7806 if (RetVT.SimpleTy != MVT::f64) 7807 return 0; 7808 if ((Subtarget->hasAVX512())) { 7809 return fastEmitInst_rr(X86::VSUBSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7810 } 7811 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 7812 return fastEmitInst_rr(X86::SUBSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7813 } 7814 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 7815 return fastEmitInst_rr(X86::VSUBSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7816 } 7817 if ((!Subtarget->hasSSE2())) { 7818 return fastEmitInst_rr(X86::SUB_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7819 } 7820 return 0; 7821} 7822 7823unsigned fastEmit_ISD_FSUB_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7824 if (RetVT.SimpleTy != MVT::f80) 7825 return 0; 7826 return fastEmitInst_rr(X86::SUB_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7827} 7828 7829unsigned fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7830 if (RetVT.SimpleTy != MVT::v4f32) 7831 return 0; 7832 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7833 return fastEmitInst_rr(X86::VSUBPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7834 } 7835 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 7836 return fastEmitInst_rr(X86::SUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7837 } 7838 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7839 return fastEmitInst_rr(X86::VSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7840 } 7841 return 0; 7842} 7843 7844unsigned fastEmit_ISD_FSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7845 if (RetVT.SimpleTy != MVT::v8f32) 7846 return 0; 7847 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7848 return fastEmitInst_rr(X86::VSUBPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7849 } 7850 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7851 return fastEmitInst_rr(X86::VSUBPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7852 } 7853 return 0; 7854} 7855 7856unsigned fastEmit_ISD_FSUB_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7857 if (RetVT.SimpleTy != MVT::v16f32) 7858 return 0; 7859 if ((Subtarget->hasAVX512())) { 7860 return fastEmitInst_rr(X86::VSUBPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7861 } 7862 return 0; 7863} 7864 7865unsigned fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7866 if (RetVT.SimpleTy != MVT::v2f64) 7867 return 0; 7868 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7869 return fastEmitInst_rr(X86::VSUBPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7870 } 7871 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 7872 return fastEmitInst_rr(X86::SUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7873 } 7874 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7875 return fastEmitInst_rr(X86::VSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7876 } 7877 return 0; 7878} 7879 7880unsigned fastEmit_ISD_FSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7881 if (RetVT.SimpleTy != MVT::v4f64) 7882 return 0; 7883 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7884 return fastEmitInst_rr(X86::VSUBPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7885 } 7886 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7887 return fastEmitInst_rr(X86::VSUBPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7888 } 7889 return 0; 7890} 7891 7892unsigned fastEmit_ISD_FSUB_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7893 if (RetVT.SimpleTy != MVT::v8f64) 7894 return 0; 7895 if ((Subtarget->hasAVX512())) { 7896 return fastEmitInst_rr(X86::VSUBPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7897 } 7898 return 0; 7899} 7900 7901unsigned fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7902 switch (VT.SimpleTy) { 7903 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7904 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7905 case MVT::f80: return fastEmit_ISD_FSUB_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7906 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7907 case MVT::v8f32: return fastEmit_ISD_FSUB_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7908 case MVT::v16f32: return fastEmit_ISD_FSUB_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7909 case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7910 case MVT::v4f64: return fastEmit_ISD_FSUB_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7911 case MVT::v8f64: return fastEmit_ISD_FSUB_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 7912 default: return 0; 7913 } 7914} 7915 7916// FastEmit functions for ISD::MUL. 7917 7918unsigned fastEmit_ISD_MUL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7919 if (RetVT.SimpleTy != MVT::i8) 7920 return 0; 7921 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::AL).addReg(Op0); 7922 return fastEmitInst_r(X86::MUL8r, &X86::GR8RegClass, Op1, Op1IsKill); 7923} 7924 7925unsigned fastEmit_ISD_MUL_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7926 if (RetVT.SimpleTy != MVT::i16) 7927 return 0; 7928 return fastEmitInst_rr(X86::IMUL16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7929} 7930 7931unsigned fastEmit_ISD_MUL_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7932 if (RetVT.SimpleTy != MVT::i32) 7933 return 0; 7934 return fastEmitInst_rr(X86::IMUL32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7935} 7936 7937unsigned fastEmit_ISD_MUL_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7938 if (RetVT.SimpleTy != MVT::i64) 7939 return 0; 7940 return fastEmitInst_rr(X86::IMUL64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7941} 7942 7943unsigned fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7944 if (RetVT.SimpleTy != MVT::v8i16) 7945 return 0; 7946 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 7947 return fastEmitInst_rr(X86::VPMULLWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7948 } 7949 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 7950 return fastEmitInst_rr(X86::PMULLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7951 } 7952 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 7953 return fastEmitInst_rr(X86::VPMULLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7954 } 7955 return 0; 7956} 7957 7958unsigned fastEmit_ISD_MUL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7959 if (RetVT.SimpleTy != MVT::v16i16) 7960 return 0; 7961 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 7962 return fastEmitInst_rr(X86::VPMULLWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7963 } 7964 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 7965 return fastEmitInst_rr(X86::VPMULLWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7966 } 7967 return 0; 7968} 7969 7970unsigned fastEmit_ISD_MUL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7971 if (RetVT.SimpleTy != MVT::v32i16) 7972 return 0; 7973 if ((Subtarget->hasBWI())) { 7974 return fastEmitInst_rr(X86::VPMULLWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7975 } 7976 return 0; 7977} 7978 7979unsigned fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7980 if (RetVT.SimpleTy != MVT::v4i32) 7981 return 0; 7982 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7983 return fastEmitInst_rr(X86::VPMULLDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7984 } 7985 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 7986 return fastEmitInst_rr(X86::PMULLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7987 } 7988 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 7989 return fastEmitInst_rr(X86::VPMULLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7990 } 7991 return 0; 7992} 7993 7994unsigned fastEmit_ISD_MUL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 7995 if (RetVT.SimpleTy != MVT::v8i32) 7996 return 0; 7997 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 7998 return fastEmitInst_rr(X86::VPMULLDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 7999 } 8000 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 8001 return fastEmitInst_rr(X86::VPMULLDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8002 } 8003 return 0; 8004} 8005 8006unsigned fastEmit_ISD_MUL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8007 if (RetVT.SimpleTy != MVT::v16i32) 8008 return 0; 8009 if ((Subtarget->hasAVX512())) { 8010 return fastEmitInst_rr(X86::VPMULLDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8011 } 8012 return 0; 8013} 8014 8015unsigned fastEmit_ISD_MUL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8016 if (RetVT.SimpleTy != MVT::v2i64) 8017 return 0; 8018 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 8019 return fastEmitInst_rr(X86::VPMULLQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8020 } 8021 return 0; 8022} 8023 8024unsigned fastEmit_ISD_MUL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8025 if (RetVT.SimpleTy != MVT::v4i64) 8026 return 0; 8027 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) { 8028 return fastEmitInst_rr(X86::VPMULLQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8029 } 8030 return 0; 8031} 8032 8033unsigned fastEmit_ISD_MUL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8034 if (RetVT.SimpleTy != MVT::v8i64) 8035 return 0; 8036 if ((Subtarget->hasDQI())) { 8037 return fastEmitInst_rr(X86::VPMULLQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8038 } 8039 return 0; 8040} 8041 8042unsigned fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8043 switch (VT.SimpleTy) { 8044 case MVT::i8: return fastEmit_ISD_MUL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8045 case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8046 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8047 case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8048 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8049 case MVT::v16i16: return fastEmit_ISD_MUL_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8050 case MVT::v32i16: return fastEmit_ISD_MUL_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8051 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8052 case MVT::v8i32: return fastEmit_ISD_MUL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8053 case MVT::v16i32: return fastEmit_ISD_MUL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8054 case MVT::v2i64: return fastEmit_ISD_MUL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8055 case MVT::v4i64: return fastEmit_ISD_MUL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8056 case MVT::v8i64: return fastEmit_ISD_MUL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8057 default: return 0; 8058 } 8059} 8060 8061// FastEmit functions for ISD::MULHS. 8062 8063unsigned fastEmit_ISD_MULHS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8064 if (RetVT.SimpleTy != MVT::v8i16) 8065 return 0; 8066 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 8067 return fastEmitInst_rr(X86::VPMULHWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8068 } 8069 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 8070 return fastEmitInst_rr(X86::PMULHWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8071 } 8072 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 8073 return fastEmitInst_rr(X86::VPMULHWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8074 } 8075 return 0; 8076} 8077 8078unsigned fastEmit_ISD_MULHS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8079 if (RetVT.SimpleTy != MVT::v16i16) 8080 return 0; 8081 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 8082 return fastEmitInst_rr(X86::VPMULHWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8083 } 8084 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 8085 return fastEmitInst_rr(X86::VPMULHWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8086 } 8087 return 0; 8088} 8089 8090unsigned fastEmit_ISD_MULHS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8091 if (RetVT.SimpleTy != MVT::v32i16) 8092 return 0; 8093 if ((Subtarget->hasBWI())) { 8094 return fastEmitInst_rr(X86::VPMULHWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8095 } 8096 return 0; 8097} 8098 8099unsigned fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8100 switch (VT.SimpleTy) { 8101 case MVT::v8i16: return fastEmit_ISD_MULHS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8102 case MVT::v16i16: return fastEmit_ISD_MULHS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8103 case MVT::v32i16: return fastEmit_ISD_MULHS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8104 default: return 0; 8105 } 8106} 8107 8108// FastEmit functions for ISD::MULHU. 8109 8110unsigned fastEmit_ISD_MULHU_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8111 if (RetVT.SimpleTy != MVT::v8i16) 8112 return 0; 8113 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 8114 return fastEmitInst_rr(X86::VPMULHUWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8115 } 8116 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 8117 return fastEmitInst_rr(X86::PMULHUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8118 } 8119 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 8120 return fastEmitInst_rr(X86::VPMULHUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8121 } 8122 return 0; 8123} 8124 8125unsigned fastEmit_ISD_MULHU_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8126 if (RetVT.SimpleTy != MVT::v16i16) 8127 return 0; 8128 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 8129 return fastEmitInst_rr(X86::VPMULHUWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8130 } 8131 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 8132 return fastEmitInst_rr(X86::VPMULHUWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8133 } 8134 return 0; 8135} 8136 8137unsigned fastEmit_ISD_MULHU_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8138 if (RetVT.SimpleTy != MVT::v32i16) 8139 return 0; 8140 if ((Subtarget->hasBWI())) { 8141 return fastEmitInst_rr(X86::VPMULHUWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8142 } 8143 return 0; 8144} 8145 8146unsigned fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8147 switch (VT.SimpleTy) { 8148 case MVT::v8i16: return fastEmit_ISD_MULHU_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8149 case MVT::v16i16: return fastEmit_ISD_MULHU_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8150 case MVT::v32i16: return fastEmit_ISD_MULHU_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8151 default: return 0; 8152 } 8153} 8154 8155// FastEmit functions for ISD::OR. 8156 8157unsigned fastEmit_ISD_OR_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8158 if (RetVT.SimpleTy != MVT::i8) 8159 return 0; 8160 return fastEmitInst_rr(X86::OR8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8161} 8162 8163unsigned fastEmit_ISD_OR_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8164 if (RetVT.SimpleTy != MVT::i16) 8165 return 0; 8166 return fastEmitInst_rr(X86::OR16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8167} 8168 8169unsigned fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8170 if (RetVT.SimpleTy != MVT::i32) 8171 return 0; 8172 return fastEmitInst_rr(X86::OR32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8173} 8174 8175unsigned fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8176 if (RetVT.SimpleTy != MVT::i64) 8177 return 0; 8178 return fastEmitInst_rr(X86::OR64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8179} 8180 8181unsigned fastEmit_ISD_OR_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8182 if (RetVT.SimpleTy != MVT::v8i1) 8183 return 0; 8184 if ((Subtarget->hasDQI())) { 8185 return fastEmitInst_rr(X86::KORBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8186 } 8187 return 0; 8188} 8189 8190unsigned fastEmit_ISD_OR_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8191 if (RetVT.SimpleTy != MVT::v16i1) 8192 return 0; 8193 if ((Subtarget->hasAVX512())) { 8194 return fastEmitInst_rr(X86::KORWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8195 } 8196 return 0; 8197} 8198 8199unsigned fastEmit_ISD_OR_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8200 if (RetVT.SimpleTy != MVT::v32i1) 8201 return 0; 8202 if ((Subtarget->hasBWI())) { 8203 return fastEmitInst_rr(X86::KORDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8204 } 8205 return 0; 8206} 8207 8208unsigned fastEmit_ISD_OR_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8209 if (RetVT.SimpleTy != MVT::v64i1) 8210 return 0; 8211 if ((Subtarget->hasBWI())) { 8212 return fastEmitInst_rr(X86::KORQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8213 } 8214 return 0; 8215} 8216 8217unsigned fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8218 if (RetVT.SimpleTy != MVT::v16i8) 8219 return 0; 8220 if ((Subtarget->hasVLX())) { 8221 return fastEmitInst_rr(X86::VPORQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8222 } 8223 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 8224 return fastEmitInst_rr(X86::PORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8225 } 8226 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 8227 return fastEmitInst_rr(X86::VPORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8228 } 8229 return 0; 8230} 8231 8232unsigned fastEmit_ISD_OR_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8233 if (RetVT.SimpleTy != MVT::v32i8) 8234 return 0; 8235 if ((Subtarget->hasVLX())) { 8236 return fastEmitInst_rr(X86::VPORQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8237 } 8238 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 8239 return fastEmitInst_rr(X86::VORPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8240 } 8241 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 8242 return fastEmitInst_rr(X86::VPORYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8243 } 8244 return 0; 8245} 8246 8247unsigned fastEmit_ISD_OR_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8248 if (RetVT.SimpleTy != MVT::v64i8) 8249 return 0; 8250 if ((Subtarget->hasAVX512())) { 8251 return fastEmitInst_rr(X86::VPORQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8252 } 8253 return 0; 8254} 8255 8256unsigned fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8257 if (RetVT.SimpleTy != MVT::v8i16) 8258 return 0; 8259 if ((Subtarget->hasVLX())) { 8260 return fastEmitInst_rr(X86::VPORQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8261 } 8262 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 8263 return fastEmitInst_rr(X86::PORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8264 } 8265 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 8266 return fastEmitInst_rr(X86::VPORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8267 } 8268 return 0; 8269} 8270 8271unsigned fastEmit_ISD_OR_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8272 if (RetVT.SimpleTy != MVT::v16i16) 8273 return 0; 8274 if ((Subtarget->hasVLX())) { 8275 return fastEmitInst_rr(X86::VPORQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8276 } 8277 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 8278 return fastEmitInst_rr(X86::VORPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8279 } 8280 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 8281 return fastEmitInst_rr(X86::VPORYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8282 } 8283 return 0; 8284} 8285 8286unsigned fastEmit_ISD_OR_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8287 if (RetVT.SimpleTy != MVT::v32i16) 8288 return 0; 8289 if ((Subtarget->hasAVX512())) { 8290 return fastEmitInst_rr(X86::VPORQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8291 } 8292 return 0; 8293} 8294 8295unsigned fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8296 if (RetVT.SimpleTy != MVT::v4i32) 8297 return 0; 8298 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 8299 return fastEmitInst_rr(X86::PORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8300 } 8301 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 8302 return fastEmitInst_rr(X86::VPORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8303 } 8304 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8305 return fastEmitInst_rr(X86::VPORDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8306 } 8307 return 0; 8308} 8309 8310unsigned fastEmit_ISD_OR_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8311 if (RetVT.SimpleTy != MVT::v8i32) 8312 return 0; 8313 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 8314 return fastEmitInst_rr(X86::VORPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8315 } 8316 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 8317 return fastEmitInst_rr(X86::VPORYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8318 } 8319 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8320 return fastEmitInst_rr(X86::VPORDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8321 } 8322 return 0; 8323} 8324 8325unsigned fastEmit_ISD_OR_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8326 if (RetVT.SimpleTy != MVT::v16i32) 8327 return 0; 8328 if ((Subtarget->hasAVX512())) { 8329 return fastEmitInst_rr(X86::VPORDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8330 } 8331 return 0; 8332} 8333 8334unsigned fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8335 if (RetVT.SimpleTy != MVT::v2i64) 8336 return 0; 8337 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8338 return fastEmitInst_rr(X86::VPORQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8339 } 8340 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 8341 return fastEmitInst_rr(X86::PORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8342 } 8343 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 8344 return fastEmitInst_rr(X86::VPORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8345 } 8346 return 0; 8347} 8348 8349unsigned fastEmit_ISD_OR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8350 if (RetVT.SimpleTy != MVT::v4i64) 8351 return 0; 8352 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 8353 return fastEmitInst_rr(X86::VORPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8354 } 8355 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8356 return fastEmitInst_rr(X86::VPORQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8357 } 8358 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 8359 return fastEmitInst_rr(X86::VPORYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8360 } 8361 return 0; 8362} 8363 8364unsigned fastEmit_ISD_OR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8365 if (RetVT.SimpleTy != MVT::v8i64) 8366 return 0; 8367 if ((Subtarget->hasAVX512())) { 8368 return fastEmitInst_rr(X86::VPORQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8369 } 8370 return 0; 8371} 8372 8373unsigned fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8374 switch (VT.SimpleTy) { 8375 case MVT::i8: return fastEmit_ISD_OR_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8376 case MVT::i16: return fastEmit_ISD_OR_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8377 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8378 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8379 case MVT::v8i1: return fastEmit_ISD_OR_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8380 case MVT::v16i1: return fastEmit_ISD_OR_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8381 case MVT::v32i1: return fastEmit_ISD_OR_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8382 case MVT::v64i1: return fastEmit_ISD_OR_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8383 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8384 case MVT::v32i8: return fastEmit_ISD_OR_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8385 case MVT::v64i8: return fastEmit_ISD_OR_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8386 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8387 case MVT::v16i16: return fastEmit_ISD_OR_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8388 case MVT::v32i16: return fastEmit_ISD_OR_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8389 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8390 case MVT::v8i32: return fastEmit_ISD_OR_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8391 case MVT::v16i32: return fastEmit_ISD_OR_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8392 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8393 case MVT::v4i64: return fastEmit_ISD_OR_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8394 case MVT::v8i64: return fastEmit_ISD_OR_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8395 default: return 0; 8396 } 8397} 8398 8399// FastEmit functions for ISD::ROTL. 8400 8401unsigned fastEmit_ISD_ROTL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8402 if (RetVT.SimpleTy != MVT::i8) 8403 return 0; 8404 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1); 8405 return fastEmitInst_r(X86::ROL8rCL, &X86::GR8RegClass, Op0, Op0IsKill); 8406} 8407 8408unsigned fastEmit_ISD_ROTL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8409 if (RetVT.SimpleTy != MVT::v16i8) 8410 return 0; 8411 if ((Subtarget->hasXOP())) { 8412 return fastEmitInst_rr(X86::VPROTBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8413 } 8414 return 0; 8415} 8416 8417unsigned fastEmit_ISD_ROTL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8418 if (RetVT.SimpleTy != MVT::v8i16) 8419 return 0; 8420 if ((Subtarget->hasXOP())) { 8421 return fastEmitInst_rr(X86::VPROTWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8422 } 8423 return 0; 8424} 8425 8426unsigned fastEmit_ISD_ROTL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8427 if (RetVT.SimpleTy != MVT::v4i32) 8428 return 0; 8429 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8430 return fastEmitInst_rr(X86::VPROLVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8431 } 8432 if ((Subtarget->hasXOP())) { 8433 return fastEmitInst_rr(X86::VPROTDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8434 } 8435 return 0; 8436} 8437 8438unsigned fastEmit_ISD_ROTL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8439 if (RetVT.SimpleTy != MVT::v8i32) 8440 return 0; 8441 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8442 return fastEmitInst_rr(X86::VPROLVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8443 } 8444 return 0; 8445} 8446 8447unsigned fastEmit_ISD_ROTL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8448 if (RetVT.SimpleTy != MVT::v16i32) 8449 return 0; 8450 if ((Subtarget->hasAVX512())) { 8451 return fastEmitInst_rr(X86::VPROLVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8452 } 8453 return 0; 8454} 8455 8456unsigned fastEmit_ISD_ROTL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8457 if (RetVT.SimpleTy != MVT::v2i64) 8458 return 0; 8459 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8460 return fastEmitInst_rr(X86::VPROLVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8461 } 8462 if ((Subtarget->hasXOP())) { 8463 return fastEmitInst_rr(X86::VPROTQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8464 } 8465 return 0; 8466} 8467 8468unsigned fastEmit_ISD_ROTL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8469 if (RetVT.SimpleTy != MVT::v4i64) 8470 return 0; 8471 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8472 return fastEmitInst_rr(X86::VPROLVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8473 } 8474 return 0; 8475} 8476 8477unsigned fastEmit_ISD_ROTL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8478 if (RetVT.SimpleTy != MVT::v8i64) 8479 return 0; 8480 if ((Subtarget->hasAVX512())) { 8481 return fastEmitInst_rr(X86::VPROLVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8482 } 8483 return 0; 8484} 8485 8486unsigned fastEmit_ISD_ROTL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8487 switch (VT.SimpleTy) { 8488 case MVT::i8: return fastEmit_ISD_ROTL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8489 case MVT::v16i8: return fastEmit_ISD_ROTL_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8490 case MVT::v8i16: return fastEmit_ISD_ROTL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8491 case MVT::v4i32: return fastEmit_ISD_ROTL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8492 case MVT::v8i32: return fastEmit_ISD_ROTL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8493 case MVT::v16i32: return fastEmit_ISD_ROTL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8494 case MVT::v2i64: return fastEmit_ISD_ROTL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8495 case MVT::v4i64: return fastEmit_ISD_ROTL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8496 case MVT::v8i64: return fastEmit_ISD_ROTL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8497 default: return 0; 8498 } 8499} 8500 8501// FastEmit functions for ISD::ROTR. 8502 8503unsigned fastEmit_ISD_ROTR_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8504 if (RetVT.SimpleTy != MVT::i8) 8505 return 0; 8506 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1); 8507 return fastEmitInst_r(X86::ROR8rCL, &X86::GR8RegClass, Op0, Op0IsKill); 8508} 8509 8510unsigned fastEmit_ISD_ROTR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8511 if (RetVT.SimpleTy != MVT::v4i32) 8512 return 0; 8513 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8514 return fastEmitInst_rr(X86::VPRORVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8515 } 8516 return 0; 8517} 8518 8519unsigned fastEmit_ISD_ROTR_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8520 if (RetVT.SimpleTy != MVT::v8i32) 8521 return 0; 8522 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8523 return fastEmitInst_rr(X86::VPRORVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8524 } 8525 return 0; 8526} 8527 8528unsigned fastEmit_ISD_ROTR_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8529 if (RetVT.SimpleTy != MVT::v16i32) 8530 return 0; 8531 if ((Subtarget->hasAVX512())) { 8532 return fastEmitInst_rr(X86::VPRORVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8533 } 8534 return 0; 8535} 8536 8537unsigned fastEmit_ISD_ROTR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8538 if (RetVT.SimpleTy != MVT::v2i64) 8539 return 0; 8540 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8541 return fastEmitInst_rr(X86::VPRORVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8542 } 8543 return 0; 8544} 8545 8546unsigned fastEmit_ISD_ROTR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8547 if (RetVT.SimpleTy != MVT::v4i64) 8548 return 0; 8549 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8550 return fastEmitInst_rr(X86::VPRORVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8551 } 8552 return 0; 8553} 8554 8555unsigned fastEmit_ISD_ROTR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8556 if (RetVT.SimpleTy != MVT::v8i64) 8557 return 0; 8558 if ((Subtarget->hasAVX512())) { 8559 return fastEmitInst_rr(X86::VPRORVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8560 } 8561 return 0; 8562} 8563 8564unsigned fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8565 switch (VT.SimpleTy) { 8566 case MVT::i8: return fastEmit_ISD_ROTR_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8567 case MVT::v4i32: return fastEmit_ISD_ROTR_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8568 case MVT::v8i32: return fastEmit_ISD_ROTR_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8569 case MVT::v16i32: return fastEmit_ISD_ROTR_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8570 case MVT::v2i64: return fastEmit_ISD_ROTR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8571 case MVT::v4i64: return fastEmit_ISD_ROTR_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8572 case MVT::v8i64: return fastEmit_ISD_ROTR_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8573 default: return 0; 8574 } 8575} 8576 8577// FastEmit functions for ISD::SADDSAT. 8578 8579unsigned fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8580 if (RetVT.SimpleTy != MVT::v16i8) 8581 return 0; 8582 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 8583 return fastEmitInst_rr(X86::VPADDSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8584 } 8585 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 8586 return fastEmitInst_rr(X86::PADDSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8587 } 8588 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 8589 return fastEmitInst_rr(X86::VPADDSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8590 } 8591 return 0; 8592} 8593 8594unsigned fastEmit_ISD_SADDSAT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8595 if (RetVT.SimpleTy != MVT::v32i8) 8596 return 0; 8597 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 8598 return fastEmitInst_rr(X86::VPADDSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8599 } 8600 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 8601 return fastEmitInst_rr(X86::VPADDSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8602 } 8603 return 0; 8604} 8605 8606unsigned fastEmit_ISD_SADDSAT_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8607 if (RetVT.SimpleTy != MVT::v64i8) 8608 return 0; 8609 if ((Subtarget->hasBWI())) { 8610 return fastEmitInst_rr(X86::VPADDSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8611 } 8612 return 0; 8613} 8614 8615unsigned fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8616 if (RetVT.SimpleTy != MVT::v8i16) 8617 return 0; 8618 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 8619 return fastEmitInst_rr(X86::VPADDSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8620 } 8621 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 8622 return fastEmitInst_rr(X86::PADDSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8623 } 8624 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 8625 return fastEmitInst_rr(X86::VPADDSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8626 } 8627 return 0; 8628} 8629 8630unsigned fastEmit_ISD_SADDSAT_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8631 if (RetVT.SimpleTy != MVT::v16i16) 8632 return 0; 8633 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 8634 return fastEmitInst_rr(X86::VPADDSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8635 } 8636 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 8637 return fastEmitInst_rr(X86::VPADDSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8638 } 8639 return 0; 8640} 8641 8642unsigned fastEmit_ISD_SADDSAT_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8643 if (RetVT.SimpleTy != MVT::v32i16) 8644 return 0; 8645 if ((Subtarget->hasBWI())) { 8646 return fastEmitInst_rr(X86::VPADDSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8647 } 8648 return 0; 8649} 8650 8651unsigned fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8652 switch (VT.SimpleTy) { 8653 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8654 case MVT::v32i8: return fastEmit_ISD_SADDSAT_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8655 case MVT::v64i8: return fastEmit_ISD_SADDSAT_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8656 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8657 case MVT::v16i16: return fastEmit_ISD_SADDSAT_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8658 case MVT::v32i16: return fastEmit_ISD_SADDSAT_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8659 default: return 0; 8660 } 8661} 8662 8663// FastEmit functions for ISD::SHL. 8664 8665unsigned fastEmit_ISD_SHL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8666 if (RetVT.SimpleTy != MVT::i8) 8667 return 0; 8668 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1); 8669 return fastEmitInst_r(X86::SHL8rCL, &X86::GR8RegClass, Op0, Op0IsKill); 8670} 8671 8672unsigned fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8673 switch (VT.SimpleTy) { 8674 case MVT::i8: return fastEmit_ISD_SHL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8675 default: return 0; 8676 } 8677} 8678 8679// FastEmit functions for ISD::SMAX. 8680 8681unsigned fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8682 if (RetVT.SimpleTy != MVT::v16i8) 8683 return 0; 8684 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 8685 return fastEmitInst_rr(X86::VPMAXSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8686 } 8687 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 8688 return fastEmitInst_rr(X86::PMAXSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8689 } 8690 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 8691 return fastEmitInst_rr(X86::VPMAXSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8692 } 8693 return 0; 8694} 8695 8696unsigned fastEmit_ISD_SMAX_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8697 if (RetVT.SimpleTy != MVT::v32i8) 8698 return 0; 8699 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 8700 return fastEmitInst_rr(X86::VPMAXSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8701 } 8702 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 8703 return fastEmitInst_rr(X86::VPMAXSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8704 } 8705 return 0; 8706} 8707 8708unsigned fastEmit_ISD_SMAX_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8709 if (RetVT.SimpleTy != MVT::v64i8) 8710 return 0; 8711 if ((Subtarget->hasBWI())) { 8712 return fastEmitInst_rr(X86::VPMAXSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8713 } 8714 return 0; 8715} 8716 8717unsigned fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8718 if (RetVT.SimpleTy != MVT::v8i16) 8719 return 0; 8720 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 8721 return fastEmitInst_rr(X86::VPMAXSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8722 } 8723 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 8724 return fastEmitInst_rr(X86::PMAXSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8725 } 8726 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 8727 return fastEmitInst_rr(X86::VPMAXSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8728 } 8729 return 0; 8730} 8731 8732unsigned fastEmit_ISD_SMAX_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8733 if (RetVT.SimpleTy != MVT::v16i16) 8734 return 0; 8735 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 8736 return fastEmitInst_rr(X86::VPMAXSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8737 } 8738 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 8739 return fastEmitInst_rr(X86::VPMAXSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8740 } 8741 return 0; 8742} 8743 8744unsigned fastEmit_ISD_SMAX_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8745 if (RetVT.SimpleTy != MVT::v32i16) 8746 return 0; 8747 if ((Subtarget->hasBWI())) { 8748 return fastEmitInst_rr(X86::VPMAXSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8749 } 8750 return 0; 8751} 8752 8753unsigned fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8754 if (RetVT.SimpleTy != MVT::v4i32) 8755 return 0; 8756 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8757 return fastEmitInst_rr(X86::VPMAXSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8758 } 8759 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 8760 return fastEmitInst_rr(X86::PMAXSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8761 } 8762 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 8763 return fastEmitInst_rr(X86::VPMAXSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8764 } 8765 return 0; 8766} 8767 8768unsigned fastEmit_ISD_SMAX_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8769 if (RetVT.SimpleTy != MVT::v8i32) 8770 return 0; 8771 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8772 return fastEmitInst_rr(X86::VPMAXSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8773 } 8774 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 8775 return fastEmitInst_rr(X86::VPMAXSDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8776 } 8777 return 0; 8778} 8779 8780unsigned fastEmit_ISD_SMAX_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8781 if (RetVT.SimpleTy != MVT::v16i32) 8782 return 0; 8783 if ((Subtarget->hasAVX512())) { 8784 return fastEmitInst_rr(X86::VPMAXSDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8785 } 8786 return 0; 8787} 8788 8789unsigned fastEmit_ISD_SMAX_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8790 if (RetVT.SimpleTy != MVT::v2i64) 8791 return 0; 8792 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8793 return fastEmitInst_rr(X86::VPMAXSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8794 } 8795 return 0; 8796} 8797 8798unsigned fastEmit_ISD_SMAX_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8799 if (RetVT.SimpleTy != MVT::v4i64) 8800 return 0; 8801 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8802 return fastEmitInst_rr(X86::VPMAXSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8803 } 8804 return 0; 8805} 8806 8807unsigned fastEmit_ISD_SMAX_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8808 if (RetVT.SimpleTy != MVT::v8i64) 8809 return 0; 8810 if ((Subtarget->hasAVX512())) { 8811 return fastEmitInst_rr(X86::VPMAXSQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8812 } 8813 return 0; 8814} 8815 8816unsigned fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8817 switch (VT.SimpleTy) { 8818 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8819 case MVT::v32i8: return fastEmit_ISD_SMAX_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8820 case MVT::v64i8: return fastEmit_ISD_SMAX_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8821 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8822 case MVT::v16i16: return fastEmit_ISD_SMAX_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8823 case MVT::v32i16: return fastEmit_ISD_SMAX_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8824 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8825 case MVT::v8i32: return fastEmit_ISD_SMAX_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8826 case MVT::v16i32: return fastEmit_ISD_SMAX_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8827 case MVT::v2i64: return fastEmit_ISD_SMAX_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8828 case MVT::v4i64: return fastEmit_ISD_SMAX_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8829 case MVT::v8i64: return fastEmit_ISD_SMAX_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8830 default: return 0; 8831 } 8832} 8833 8834// FastEmit functions for ISD::SMIN. 8835 8836unsigned fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8837 if (RetVT.SimpleTy != MVT::v16i8) 8838 return 0; 8839 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 8840 return fastEmitInst_rr(X86::VPMINSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8841 } 8842 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 8843 return fastEmitInst_rr(X86::PMINSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8844 } 8845 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 8846 return fastEmitInst_rr(X86::VPMINSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8847 } 8848 return 0; 8849} 8850 8851unsigned fastEmit_ISD_SMIN_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8852 if (RetVT.SimpleTy != MVT::v32i8) 8853 return 0; 8854 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 8855 return fastEmitInst_rr(X86::VPMINSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8856 } 8857 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 8858 return fastEmitInst_rr(X86::VPMINSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8859 } 8860 return 0; 8861} 8862 8863unsigned fastEmit_ISD_SMIN_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8864 if (RetVT.SimpleTy != MVT::v64i8) 8865 return 0; 8866 if ((Subtarget->hasBWI())) { 8867 return fastEmitInst_rr(X86::VPMINSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8868 } 8869 return 0; 8870} 8871 8872unsigned fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8873 if (RetVT.SimpleTy != MVT::v8i16) 8874 return 0; 8875 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 8876 return fastEmitInst_rr(X86::VPMINSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8877 } 8878 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 8879 return fastEmitInst_rr(X86::PMINSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8880 } 8881 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 8882 return fastEmitInst_rr(X86::VPMINSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8883 } 8884 return 0; 8885} 8886 8887unsigned fastEmit_ISD_SMIN_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8888 if (RetVT.SimpleTy != MVT::v16i16) 8889 return 0; 8890 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 8891 return fastEmitInst_rr(X86::VPMINSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8892 } 8893 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 8894 return fastEmitInst_rr(X86::VPMINSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8895 } 8896 return 0; 8897} 8898 8899unsigned fastEmit_ISD_SMIN_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8900 if (RetVT.SimpleTy != MVT::v32i16) 8901 return 0; 8902 if ((Subtarget->hasBWI())) { 8903 return fastEmitInst_rr(X86::VPMINSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8904 } 8905 return 0; 8906} 8907 8908unsigned fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8909 if (RetVT.SimpleTy != MVT::v4i32) 8910 return 0; 8911 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8912 return fastEmitInst_rr(X86::VPMINSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8913 } 8914 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 8915 return fastEmitInst_rr(X86::PMINSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8916 } 8917 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 8918 return fastEmitInst_rr(X86::VPMINSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8919 } 8920 return 0; 8921} 8922 8923unsigned fastEmit_ISD_SMIN_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8924 if (RetVT.SimpleTy != MVT::v8i32) 8925 return 0; 8926 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8927 return fastEmitInst_rr(X86::VPMINSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8928 } 8929 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 8930 return fastEmitInst_rr(X86::VPMINSDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8931 } 8932 return 0; 8933} 8934 8935unsigned fastEmit_ISD_SMIN_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8936 if (RetVT.SimpleTy != MVT::v16i32) 8937 return 0; 8938 if ((Subtarget->hasAVX512())) { 8939 return fastEmitInst_rr(X86::VPMINSDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8940 } 8941 return 0; 8942} 8943 8944unsigned fastEmit_ISD_SMIN_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8945 if (RetVT.SimpleTy != MVT::v2i64) 8946 return 0; 8947 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8948 return fastEmitInst_rr(X86::VPMINSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8949 } 8950 return 0; 8951} 8952 8953unsigned fastEmit_ISD_SMIN_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8954 if (RetVT.SimpleTy != MVT::v4i64) 8955 return 0; 8956 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 8957 return fastEmitInst_rr(X86::VPMINSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8958 } 8959 return 0; 8960} 8961 8962unsigned fastEmit_ISD_SMIN_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8963 if (RetVT.SimpleTy != MVT::v8i64) 8964 return 0; 8965 if ((Subtarget->hasAVX512())) { 8966 return fastEmitInst_rr(X86::VPMINSQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 8967 } 8968 return 0; 8969} 8970 8971unsigned fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8972 switch (VT.SimpleTy) { 8973 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8974 case MVT::v32i8: return fastEmit_ISD_SMIN_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8975 case MVT::v64i8: return fastEmit_ISD_SMIN_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8976 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8977 case MVT::v16i16: return fastEmit_ISD_SMIN_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8978 case MVT::v32i16: return fastEmit_ISD_SMIN_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8979 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8980 case MVT::v8i32: return fastEmit_ISD_SMIN_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8981 case MVT::v16i32: return fastEmit_ISD_SMIN_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8982 case MVT::v2i64: return fastEmit_ISD_SMIN_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8983 case MVT::v4i64: return fastEmit_ISD_SMIN_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8984 case MVT::v8i64: return fastEmit_ISD_SMIN_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 8985 default: return 0; 8986 } 8987} 8988 8989// FastEmit functions for ISD::SRA. 8990 8991unsigned fastEmit_ISD_SRA_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8992 if (RetVT.SimpleTy != MVT::i8) 8993 return 0; 8994 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1); 8995 return fastEmitInst_r(X86::SAR8rCL, &X86::GR8RegClass, Op0, Op0IsKill); 8996} 8997 8998unsigned fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 8999 switch (VT.SimpleTy) { 9000 case MVT::i8: return fastEmit_ISD_SRA_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9001 default: return 0; 9002 } 9003} 9004 9005// FastEmit functions for ISD::SRL. 9006 9007unsigned fastEmit_ISD_SRL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9008 if (RetVT.SimpleTy != MVT::i8) 9009 return 0; 9010 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1); 9011 return fastEmitInst_r(X86::SHR8rCL, &X86::GR8RegClass, Op0, Op0IsKill); 9012} 9013 9014unsigned fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9015 switch (VT.SimpleTy) { 9016 case MVT::i8: return fastEmit_ISD_SRL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9017 default: return 0; 9018 } 9019} 9020 9021// FastEmit functions for ISD::SSUBSAT. 9022 9023unsigned fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9024 if (RetVT.SimpleTy != MVT::v16i8) 9025 return 0; 9026 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 9027 return fastEmitInst_rr(X86::VPSUBSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9028 } 9029 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9030 return fastEmitInst_rr(X86::PSUBSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9031 } 9032 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 9033 return fastEmitInst_rr(X86::VPSUBSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9034 } 9035 return 0; 9036} 9037 9038unsigned fastEmit_ISD_SSUBSAT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9039 if (RetVT.SimpleTy != MVT::v32i8) 9040 return 0; 9041 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 9042 return fastEmitInst_rr(X86::VPSUBSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9043 } 9044 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 9045 return fastEmitInst_rr(X86::VPSUBSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9046 } 9047 return 0; 9048} 9049 9050unsigned fastEmit_ISD_SSUBSAT_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9051 if (RetVT.SimpleTy != MVT::v64i8) 9052 return 0; 9053 if ((Subtarget->hasBWI())) { 9054 return fastEmitInst_rr(X86::VPSUBSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9055 } 9056 return 0; 9057} 9058 9059unsigned fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9060 if (RetVT.SimpleTy != MVT::v8i16) 9061 return 0; 9062 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 9063 return fastEmitInst_rr(X86::VPSUBSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9064 } 9065 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9066 return fastEmitInst_rr(X86::PSUBSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9067 } 9068 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 9069 return fastEmitInst_rr(X86::VPSUBSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9070 } 9071 return 0; 9072} 9073 9074unsigned fastEmit_ISD_SSUBSAT_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9075 if (RetVT.SimpleTy != MVT::v16i16) 9076 return 0; 9077 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 9078 return fastEmitInst_rr(X86::VPSUBSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9079 } 9080 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 9081 return fastEmitInst_rr(X86::VPSUBSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9082 } 9083 return 0; 9084} 9085 9086unsigned fastEmit_ISD_SSUBSAT_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9087 if (RetVT.SimpleTy != MVT::v32i16) 9088 return 0; 9089 if ((Subtarget->hasBWI())) { 9090 return fastEmitInst_rr(X86::VPSUBSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9091 } 9092 return 0; 9093} 9094 9095unsigned fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9096 switch (VT.SimpleTy) { 9097 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9098 case MVT::v32i8: return fastEmit_ISD_SSUBSAT_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9099 case MVT::v64i8: return fastEmit_ISD_SSUBSAT_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9100 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9101 case MVT::v16i16: return fastEmit_ISD_SSUBSAT_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9102 case MVT::v32i16: return fastEmit_ISD_SSUBSAT_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9103 default: return 0; 9104 } 9105} 9106 9107// FastEmit functions for ISD::STRICT_FADD. 9108 9109unsigned fastEmit_ISD_STRICT_FADD_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9110 if (RetVT.SimpleTy != MVT::f32) 9111 return 0; 9112 if ((Subtarget->hasAVX512())) { 9113 return fastEmitInst_rr(X86::VADDSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9114 } 9115 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 9116 return fastEmitInst_rr(X86::ADDSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9117 } 9118 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 9119 return fastEmitInst_rr(X86::VADDSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9120 } 9121 if ((!Subtarget->hasSSE1())) { 9122 return fastEmitInst_rr(X86::ADD_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9123 } 9124 return 0; 9125} 9126 9127unsigned fastEmit_ISD_STRICT_FADD_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9128 if (RetVT.SimpleTy != MVT::f64) 9129 return 0; 9130 if ((Subtarget->hasAVX512())) { 9131 return fastEmitInst_rr(X86::VADDSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9132 } 9133 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9134 return fastEmitInst_rr(X86::ADDSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9135 } 9136 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 9137 return fastEmitInst_rr(X86::VADDSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9138 } 9139 if ((!Subtarget->hasSSE2())) { 9140 return fastEmitInst_rr(X86::ADD_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9141 } 9142 return 0; 9143} 9144 9145unsigned fastEmit_ISD_STRICT_FADD_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9146 if (RetVT.SimpleTy != MVT::f80) 9147 return 0; 9148 return fastEmitInst_rr(X86::ADD_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9149} 9150 9151unsigned fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9152 if (RetVT.SimpleTy != MVT::v4f32) 9153 return 0; 9154 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9155 return fastEmitInst_rr(X86::VADDPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9156 } 9157 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 9158 return fastEmitInst_rr(X86::ADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9159 } 9160 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9161 return fastEmitInst_rr(X86::VADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9162 } 9163 return 0; 9164} 9165 9166unsigned fastEmit_ISD_STRICT_FADD_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9167 if (RetVT.SimpleTy != MVT::v8f32) 9168 return 0; 9169 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9170 return fastEmitInst_rr(X86::VADDPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9171 } 9172 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9173 return fastEmitInst_rr(X86::VADDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9174 } 9175 return 0; 9176} 9177 9178unsigned fastEmit_ISD_STRICT_FADD_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9179 if (RetVT.SimpleTy != MVT::v16f32) 9180 return 0; 9181 if ((Subtarget->hasAVX512())) { 9182 return fastEmitInst_rr(X86::VADDPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9183 } 9184 return 0; 9185} 9186 9187unsigned fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9188 if (RetVT.SimpleTy != MVT::v2f64) 9189 return 0; 9190 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9191 return fastEmitInst_rr(X86::VADDPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9192 } 9193 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9194 return fastEmitInst_rr(X86::ADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9195 } 9196 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9197 return fastEmitInst_rr(X86::VADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9198 } 9199 return 0; 9200} 9201 9202unsigned fastEmit_ISD_STRICT_FADD_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9203 if (RetVT.SimpleTy != MVT::v4f64) 9204 return 0; 9205 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9206 return fastEmitInst_rr(X86::VADDPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9207 } 9208 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9209 return fastEmitInst_rr(X86::VADDPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9210 } 9211 return 0; 9212} 9213 9214unsigned fastEmit_ISD_STRICT_FADD_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9215 if (RetVT.SimpleTy != MVT::v8f64) 9216 return 0; 9217 if ((Subtarget->hasAVX512())) { 9218 return fastEmitInst_rr(X86::VADDPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9219 } 9220 return 0; 9221} 9222 9223unsigned fastEmit_ISD_STRICT_FADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9224 switch (VT.SimpleTy) { 9225 case MVT::f32: return fastEmit_ISD_STRICT_FADD_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9226 case MVT::f64: return fastEmit_ISD_STRICT_FADD_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9227 case MVT::f80: return fastEmit_ISD_STRICT_FADD_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9228 case MVT::v4f32: return fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9229 case MVT::v8f32: return fastEmit_ISD_STRICT_FADD_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9230 case MVT::v16f32: return fastEmit_ISD_STRICT_FADD_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9231 case MVT::v2f64: return fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9232 case MVT::v4f64: return fastEmit_ISD_STRICT_FADD_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9233 case MVT::v8f64: return fastEmit_ISD_STRICT_FADD_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9234 default: return 0; 9235 } 9236} 9237 9238// FastEmit functions for ISD::STRICT_FDIV. 9239 9240unsigned fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9241 if (RetVT.SimpleTy != MVT::f32) 9242 return 0; 9243 if ((Subtarget->hasAVX512())) { 9244 return fastEmitInst_rr(X86::VDIVSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9245 } 9246 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 9247 return fastEmitInst_rr(X86::DIVSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9248 } 9249 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 9250 return fastEmitInst_rr(X86::VDIVSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9251 } 9252 if ((!Subtarget->hasSSE1())) { 9253 return fastEmitInst_rr(X86::DIV_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9254 } 9255 return 0; 9256} 9257 9258unsigned fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9259 if (RetVT.SimpleTy != MVT::f64) 9260 return 0; 9261 if ((Subtarget->hasAVX512())) { 9262 return fastEmitInst_rr(X86::VDIVSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9263 } 9264 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9265 return fastEmitInst_rr(X86::DIVSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9266 } 9267 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 9268 return fastEmitInst_rr(X86::VDIVSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9269 } 9270 if ((!Subtarget->hasSSE2())) { 9271 return fastEmitInst_rr(X86::DIV_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9272 } 9273 return 0; 9274} 9275 9276unsigned fastEmit_ISD_STRICT_FDIV_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9277 if (RetVT.SimpleTy != MVT::f80) 9278 return 0; 9279 return fastEmitInst_rr(X86::DIV_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9280} 9281 9282unsigned fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9283 if (RetVT.SimpleTy != MVT::v4f32) 9284 return 0; 9285 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9286 return fastEmitInst_rr(X86::VDIVPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9287 } 9288 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 9289 return fastEmitInst_rr(X86::DIVPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9290 } 9291 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9292 return fastEmitInst_rr(X86::VDIVPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9293 } 9294 return 0; 9295} 9296 9297unsigned fastEmit_ISD_STRICT_FDIV_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9298 if (RetVT.SimpleTy != MVT::v8f32) 9299 return 0; 9300 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9301 return fastEmitInst_rr(X86::VDIVPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9302 } 9303 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9304 return fastEmitInst_rr(X86::VDIVPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9305 } 9306 return 0; 9307} 9308 9309unsigned fastEmit_ISD_STRICT_FDIV_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9310 if (RetVT.SimpleTy != MVT::v16f32) 9311 return 0; 9312 if ((Subtarget->hasAVX512())) { 9313 return fastEmitInst_rr(X86::VDIVPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9314 } 9315 return 0; 9316} 9317 9318unsigned fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9319 if (RetVT.SimpleTy != MVT::v2f64) 9320 return 0; 9321 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9322 return fastEmitInst_rr(X86::VDIVPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9323 } 9324 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9325 return fastEmitInst_rr(X86::DIVPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9326 } 9327 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9328 return fastEmitInst_rr(X86::VDIVPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9329 } 9330 return 0; 9331} 9332 9333unsigned fastEmit_ISD_STRICT_FDIV_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9334 if (RetVT.SimpleTy != MVT::v4f64) 9335 return 0; 9336 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9337 return fastEmitInst_rr(X86::VDIVPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9338 } 9339 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9340 return fastEmitInst_rr(X86::VDIVPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9341 } 9342 return 0; 9343} 9344 9345unsigned fastEmit_ISD_STRICT_FDIV_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9346 if (RetVT.SimpleTy != MVT::v8f64) 9347 return 0; 9348 if ((Subtarget->hasAVX512())) { 9349 return fastEmitInst_rr(X86::VDIVPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9350 } 9351 return 0; 9352} 9353 9354unsigned fastEmit_ISD_STRICT_FDIV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9355 switch (VT.SimpleTy) { 9356 case MVT::f32: return fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9357 case MVT::f64: return fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9358 case MVT::f80: return fastEmit_ISD_STRICT_FDIV_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9359 case MVT::v4f32: return fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9360 case MVT::v8f32: return fastEmit_ISD_STRICT_FDIV_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9361 case MVT::v16f32: return fastEmit_ISD_STRICT_FDIV_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9362 case MVT::v2f64: return fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9363 case MVT::v4f64: return fastEmit_ISD_STRICT_FDIV_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9364 case MVT::v8f64: return fastEmit_ISD_STRICT_FDIV_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9365 default: return 0; 9366 } 9367} 9368 9369// FastEmit functions for ISD::STRICT_FMUL. 9370 9371unsigned fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9372 if (RetVT.SimpleTy != MVT::f32) 9373 return 0; 9374 if ((Subtarget->hasAVX512())) { 9375 return fastEmitInst_rr(X86::VMULSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9376 } 9377 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 9378 return fastEmitInst_rr(X86::MULSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9379 } 9380 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 9381 return fastEmitInst_rr(X86::VMULSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9382 } 9383 if ((!Subtarget->hasSSE1())) { 9384 return fastEmitInst_rr(X86::MUL_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9385 } 9386 return 0; 9387} 9388 9389unsigned fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9390 if (RetVT.SimpleTy != MVT::f64) 9391 return 0; 9392 if ((Subtarget->hasAVX512())) { 9393 return fastEmitInst_rr(X86::VMULSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9394 } 9395 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9396 return fastEmitInst_rr(X86::MULSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9397 } 9398 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 9399 return fastEmitInst_rr(X86::VMULSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9400 } 9401 if ((!Subtarget->hasSSE2())) { 9402 return fastEmitInst_rr(X86::MUL_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9403 } 9404 return 0; 9405} 9406 9407unsigned fastEmit_ISD_STRICT_FMUL_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9408 if (RetVT.SimpleTy != MVT::f80) 9409 return 0; 9410 return fastEmitInst_rr(X86::MUL_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9411} 9412 9413unsigned fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9414 if (RetVT.SimpleTy != MVT::v4f32) 9415 return 0; 9416 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9417 return fastEmitInst_rr(X86::VMULPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9418 } 9419 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 9420 return fastEmitInst_rr(X86::MULPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9421 } 9422 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9423 return fastEmitInst_rr(X86::VMULPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9424 } 9425 return 0; 9426} 9427 9428unsigned fastEmit_ISD_STRICT_FMUL_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9429 if (RetVT.SimpleTy != MVT::v8f32) 9430 return 0; 9431 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9432 return fastEmitInst_rr(X86::VMULPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9433 } 9434 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9435 return fastEmitInst_rr(X86::VMULPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9436 } 9437 return 0; 9438} 9439 9440unsigned fastEmit_ISD_STRICT_FMUL_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9441 if (RetVT.SimpleTy != MVT::v16f32) 9442 return 0; 9443 if ((Subtarget->hasAVX512())) { 9444 return fastEmitInst_rr(X86::VMULPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9445 } 9446 return 0; 9447} 9448 9449unsigned fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9450 if (RetVT.SimpleTy != MVT::v2f64) 9451 return 0; 9452 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9453 return fastEmitInst_rr(X86::VMULPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9454 } 9455 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9456 return fastEmitInst_rr(X86::MULPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9457 } 9458 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9459 return fastEmitInst_rr(X86::VMULPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9460 } 9461 return 0; 9462} 9463 9464unsigned fastEmit_ISD_STRICT_FMUL_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9465 if (RetVT.SimpleTy != MVT::v4f64) 9466 return 0; 9467 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9468 return fastEmitInst_rr(X86::VMULPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9469 } 9470 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9471 return fastEmitInst_rr(X86::VMULPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9472 } 9473 return 0; 9474} 9475 9476unsigned fastEmit_ISD_STRICT_FMUL_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9477 if (RetVT.SimpleTy != MVT::v8f64) 9478 return 0; 9479 if ((Subtarget->hasAVX512())) { 9480 return fastEmitInst_rr(X86::VMULPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9481 } 9482 return 0; 9483} 9484 9485unsigned fastEmit_ISD_STRICT_FMUL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9486 switch (VT.SimpleTy) { 9487 case MVT::f32: return fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9488 case MVT::f64: return fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9489 case MVT::f80: return fastEmit_ISD_STRICT_FMUL_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9490 case MVT::v4f32: return fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9491 case MVT::v8f32: return fastEmit_ISD_STRICT_FMUL_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9492 case MVT::v16f32: return fastEmit_ISD_STRICT_FMUL_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9493 case MVT::v2f64: return fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9494 case MVT::v4f64: return fastEmit_ISD_STRICT_FMUL_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9495 case MVT::v8f64: return fastEmit_ISD_STRICT_FMUL_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9496 default: return 0; 9497 } 9498} 9499 9500// FastEmit functions for ISD::STRICT_FSUB. 9501 9502unsigned fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9503 if (RetVT.SimpleTy != MVT::f32) 9504 return 0; 9505 if ((Subtarget->hasAVX512())) { 9506 return fastEmitInst_rr(X86::VSUBSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9507 } 9508 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 9509 return fastEmitInst_rr(X86::SUBSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9510 } 9511 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 9512 return fastEmitInst_rr(X86::VSUBSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9513 } 9514 if ((!Subtarget->hasSSE1())) { 9515 return fastEmitInst_rr(X86::SUB_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9516 } 9517 return 0; 9518} 9519 9520unsigned fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9521 if (RetVT.SimpleTy != MVT::f64) 9522 return 0; 9523 if ((Subtarget->hasAVX512())) { 9524 return fastEmitInst_rr(X86::VSUBSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9525 } 9526 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9527 return fastEmitInst_rr(X86::SUBSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9528 } 9529 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 9530 return fastEmitInst_rr(X86::VSUBSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9531 } 9532 if ((!Subtarget->hasSSE2())) { 9533 return fastEmitInst_rr(X86::SUB_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9534 } 9535 return 0; 9536} 9537 9538unsigned fastEmit_ISD_STRICT_FSUB_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9539 if (RetVT.SimpleTy != MVT::f80) 9540 return 0; 9541 return fastEmitInst_rr(X86::SUB_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9542} 9543 9544unsigned fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9545 if (RetVT.SimpleTy != MVT::v4f32) 9546 return 0; 9547 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9548 return fastEmitInst_rr(X86::VSUBPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9549 } 9550 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 9551 return fastEmitInst_rr(X86::SUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9552 } 9553 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9554 return fastEmitInst_rr(X86::VSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9555 } 9556 return 0; 9557} 9558 9559unsigned fastEmit_ISD_STRICT_FSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9560 if (RetVT.SimpleTy != MVT::v8f32) 9561 return 0; 9562 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9563 return fastEmitInst_rr(X86::VSUBPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9564 } 9565 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9566 return fastEmitInst_rr(X86::VSUBPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9567 } 9568 return 0; 9569} 9570 9571unsigned fastEmit_ISD_STRICT_FSUB_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9572 if (RetVT.SimpleTy != MVT::v16f32) 9573 return 0; 9574 if ((Subtarget->hasAVX512())) { 9575 return fastEmitInst_rr(X86::VSUBPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9576 } 9577 return 0; 9578} 9579 9580unsigned fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9581 if (RetVT.SimpleTy != MVT::v2f64) 9582 return 0; 9583 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9584 return fastEmitInst_rr(X86::VSUBPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9585 } 9586 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9587 return fastEmitInst_rr(X86::SUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9588 } 9589 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9590 return fastEmitInst_rr(X86::VSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9591 } 9592 return 0; 9593} 9594 9595unsigned fastEmit_ISD_STRICT_FSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9596 if (RetVT.SimpleTy != MVT::v4f64) 9597 return 0; 9598 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9599 return fastEmitInst_rr(X86::VSUBPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9600 } 9601 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9602 return fastEmitInst_rr(X86::VSUBPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9603 } 9604 return 0; 9605} 9606 9607unsigned fastEmit_ISD_STRICT_FSUB_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9608 if (RetVT.SimpleTy != MVT::v8f64) 9609 return 0; 9610 if ((Subtarget->hasAVX512())) { 9611 return fastEmitInst_rr(X86::VSUBPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9612 } 9613 return 0; 9614} 9615 9616unsigned fastEmit_ISD_STRICT_FSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9617 switch (VT.SimpleTy) { 9618 case MVT::f32: return fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9619 case MVT::f64: return fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9620 case MVT::f80: return fastEmit_ISD_STRICT_FSUB_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9621 case MVT::v4f32: return fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9622 case MVT::v8f32: return fastEmit_ISD_STRICT_FSUB_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9623 case MVT::v16f32: return fastEmit_ISD_STRICT_FSUB_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9624 case MVT::v2f64: return fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9625 case MVT::v4f64: return fastEmit_ISD_STRICT_FSUB_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9626 case MVT::v8f64: return fastEmit_ISD_STRICT_FSUB_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9627 default: return 0; 9628 } 9629} 9630 9631// FastEmit functions for ISD::SUB. 9632 9633unsigned fastEmit_ISD_SUB_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9634 if (RetVT.SimpleTy != MVT::i8) 9635 return 0; 9636 return fastEmitInst_rr(X86::SUB8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9637} 9638 9639unsigned fastEmit_ISD_SUB_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9640 if (RetVT.SimpleTy != MVT::i16) 9641 return 0; 9642 return fastEmitInst_rr(X86::SUB16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9643} 9644 9645unsigned fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9646 if (RetVT.SimpleTy != MVT::i32) 9647 return 0; 9648 return fastEmitInst_rr(X86::SUB32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9649} 9650 9651unsigned fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9652 if (RetVT.SimpleTy != MVT::i64) 9653 return 0; 9654 return fastEmitInst_rr(X86::SUB64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9655} 9656 9657unsigned fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9658 if (RetVT.SimpleTy != MVT::v16i8) 9659 return 0; 9660 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 9661 return fastEmitInst_rr(X86::VPSUBBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9662 } 9663 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9664 return fastEmitInst_rr(X86::PSUBBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9665 } 9666 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 9667 return fastEmitInst_rr(X86::VPSUBBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9668 } 9669 return 0; 9670} 9671 9672unsigned fastEmit_ISD_SUB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9673 if (RetVT.SimpleTy != MVT::v32i8) 9674 return 0; 9675 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 9676 return fastEmitInst_rr(X86::VPSUBBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9677 } 9678 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 9679 return fastEmitInst_rr(X86::VPSUBBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9680 } 9681 return 0; 9682} 9683 9684unsigned fastEmit_ISD_SUB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9685 if (RetVT.SimpleTy != MVT::v64i8) 9686 return 0; 9687 if ((Subtarget->hasBWI())) { 9688 return fastEmitInst_rr(X86::VPSUBBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9689 } 9690 return 0; 9691} 9692 9693unsigned fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9694 if (RetVT.SimpleTy != MVT::v8i16) 9695 return 0; 9696 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 9697 return fastEmitInst_rr(X86::VPSUBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9698 } 9699 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9700 return fastEmitInst_rr(X86::PSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9701 } 9702 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 9703 return fastEmitInst_rr(X86::VPSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9704 } 9705 return 0; 9706} 9707 9708unsigned fastEmit_ISD_SUB_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9709 if (RetVT.SimpleTy != MVT::v16i16) 9710 return 0; 9711 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 9712 return fastEmitInst_rr(X86::VPSUBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9713 } 9714 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 9715 return fastEmitInst_rr(X86::VPSUBWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9716 } 9717 return 0; 9718} 9719 9720unsigned fastEmit_ISD_SUB_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9721 if (RetVT.SimpleTy != MVT::v32i16) 9722 return 0; 9723 if ((Subtarget->hasBWI())) { 9724 return fastEmitInst_rr(X86::VPSUBWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9725 } 9726 return 0; 9727} 9728 9729unsigned fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9730 if (RetVT.SimpleTy != MVT::v4i32) 9731 return 0; 9732 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9733 return fastEmitInst_rr(X86::VPSUBDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9734 } 9735 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9736 return fastEmitInst_rr(X86::PSUBDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9737 } 9738 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9739 return fastEmitInst_rr(X86::VPSUBDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9740 } 9741 return 0; 9742} 9743 9744unsigned fastEmit_ISD_SUB_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9745 if (RetVT.SimpleTy != MVT::v8i32) 9746 return 0; 9747 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9748 return fastEmitInst_rr(X86::VPSUBDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9749 } 9750 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 9751 return fastEmitInst_rr(X86::VPSUBDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9752 } 9753 return 0; 9754} 9755 9756unsigned fastEmit_ISD_SUB_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9757 if (RetVT.SimpleTy != MVT::v16i32) 9758 return 0; 9759 if ((Subtarget->hasAVX512())) { 9760 return fastEmitInst_rr(X86::VPSUBDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9761 } 9762 return 0; 9763} 9764 9765unsigned fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9766 if (RetVT.SimpleTy != MVT::v2i64) 9767 return 0; 9768 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9769 return fastEmitInst_rr(X86::VPSUBQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9770 } 9771 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9772 return fastEmitInst_rr(X86::PSUBQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9773 } 9774 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9775 return fastEmitInst_rr(X86::VPSUBQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9776 } 9777 return 0; 9778} 9779 9780unsigned fastEmit_ISD_SUB_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9781 if (RetVT.SimpleTy != MVT::v4i64) 9782 return 0; 9783 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9784 return fastEmitInst_rr(X86::VPSUBQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9785 } 9786 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 9787 return fastEmitInst_rr(X86::VPSUBQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9788 } 9789 return 0; 9790} 9791 9792unsigned fastEmit_ISD_SUB_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9793 if (RetVT.SimpleTy != MVT::v8i64) 9794 return 0; 9795 if ((Subtarget->hasAVX512())) { 9796 return fastEmitInst_rr(X86::VPSUBQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9797 } 9798 return 0; 9799} 9800 9801unsigned fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9802 switch (VT.SimpleTy) { 9803 case MVT::i8: return fastEmit_ISD_SUB_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9804 case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9805 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9806 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9807 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9808 case MVT::v32i8: return fastEmit_ISD_SUB_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9809 case MVT::v64i8: return fastEmit_ISD_SUB_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9810 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9811 case MVT::v16i16: return fastEmit_ISD_SUB_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9812 case MVT::v32i16: return fastEmit_ISD_SUB_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9813 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9814 case MVT::v8i32: return fastEmit_ISD_SUB_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9815 case MVT::v16i32: return fastEmit_ISD_SUB_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9816 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9817 case MVT::v4i64: return fastEmit_ISD_SUB_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9818 case MVT::v8i64: return fastEmit_ISD_SUB_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9819 default: return 0; 9820 } 9821} 9822 9823// FastEmit functions for ISD::UADDSAT. 9824 9825unsigned fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9826 if (RetVT.SimpleTy != MVT::v16i8) 9827 return 0; 9828 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 9829 return fastEmitInst_rr(X86::VPADDUSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9830 } 9831 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9832 return fastEmitInst_rr(X86::PADDUSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9833 } 9834 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 9835 return fastEmitInst_rr(X86::VPADDUSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9836 } 9837 return 0; 9838} 9839 9840unsigned fastEmit_ISD_UADDSAT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9841 if (RetVT.SimpleTy != MVT::v32i8) 9842 return 0; 9843 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 9844 return fastEmitInst_rr(X86::VPADDUSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9845 } 9846 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 9847 return fastEmitInst_rr(X86::VPADDUSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9848 } 9849 return 0; 9850} 9851 9852unsigned fastEmit_ISD_UADDSAT_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9853 if (RetVT.SimpleTy != MVT::v64i8) 9854 return 0; 9855 if ((Subtarget->hasBWI())) { 9856 return fastEmitInst_rr(X86::VPADDUSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9857 } 9858 return 0; 9859} 9860 9861unsigned fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9862 if (RetVT.SimpleTy != MVT::v8i16) 9863 return 0; 9864 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 9865 return fastEmitInst_rr(X86::VPADDUSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9866 } 9867 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9868 return fastEmitInst_rr(X86::PADDUSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9869 } 9870 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 9871 return fastEmitInst_rr(X86::VPADDUSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9872 } 9873 return 0; 9874} 9875 9876unsigned fastEmit_ISD_UADDSAT_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9877 if (RetVT.SimpleTy != MVT::v16i16) 9878 return 0; 9879 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 9880 return fastEmitInst_rr(X86::VPADDUSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9881 } 9882 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 9883 return fastEmitInst_rr(X86::VPADDUSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9884 } 9885 return 0; 9886} 9887 9888unsigned fastEmit_ISD_UADDSAT_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9889 if (RetVT.SimpleTy != MVT::v32i16) 9890 return 0; 9891 if ((Subtarget->hasBWI())) { 9892 return fastEmitInst_rr(X86::VPADDUSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9893 } 9894 return 0; 9895} 9896 9897unsigned fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9898 switch (VT.SimpleTy) { 9899 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9900 case MVT::v32i8: return fastEmit_ISD_UADDSAT_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9901 case MVT::v64i8: return fastEmit_ISD_UADDSAT_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9902 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9903 case MVT::v16i16: return fastEmit_ISD_UADDSAT_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9904 case MVT::v32i16: return fastEmit_ISD_UADDSAT_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 9905 default: return 0; 9906 } 9907} 9908 9909// FastEmit functions for ISD::UMAX. 9910 9911unsigned fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9912 if (RetVT.SimpleTy != MVT::v16i8) 9913 return 0; 9914 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 9915 return fastEmitInst_rr(X86::VPMAXUBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9916 } 9917 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 9918 return fastEmitInst_rr(X86::PMAXUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9919 } 9920 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 9921 return fastEmitInst_rr(X86::VPMAXUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9922 } 9923 return 0; 9924} 9925 9926unsigned fastEmit_ISD_UMAX_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9927 if (RetVT.SimpleTy != MVT::v32i8) 9928 return 0; 9929 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 9930 return fastEmitInst_rr(X86::VPMAXUBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9931 } 9932 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 9933 return fastEmitInst_rr(X86::VPMAXUBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9934 } 9935 return 0; 9936} 9937 9938unsigned fastEmit_ISD_UMAX_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9939 if (RetVT.SimpleTy != MVT::v64i8) 9940 return 0; 9941 if ((Subtarget->hasBWI())) { 9942 return fastEmitInst_rr(X86::VPMAXUBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9943 } 9944 return 0; 9945} 9946 9947unsigned fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9948 if (RetVT.SimpleTy != MVT::v8i16) 9949 return 0; 9950 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 9951 return fastEmitInst_rr(X86::VPMAXUWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9952 } 9953 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 9954 return fastEmitInst_rr(X86::PMAXUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9955 } 9956 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 9957 return fastEmitInst_rr(X86::VPMAXUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9958 } 9959 return 0; 9960} 9961 9962unsigned fastEmit_ISD_UMAX_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9963 if (RetVT.SimpleTy != MVT::v16i16) 9964 return 0; 9965 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 9966 return fastEmitInst_rr(X86::VPMAXUWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9967 } 9968 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 9969 return fastEmitInst_rr(X86::VPMAXUWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9970 } 9971 return 0; 9972} 9973 9974unsigned fastEmit_ISD_UMAX_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9975 if (RetVT.SimpleTy != MVT::v32i16) 9976 return 0; 9977 if ((Subtarget->hasBWI())) { 9978 return fastEmitInst_rr(X86::VPMAXUWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9979 } 9980 return 0; 9981} 9982 9983unsigned fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9984 if (RetVT.SimpleTy != MVT::v4i32) 9985 return 0; 9986 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 9987 return fastEmitInst_rr(X86::VPMAXUDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9988 } 9989 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 9990 return fastEmitInst_rr(X86::PMAXUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9991 } 9992 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 9993 return fastEmitInst_rr(X86::VPMAXUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 9994 } 9995 return 0; 9996} 9997 9998unsigned fastEmit_ISD_UMAX_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 9999 if (RetVT.SimpleTy != MVT::v8i32) 10000 return 0; 10001 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10002 return fastEmitInst_rr(X86::VPMAXUDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10003 } 10004 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 10005 return fastEmitInst_rr(X86::VPMAXUDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10006 } 10007 return 0; 10008} 10009 10010unsigned fastEmit_ISD_UMAX_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10011 if (RetVT.SimpleTy != MVT::v16i32) 10012 return 0; 10013 if ((Subtarget->hasAVX512())) { 10014 return fastEmitInst_rr(X86::VPMAXUDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10015 } 10016 return 0; 10017} 10018 10019unsigned fastEmit_ISD_UMAX_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10020 if (RetVT.SimpleTy != MVT::v2i64) 10021 return 0; 10022 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10023 return fastEmitInst_rr(X86::VPMAXUQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10024 } 10025 return 0; 10026} 10027 10028unsigned fastEmit_ISD_UMAX_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10029 if (RetVT.SimpleTy != MVT::v4i64) 10030 return 0; 10031 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10032 return fastEmitInst_rr(X86::VPMAXUQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10033 } 10034 return 0; 10035} 10036 10037unsigned fastEmit_ISD_UMAX_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10038 if (RetVT.SimpleTy != MVT::v8i64) 10039 return 0; 10040 if ((Subtarget->hasAVX512())) { 10041 return fastEmitInst_rr(X86::VPMAXUQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10042 } 10043 return 0; 10044} 10045 10046unsigned fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10047 switch (VT.SimpleTy) { 10048 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10049 case MVT::v32i8: return fastEmit_ISD_UMAX_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10050 case MVT::v64i8: return fastEmit_ISD_UMAX_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10051 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10052 case MVT::v16i16: return fastEmit_ISD_UMAX_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10053 case MVT::v32i16: return fastEmit_ISD_UMAX_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10054 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10055 case MVT::v8i32: return fastEmit_ISD_UMAX_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10056 case MVT::v16i32: return fastEmit_ISD_UMAX_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10057 case MVT::v2i64: return fastEmit_ISD_UMAX_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10058 case MVT::v4i64: return fastEmit_ISD_UMAX_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10059 case MVT::v8i64: return fastEmit_ISD_UMAX_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10060 default: return 0; 10061 } 10062} 10063 10064// FastEmit functions for ISD::UMIN. 10065 10066unsigned fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10067 if (RetVT.SimpleTy != MVT::v16i8) 10068 return 0; 10069 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 10070 return fastEmitInst_rr(X86::VPMINUBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10071 } 10072 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 10073 return fastEmitInst_rr(X86::PMINUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10074 } 10075 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 10076 return fastEmitInst_rr(X86::VPMINUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10077 } 10078 return 0; 10079} 10080 10081unsigned fastEmit_ISD_UMIN_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10082 if (RetVT.SimpleTy != MVT::v32i8) 10083 return 0; 10084 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 10085 return fastEmitInst_rr(X86::VPMINUBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10086 } 10087 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 10088 return fastEmitInst_rr(X86::VPMINUBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10089 } 10090 return 0; 10091} 10092 10093unsigned fastEmit_ISD_UMIN_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10094 if (RetVT.SimpleTy != MVT::v64i8) 10095 return 0; 10096 if ((Subtarget->hasBWI())) { 10097 return fastEmitInst_rr(X86::VPMINUBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10098 } 10099 return 0; 10100} 10101 10102unsigned fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10103 if (RetVT.SimpleTy != MVT::v8i16) 10104 return 0; 10105 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 10106 return fastEmitInst_rr(X86::VPMINUWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10107 } 10108 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 10109 return fastEmitInst_rr(X86::PMINUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10110 } 10111 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 10112 return fastEmitInst_rr(X86::VPMINUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10113 } 10114 return 0; 10115} 10116 10117unsigned fastEmit_ISD_UMIN_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10118 if (RetVT.SimpleTy != MVT::v16i16) 10119 return 0; 10120 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 10121 return fastEmitInst_rr(X86::VPMINUWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10122 } 10123 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 10124 return fastEmitInst_rr(X86::VPMINUWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10125 } 10126 return 0; 10127} 10128 10129unsigned fastEmit_ISD_UMIN_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10130 if (RetVT.SimpleTy != MVT::v32i16) 10131 return 0; 10132 if ((Subtarget->hasBWI())) { 10133 return fastEmitInst_rr(X86::VPMINUWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10134 } 10135 return 0; 10136} 10137 10138unsigned fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10139 if (RetVT.SimpleTy != MVT::v4i32) 10140 return 0; 10141 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10142 return fastEmitInst_rr(X86::VPMINUDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10143 } 10144 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 10145 return fastEmitInst_rr(X86::PMINUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10146 } 10147 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 10148 return fastEmitInst_rr(X86::VPMINUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10149 } 10150 return 0; 10151} 10152 10153unsigned fastEmit_ISD_UMIN_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10154 if (RetVT.SimpleTy != MVT::v8i32) 10155 return 0; 10156 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10157 return fastEmitInst_rr(X86::VPMINUDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10158 } 10159 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 10160 return fastEmitInst_rr(X86::VPMINUDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10161 } 10162 return 0; 10163} 10164 10165unsigned fastEmit_ISD_UMIN_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10166 if (RetVT.SimpleTy != MVT::v16i32) 10167 return 0; 10168 if ((Subtarget->hasAVX512())) { 10169 return fastEmitInst_rr(X86::VPMINUDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10170 } 10171 return 0; 10172} 10173 10174unsigned fastEmit_ISD_UMIN_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10175 if (RetVT.SimpleTy != MVT::v2i64) 10176 return 0; 10177 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10178 return fastEmitInst_rr(X86::VPMINUQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10179 } 10180 return 0; 10181} 10182 10183unsigned fastEmit_ISD_UMIN_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10184 if (RetVT.SimpleTy != MVT::v4i64) 10185 return 0; 10186 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10187 return fastEmitInst_rr(X86::VPMINUQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10188 } 10189 return 0; 10190} 10191 10192unsigned fastEmit_ISD_UMIN_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10193 if (RetVT.SimpleTy != MVT::v8i64) 10194 return 0; 10195 if ((Subtarget->hasAVX512())) { 10196 return fastEmitInst_rr(X86::VPMINUQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10197 } 10198 return 0; 10199} 10200 10201unsigned fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10202 switch (VT.SimpleTy) { 10203 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10204 case MVT::v32i8: return fastEmit_ISD_UMIN_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10205 case MVT::v64i8: return fastEmit_ISD_UMIN_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10206 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10207 case MVT::v16i16: return fastEmit_ISD_UMIN_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10208 case MVT::v32i16: return fastEmit_ISD_UMIN_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10209 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10210 case MVT::v8i32: return fastEmit_ISD_UMIN_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10211 case MVT::v16i32: return fastEmit_ISD_UMIN_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10212 case MVT::v2i64: return fastEmit_ISD_UMIN_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10213 case MVT::v4i64: return fastEmit_ISD_UMIN_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10214 case MVT::v8i64: return fastEmit_ISD_UMIN_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10215 default: return 0; 10216 } 10217} 10218 10219// FastEmit functions for ISD::USUBSAT. 10220 10221unsigned fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10222 if (RetVT.SimpleTy != MVT::v16i8) 10223 return 0; 10224 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 10225 return fastEmitInst_rr(X86::VPSUBUSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10226 } 10227 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 10228 return fastEmitInst_rr(X86::PSUBUSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10229 } 10230 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 10231 return fastEmitInst_rr(X86::VPSUBUSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10232 } 10233 return 0; 10234} 10235 10236unsigned fastEmit_ISD_USUBSAT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10237 if (RetVT.SimpleTy != MVT::v32i8) 10238 return 0; 10239 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 10240 return fastEmitInst_rr(X86::VPSUBUSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10241 } 10242 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 10243 return fastEmitInst_rr(X86::VPSUBUSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10244 } 10245 return 0; 10246} 10247 10248unsigned fastEmit_ISD_USUBSAT_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10249 if (RetVT.SimpleTy != MVT::v64i8) 10250 return 0; 10251 if ((Subtarget->hasBWI())) { 10252 return fastEmitInst_rr(X86::VPSUBUSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10253 } 10254 return 0; 10255} 10256 10257unsigned fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10258 if (RetVT.SimpleTy != MVT::v8i16) 10259 return 0; 10260 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 10261 return fastEmitInst_rr(X86::VPSUBUSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10262 } 10263 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 10264 return fastEmitInst_rr(X86::PSUBUSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10265 } 10266 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 10267 return fastEmitInst_rr(X86::VPSUBUSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10268 } 10269 return 0; 10270} 10271 10272unsigned fastEmit_ISD_USUBSAT_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10273 if (RetVT.SimpleTy != MVT::v16i16) 10274 return 0; 10275 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 10276 return fastEmitInst_rr(X86::VPSUBUSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10277 } 10278 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 10279 return fastEmitInst_rr(X86::VPSUBUSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10280 } 10281 return 0; 10282} 10283 10284unsigned fastEmit_ISD_USUBSAT_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10285 if (RetVT.SimpleTy != MVT::v32i16) 10286 return 0; 10287 if ((Subtarget->hasBWI())) { 10288 return fastEmitInst_rr(X86::VPSUBUSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10289 } 10290 return 0; 10291} 10292 10293unsigned fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10294 switch (VT.SimpleTy) { 10295 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10296 case MVT::v32i8: return fastEmit_ISD_USUBSAT_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10297 case MVT::v64i8: return fastEmit_ISD_USUBSAT_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10298 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10299 case MVT::v16i16: return fastEmit_ISD_USUBSAT_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10300 case MVT::v32i16: return fastEmit_ISD_USUBSAT_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10301 default: return 0; 10302 } 10303} 10304 10305// FastEmit functions for ISD::XOR. 10306 10307unsigned fastEmit_ISD_XOR_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10308 if (RetVT.SimpleTy != MVT::i8) 10309 return 0; 10310 return fastEmitInst_rr(X86::XOR8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10311} 10312 10313unsigned fastEmit_ISD_XOR_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10314 if (RetVT.SimpleTy != MVT::i16) 10315 return 0; 10316 return fastEmitInst_rr(X86::XOR16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10317} 10318 10319unsigned fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10320 if (RetVT.SimpleTy != MVT::i32) 10321 return 0; 10322 return fastEmitInst_rr(X86::XOR32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10323} 10324 10325unsigned fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10326 if (RetVT.SimpleTy != MVT::i64) 10327 return 0; 10328 return fastEmitInst_rr(X86::XOR64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10329} 10330 10331unsigned fastEmit_ISD_XOR_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10332 if (RetVT.SimpleTy != MVT::v8i1) 10333 return 0; 10334 if ((Subtarget->hasDQI())) { 10335 return fastEmitInst_rr(X86::KXORBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10336 } 10337 return 0; 10338} 10339 10340unsigned fastEmit_ISD_XOR_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10341 if (RetVT.SimpleTy != MVT::v16i1) 10342 return 0; 10343 if ((Subtarget->hasAVX512())) { 10344 return fastEmitInst_rr(X86::KXORWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10345 } 10346 return 0; 10347} 10348 10349unsigned fastEmit_ISD_XOR_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10350 if (RetVT.SimpleTy != MVT::v32i1) 10351 return 0; 10352 if ((Subtarget->hasBWI())) { 10353 return fastEmitInst_rr(X86::KXORDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10354 } 10355 return 0; 10356} 10357 10358unsigned fastEmit_ISD_XOR_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10359 if (RetVT.SimpleTy != MVT::v64i1) 10360 return 0; 10361 if ((Subtarget->hasBWI())) { 10362 return fastEmitInst_rr(X86::KXORQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10363 } 10364 return 0; 10365} 10366 10367unsigned fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10368 if (RetVT.SimpleTy != MVT::v16i8) 10369 return 0; 10370 if ((Subtarget->hasVLX())) { 10371 return fastEmitInst_rr(X86::VPXORQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10372 } 10373 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 10374 return fastEmitInst_rr(X86::PXORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10375 } 10376 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 10377 return fastEmitInst_rr(X86::VPXORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10378 } 10379 return 0; 10380} 10381 10382unsigned fastEmit_ISD_XOR_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10383 if (RetVT.SimpleTy != MVT::v32i8) 10384 return 0; 10385 if ((Subtarget->hasVLX())) { 10386 return fastEmitInst_rr(X86::VPXORQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10387 } 10388 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 10389 return fastEmitInst_rr(X86::VXORPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10390 } 10391 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 10392 return fastEmitInst_rr(X86::VPXORYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10393 } 10394 return 0; 10395} 10396 10397unsigned fastEmit_ISD_XOR_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10398 if (RetVT.SimpleTy != MVT::v64i8) 10399 return 0; 10400 if ((Subtarget->hasAVX512())) { 10401 return fastEmitInst_rr(X86::VPXORQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10402 } 10403 return 0; 10404} 10405 10406unsigned fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10407 if (RetVT.SimpleTy != MVT::v8i16) 10408 return 0; 10409 if ((Subtarget->hasVLX())) { 10410 return fastEmitInst_rr(X86::VPXORQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10411 } 10412 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 10413 return fastEmitInst_rr(X86::PXORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10414 } 10415 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 10416 return fastEmitInst_rr(X86::VPXORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10417 } 10418 return 0; 10419} 10420 10421unsigned fastEmit_ISD_XOR_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10422 if (RetVT.SimpleTy != MVT::v16i16) 10423 return 0; 10424 if ((Subtarget->hasVLX())) { 10425 return fastEmitInst_rr(X86::VPXORQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10426 } 10427 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 10428 return fastEmitInst_rr(X86::VXORPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10429 } 10430 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 10431 return fastEmitInst_rr(X86::VPXORYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10432 } 10433 return 0; 10434} 10435 10436unsigned fastEmit_ISD_XOR_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10437 if (RetVT.SimpleTy != MVT::v32i16) 10438 return 0; 10439 if ((Subtarget->hasAVX512())) { 10440 return fastEmitInst_rr(X86::VPXORQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10441 } 10442 return 0; 10443} 10444 10445unsigned fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10446 if (RetVT.SimpleTy != MVT::v4i32) 10447 return 0; 10448 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 10449 return fastEmitInst_rr(X86::PXORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10450 } 10451 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 10452 return fastEmitInst_rr(X86::VPXORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10453 } 10454 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10455 return fastEmitInst_rr(X86::VPXORDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10456 } 10457 return 0; 10458} 10459 10460unsigned fastEmit_ISD_XOR_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10461 if (RetVT.SimpleTy != MVT::v8i32) 10462 return 0; 10463 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 10464 return fastEmitInst_rr(X86::VXORPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10465 } 10466 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 10467 return fastEmitInst_rr(X86::VPXORYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10468 } 10469 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10470 return fastEmitInst_rr(X86::VPXORDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10471 } 10472 return 0; 10473} 10474 10475unsigned fastEmit_ISD_XOR_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10476 if (RetVT.SimpleTy != MVT::v16i32) 10477 return 0; 10478 if ((Subtarget->hasAVX512())) { 10479 return fastEmitInst_rr(X86::VPXORDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10480 } 10481 return 0; 10482} 10483 10484unsigned fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10485 if (RetVT.SimpleTy != MVT::v2i64) 10486 return 0; 10487 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10488 return fastEmitInst_rr(X86::VPXORQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10489 } 10490 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 10491 return fastEmitInst_rr(X86::PXORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10492 } 10493 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 10494 return fastEmitInst_rr(X86::VPXORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10495 } 10496 return 0; 10497} 10498 10499unsigned fastEmit_ISD_XOR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10500 if (RetVT.SimpleTy != MVT::v4i64) 10501 return 0; 10502 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 10503 return fastEmitInst_rr(X86::VXORPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10504 } 10505 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10506 return fastEmitInst_rr(X86::VPXORQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10507 } 10508 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 10509 return fastEmitInst_rr(X86::VPXORYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10510 } 10511 return 0; 10512} 10513 10514unsigned fastEmit_ISD_XOR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10515 if (RetVT.SimpleTy != MVT::v8i64) 10516 return 0; 10517 if ((Subtarget->hasAVX512())) { 10518 return fastEmitInst_rr(X86::VPXORQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10519 } 10520 return 0; 10521} 10522 10523unsigned fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10524 switch (VT.SimpleTy) { 10525 case MVT::i8: return fastEmit_ISD_XOR_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10526 case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10527 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10528 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10529 case MVT::v8i1: return fastEmit_ISD_XOR_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10530 case MVT::v16i1: return fastEmit_ISD_XOR_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10531 case MVT::v32i1: return fastEmit_ISD_XOR_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10532 case MVT::v64i1: return fastEmit_ISD_XOR_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10533 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10534 case MVT::v32i8: return fastEmit_ISD_XOR_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10535 case MVT::v64i8: return fastEmit_ISD_XOR_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10536 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10537 case MVT::v16i16: return fastEmit_ISD_XOR_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10538 case MVT::v32i16: return fastEmit_ISD_XOR_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10539 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10540 case MVT::v8i32: return fastEmit_ISD_XOR_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10541 case MVT::v16i32: return fastEmit_ISD_XOR_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10542 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10543 case MVT::v4i64: return fastEmit_ISD_XOR_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10544 case MVT::v8i64: return fastEmit_ISD_XOR_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10545 default: return 0; 10546 } 10547} 10548 10549// FastEmit functions for X86ISD::ADDSUB. 10550 10551unsigned fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10552 if (RetVT.SimpleTy != MVT::v4f32) 10553 return 0; 10554 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) { 10555 return fastEmitInst_rr(X86::ADDSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10556 } 10557 if ((Subtarget->hasAVX())) { 10558 return fastEmitInst_rr(X86::VADDSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10559 } 10560 return 0; 10561} 10562 10563unsigned fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10564 if (RetVT.SimpleTy != MVT::v8f32) 10565 return 0; 10566 if ((Subtarget->hasAVX())) { 10567 return fastEmitInst_rr(X86::VADDSUBPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10568 } 10569 return 0; 10570} 10571 10572unsigned fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10573 if (RetVT.SimpleTy != MVT::v2f64) 10574 return 0; 10575 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) { 10576 return fastEmitInst_rr(X86::ADDSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10577 } 10578 if ((Subtarget->hasAVX())) { 10579 return fastEmitInst_rr(X86::VADDSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10580 } 10581 return 0; 10582} 10583 10584unsigned fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10585 if (RetVT.SimpleTy != MVT::v4f64) 10586 return 0; 10587 if ((Subtarget->hasAVX())) { 10588 return fastEmitInst_rr(X86::VADDSUBPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10589 } 10590 return 0; 10591} 10592 10593unsigned fastEmit_X86ISD_ADDSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10594 switch (VT.SimpleTy) { 10595 case MVT::v4f32: return fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10596 case MVT::v8f32: return fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10597 case MVT::v2f64: return fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10598 case MVT::v4f64: return fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10599 default: return 0; 10600 } 10601} 10602 10603// FastEmit functions for X86ISD::ANDNP. 10604 10605unsigned fastEmit_X86ISD_ANDNP_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10606 if (RetVT.SimpleTy != MVT::v16i8) 10607 return 0; 10608 if ((Subtarget->hasVLX())) { 10609 return fastEmitInst_rr(X86::VPANDNQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10610 } 10611 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 10612 return fastEmitInst_rr(X86::PANDNrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10613 } 10614 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 10615 return fastEmitInst_rr(X86::VPANDNrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10616 } 10617 return 0; 10618} 10619 10620unsigned fastEmit_X86ISD_ANDNP_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10621 if (RetVT.SimpleTy != MVT::v32i8) 10622 return 0; 10623 if ((Subtarget->hasVLX())) { 10624 return fastEmitInst_rr(X86::VPANDNQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10625 } 10626 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 10627 return fastEmitInst_rr(X86::VANDNPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10628 } 10629 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 10630 return fastEmitInst_rr(X86::VPANDNYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10631 } 10632 return 0; 10633} 10634 10635unsigned fastEmit_X86ISD_ANDNP_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10636 if (RetVT.SimpleTy != MVT::v64i8) 10637 return 0; 10638 if ((Subtarget->hasAVX512())) { 10639 return fastEmitInst_rr(X86::VPANDNQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10640 } 10641 return 0; 10642} 10643 10644unsigned fastEmit_X86ISD_ANDNP_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10645 if (RetVT.SimpleTy != MVT::v8i16) 10646 return 0; 10647 if ((Subtarget->hasVLX())) { 10648 return fastEmitInst_rr(X86::VPANDNQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10649 } 10650 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 10651 return fastEmitInst_rr(X86::PANDNrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10652 } 10653 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 10654 return fastEmitInst_rr(X86::VPANDNrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10655 } 10656 return 0; 10657} 10658 10659unsigned fastEmit_X86ISD_ANDNP_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10660 if (RetVT.SimpleTy != MVT::v16i16) 10661 return 0; 10662 if ((Subtarget->hasVLX())) { 10663 return fastEmitInst_rr(X86::VPANDNQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10664 } 10665 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 10666 return fastEmitInst_rr(X86::VANDNPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10667 } 10668 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 10669 return fastEmitInst_rr(X86::VPANDNYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10670 } 10671 return 0; 10672} 10673 10674unsigned fastEmit_X86ISD_ANDNP_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10675 if (RetVT.SimpleTy != MVT::v32i16) 10676 return 0; 10677 if ((Subtarget->hasAVX512())) { 10678 return fastEmitInst_rr(X86::VPANDNQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10679 } 10680 return 0; 10681} 10682 10683unsigned fastEmit_X86ISD_ANDNP_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10684 if (RetVT.SimpleTy != MVT::v4i32) 10685 return 0; 10686 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 10687 return fastEmitInst_rr(X86::PANDNrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10688 } 10689 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 10690 return fastEmitInst_rr(X86::VPANDNrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10691 } 10692 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10693 return fastEmitInst_rr(X86::VPANDNDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10694 } 10695 return 0; 10696} 10697 10698unsigned fastEmit_X86ISD_ANDNP_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10699 if (RetVT.SimpleTy != MVT::v8i32) 10700 return 0; 10701 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 10702 return fastEmitInst_rr(X86::VANDNPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10703 } 10704 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 10705 return fastEmitInst_rr(X86::VPANDNYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10706 } 10707 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10708 return fastEmitInst_rr(X86::VPANDNDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10709 } 10710 return 0; 10711} 10712 10713unsigned fastEmit_X86ISD_ANDNP_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10714 if (RetVT.SimpleTy != MVT::v16i32) 10715 return 0; 10716 if ((Subtarget->hasAVX512())) { 10717 return fastEmitInst_rr(X86::VPANDNDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10718 } 10719 return 0; 10720} 10721 10722unsigned fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10723 if (RetVT.SimpleTy != MVT::v2i64) 10724 return 0; 10725 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10726 return fastEmitInst_rr(X86::VPANDNQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10727 } 10728 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 10729 return fastEmitInst_rr(X86::PANDNrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10730 } 10731 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 10732 return fastEmitInst_rr(X86::VPANDNrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10733 } 10734 return 0; 10735} 10736 10737unsigned fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10738 if (RetVT.SimpleTy != MVT::v4i64) 10739 return 0; 10740 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 10741 return fastEmitInst_rr(X86::VANDNPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10742 } 10743 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 10744 return fastEmitInst_rr(X86::VPANDNQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10745 } 10746 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 10747 return fastEmitInst_rr(X86::VPANDNYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10748 } 10749 return 0; 10750} 10751 10752unsigned fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10753 if (RetVT.SimpleTy != MVT::v8i64) 10754 return 0; 10755 if ((Subtarget->hasAVX512())) { 10756 return fastEmitInst_rr(X86::VPANDNQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10757 } 10758 return 0; 10759} 10760 10761unsigned fastEmit_X86ISD_ANDNP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10762 switch (VT.SimpleTy) { 10763 case MVT::v16i8: return fastEmit_X86ISD_ANDNP_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10764 case MVT::v32i8: return fastEmit_X86ISD_ANDNP_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10765 case MVT::v64i8: return fastEmit_X86ISD_ANDNP_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10766 case MVT::v8i16: return fastEmit_X86ISD_ANDNP_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10767 case MVT::v16i16: return fastEmit_X86ISD_ANDNP_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10768 case MVT::v32i16: return fastEmit_X86ISD_ANDNP_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10769 case MVT::v4i32: return fastEmit_X86ISD_ANDNP_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10770 case MVT::v8i32: return fastEmit_X86ISD_ANDNP_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10771 case MVT::v16i32: return fastEmit_X86ISD_ANDNP_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10772 case MVT::v2i64: return fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10773 case MVT::v4i64: return fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10774 case MVT::v8i64: return fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10775 default: return 0; 10776 } 10777} 10778 10779// FastEmit functions for X86ISD::AVG. 10780 10781unsigned fastEmit_X86ISD_AVG_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10782 if (RetVT.SimpleTy != MVT::v16i8) 10783 return 0; 10784 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 10785 return fastEmitInst_rr(X86::VPAVGBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10786 } 10787 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 10788 return fastEmitInst_rr(X86::PAVGBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10789 } 10790 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 10791 return fastEmitInst_rr(X86::VPAVGBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10792 } 10793 return 0; 10794} 10795 10796unsigned fastEmit_X86ISD_AVG_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10797 if (RetVT.SimpleTy != MVT::v32i8) 10798 return 0; 10799 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 10800 return fastEmitInst_rr(X86::VPAVGBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10801 } 10802 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 10803 return fastEmitInst_rr(X86::VPAVGBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10804 } 10805 return 0; 10806} 10807 10808unsigned fastEmit_X86ISD_AVG_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10809 if (RetVT.SimpleTy != MVT::v64i8) 10810 return 0; 10811 if ((Subtarget->hasBWI())) { 10812 return fastEmitInst_rr(X86::VPAVGBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10813 } 10814 return 0; 10815} 10816 10817unsigned fastEmit_X86ISD_AVG_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10818 if (RetVT.SimpleTy != MVT::v8i16) 10819 return 0; 10820 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 10821 return fastEmitInst_rr(X86::VPAVGWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10822 } 10823 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 10824 return fastEmitInst_rr(X86::PAVGWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10825 } 10826 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 10827 return fastEmitInst_rr(X86::VPAVGWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10828 } 10829 return 0; 10830} 10831 10832unsigned fastEmit_X86ISD_AVG_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10833 if (RetVT.SimpleTy != MVT::v16i16) 10834 return 0; 10835 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 10836 return fastEmitInst_rr(X86::VPAVGWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10837 } 10838 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 10839 return fastEmitInst_rr(X86::VPAVGWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10840 } 10841 return 0; 10842} 10843 10844unsigned fastEmit_X86ISD_AVG_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10845 if (RetVT.SimpleTy != MVT::v32i16) 10846 return 0; 10847 if ((Subtarget->hasBWI())) { 10848 return fastEmitInst_rr(X86::VPAVGWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10849 } 10850 return 0; 10851} 10852 10853unsigned fastEmit_X86ISD_AVG_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10854 switch (VT.SimpleTy) { 10855 case MVT::v16i8: return fastEmit_X86ISD_AVG_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10856 case MVT::v32i8: return fastEmit_X86ISD_AVG_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10857 case MVT::v64i8: return fastEmit_X86ISD_AVG_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10858 case MVT::v8i16: return fastEmit_X86ISD_AVG_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10859 case MVT::v16i16: return fastEmit_X86ISD_AVG_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10860 case MVT::v32i16: return fastEmit_X86ISD_AVG_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10861 default: return 0; 10862 } 10863} 10864 10865// FastEmit functions for X86ISD::BEXTR. 10866 10867unsigned fastEmit_X86ISD_BEXTR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10868 if (RetVT.SimpleTy != MVT::i32) 10869 return 0; 10870 if ((Subtarget->hasBMI())) { 10871 return fastEmitInst_rr(X86::BEXTR32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10872 } 10873 return 0; 10874} 10875 10876unsigned fastEmit_X86ISD_BEXTR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10877 if (RetVT.SimpleTy != MVT::i64) 10878 return 0; 10879 if ((Subtarget->hasBMI())) { 10880 return fastEmitInst_rr(X86::BEXTR64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10881 } 10882 return 0; 10883} 10884 10885unsigned fastEmit_X86ISD_BEXTR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10886 switch (VT.SimpleTy) { 10887 case MVT::i32: return fastEmit_X86ISD_BEXTR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10888 case MVT::i64: return fastEmit_X86ISD_BEXTR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10889 default: return 0; 10890 } 10891} 10892 10893// FastEmit functions for X86ISD::BT. 10894 10895unsigned fastEmit_X86ISD_BT_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10896 if (RetVT.SimpleTy != MVT::i32) 10897 return 0; 10898 return fastEmitInst_rr(X86::BT16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10899} 10900 10901unsigned fastEmit_X86ISD_BT_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10902 if (RetVT.SimpleTy != MVT::i32) 10903 return 0; 10904 return fastEmitInst_rr(X86::BT32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10905} 10906 10907unsigned fastEmit_X86ISD_BT_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10908 if (RetVT.SimpleTy != MVT::i32) 10909 return 0; 10910 return fastEmitInst_rr(X86::BT64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10911} 10912 10913unsigned fastEmit_X86ISD_BT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10914 switch (VT.SimpleTy) { 10915 case MVT::i16: return fastEmit_X86ISD_BT_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10916 case MVT::i32: return fastEmit_X86ISD_BT_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10917 case MVT::i64: return fastEmit_X86ISD_BT_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10918 default: return 0; 10919 } 10920} 10921 10922// FastEmit functions for X86ISD::BZHI. 10923 10924unsigned fastEmit_X86ISD_BZHI_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10925 if (RetVT.SimpleTy != MVT::i32) 10926 return 0; 10927 if ((Subtarget->hasBMI2())) { 10928 return fastEmitInst_rr(X86::BZHI32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10929 } 10930 return 0; 10931} 10932 10933unsigned fastEmit_X86ISD_BZHI_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10934 if (RetVT.SimpleTy != MVT::i64) 10935 return 0; 10936 if ((Subtarget->hasBMI2())) { 10937 return fastEmitInst_rr(X86::BZHI64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10938 } 10939 return 0; 10940} 10941 10942unsigned fastEmit_X86ISD_BZHI_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10943 switch (VT.SimpleTy) { 10944 case MVT::i32: return fastEmit_X86ISD_BZHI_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10945 case MVT::i64: return fastEmit_X86ISD_BZHI_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 10946 default: return 0; 10947 } 10948} 10949 10950// FastEmit functions for X86ISD::CMP. 10951 10952unsigned fastEmit_X86ISD_CMP_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10953 if (RetVT.SimpleTy != MVT::i32) 10954 return 0; 10955 return fastEmitInst_rr(X86::CMP8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10956} 10957 10958unsigned fastEmit_X86ISD_CMP_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10959 if (RetVT.SimpleTy != MVT::i32) 10960 return 0; 10961 return fastEmitInst_rr(X86::CMP16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10962} 10963 10964unsigned fastEmit_X86ISD_CMP_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10965 if (RetVT.SimpleTy != MVT::i32) 10966 return 0; 10967 return fastEmitInst_rr(X86::CMP32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10968} 10969 10970unsigned fastEmit_X86ISD_CMP_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10971 if (RetVT.SimpleTy != MVT::i32) 10972 return 0; 10973 return fastEmitInst_rr(X86::CMP64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10974} 10975 10976unsigned fastEmit_X86ISD_CMP_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10977 if (RetVT.SimpleTy != MVT::i32) 10978 return 0; 10979 if ((Subtarget->hasAVX512())) { 10980 return fastEmitInst_rr(X86::VUCOMISSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10981 } 10982 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 10983 return fastEmitInst_rr(X86::UCOMISSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10984 } 10985 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 10986 return fastEmitInst_rr(X86::VUCOMISSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10987 } 10988 if ((!Subtarget->hasSSE1()) && (Subtarget->hasCMov())) { 10989 return fastEmitInst_rr(X86::UCOM_FpIr32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10990 } 10991 return 0; 10992} 10993 10994unsigned fastEmit_X86ISD_CMP_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 10995 if (RetVT.SimpleTy != MVT::i32) 10996 return 0; 10997 if ((Subtarget->hasAVX512())) { 10998 return fastEmitInst_rr(X86::VUCOMISDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 10999 } 11000 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 11001 return fastEmitInst_rr(X86::UCOMISDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11002 } 11003 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 11004 return fastEmitInst_rr(X86::VUCOMISDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11005 } 11006 if ((!Subtarget->hasSSE2()) && (Subtarget->hasCMov())) { 11007 return fastEmitInst_rr(X86::UCOM_FpIr64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11008 } 11009 return 0; 11010} 11011 11012unsigned fastEmit_X86ISD_CMP_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11013 if (RetVT.SimpleTy != MVT::i32) 11014 return 0; 11015 if ((Subtarget->hasCMov())) { 11016 return fastEmitInst_rr(X86::UCOM_FpIr80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11017 } 11018 return 0; 11019} 11020 11021unsigned fastEmit_X86ISD_CMP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11022 switch (VT.SimpleTy) { 11023 case MVT::i8: return fastEmit_X86ISD_CMP_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11024 case MVT::i16: return fastEmit_X86ISD_CMP_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11025 case MVT::i32: return fastEmit_X86ISD_CMP_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11026 case MVT::i64: return fastEmit_X86ISD_CMP_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11027 case MVT::f32: return fastEmit_X86ISD_CMP_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11028 case MVT::f64: return fastEmit_X86ISD_CMP_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11029 case MVT::f80: return fastEmit_X86ISD_CMP_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11030 default: return 0; 11031 } 11032} 11033 11034// FastEmit functions for X86ISD::COMI. 11035 11036unsigned fastEmit_X86ISD_COMI_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11037 if (RetVT.SimpleTy != MVT::i32) 11038 return 0; 11039 if ((Subtarget->hasAVX512())) { 11040 return fastEmitInst_rr(X86::VCOMISSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11041 } 11042 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 11043 return fastEmitInst_rr(X86::COMISSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11044 } 11045 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 11046 return fastEmitInst_rr(X86::VCOMISSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11047 } 11048 return 0; 11049} 11050 11051unsigned fastEmit_X86ISD_COMI_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11052 if (RetVT.SimpleTy != MVT::i32) 11053 return 0; 11054 if ((Subtarget->hasAVX512())) { 11055 return fastEmitInst_rr(X86::VCOMISDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11056 } 11057 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 11058 return fastEmitInst_rr(X86::COMISDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11059 } 11060 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 11061 return fastEmitInst_rr(X86::VCOMISDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11062 } 11063 return 0; 11064} 11065 11066unsigned fastEmit_X86ISD_COMI_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11067 switch (VT.SimpleTy) { 11068 case MVT::v4f32: return fastEmit_X86ISD_COMI_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11069 case MVT::v2f64: return fastEmit_X86ISD_COMI_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11070 default: return 0; 11071 } 11072} 11073 11074// FastEmit functions for X86ISD::CVTNE2PS2BF16. 11075 11076unsigned fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11077 if (RetVT.SimpleTy != MVT::v8i16) 11078 return 0; 11079 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) { 11080 return fastEmitInst_rr(X86::VCVTNE2PS2BF16Z128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11081 } 11082 return 0; 11083} 11084 11085unsigned fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11086 if (RetVT.SimpleTy != MVT::v16i16) 11087 return 0; 11088 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) { 11089 return fastEmitInst_rr(X86::VCVTNE2PS2BF16Z256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11090 } 11091 return 0; 11092} 11093 11094unsigned fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11095 if (RetVT.SimpleTy != MVT::v32i16) 11096 return 0; 11097 if ((Subtarget->hasBF16())) { 11098 return fastEmitInst_rr(X86::VCVTNE2PS2BF16Zrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11099 } 11100 return 0; 11101} 11102 11103unsigned fastEmit_X86ISD_CVTNE2PS2BF16_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11104 switch (VT.SimpleTy) { 11105 case MVT::v4f32: return fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11106 case MVT::v8f32: return fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11107 case MVT::v16f32: return fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11108 default: return 0; 11109 } 11110} 11111 11112// FastEmit functions for X86ISD::FADDS. 11113 11114unsigned fastEmit_X86ISD_FADDS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11115 if (RetVT.SimpleTy != MVT::v4f32) 11116 return 0; 11117 if ((Subtarget->hasAVX512())) { 11118 return fastEmitInst_rr(X86::VADDSSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11119 } 11120 return 0; 11121} 11122 11123unsigned fastEmit_X86ISD_FADDS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11124 if (RetVT.SimpleTy != MVT::v2f64) 11125 return 0; 11126 if ((Subtarget->hasAVX512())) { 11127 return fastEmitInst_rr(X86::VADDSDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11128 } 11129 return 0; 11130} 11131 11132unsigned fastEmit_X86ISD_FADDS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11133 switch (VT.SimpleTy) { 11134 case MVT::v4f32: return fastEmit_X86ISD_FADDS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11135 case MVT::v2f64: return fastEmit_X86ISD_FADDS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11136 default: return 0; 11137 } 11138} 11139 11140// FastEmit functions for X86ISD::FAND. 11141 11142unsigned fastEmit_X86ISD_FAND_MVT_f128_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11143 if (RetVT.SimpleTy != MVT::f128) 11144 return 0; 11145 if ((Subtarget->hasVLX())) { 11146 return fastEmitInst_rr(X86::VANDPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11147 } 11148 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11149 return fastEmitInst_rr(X86::VANDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11150 } 11151 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 11152 return fastEmitInst_rr(X86::ANDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11153 } 11154 return 0; 11155} 11156 11157unsigned fastEmit_X86ISD_FAND_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11158 if (RetVT.SimpleTy != MVT::v4f32) 11159 return 0; 11160 return fastEmitInst_rr(X86::ANDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11161} 11162 11163unsigned fastEmit_X86ISD_FAND_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11164 switch (VT.SimpleTy) { 11165 case MVT::f128: return fastEmit_X86ISD_FAND_MVT_f128_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11166 case MVT::v4f32: return fastEmit_X86ISD_FAND_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11167 default: return 0; 11168 } 11169} 11170 11171// FastEmit functions for X86ISD::FANDN. 11172 11173unsigned fastEmit_X86ISD_FANDN_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11174 if (RetVT.SimpleTy != MVT::v4f32) 11175 return 0; 11176 return fastEmitInst_rr(X86::ANDNPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11177} 11178 11179unsigned fastEmit_X86ISD_FANDN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11180 switch (VT.SimpleTy) { 11181 case MVT::v4f32: return fastEmit_X86ISD_FANDN_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11182 default: return 0; 11183 } 11184} 11185 11186// FastEmit functions for X86ISD::FDIVS. 11187 11188unsigned fastEmit_X86ISD_FDIVS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11189 if (RetVT.SimpleTy != MVT::v4f32) 11190 return 0; 11191 if ((Subtarget->hasAVX512())) { 11192 return fastEmitInst_rr(X86::VDIVSSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11193 } 11194 return 0; 11195} 11196 11197unsigned fastEmit_X86ISD_FDIVS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11198 if (RetVT.SimpleTy != MVT::v2f64) 11199 return 0; 11200 if ((Subtarget->hasAVX512())) { 11201 return fastEmitInst_rr(X86::VDIVSDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11202 } 11203 return 0; 11204} 11205 11206unsigned fastEmit_X86ISD_FDIVS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11207 switch (VT.SimpleTy) { 11208 case MVT::v4f32: return fastEmit_X86ISD_FDIVS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11209 case MVT::v2f64: return fastEmit_X86ISD_FDIVS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11210 default: return 0; 11211 } 11212} 11213 11214// FastEmit functions for X86ISD::FGETEXPS. 11215 11216unsigned fastEmit_X86ISD_FGETEXPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11217 if (RetVT.SimpleTy != MVT::v4f32) 11218 return 0; 11219 if ((Subtarget->hasAVX512())) { 11220 return fastEmitInst_rr(X86::VGETEXPSSZr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11221 } 11222 return 0; 11223} 11224 11225unsigned fastEmit_X86ISD_FGETEXPS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11226 if (RetVT.SimpleTy != MVT::v2f64) 11227 return 0; 11228 if ((Subtarget->hasAVX512())) { 11229 return fastEmitInst_rr(X86::VGETEXPSDZr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11230 } 11231 return 0; 11232} 11233 11234unsigned fastEmit_X86ISD_FGETEXPS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11235 switch (VT.SimpleTy) { 11236 case MVT::v4f32: return fastEmit_X86ISD_FGETEXPS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11237 case MVT::v2f64: return fastEmit_X86ISD_FGETEXPS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11238 default: return 0; 11239 } 11240} 11241 11242// FastEmit functions for X86ISD::FGETEXPS_SAE. 11243 11244unsigned fastEmit_X86ISD_FGETEXPS_SAE_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11245 if (RetVT.SimpleTy != MVT::v4f32) 11246 return 0; 11247 if ((Subtarget->hasAVX512())) { 11248 return fastEmitInst_rr(X86::VGETEXPSSZrb, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11249 } 11250 return 0; 11251} 11252 11253unsigned fastEmit_X86ISD_FGETEXPS_SAE_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11254 if (RetVT.SimpleTy != MVT::v2f64) 11255 return 0; 11256 if ((Subtarget->hasAVX512())) { 11257 return fastEmitInst_rr(X86::VGETEXPSDZrb, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11258 } 11259 return 0; 11260} 11261 11262unsigned fastEmit_X86ISD_FGETEXPS_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11263 switch (VT.SimpleTy) { 11264 case MVT::v4f32: return fastEmit_X86ISD_FGETEXPS_SAE_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11265 case MVT::v2f64: return fastEmit_X86ISD_FGETEXPS_SAE_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11266 default: return 0; 11267 } 11268} 11269 11270// FastEmit functions for X86ISD::FHADD. 11271 11272unsigned fastEmit_X86ISD_FHADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11273 if (RetVT.SimpleTy != MVT::v4f32) 11274 return 0; 11275 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) { 11276 return fastEmitInst_rr(X86::HADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11277 } 11278 if ((Subtarget->hasAVX())) { 11279 return fastEmitInst_rr(X86::VHADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11280 } 11281 return 0; 11282} 11283 11284unsigned fastEmit_X86ISD_FHADD_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11285 if (RetVT.SimpleTy != MVT::v8f32) 11286 return 0; 11287 if ((Subtarget->hasAVX())) { 11288 return fastEmitInst_rr(X86::VHADDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11289 } 11290 return 0; 11291} 11292 11293unsigned fastEmit_X86ISD_FHADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11294 if (RetVT.SimpleTy != MVT::v2f64) 11295 return 0; 11296 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) { 11297 return fastEmitInst_rr(X86::HADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11298 } 11299 if ((Subtarget->hasAVX())) { 11300 return fastEmitInst_rr(X86::VHADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11301 } 11302 return 0; 11303} 11304 11305unsigned fastEmit_X86ISD_FHADD_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11306 if (RetVT.SimpleTy != MVT::v4f64) 11307 return 0; 11308 if ((Subtarget->hasAVX())) { 11309 return fastEmitInst_rr(X86::VHADDPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11310 } 11311 return 0; 11312} 11313 11314unsigned fastEmit_X86ISD_FHADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11315 switch (VT.SimpleTy) { 11316 case MVT::v4f32: return fastEmit_X86ISD_FHADD_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11317 case MVT::v8f32: return fastEmit_X86ISD_FHADD_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11318 case MVT::v2f64: return fastEmit_X86ISD_FHADD_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11319 case MVT::v4f64: return fastEmit_X86ISD_FHADD_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11320 default: return 0; 11321 } 11322} 11323 11324// FastEmit functions for X86ISD::FHSUB. 11325 11326unsigned fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11327 if (RetVT.SimpleTy != MVT::v4f32) 11328 return 0; 11329 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) { 11330 return fastEmitInst_rr(X86::HSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11331 } 11332 if ((Subtarget->hasAVX())) { 11333 return fastEmitInst_rr(X86::VHSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11334 } 11335 return 0; 11336} 11337 11338unsigned fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11339 if (RetVT.SimpleTy != MVT::v8f32) 11340 return 0; 11341 if ((Subtarget->hasAVX())) { 11342 return fastEmitInst_rr(X86::VHSUBPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11343 } 11344 return 0; 11345} 11346 11347unsigned fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11348 if (RetVT.SimpleTy != MVT::v2f64) 11349 return 0; 11350 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) { 11351 return fastEmitInst_rr(X86::HSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11352 } 11353 if ((Subtarget->hasAVX())) { 11354 return fastEmitInst_rr(X86::VHSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11355 } 11356 return 0; 11357} 11358 11359unsigned fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11360 if (RetVT.SimpleTy != MVT::v4f64) 11361 return 0; 11362 if ((Subtarget->hasAVX())) { 11363 return fastEmitInst_rr(X86::VHSUBPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11364 } 11365 return 0; 11366} 11367 11368unsigned fastEmit_X86ISD_FHSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11369 switch (VT.SimpleTy) { 11370 case MVT::v4f32: return fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11371 case MVT::v8f32: return fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11372 case MVT::v2f64: return fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11373 case MVT::v4f64: return fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11374 default: return 0; 11375 } 11376} 11377 11378// FastEmit functions for X86ISD::FMAX. 11379 11380unsigned fastEmit_X86ISD_FMAX_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11381 if (RetVT.SimpleTy != MVT::f32) 11382 return 0; 11383 if ((Subtarget->hasAVX512())) { 11384 return fastEmitInst_rr(X86::VMAXSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11385 } 11386 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 11387 return fastEmitInst_rr(X86::MAXSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11388 } 11389 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 11390 return fastEmitInst_rr(X86::VMAXSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11391 } 11392 return 0; 11393} 11394 11395unsigned fastEmit_X86ISD_FMAX_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11396 if (RetVT.SimpleTy != MVT::f64) 11397 return 0; 11398 if ((Subtarget->hasAVX512())) { 11399 return fastEmitInst_rr(X86::VMAXSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11400 } 11401 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 11402 return fastEmitInst_rr(X86::MAXSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11403 } 11404 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 11405 return fastEmitInst_rr(X86::VMAXSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11406 } 11407 return 0; 11408} 11409 11410unsigned fastEmit_X86ISD_FMAX_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11411 if (RetVT.SimpleTy != MVT::v4f32) 11412 return 0; 11413 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11414 return fastEmitInst_rr(X86::VMAXPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11415 } 11416 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 11417 return fastEmitInst_rr(X86::MAXPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11418 } 11419 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11420 return fastEmitInst_rr(X86::VMAXPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11421 } 11422 return 0; 11423} 11424 11425unsigned fastEmit_X86ISD_FMAX_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11426 if (RetVT.SimpleTy != MVT::v8f32) 11427 return 0; 11428 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11429 return fastEmitInst_rr(X86::VMAXPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11430 } 11431 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11432 return fastEmitInst_rr(X86::VMAXPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11433 } 11434 return 0; 11435} 11436 11437unsigned fastEmit_X86ISD_FMAX_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11438 if (RetVT.SimpleTy != MVT::v16f32) 11439 return 0; 11440 if ((Subtarget->hasAVX512())) { 11441 return fastEmitInst_rr(X86::VMAXPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11442 } 11443 return 0; 11444} 11445 11446unsigned fastEmit_X86ISD_FMAX_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11447 if (RetVT.SimpleTy != MVT::v2f64) 11448 return 0; 11449 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11450 return fastEmitInst_rr(X86::VMAXPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11451 } 11452 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 11453 return fastEmitInst_rr(X86::MAXPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11454 } 11455 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11456 return fastEmitInst_rr(X86::VMAXPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11457 } 11458 return 0; 11459} 11460 11461unsigned fastEmit_X86ISD_FMAX_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11462 if (RetVT.SimpleTy != MVT::v4f64) 11463 return 0; 11464 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11465 return fastEmitInst_rr(X86::VMAXPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11466 } 11467 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11468 return fastEmitInst_rr(X86::VMAXPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11469 } 11470 return 0; 11471} 11472 11473unsigned fastEmit_X86ISD_FMAX_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11474 if (RetVT.SimpleTy != MVT::v8f64) 11475 return 0; 11476 if ((Subtarget->hasAVX512())) { 11477 return fastEmitInst_rr(X86::VMAXPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11478 } 11479 return 0; 11480} 11481 11482unsigned fastEmit_X86ISD_FMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11483 switch (VT.SimpleTy) { 11484 case MVT::f32: return fastEmit_X86ISD_FMAX_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11485 case MVT::f64: return fastEmit_X86ISD_FMAX_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11486 case MVT::v4f32: return fastEmit_X86ISD_FMAX_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11487 case MVT::v8f32: return fastEmit_X86ISD_FMAX_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11488 case MVT::v16f32: return fastEmit_X86ISD_FMAX_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11489 case MVT::v2f64: return fastEmit_X86ISD_FMAX_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11490 case MVT::v4f64: return fastEmit_X86ISD_FMAX_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11491 case MVT::v8f64: return fastEmit_X86ISD_FMAX_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11492 default: return 0; 11493 } 11494} 11495 11496// FastEmit functions for X86ISD::FMAXC. 11497 11498unsigned fastEmit_X86ISD_FMAXC_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11499 if (RetVT.SimpleTy != MVT::f32) 11500 return 0; 11501 if ((Subtarget->hasAVX512())) { 11502 return fastEmitInst_rr(X86::VMAXCSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11503 } 11504 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 11505 return fastEmitInst_rr(X86::MAXCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11506 } 11507 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 11508 return fastEmitInst_rr(X86::VMAXCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11509 } 11510 return 0; 11511} 11512 11513unsigned fastEmit_X86ISD_FMAXC_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11514 if (RetVT.SimpleTy != MVT::f64) 11515 return 0; 11516 if ((Subtarget->hasAVX512())) { 11517 return fastEmitInst_rr(X86::VMAXCSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11518 } 11519 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 11520 return fastEmitInst_rr(X86::MAXCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11521 } 11522 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 11523 return fastEmitInst_rr(X86::VMAXCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11524 } 11525 return 0; 11526} 11527 11528unsigned fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11529 if (RetVT.SimpleTy != MVT::v4f32) 11530 return 0; 11531 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11532 return fastEmitInst_rr(X86::VMAXCPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11533 } 11534 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 11535 return fastEmitInst_rr(X86::MAXCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11536 } 11537 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11538 return fastEmitInst_rr(X86::VMAXCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11539 } 11540 return 0; 11541} 11542 11543unsigned fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11544 if (RetVT.SimpleTy != MVT::v8f32) 11545 return 0; 11546 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11547 return fastEmitInst_rr(X86::VMAXCPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11548 } 11549 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11550 return fastEmitInst_rr(X86::VMAXCPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11551 } 11552 return 0; 11553} 11554 11555unsigned fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11556 if (RetVT.SimpleTy != MVT::v16f32) 11557 return 0; 11558 if ((Subtarget->hasAVX512())) { 11559 return fastEmitInst_rr(X86::VMAXCPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11560 } 11561 return 0; 11562} 11563 11564unsigned fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11565 if (RetVT.SimpleTy != MVT::v2f64) 11566 return 0; 11567 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11568 return fastEmitInst_rr(X86::VMAXCPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11569 } 11570 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 11571 return fastEmitInst_rr(X86::MAXCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11572 } 11573 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11574 return fastEmitInst_rr(X86::VMAXCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11575 } 11576 return 0; 11577} 11578 11579unsigned fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11580 if (RetVT.SimpleTy != MVT::v4f64) 11581 return 0; 11582 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11583 return fastEmitInst_rr(X86::VMAXCPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11584 } 11585 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11586 return fastEmitInst_rr(X86::VMAXCPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11587 } 11588 return 0; 11589} 11590 11591unsigned fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11592 if (RetVT.SimpleTy != MVT::v8f64) 11593 return 0; 11594 if ((Subtarget->hasAVX512())) { 11595 return fastEmitInst_rr(X86::VMAXCPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11596 } 11597 return 0; 11598} 11599 11600unsigned fastEmit_X86ISD_FMAXC_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11601 switch (VT.SimpleTy) { 11602 case MVT::f32: return fastEmit_X86ISD_FMAXC_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11603 case MVT::f64: return fastEmit_X86ISD_FMAXC_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11604 case MVT::v4f32: return fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11605 case MVT::v8f32: return fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11606 case MVT::v16f32: return fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11607 case MVT::v2f64: return fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11608 case MVT::v4f64: return fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11609 case MVT::v8f64: return fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11610 default: return 0; 11611 } 11612} 11613 11614// FastEmit functions for X86ISD::FMAXS. 11615 11616unsigned fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11617 if (RetVT.SimpleTy != MVT::v4f32) 11618 return 0; 11619 if ((Subtarget->hasAVX512())) { 11620 return fastEmitInst_rr(X86::VMAXSSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11621 } 11622 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 11623 return fastEmitInst_rr(X86::MAXSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11624 } 11625 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 11626 return fastEmitInst_rr(X86::VMAXSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11627 } 11628 return 0; 11629} 11630 11631unsigned fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11632 if (RetVT.SimpleTy != MVT::v2f64) 11633 return 0; 11634 if ((Subtarget->hasAVX512())) { 11635 return fastEmitInst_rr(X86::VMAXSDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11636 } 11637 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 11638 return fastEmitInst_rr(X86::MAXSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11639 } 11640 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 11641 return fastEmitInst_rr(X86::VMAXSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11642 } 11643 return 0; 11644} 11645 11646unsigned fastEmit_X86ISD_FMAXS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11647 switch (VT.SimpleTy) { 11648 case MVT::v4f32: return fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11649 case MVT::v2f64: return fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11650 default: return 0; 11651 } 11652} 11653 11654// FastEmit functions for X86ISD::FMAXS_SAE. 11655 11656unsigned fastEmit_X86ISD_FMAXS_SAE_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11657 if (RetVT.SimpleTy != MVT::v4f32) 11658 return 0; 11659 if ((Subtarget->hasAVX512())) { 11660 return fastEmitInst_rr(X86::VMAXSSZrrb_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11661 } 11662 return 0; 11663} 11664 11665unsigned fastEmit_X86ISD_FMAXS_SAE_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11666 if (RetVT.SimpleTy != MVT::v2f64) 11667 return 0; 11668 if ((Subtarget->hasAVX512())) { 11669 return fastEmitInst_rr(X86::VMAXSDZrrb_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11670 } 11671 return 0; 11672} 11673 11674unsigned fastEmit_X86ISD_FMAXS_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11675 switch (VT.SimpleTy) { 11676 case MVT::v4f32: return fastEmit_X86ISD_FMAXS_SAE_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11677 case MVT::v2f64: return fastEmit_X86ISD_FMAXS_SAE_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11678 default: return 0; 11679 } 11680} 11681 11682// FastEmit functions for X86ISD::FMAX_SAE. 11683 11684unsigned fastEmit_X86ISD_FMAX_SAE_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11685 if (RetVT.SimpleTy != MVT::v16f32) 11686 return 0; 11687 if ((Subtarget->hasAVX512())) { 11688 return fastEmitInst_rr(X86::VMAXPSZrrb, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11689 } 11690 return 0; 11691} 11692 11693unsigned fastEmit_X86ISD_FMAX_SAE_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11694 if (RetVT.SimpleTy != MVT::v8f64) 11695 return 0; 11696 if ((Subtarget->hasAVX512())) { 11697 return fastEmitInst_rr(X86::VMAXPDZrrb, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11698 } 11699 return 0; 11700} 11701 11702unsigned fastEmit_X86ISD_FMAX_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11703 switch (VT.SimpleTy) { 11704 case MVT::v16f32: return fastEmit_X86ISD_FMAX_SAE_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11705 case MVT::v8f64: return fastEmit_X86ISD_FMAX_SAE_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11706 default: return 0; 11707 } 11708} 11709 11710// FastEmit functions for X86ISD::FMIN. 11711 11712unsigned fastEmit_X86ISD_FMIN_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11713 if (RetVT.SimpleTy != MVT::f32) 11714 return 0; 11715 if ((Subtarget->hasAVX512())) { 11716 return fastEmitInst_rr(X86::VMINSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11717 } 11718 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 11719 return fastEmitInst_rr(X86::MINSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11720 } 11721 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 11722 return fastEmitInst_rr(X86::VMINSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11723 } 11724 return 0; 11725} 11726 11727unsigned fastEmit_X86ISD_FMIN_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11728 if (RetVT.SimpleTy != MVT::f64) 11729 return 0; 11730 if ((Subtarget->hasAVX512())) { 11731 return fastEmitInst_rr(X86::VMINSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11732 } 11733 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 11734 return fastEmitInst_rr(X86::MINSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11735 } 11736 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 11737 return fastEmitInst_rr(X86::VMINSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11738 } 11739 return 0; 11740} 11741 11742unsigned fastEmit_X86ISD_FMIN_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11743 if (RetVT.SimpleTy != MVT::v4f32) 11744 return 0; 11745 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11746 return fastEmitInst_rr(X86::VMINPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11747 } 11748 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 11749 return fastEmitInst_rr(X86::MINPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11750 } 11751 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11752 return fastEmitInst_rr(X86::VMINPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11753 } 11754 return 0; 11755} 11756 11757unsigned fastEmit_X86ISD_FMIN_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11758 if (RetVT.SimpleTy != MVT::v8f32) 11759 return 0; 11760 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11761 return fastEmitInst_rr(X86::VMINPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11762 } 11763 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11764 return fastEmitInst_rr(X86::VMINPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11765 } 11766 return 0; 11767} 11768 11769unsigned fastEmit_X86ISD_FMIN_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11770 if (RetVT.SimpleTy != MVT::v16f32) 11771 return 0; 11772 if ((Subtarget->hasAVX512())) { 11773 return fastEmitInst_rr(X86::VMINPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11774 } 11775 return 0; 11776} 11777 11778unsigned fastEmit_X86ISD_FMIN_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11779 if (RetVT.SimpleTy != MVT::v2f64) 11780 return 0; 11781 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11782 return fastEmitInst_rr(X86::VMINPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11783 } 11784 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 11785 return fastEmitInst_rr(X86::MINPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11786 } 11787 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11788 return fastEmitInst_rr(X86::VMINPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11789 } 11790 return 0; 11791} 11792 11793unsigned fastEmit_X86ISD_FMIN_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11794 if (RetVT.SimpleTy != MVT::v4f64) 11795 return 0; 11796 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11797 return fastEmitInst_rr(X86::VMINPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11798 } 11799 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11800 return fastEmitInst_rr(X86::VMINPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11801 } 11802 return 0; 11803} 11804 11805unsigned fastEmit_X86ISD_FMIN_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11806 if (RetVT.SimpleTy != MVT::v8f64) 11807 return 0; 11808 if ((Subtarget->hasAVX512())) { 11809 return fastEmitInst_rr(X86::VMINPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11810 } 11811 return 0; 11812} 11813 11814unsigned fastEmit_X86ISD_FMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11815 switch (VT.SimpleTy) { 11816 case MVT::f32: return fastEmit_X86ISD_FMIN_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11817 case MVT::f64: return fastEmit_X86ISD_FMIN_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11818 case MVT::v4f32: return fastEmit_X86ISD_FMIN_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11819 case MVT::v8f32: return fastEmit_X86ISD_FMIN_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11820 case MVT::v16f32: return fastEmit_X86ISD_FMIN_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11821 case MVT::v2f64: return fastEmit_X86ISD_FMIN_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11822 case MVT::v4f64: return fastEmit_X86ISD_FMIN_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11823 case MVT::v8f64: return fastEmit_X86ISD_FMIN_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11824 default: return 0; 11825 } 11826} 11827 11828// FastEmit functions for X86ISD::FMINC. 11829 11830unsigned fastEmit_X86ISD_FMINC_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11831 if (RetVT.SimpleTy != MVT::f32) 11832 return 0; 11833 if ((Subtarget->hasAVX512())) { 11834 return fastEmitInst_rr(X86::VMINCSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11835 } 11836 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 11837 return fastEmitInst_rr(X86::MINCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11838 } 11839 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 11840 return fastEmitInst_rr(X86::VMINCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11841 } 11842 return 0; 11843} 11844 11845unsigned fastEmit_X86ISD_FMINC_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11846 if (RetVT.SimpleTy != MVT::f64) 11847 return 0; 11848 if ((Subtarget->hasAVX512())) { 11849 return fastEmitInst_rr(X86::VMINCSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11850 } 11851 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 11852 return fastEmitInst_rr(X86::MINCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11853 } 11854 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 11855 return fastEmitInst_rr(X86::VMINCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11856 } 11857 return 0; 11858} 11859 11860unsigned fastEmit_X86ISD_FMINC_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11861 if (RetVT.SimpleTy != MVT::v4f32) 11862 return 0; 11863 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11864 return fastEmitInst_rr(X86::VMINCPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11865 } 11866 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 11867 return fastEmitInst_rr(X86::MINCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11868 } 11869 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11870 return fastEmitInst_rr(X86::VMINCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11871 } 11872 return 0; 11873} 11874 11875unsigned fastEmit_X86ISD_FMINC_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11876 if (RetVT.SimpleTy != MVT::v8f32) 11877 return 0; 11878 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11879 return fastEmitInst_rr(X86::VMINCPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11880 } 11881 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11882 return fastEmitInst_rr(X86::VMINCPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11883 } 11884 return 0; 11885} 11886 11887unsigned fastEmit_X86ISD_FMINC_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11888 if (RetVT.SimpleTy != MVT::v16f32) 11889 return 0; 11890 if ((Subtarget->hasAVX512())) { 11891 return fastEmitInst_rr(X86::VMINCPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11892 } 11893 return 0; 11894} 11895 11896unsigned fastEmit_X86ISD_FMINC_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11897 if (RetVT.SimpleTy != MVT::v2f64) 11898 return 0; 11899 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11900 return fastEmitInst_rr(X86::VMINCPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11901 } 11902 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 11903 return fastEmitInst_rr(X86::MINCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11904 } 11905 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11906 return fastEmitInst_rr(X86::VMINCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11907 } 11908 return 0; 11909} 11910 11911unsigned fastEmit_X86ISD_FMINC_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11912 if (RetVT.SimpleTy != MVT::v4f64) 11913 return 0; 11914 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 11915 return fastEmitInst_rr(X86::VMINCPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11916 } 11917 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 11918 return fastEmitInst_rr(X86::VMINCPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11919 } 11920 return 0; 11921} 11922 11923unsigned fastEmit_X86ISD_FMINC_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11924 if (RetVT.SimpleTy != MVT::v8f64) 11925 return 0; 11926 if ((Subtarget->hasAVX512())) { 11927 return fastEmitInst_rr(X86::VMINCPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11928 } 11929 return 0; 11930} 11931 11932unsigned fastEmit_X86ISD_FMINC_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11933 switch (VT.SimpleTy) { 11934 case MVT::f32: return fastEmit_X86ISD_FMINC_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11935 case MVT::f64: return fastEmit_X86ISD_FMINC_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11936 case MVT::v4f32: return fastEmit_X86ISD_FMINC_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11937 case MVT::v8f32: return fastEmit_X86ISD_FMINC_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11938 case MVT::v16f32: return fastEmit_X86ISD_FMINC_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11939 case MVT::v2f64: return fastEmit_X86ISD_FMINC_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11940 case MVT::v4f64: return fastEmit_X86ISD_FMINC_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11941 case MVT::v8f64: return fastEmit_X86ISD_FMINC_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11942 default: return 0; 11943 } 11944} 11945 11946// FastEmit functions for X86ISD::FMINS. 11947 11948unsigned fastEmit_X86ISD_FMINS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11949 if (RetVT.SimpleTy != MVT::v4f32) 11950 return 0; 11951 if ((Subtarget->hasAVX512())) { 11952 return fastEmitInst_rr(X86::VMINSSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11953 } 11954 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 11955 return fastEmitInst_rr(X86::MINSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11956 } 11957 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 11958 return fastEmitInst_rr(X86::VMINSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11959 } 11960 return 0; 11961} 11962 11963unsigned fastEmit_X86ISD_FMINS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11964 if (RetVT.SimpleTy != MVT::v2f64) 11965 return 0; 11966 if ((Subtarget->hasAVX512())) { 11967 return fastEmitInst_rr(X86::VMINSDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11968 } 11969 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 11970 return fastEmitInst_rr(X86::MINSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11971 } 11972 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 11973 return fastEmitInst_rr(X86::VMINSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11974 } 11975 return 0; 11976} 11977 11978unsigned fastEmit_X86ISD_FMINS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11979 switch (VT.SimpleTy) { 11980 case MVT::v4f32: return fastEmit_X86ISD_FMINS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11981 case MVT::v2f64: return fastEmit_X86ISD_FMINS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 11982 default: return 0; 11983 } 11984} 11985 11986// FastEmit functions for X86ISD::FMINS_SAE. 11987 11988unsigned fastEmit_X86ISD_FMINS_SAE_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11989 if (RetVT.SimpleTy != MVT::v4f32) 11990 return 0; 11991 if ((Subtarget->hasAVX512())) { 11992 return fastEmitInst_rr(X86::VMINSSZrrb_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 11993 } 11994 return 0; 11995} 11996 11997unsigned fastEmit_X86ISD_FMINS_SAE_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 11998 if (RetVT.SimpleTy != MVT::v2f64) 11999 return 0; 12000 if ((Subtarget->hasAVX512())) { 12001 return fastEmitInst_rr(X86::VMINSDZrrb_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12002 } 12003 return 0; 12004} 12005 12006unsigned fastEmit_X86ISD_FMINS_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12007 switch (VT.SimpleTy) { 12008 case MVT::v4f32: return fastEmit_X86ISD_FMINS_SAE_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12009 case MVT::v2f64: return fastEmit_X86ISD_FMINS_SAE_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12010 default: return 0; 12011 } 12012} 12013 12014// FastEmit functions for X86ISD::FMIN_SAE. 12015 12016unsigned fastEmit_X86ISD_FMIN_SAE_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12017 if (RetVT.SimpleTy != MVT::v16f32) 12018 return 0; 12019 if ((Subtarget->hasAVX512())) { 12020 return fastEmitInst_rr(X86::VMINPSZrrb, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12021 } 12022 return 0; 12023} 12024 12025unsigned fastEmit_X86ISD_FMIN_SAE_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12026 if (RetVT.SimpleTy != MVT::v8f64) 12027 return 0; 12028 if ((Subtarget->hasAVX512())) { 12029 return fastEmitInst_rr(X86::VMINPDZrrb, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12030 } 12031 return 0; 12032} 12033 12034unsigned fastEmit_X86ISD_FMIN_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12035 switch (VT.SimpleTy) { 12036 case MVT::v16f32: return fastEmit_X86ISD_FMIN_SAE_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12037 case MVT::v8f64: return fastEmit_X86ISD_FMIN_SAE_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12038 default: return 0; 12039 } 12040} 12041 12042// FastEmit functions for X86ISD::FMULS. 12043 12044unsigned fastEmit_X86ISD_FMULS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12045 if (RetVT.SimpleTy != MVT::v4f32) 12046 return 0; 12047 if ((Subtarget->hasAVX512())) { 12048 return fastEmitInst_rr(X86::VMULSSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12049 } 12050 return 0; 12051} 12052 12053unsigned fastEmit_X86ISD_FMULS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12054 if (RetVT.SimpleTy != MVT::v2f64) 12055 return 0; 12056 if ((Subtarget->hasAVX512())) { 12057 return fastEmitInst_rr(X86::VMULSDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12058 } 12059 return 0; 12060} 12061 12062unsigned fastEmit_X86ISD_FMULS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12063 switch (VT.SimpleTy) { 12064 case MVT::v4f32: return fastEmit_X86ISD_FMULS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12065 case MVT::v2f64: return fastEmit_X86ISD_FMULS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12066 default: return 0; 12067 } 12068} 12069 12070// FastEmit functions for X86ISD::FOR. 12071 12072unsigned fastEmit_X86ISD_FOR_MVT_f128_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12073 if (RetVT.SimpleTy != MVT::f128) 12074 return 0; 12075 if ((Subtarget->hasVLX())) { 12076 return fastEmitInst_rr(X86::VORPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12077 } 12078 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 12079 return fastEmitInst_rr(X86::VORPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12080 } 12081 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 12082 return fastEmitInst_rr(X86::ORPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12083 } 12084 return 0; 12085} 12086 12087unsigned fastEmit_X86ISD_FOR_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12088 if (RetVT.SimpleTy != MVT::v4f32) 12089 return 0; 12090 return fastEmitInst_rr(X86::ORPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12091} 12092 12093unsigned fastEmit_X86ISD_FOR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12094 switch (VT.SimpleTy) { 12095 case MVT::f128: return fastEmit_X86ISD_FOR_MVT_f128_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12096 case MVT::v4f32: return fastEmit_X86ISD_FOR_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12097 default: return 0; 12098 } 12099} 12100 12101// FastEmit functions for X86ISD::FSQRTS. 12102 12103unsigned fastEmit_X86ISD_FSQRTS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12104 if (RetVT.SimpleTy != MVT::v4f32) 12105 return 0; 12106 if ((Subtarget->hasAVX512())) { 12107 return fastEmitInst_rr(X86::VSQRTSSZr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12108 } 12109 return 0; 12110} 12111 12112unsigned fastEmit_X86ISD_FSQRTS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12113 if (RetVT.SimpleTy != MVT::v2f64) 12114 return 0; 12115 if ((Subtarget->hasAVX512())) { 12116 return fastEmitInst_rr(X86::VSQRTSDZr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12117 } 12118 return 0; 12119} 12120 12121unsigned fastEmit_X86ISD_FSQRTS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12122 switch (VT.SimpleTy) { 12123 case MVT::v4f32: return fastEmit_X86ISD_FSQRTS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12124 case MVT::v2f64: return fastEmit_X86ISD_FSQRTS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12125 default: return 0; 12126 } 12127} 12128 12129// FastEmit functions for X86ISD::FSUBS. 12130 12131unsigned fastEmit_X86ISD_FSUBS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12132 if (RetVT.SimpleTy != MVT::v4f32) 12133 return 0; 12134 if ((Subtarget->hasAVX512())) { 12135 return fastEmitInst_rr(X86::VSUBSSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12136 } 12137 return 0; 12138} 12139 12140unsigned fastEmit_X86ISD_FSUBS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12141 if (RetVT.SimpleTy != MVT::v2f64) 12142 return 0; 12143 if ((Subtarget->hasAVX512())) { 12144 return fastEmitInst_rr(X86::VSUBSDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12145 } 12146 return 0; 12147} 12148 12149unsigned fastEmit_X86ISD_FSUBS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12150 switch (VT.SimpleTy) { 12151 case MVT::v4f32: return fastEmit_X86ISD_FSUBS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12152 case MVT::v2f64: return fastEmit_X86ISD_FSUBS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12153 default: return 0; 12154 } 12155} 12156 12157// FastEmit functions for X86ISD::FXOR. 12158 12159unsigned fastEmit_X86ISD_FXOR_MVT_f128_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12160 if (RetVT.SimpleTy != MVT::f128) 12161 return 0; 12162 if ((Subtarget->hasVLX())) { 12163 return fastEmitInst_rr(X86::VXORPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12164 } 12165 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 12166 return fastEmitInst_rr(X86::VXORPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12167 } 12168 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 12169 return fastEmitInst_rr(X86::XORPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12170 } 12171 return 0; 12172} 12173 12174unsigned fastEmit_X86ISD_FXOR_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12175 if (RetVT.SimpleTy != MVT::v4f32) 12176 return 0; 12177 return fastEmitInst_rr(X86::XORPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12178} 12179 12180unsigned fastEmit_X86ISD_FXOR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12181 switch (VT.SimpleTy) { 12182 case MVT::f128: return fastEmit_X86ISD_FXOR_MVT_f128_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12183 case MVT::v4f32: return fastEmit_X86ISD_FXOR_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12184 default: return 0; 12185 } 12186} 12187 12188// FastEmit functions for X86ISD::GF2P8MULB. 12189 12190unsigned fastEmit_X86ISD_GF2P8MULB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12191 if (RetVT.SimpleTy != MVT::v16i8) 12192 return 0; 12193 if ((Subtarget->hasBWI()) && (Subtarget->hasGFNI()) && (Subtarget->hasVLX())) { 12194 return fastEmitInst_rr(X86::VGF2P8MULBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12195 } 12196 if ((Subtarget->hasAVX()) && (Subtarget->hasGFNI()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 12197 return fastEmitInst_rr(X86::VGF2P8MULBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12198 } 12199 if ((Subtarget->hasGFNI()) && (Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 12200 return fastEmitInst_rr(X86::GF2P8MULBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12201 } 12202 return 0; 12203} 12204 12205unsigned fastEmit_X86ISD_GF2P8MULB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12206 if (RetVT.SimpleTy != MVT::v32i8) 12207 return 0; 12208 if ((Subtarget->hasBWI()) && (Subtarget->hasGFNI()) && (Subtarget->hasVLX())) { 12209 return fastEmitInst_rr(X86::VGF2P8MULBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12210 } 12211 if ((Subtarget->hasAVX()) && (Subtarget->hasGFNI()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 12212 return fastEmitInst_rr(X86::VGF2P8MULBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12213 } 12214 return 0; 12215} 12216 12217unsigned fastEmit_X86ISD_GF2P8MULB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12218 if (RetVT.SimpleTy != MVT::v64i8) 12219 return 0; 12220 if ((Subtarget->hasAVX512()) && (Subtarget->hasBWI()) && (Subtarget->hasGFNI())) { 12221 return fastEmitInst_rr(X86::VGF2P8MULBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12222 } 12223 return 0; 12224} 12225 12226unsigned fastEmit_X86ISD_GF2P8MULB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12227 switch (VT.SimpleTy) { 12228 case MVT::v16i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12229 case MVT::v32i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12230 case MVT::v64i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12231 default: return 0; 12232 } 12233} 12234 12235// FastEmit functions for X86ISD::HADD. 12236 12237unsigned fastEmit_X86ISD_HADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12238 if (RetVT.SimpleTy != MVT::v8i16) 12239 return 0; 12240 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) { 12241 return fastEmitInst_rr(X86::PHADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12242 } 12243 if ((Subtarget->hasAVX())) { 12244 return fastEmitInst_rr(X86::VPHADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12245 } 12246 return 0; 12247} 12248 12249unsigned fastEmit_X86ISD_HADD_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12250 if (RetVT.SimpleTy != MVT::v16i16) 12251 return 0; 12252 if ((Subtarget->hasAVX2())) { 12253 return fastEmitInst_rr(X86::VPHADDWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12254 } 12255 return 0; 12256} 12257 12258unsigned fastEmit_X86ISD_HADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12259 if (RetVT.SimpleTy != MVT::v4i32) 12260 return 0; 12261 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) { 12262 return fastEmitInst_rr(X86::PHADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12263 } 12264 if ((Subtarget->hasAVX())) { 12265 return fastEmitInst_rr(X86::VPHADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12266 } 12267 return 0; 12268} 12269 12270unsigned fastEmit_X86ISD_HADD_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12271 if (RetVT.SimpleTy != MVT::v8i32) 12272 return 0; 12273 if ((Subtarget->hasAVX2())) { 12274 return fastEmitInst_rr(X86::VPHADDDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12275 } 12276 return 0; 12277} 12278 12279unsigned fastEmit_X86ISD_HADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12280 switch (VT.SimpleTy) { 12281 case MVT::v8i16: return fastEmit_X86ISD_HADD_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12282 case MVT::v16i16: return fastEmit_X86ISD_HADD_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12283 case MVT::v4i32: return fastEmit_X86ISD_HADD_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12284 case MVT::v8i32: return fastEmit_X86ISD_HADD_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12285 default: return 0; 12286 } 12287} 12288 12289// FastEmit functions for X86ISD::HSUB. 12290 12291unsigned fastEmit_X86ISD_HSUB_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12292 if (RetVT.SimpleTy != MVT::v8i16) 12293 return 0; 12294 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) { 12295 return fastEmitInst_rr(X86::PHSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12296 } 12297 if ((Subtarget->hasAVX())) { 12298 return fastEmitInst_rr(X86::VPHSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12299 } 12300 return 0; 12301} 12302 12303unsigned fastEmit_X86ISD_HSUB_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12304 if (RetVT.SimpleTy != MVT::v16i16) 12305 return 0; 12306 if ((Subtarget->hasAVX2())) { 12307 return fastEmitInst_rr(X86::VPHSUBWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12308 } 12309 return 0; 12310} 12311 12312unsigned fastEmit_X86ISD_HSUB_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12313 if (RetVT.SimpleTy != MVT::v4i32) 12314 return 0; 12315 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) { 12316 return fastEmitInst_rr(X86::PHSUBDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12317 } 12318 if ((Subtarget->hasAVX())) { 12319 return fastEmitInst_rr(X86::VPHSUBDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12320 } 12321 return 0; 12322} 12323 12324unsigned fastEmit_X86ISD_HSUB_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12325 if (RetVT.SimpleTy != MVT::v8i32) 12326 return 0; 12327 if ((Subtarget->hasAVX2())) { 12328 return fastEmitInst_rr(X86::VPHSUBDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12329 } 12330 return 0; 12331} 12332 12333unsigned fastEmit_X86ISD_HSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12334 switch (VT.SimpleTy) { 12335 case MVT::v8i16: return fastEmit_X86ISD_HSUB_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12336 case MVT::v16i16: return fastEmit_X86ISD_HSUB_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12337 case MVT::v4i32: return fastEmit_X86ISD_HSUB_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12338 case MVT::v8i32: return fastEmit_X86ISD_HSUB_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12339 default: return 0; 12340 } 12341} 12342 12343// FastEmit functions for X86ISD::KADD. 12344 12345unsigned fastEmit_X86ISD_KADD_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12346 if (RetVT.SimpleTy != MVT::v8i1) 12347 return 0; 12348 if ((Subtarget->hasDQI())) { 12349 return fastEmitInst_rr(X86::KADDBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12350 } 12351 return 0; 12352} 12353 12354unsigned fastEmit_X86ISD_KADD_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12355 if (RetVT.SimpleTy != MVT::v16i1) 12356 return 0; 12357 if ((Subtarget->hasDQI())) { 12358 return fastEmitInst_rr(X86::KADDWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12359 } 12360 return 0; 12361} 12362 12363unsigned fastEmit_X86ISD_KADD_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12364 if (RetVT.SimpleTy != MVT::v32i1) 12365 return 0; 12366 if ((Subtarget->hasBWI())) { 12367 return fastEmitInst_rr(X86::KADDDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12368 } 12369 return 0; 12370} 12371 12372unsigned fastEmit_X86ISD_KADD_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12373 if (RetVT.SimpleTy != MVT::v64i1) 12374 return 0; 12375 if ((Subtarget->hasBWI())) { 12376 return fastEmitInst_rr(X86::KADDQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12377 } 12378 return 0; 12379} 12380 12381unsigned fastEmit_X86ISD_KADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12382 switch (VT.SimpleTy) { 12383 case MVT::v8i1: return fastEmit_X86ISD_KADD_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12384 case MVT::v16i1: return fastEmit_X86ISD_KADD_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12385 case MVT::v32i1: return fastEmit_X86ISD_KADD_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12386 case MVT::v64i1: return fastEmit_X86ISD_KADD_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12387 default: return 0; 12388 } 12389} 12390 12391// FastEmit functions for X86ISD::KORTEST. 12392 12393unsigned fastEmit_X86ISD_KORTEST_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12394 if (RetVT.SimpleTy != MVT::i32) 12395 return 0; 12396 if ((Subtarget->hasDQI())) { 12397 return fastEmitInst_rr(X86::KORTESTBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12398 } 12399 return 0; 12400} 12401 12402unsigned fastEmit_X86ISD_KORTEST_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12403 if (RetVT.SimpleTy != MVT::i32) 12404 return 0; 12405 if ((Subtarget->hasAVX512())) { 12406 return fastEmitInst_rr(X86::KORTESTWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12407 } 12408 return 0; 12409} 12410 12411unsigned fastEmit_X86ISD_KORTEST_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12412 if (RetVT.SimpleTy != MVT::i32) 12413 return 0; 12414 if ((Subtarget->hasBWI())) { 12415 return fastEmitInst_rr(X86::KORTESTDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12416 } 12417 return 0; 12418} 12419 12420unsigned fastEmit_X86ISD_KORTEST_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12421 if (RetVT.SimpleTy != MVT::i32) 12422 return 0; 12423 if ((Subtarget->hasBWI())) { 12424 return fastEmitInst_rr(X86::KORTESTQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12425 } 12426 return 0; 12427} 12428 12429unsigned fastEmit_X86ISD_KORTEST_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12430 switch (VT.SimpleTy) { 12431 case MVT::v8i1: return fastEmit_X86ISD_KORTEST_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12432 case MVT::v16i1: return fastEmit_X86ISD_KORTEST_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12433 case MVT::v32i1: return fastEmit_X86ISD_KORTEST_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12434 case MVT::v64i1: return fastEmit_X86ISD_KORTEST_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12435 default: return 0; 12436 } 12437} 12438 12439// FastEmit functions for X86ISD::KTEST. 12440 12441unsigned fastEmit_X86ISD_KTEST_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12442 if (RetVT.SimpleTy != MVT::i32) 12443 return 0; 12444 if ((Subtarget->hasDQI())) { 12445 return fastEmitInst_rr(X86::KTESTBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12446 } 12447 return 0; 12448} 12449 12450unsigned fastEmit_X86ISD_KTEST_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12451 if (RetVT.SimpleTy != MVT::i32) 12452 return 0; 12453 if ((Subtarget->hasDQI())) { 12454 return fastEmitInst_rr(X86::KTESTWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12455 } 12456 return 0; 12457} 12458 12459unsigned fastEmit_X86ISD_KTEST_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12460 if (RetVT.SimpleTy != MVT::i32) 12461 return 0; 12462 if ((Subtarget->hasBWI())) { 12463 return fastEmitInst_rr(X86::KTESTDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12464 } 12465 return 0; 12466} 12467 12468unsigned fastEmit_X86ISD_KTEST_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12469 if (RetVT.SimpleTy != MVT::i32) 12470 return 0; 12471 if ((Subtarget->hasBWI())) { 12472 return fastEmitInst_rr(X86::KTESTQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12473 } 12474 return 0; 12475} 12476 12477unsigned fastEmit_X86ISD_KTEST_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12478 switch (VT.SimpleTy) { 12479 case MVT::v8i1: return fastEmit_X86ISD_KTEST_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12480 case MVT::v16i1: return fastEmit_X86ISD_KTEST_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12481 case MVT::v32i1: return fastEmit_X86ISD_KTEST_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12482 case MVT::v64i1: return fastEmit_X86ISD_KTEST_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12483 default: return 0; 12484 } 12485} 12486 12487// FastEmit functions for X86ISD::MOVHLPS. 12488 12489unsigned fastEmit_X86ISD_MOVHLPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12490 if (RetVT.SimpleTy != MVT::v4f32) 12491 return 0; 12492 if ((Subtarget->hasAVX512())) { 12493 return fastEmitInst_rr(X86::VMOVHLPSZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12494 } 12495 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 12496 return fastEmitInst_rr(X86::MOVHLPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12497 } 12498 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 12499 return fastEmitInst_rr(X86::VMOVHLPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12500 } 12501 return 0; 12502} 12503 12504unsigned fastEmit_X86ISD_MOVHLPS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12505 switch (VT.SimpleTy) { 12506 case MVT::v4f32: return fastEmit_X86ISD_MOVHLPS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12507 default: return 0; 12508 } 12509} 12510 12511// FastEmit functions for X86ISD::MOVLHPS. 12512 12513unsigned fastEmit_X86ISD_MOVLHPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12514 if (RetVT.SimpleTy != MVT::v4f32) 12515 return 0; 12516 if ((Subtarget->hasAVX512())) { 12517 return fastEmitInst_rr(X86::VMOVLHPSZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12518 } 12519 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 12520 return fastEmitInst_rr(X86::MOVLHPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12521 } 12522 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 12523 return fastEmitInst_rr(X86::VMOVLHPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12524 } 12525 return 0; 12526} 12527 12528unsigned fastEmit_X86ISD_MOVLHPS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12529 switch (VT.SimpleTy) { 12530 case MVT::v4f32: return fastEmit_X86ISD_MOVLHPS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12531 default: return 0; 12532 } 12533} 12534 12535// FastEmit functions for X86ISD::MOVSD. 12536 12537unsigned fastEmit_X86ISD_MOVSD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12538 if (RetVT.SimpleTy != MVT::v2f64) 12539 return 0; 12540 if ((Subtarget->hasAVX512()) && (shouldOptForSize(MF))) { 12541 return fastEmitInst_rr(X86::VMOVSDZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12542 } 12543 if ((shouldOptForSize(MF) || !Subtarget->hasSSE41()) && (Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 12544 return fastEmitInst_rr(X86::MOVSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12545 } 12546 if ((shouldOptForSize(MF)) && (Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 12547 return fastEmitInst_rr(X86::VMOVSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12548 } 12549 return 0; 12550} 12551 12552unsigned fastEmit_X86ISD_MOVSD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12553 switch (VT.SimpleTy) { 12554 case MVT::v2f64: return fastEmit_X86ISD_MOVSD_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12555 default: return 0; 12556 } 12557} 12558 12559// FastEmit functions for X86ISD::MOVSS. 12560 12561unsigned fastEmit_X86ISD_MOVSS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12562 if (RetVT.SimpleTy != MVT::v4f32) 12563 return 0; 12564 if ((Subtarget->hasAVX512()) && (shouldOptForSize(MF))) { 12565 return fastEmitInst_rr(X86::VMOVSSZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12566 } 12567 if ((shouldOptForSize(MF) || !Subtarget->hasSSE41()) && (Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 12568 return fastEmitInst_rr(X86::MOVSSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12569 } 12570 if ((shouldOptForSize(MF)) && (Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 12571 return fastEmitInst_rr(X86::VMOVSSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12572 } 12573 return 0; 12574} 12575 12576unsigned fastEmit_X86ISD_MOVSS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12577 switch (VT.SimpleTy) { 12578 case MVT::v4f32: return fastEmit_X86ISD_MOVSS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12579 default: return 0; 12580 } 12581} 12582 12583// FastEmit functions for X86ISD::MULHRS. 12584 12585unsigned fastEmit_X86ISD_MULHRS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12586 if (RetVT.SimpleTy != MVT::v8i16) 12587 return 0; 12588 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 12589 return fastEmitInst_rr(X86::VPMULHRSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12590 } 12591 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) { 12592 return fastEmitInst_rr(X86::PMULHRSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12593 } 12594 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 12595 return fastEmitInst_rr(X86::VPMULHRSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12596 } 12597 return 0; 12598} 12599 12600unsigned fastEmit_X86ISD_MULHRS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12601 if (RetVT.SimpleTy != MVT::v16i16) 12602 return 0; 12603 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 12604 return fastEmitInst_rr(X86::VPMULHRSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12605 } 12606 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 12607 return fastEmitInst_rr(X86::VPMULHRSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12608 } 12609 return 0; 12610} 12611 12612unsigned fastEmit_X86ISD_MULHRS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12613 if (RetVT.SimpleTy != MVT::v32i16) 12614 return 0; 12615 if ((Subtarget->hasBWI())) { 12616 return fastEmitInst_rr(X86::VPMULHRSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12617 } 12618 return 0; 12619} 12620 12621unsigned fastEmit_X86ISD_MULHRS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12622 switch (VT.SimpleTy) { 12623 case MVT::v8i16: return fastEmit_X86ISD_MULHRS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12624 case MVT::v16i16: return fastEmit_X86ISD_MULHRS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12625 case MVT::v32i16: return fastEmit_X86ISD_MULHRS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12626 default: return 0; 12627 } 12628} 12629 12630// FastEmit functions for X86ISD::MULTISHIFT. 12631 12632unsigned fastEmit_X86ISD_MULTISHIFT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12633 if (RetVT.SimpleTy != MVT::v16i8) 12634 return 0; 12635 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) { 12636 return fastEmitInst_rr(X86::VPMULTISHIFTQBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12637 } 12638 return 0; 12639} 12640 12641unsigned fastEmit_X86ISD_MULTISHIFT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12642 if (RetVT.SimpleTy != MVT::v32i8) 12643 return 0; 12644 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) { 12645 return fastEmitInst_rr(X86::VPMULTISHIFTQBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12646 } 12647 return 0; 12648} 12649 12650unsigned fastEmit_X86ISD_MULTISHIFT_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12651 if (RetVT.SimpleTy != MVT::v64i8) 12652 return 0; 12653 if ((Subtarget->hasVBMI())) { 12654 return fastEmitInst_rr(X86::VPMULTISHIFTQBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12655 } 12656 return 0; 12657} 12658 12659unsigned fastEmit_X86ISD_MULTISHIFT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12660 switch (VT.SimpleTy) { 12661 case MVT::v16i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12662 case MVT::v32i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12663 case MVT::v64i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12664 default: return 0; 12665 } 12666} 12667 12668// FastEmit functions for X86ISD::PACKSS. 12669 12670unsigned fastEmit_X86ISD_PACKSS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12671 if (RetVT.SimpleTy != MVT::v16i8) 12672 return 0; 12673 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 12674 return fastEmitInst_rr(X86::VPACKSSWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12675 } 12676 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 12677 return fastEmitInst_rr(X86::PACKSSWBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12678 } 12679 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 12680 return fastEmitInst_rr(X86::VPACKSSWBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12681 } 12682 return 0; 12683} 12684 12685unsigned fastEmit_X86ISD_PACKSS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12686 if (RetVT.SimpleTy != MVT::v32i8) 12687 return 0; 12688 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 12689 return fastEmitInst_rr(X86::VPACKSSWBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12690 } 12691 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 12692 return fastEmitInst_rr(X86::VPACKSSWBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12693 } 12694 return 0; 12695} 12696 12697unsigned fastEmit_X86ISD_PACKSS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12698 if (RetVT.SimpleTy != MVT::v64i8) 12699 return 0; 12700 if ((Subtarget->hasBWI())) { 12701 return fastEmitInst_rr(X86::VPACKSSWBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12702 } 12703 return 0; 12704} 12705 12706unsigned fastEmit_X86ISD_PACKSS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12707 if (RetVT.SimpleTy != MVT::v8i16) 12708 return 0; 12709 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 12710 return fastEmitInst_rr(X86::VPACKSSDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12711 } 12712 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 12713 return fastEmitInst_rr(X86::PACKSSDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12714 } 12715 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 12716 return fastEmitInst_rr(X86::VPACKSSDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12717 } 12718 return 0; 12719} 12720 12721unsigned fastEmit_X86ISD_PACKSS_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12722 if (RetVT.SimpleTy != MVT::v16i16) 12723 return 0; 12724 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 12725 return fastEmitInst_rr(X86::VPACKSSDWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12726 } 12727 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 12728 return fastEmitInst_rr(X86::VPACKSSDWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12729 } 12730 return 0; 12731} 12732 12733unsigned fastEmit_X86ISD_PACKSS_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12734 if (RetVT.SimpleTy != MVT::v32i16) 12735 return 0; 12736 if ((Subtarget->hasBWI())) { 12737 return fastEmitInst_rr(X86::VPACKSSDWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12738 } 12739 return 0; 12740} 12741 12742unsigned fastEmit_X86ISD_PACKSS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12743 switch (VT.SimpleTy) { 12744 case MVT::v8i16: return fastEmit_X86ISD_PACKSS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12745 case MVT::v16i16: return fastEmit_X86ISD_PACKSS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12746 case MVT::v32i16: return fastEmit_X86ISD_PACKSS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12747 case MVT::v4i32: return fastEmit_X86ISD_PACKSS_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12748 case MVT::v8i32: return fastEmit_X86ISD_PACKSS_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12749 case MVT::v16i32: return fastEmit_X86ISD_PACKSS_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12750 default: return 0; 12751 } 12752} 12753 12754// FastEmit functions for X86ISD::PACKUS. 12755 12756unsigned fastEmit_X86ISD_PACKUS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12757 if (RetVT.SimpleTy != MVT::v16i8) 12758 return 0; 12759 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 12760 return fastEmitInst_rr(X86::VPACKUSWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12761 } 12762 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 12763 return fastEmitInst_rr(X86::PACKUSWBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12764 } 12765 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 12766 return fastEmitInst_rr(X86::VPACKUSWBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12767 } 12768 return 0; 12769} 12770 12771unsigned fastEmit_X86ISD_PACKUS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12772 if (RetVT.SimpleTy != MVT::v32i8) 12773 return 0; 12774 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 12775 return fastEmitInst_rr(X86::VPACKUSWBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12776 } 12777 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 12778 return fastEmitInst_rr(X86::VPACKUSWBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12779 } 12780 return 0; 12781} 12782 12783unsigned fastEmit_X86ISD_PACKUS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12784 if (RetVT.SimpleTy != MVT::v64i8) 12785 return 0; 12786 if ((Subtarget->hasBWI())) { 12787 return fastEmitInst_rr(X86::VPACKUSWBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12788 } 12789 return 0; 12790} 12791 12792unsigned fastEmit_X86ISD_PACKUS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12793 if (RetVT.SimpleTy != MVT::v8i16) 12794 return 0; 12795 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 12796 return fastEmitInst_rr(X86::VPACKUSDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12797 } 12798 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 12799 return fastEmitInst_rr(X86::PACKUSDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12800 } 12801 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 12802 return fastEmitInst_rr(X86::VPACKUSDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12803 } 12804 return 0; 12805} 12806 12807unsigned fastEmit_X86ISD_PACKUS_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12808 if (RetVT.SimpleTy != MVT::v16i16) 12809 return 0; 12810 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 12811 return fastEmitInst_rr(X86::VPACKUSDWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12812 } 12813 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 12814 return fastEmitInst_rr(X86::VPACKUSDWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12815 } 12816 return 0; 12817} 12818 12819unsigned fastEmit_X86ISD_PACKUS_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12820 if (RetVT.SimpleTy != MVT::v32i16) 12821 return 0; 12822 if ((Subtarget->hasBWI())) { 12823 return fastEmitInst_rr(X86::VPACKUSDWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12824 } 12825 return 0; 12826} 12827 12828unsigned fastEmit_X86ISD_PACKUS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12829 switch (VT.SimpleTy) { 12830 case MVT::v8i16: return fastEmit_X86ISD_PACKUS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12831 case MVT::v16i16: return fastEmit_X86ISD_PACKUS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12832 case MVT::v32i16: return fastEmit_X86ISD_PACKUS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12833 case MVT::v4i32: return fastEmit_X86ISD_PACKUS_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12834 case MVT::v8i32: return fastEmit_X86ISD_PACKUS_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12835 case MVT::v16i32: return fastEmit_X86ISD_PACKUS_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12836 default: return 0; 12837 } 12838} 12839 12840// FastEmit functions for X86ISD::PCMPEQ. 12841 12842unsigned fastEmit_X86ISD_PCMPEQ_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12843 if (RetVT.SimpleTy != MVT::v16i8) 12844 return 0; 12845 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 12846 return fastEmitInst_rr(X86::PCMPEQBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12847 } 12848 if ((Subtarget->hasAVX()) && (true)) { 12849 return fastEmitInst_rr(X86::VPCMPEQBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12850 } 12851 return 0; 12852} 12853 12854unsigned fastEmit_X86ISD_PCMPEQ_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12855 if (RetVT.SimpleTy != MVT::v32i8) 12856 return 0; 12857 if ((Subtarget->hasAVX2()) && (true)) { 12858 return fastEmitInst_rr(X86::VPCMPEQBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12859 } 12860 return 0; 12861} 12862 12863unsigned fastEmit_X86ISD_PCMPEQ_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12864 if (RetVT.SimpleTy != MVT::v8i16) 12865 return 0; 12866 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 12867 return fastEmitInst_rr(X86::PCMPEQWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12868 } 12869 if ((Subtarget->hasAVX()) && (true)) { 12870 return fastEmitInst_rr(X86::VPCMPEQWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12871 } 12872 return 0; 12873} 12874 12875unsigned fastEmit_X86ISD_PCMPEQ_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12876 if (RetVT.SimpleTy != MVT::v16i16) 12877 return 0; 12878 if ((Subtarget->hasAVX2()) && (true)) { 12879 return fastEmitInst_rr(X86::VPCMPEQWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12880 } 12881 return 0; 12882} 12883 12884unsigned fastEmit_X86ISD_PCMPEQ_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12885 if (RetVT.SimpleTy != MVT::v4i32) 12886 return 0; 12887 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 12888 return fastEmitInst_rr(X86::PCMPEQDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12889 } 12890 if ((Subtarget->hasAVX()) && (true)) { 12891 return fastEmitInst_rr(X86::VPCMPEQDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12892 } 12893 return 0; 12894} 12895 12896unsigned fastEmit_X86ISD_PCMPEQ_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12897 if (RetVT.SimpleTy != MVT::v8i32) 12898 return 0; 12899 if ((Subtarget->hasAVX2()) && (true)) { 12900 return fastEmitInst_rr(X86::VPCMPEQDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12901 } 12902 return 0; 12903} 12904 12905unsigned fastEmit_X86ISD_PCMPEQ_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12906 if (RetVT.SimpleTy != MVT::v2i64) 12907 return 0; 12908 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 12909 return fastEmitInst_rr(X86::PCMPEQQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12910 } 12911 if ((Subtarget->hasAVX())) { 12912 return fastEmitInst_rr(X86::VPCMPEQQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12913 } 12914 return 0; 12915} 12916 12917unsigned fastEmit_X86ISD_PCMPEQ_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12918 if (RetVT.SimpleTy != MVT::v4i64) 12919 return 0; 12920 if ((Subtarget->hasAVX2())) { 12921 return fastEmitInst_rr(X86::VPCMPEQQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12922 } 12923 return 0; 12924} 12925 12926unsigned fastEmit_X86ISD_PCMPEQ_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12927 switch (VT.SimpleTy) { 12928 case MVT::v16i8: return fastEmit_X86ISD_PCMPEQ_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12929 case MVT::v32i8: return fastEmit_X86ISD_PCMPEQ_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12930 case MVT::v8i16: return fastEmit_X86ISD_PCMPEQ_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12931 case MVT::v16i16: return fastEmit_X86ISD_PCMPEQ_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12932 case MVT::v4i32: return fastEmit_X86ISD_PCMPEQ_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12933 case MVT::v8i32: return fastEmit_X86ISD_PCMPEQ_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12934 case MVT::v2i64: return fastEmit_X86ISD_PCMPEQ_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12935 case MVT::v4i64: return fastEmit_X86ISD_PCMPEQ_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 12936 default: return 0; 12937 } 12938} 12939 12940// FastEmit functions for X86ISD::PCMPGT. 12941 12942unsigned fastEmit_X86ISD_PCMPGT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12943 if (RetVT.SimpleTy != MVT::v16i8) 12944 return 0; 12945 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 12946 return fastEmitInst_rr(X86::PCMPGTBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12947 } 12948 if ((Subtarget->hasAVX()) && (true)) { 12949 return fastEmitInst_rr(X86::VPCMPGTBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12950 } 12951 return 0; 12952} 12953 12954unsigned fastEmit_X86ISD_PCMPGT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12955 if (RetVT.SimpleTy != MVT::v32i8) 12956 return 0; 12957 if ((Subtarget->hasAVX2()) && (true)) { 12958 return fastEmitInst_rr(X86::VPCMPGTBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12959 } 12960 return 0; 12961} 12962 12963unsigned fastEmit_X86ISD_PCMPGT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12964 if (RetVT.SimpleTy != MVT::v8i16) 12965 return 0; 12966 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 12967 return fastEmitInst_rr(X86::PCMPGTWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12968 } 12969 if ((Subtarget->hasAVX()) && (true)) { 12970 return fastEmitInst_rr(X86::VPCMPGTWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12971 } 12972 return 0; 12973} 12974 12975unsigned fastEmit_X86ISD_PCMPGT_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12976 if (RetVT.SimpleTy != MVT::v16i16) 12977 return 0; 12978 if ((Subtarget->hasAVX2()) && (true)) { 12979 return fastEmitInst_rr(X86::VPCMPGTWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12980 } 12981 return 0; 12982} 12983 12984unsigned fastEmit_X86ISD_PCMPGT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12985 if (RetVT.SimpleTy != MVT::v4i32) 12986 return 0; 12987 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 12988 return fastEmitInst_rr(X86::PCMPGTDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12989 } 12990 if ((Subtarget->hasAVX()) && (true)) { 12991 return fastEmitInst_rr(X86::VPCMPGTDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 12992 } 12993 return 0; 12994} 12995 12996unsigned fastEmit_X86ISD_PCMPGT_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 12997 if (RetVT.SimpleTy != MVT::v8i32) 12998 return 0; 12999 if ((Subtarget->hasAVX2()) && (true)) { 13000 return fastEmitInst_rr(X86::VPCMPGTDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13001 } 13002 return 0; 13003} 13004 13005unsigned fastEmit_X86ISD_PCMPGT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13006 if (RetVT.SimpleTy != MVT::v2i64) 13007 return 0; 13008 if ((Subtarget->hasSSE42() && !Subtarget->hasAVX())) { 13009 return fastEmitInst_rr(X86::PCMPGTQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13010 } 13011 if ((Subtarget->hasAVX())) { 13012 return fastEmitInst_rr(X86::VPCMPGTQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13013 } 13014 return 0; 13015} 13016 13017unsigned fastEmit_X86ISD_PCMPGT_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13018 if (RetVT.SimpleTy != MVT::v4i64) 13019 return 0; 13020 if ((Subtarget->hasAVX2())) { 13021 return fastEmitInst_rr(X86::VPCMPGTQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13022 } 13023 return 0; 13024} 13025 13026unsigned fastEmit_X86ISD_PCMPGT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13027 switch (VT.SimpleTy) { 13028 case MVT::v16i8: return fastEmit_X86ISD_PCMPGT_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13029 case MVT::v32i8: return fastEmit_X86ISD_PCMPGT_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13030 case MVT::v8i16: return fastEmit_X86ISD_PCMPGT_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13031 case MVT::v16i16: return fastEmit_X86ISD_PCMPGT_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13032 case MVT::v4i32: return fastEmit_X86ISD_PCMPGT_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13033 case MVT::v8i32: return fastEmit_X86ISD_PCMPGT_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13034 case MVT::v2i64: return fastEmit_X86ISD_PCMPGT_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13035 case MVT::v4i64: return fastEmit_X86ISD_PCMPGT_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13036 default: return 0; 13037 } 13038} 13039 13040// FastEmit functions for X86ISD::PMULDQ. 13041 13042unsigned fastEmit_X86ISD_PMULDQ_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13043 if (RetVT.SimpleTy != MVT::v2i64) 13044 return 0; 13045 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 13046 return fastEmitInst_rr(X86::VPMULDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13047 } 13048 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 13049 return fastEmitInst_rr(X86::PMULDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13050 } 13051 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 13052 return fastEmitInst_rr(X86::VPMULDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13053 } 13054 return 0; 13055} 13056 13057unsigned fastEmit_X86ISD_PMULDQ_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13058 if (RetVT.SimpleTy != MVT::v4i64) 13059 return 0; 13060 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 13061 return fastEmitInst_rr(X86::VPMULDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13062 } 13063 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 13064 return fastEmitInst_rr(X86::VPMULDQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13065 } 13066 return 0; 13067} 13068 13069unsigned fastEmit_X86ISD_PMULDQ_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13070 if (RetVT.SimpleTy != MVT::v8i64) 13071 return 0; 13072 if ((Subtarget->hasAVX512())) { 13073 return fastEmitInst_rr(X86::VPMULDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13074 } 13075 return 0; 13076} 13077 13078unsigned fastEmit_X86ISD_PMULDQ_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13079 switch (VT.SimpleTy) { 13080 case MVT::v2i64: return fastEmit_X86ISD_PMULDQ_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13081 case MVT::v4i64: return fastEmit_X86ISD_PMULDQ_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13082 case MVT::v8i64: return fastEmit_X86ISD_PMULDQ_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13083 default: return 0; 13084 } 13085} 13086 13087// FastEmit functions for X86ISD::PMULUDQ. 13088 13089unsigned fastEmit_X86ISD_PMULUDQ_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13090 if (RetVT.SimpleTy != MVT::v2i64) 13091 return 0; 13092 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 13093 return fastEmitInst_rr(X86::VPMULUDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13094 } 13095 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 13096 return fastEmitInst_rr(X86::PMULUDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13097 } 13098 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 13099 return fastEmitInst_rr(X86::VPMULUDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13100 } 13101 return 0; 13102} 13103 13104unsigned fastEmit_X86ISD_PMULUDQ_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13105 if (RetVT.SimpleTy != MVT::v4i64) 13106 return 0; 13107 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 13108 return fastEmitInst_rr(X86::VPMULUDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13109 } 13110 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 13111 return fastEmitInst_rr(X86::VPMULUDQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13112 } 13113 return 0; 13114} 13115 13116unsigned fastEmit_X86ISD_PMULUDQ_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13117 if (RetVT.SimpleTy != MVT::v8i64) 13118 return 0; 13119 if ((Subtarget->hasAVX512())) { 13120 return fastEmitInst_rr(X86::VPMULUDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13121 } 13122 return 0; 13123} 13124 13125unsigned fastEmit_X86ISD_PMULUDQ_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13126 switch (VT.SimpleTy) { 13127 case MVT::v2i64: return fastEmit_X86ISD_PMULUDQ_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13128 case MVT::v4i64: return fastEmit_X86ISD_PMULUDQ_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13129 case MVT::v8i64: return fastEmit_X86ISD_PMULUDQ_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13130 default: return 0; 13131 } 13132} 13133 13134// FastEmit functions for X86ISD::PSADBW. 13135 13136unsigned fastEmit_X86ISD_PSADBW_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13137 if (RetVT.SimpleTy != MVT::v2i64) 13138 return 0; 13139 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 13140 return fastEmitInst_rr(X86::VPSADBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13141 } 13142 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 13143 return fastEmitInst_rr(X86::PSADBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13144 } 13145 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 13146 return fastEmitInst_rr(X86::VPSADBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13147 } 13148 return 0; 13149} 13150 13151unsigned fastEmit_X86ISD_PSADBW_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13152 if (RetVT.SimpleTy != MVT::v4i64) 13153 return 0; 13154 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 13155 return fastEmitInst_rr(X86::VPSADBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13156 } 13157 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 13158 return fastEmitInst_rr(X86::VPSADBWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13159 } 13160 return 0; 13161} 13162 13163unsigned fastEmit_X86ISD_PSADBW_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13164 if (RetVT.SimpleTy != MVT::v8i64) 13165 return 0; 13166 if ((Subtarget->hasBWI())) { 13167 return fastEmitInst_rr(X86::VPSADBWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13168 } 13169 return 0; 13170} 13171 13172unsigned fastEmit_X86ISD_PSADBW_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13173 switch (VT.SimpleTy) { 13174 case MVT::v16i8: return fastEmit_X86ISD_PSADBW_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13175 case MVT::v32i8: return fastEmit_X86ISD_PSADBW_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13176 case MVT::v64i8: return fastEmit_X86ISD_PSADBW_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13177 default: return 0; 13178 } 13179} 13180 13181// FastEmit functions for X86ISD::PSHUFB. 13182 13183unsigned fastEmit_X86ISD_PSHUFB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13184 if (RetVT.SimpleTy != MVT::v16i8) 13185 return 0; 13186 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 13187 return fastEmitInst_rr(X86::VPSHUFBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13188 } 13189 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) { 13190 return fastEmitInst_rr(X86::PSHUFBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13191 } 13192 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 13193 return fastEmitInst_rr(X86::VPSHUFBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13194 } 13195 return 0; 13196} 13197 13198unsigned fastEmit_X86ISD_PSHUFB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13199 if (RetVT.SimpleTy != MVT::v32i8) 13200 return 0; 13201 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 13202 return fastEmitInst_rr(X86::VPSHUFBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13203 } 13204 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 13205 return fastEmitInst_rr(X86::VPSHUFBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13206 } 13207 return 0; 13208} 13209 13210unsigned fastEmit_X86ISD_PSHUFB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13211 if (RetVT.SimpleTy != MVT::v64i8) 13212 return 0; 13213 if ((Subtarget->hasBWI())) { 13214 return fastEmitInst_rr(X86::VPSHUFBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13215 } 13216 return 0; 13217} 13218 13219unsigned fastEmit_X86ISD_PSHUFB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13220 switch (VT.SimpleTy) { 13221 case MVT::v16i8: return fastEmit_X86ISD_PSHUFB_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13222 case MVT::v32i8: return fastEmit_X86ISD_PSHUFB_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13223 case MVT::v64i8: return fastEmit_X86ISD_PSHUFB_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13224 default: return 0; 13225 } 13226} 13227 13228// FastEmit functions for X86ISD::PTEST. 13229 13230unsigned fastEmit_X86ISD_PTEST_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13231 if (RetVT.SimpleTy != MVT::i32) 13232 return 0; 13233 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 13234 return fastEmitInst_rr(X86::PTESTrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13235 } 13236 if ((Subtarget->hasAVX())) { 13237 return fastEmitInst_rr(X86::VPTESTrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13238 } 13239 return 0; 13240} 13241 13242unsigned fastEmit_X86ISD_PTEST_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13243 if (RetVT.SimpleTy != MVT::i32) 13244 return 0; 13245 if ((Subtarget->hasAVX())) { 13246 return fastEmitInst_rr(X86::VPTESTYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13247 } 13248 return 0; 13249} 13250 13251unsigned fastEmit_X86ISD_PTEST_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13252 switch (VT.SimpleTy) { 13253 case MVT::v2i64: return fastEmit_X86ISD_PTEST_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13254 case MVT::v4i64: return fastEmit_X86ISD_PTEST_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13255 default: return 0; 13256 } 13257} 13258 13259// FastEmit functions for X86ISD::RCP14S. 13260 13261unsigned fastEmit_X86ISD_RCP14S_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13262 if (RetVT.SimpleTy != MVT::v4f32) 13263 return 0; 13264 if ((Subtarget->hasAVX512())) { 13265 return fastEmitInst_rr(X86::VRCP14SSZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13266 } 13267 return 0; 13268} 13269 13270unsigned fastEmit_X86ISD_RCP14S_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13271 if (RetVT.SimpleTy != MVT::v2f64) 13272 return 0; 13273 if ((Subtarget->hasAVX512())) { 13274 return fastEmitInst_rr(X86::VRCP14SDZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13275 } 13276 return 0; 13277} 13278 13279unsigned fastEmit_X86ISD_RCP14S_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13280 switch (VT.SimpleTy) { 13281 case MVT::v4f32: return fastEmit_X86ISD_RCP14S_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13282 case MVT::v2f64: return fastEmit_X86ISD_RCP14S_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13283 default: return 0; 13284 } 13285} 13286 13287// FastEmit functions for X86ISD::RCP28S. 13288 13289unsigned fastEmit_X86ISD_RCP28S_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13290 if (RetVT.SimpleTy != MVT::v4f32) 13291 return 0; 13292 if ((Subtarget->hasERI())) { 13293 return fastEmitInst_rr(X86::VRCP28SSZr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13294 } 13295 return 0; 13296} 13297 13298unsigned fastEmit_X86ISD_RCP28S_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13299 if (RetVT.SimpleTy != MVT::v2f64) 13300 return 0; 13301 if ((Subtarget->hasERI())) { 13302 return fastEmitInst_rr(X86::VRCP28SDZr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13303 } 13304 return 0; 13305} 13306 13307unsigned fastEmit_X86ISD_RCP28S_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13308 switch (VT.SimpleTy) { 13309 case MVT::v4f32: return fastEmit_X86ISD_RCP28S_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13310 case MVT::v2f64: return fastEmit_X86ISD_RCP28S_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13311 default: return 0; 13312 } 13313} 13314 13315// FastEmit functions for X86ISD::RCP28S_SAE. 13316 13317unsigned fastEmit_X86ISD_RCP28S_SAE_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13318 if (RetVT.SimpleTy != MVT::v4f32) 13319 return 0; 13320 if ((Subtarget->hasERI())) { 13321 return fastEmitInst_rr(X86::VRCP28SSZrb, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13322 } 13323 return 0; 13324} 13325 13326unsigned fastEmit_X86ISD_RCP28S_SAE_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13327 if (RetVT.SimpleTy != MVT::v2f64) 13328 return 0; 13329 if ((Subtarget->hasERI())) { 13330 return fastEmitInst_rr(X86::VRCP28SDZrb, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13331 } 13332 return 0; 13333} 13334 13335unsigned fastEmit_X86ISD_RCP28S_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13336 switch (VT.SimpleTy) { 13337 case MVT::v4f32: return fastEmit_X86ISD_RCP28S_SAE_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13338 case MVT::v2f64: return fastEmit_X86ISD_RCP28S_SAE_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13339 default: return 0; 13340 } 13341} 13342 13343// FastEmit functions for X86ISD::RSQRT14S. 13344 13345unsigned fastEmit_X86ISD_RSQRT14S_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13346 if (RetVT.SimpleTy != MVT::v4f32) 13347 return 0; 13348 if ((Subtarget->hasAVX512())) { 13349 return fastEmitInst_rr(X86::VRSQRT14SSZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13350 } 13351 return 0; 13352} 13353 13354unsigned fastEmit_X86ISD_RSQRT14S_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13355 if (RetVT.SimpleTy != MVT::v2f64) 13356 return 0; 13357 if ((Subtarget->hasAVX512())) { 13358 return fastEmitInst_rr(X86::VRSQRT14SDZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13359 } 13360 return 0; 13361} 13362 13363unsigned fastEmit_X86ISD_RSQRT14S_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13364 switch (VT.SimpleTy) { 13365 case MVT::v4f32: return fastEmit_X86ISD_RSQRT14S_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13366 case MVT::v2f64: return fastEmit_X86ISD_RSQRT14S_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13367 default: return 0; 13368 } 13369} 13370 13371// FastEmit functions for X86ISD::RSQRT28S. 13372 13373unsigned fastEmit_X86ISD_RSQRT28S_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13374 if (RetVT.SimpleTy != MVT::v4f32) 13375 return 0; 13376 if ((Subtarget->hasERI())) { 13377 return fastEmitInst_rr(X86::VRSQRT28SSZr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13378 } 13379 return 0; 13380} 13381 13382unsigned fastEmit_X86ISD_RSQRT28S_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13383 if (RetVT.SimpleTy != MVT::v2f64) 13384 return 0; 13385 if ((Subtarget->hasERI())) { 13386 return fastEmitInst_rr(X86::VRSQRT28SDZr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13387 } 13388 return 0; 13389} 13390 13391unsigned fastEmit_X86ISD_RSQRT28S_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13392 switch (VT.SimpleTy) { 13393 case MVT::v4f32: return fastEmit_X86ISD_RSQRT28S_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13394 case MVT::v2f64: return fastEmit_X86ISD_RSQRT28S_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13395 default: return 0; 13396 } 13397} 13398 13399// FastEmit functions for X86ISD::RSQRT28S_SAE. 13400 13401unsigned fastEmit_X86ISD_RSQRT28S_SAE_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13402 if (RetVT.SimpleTy != MVT::v4f32) 13403 return 0; 13404 if ((Subtarget->hasERI())) { 13405 return fastEmitInst_rr(X86::VRSQRT28SSZrb, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13406 } 13407 return 0; 13408} 13409 13410unsigned fastEmit_X86ISD_RSQRT28S_SAE_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13411 if (RetVT.SimpleTy != MVT::v2f64) 13412 return 0; 13413 if ((Subtarget->hasERI())) { 13414 return fastEmitInst_rr(X86::VRSQRT28SDZrb, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13415 } 13416 return 0; 13417} 13418 13419unsigned fastEmit_X86ISD_RSQRT28S_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13420 switch (VT.SimpleTy) { 13421 case MVT::v4f32: return fastEmit_X86ISD_RSQRT28S_SAE_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13422 case MVT::v2f64: return fastEmit_X86ISD_RSQRT28S_SAE_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13423 default: return 0; 13424 } 13425} 13426 13427// FastEmit functions for X86ISD::SCALEF. 13428 13429unsigned fastEmit_X86ISD_SCALEF_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13430 if (RetVT.SimpleTy != MVT::v4f32) 13431 return 0; 13432 if ((Subtarget->hasVLX())) { 13433 return fastEmitInst_rr(X86::VSCALEFPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13434 } 13435 return 0; 13436} 13437 13438unsigned fastEmit_X86ISD_SCALEF_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13439 if (RetVT.SimpleTy != MVT::v8f32) 13440 return 0; 13441 if ((Subtarget->hasVLX())) { 13442 return fastEmitInst_rr(X86::VSCALEFPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13443 } 13444 return 0; 13445} 13446 13447unsigned fastEmit_X86ISD_SCALEF_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13448 if (RetVT.SimpleTy != MVT::v16f32) 13449 return 0; 13450 if ((Subtarget->hasAVX512())) { 13451 return fastEmitInst_rr(X86::VSCALEFPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13452 } 13453 return 0; 13454} 13455 13456unsigned fastEmit_X86ISD_SCALEF_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13457 if (RetVT.SimpleTy != MVT::v2f64) 13458 return 0; 13459 if ((Subtarget->hasVLX())) { 13460 return fastEmitInst_rr(X86::VSCALEFPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13461 } 13462 return 0; 13463} 13464 13465unsigned fastEmit_X86ISD_SCALEF_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13466 if (RetVT.SimpleTy != MVT::v4f64) 13467 return 0; 13468 if ((Subtarget->hasVLX())) { 13469 return fastEmitInst_rr(X86::VSCALEFPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13470 } 13471 return 0; 13472} 13473 13474unsigned fastEmit_X86ISD_SCALEF_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13475 if (RetVT.SimpleTy != MVT::v8f64) 13476 return 0; 13477 if ((Subtarget->hasAVX512())) { 13478 return fastEmitInst_rr(X86::VSCALEFPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13479 } 13480 return 0; 13481} 13482 13483unsigned fastEmit_X86ISD_SCALEF_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13484 switch (VT.SimpleTy) { 13485 case MVT::v4f32: return fastEmit_X86ISD_SCALEF_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13486 case MVT::v8f32: return fastEmit_X86ISD_SCALEF_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13487 case MVT::v16f32: return fastEmit_X86ISD_SCALEF_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13488 case MVT::v2f64: return fastEmit_X86ISD_SCALEF_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13489 case MVT::v4f64: return fastEmit_X86ISD_SCALEF_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13490 case MVT::v8f64: return fastEmit_X86ISD_SCALEF_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13491 default: return 0; 13492 } 13493} 13494 13495// FastEmit functions for X86ISD::SCALEFS. 13496 13497unsigned fastEmit_X86ISD_SCALEFS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13498 if (RetVT.SimpleTy != MVT::v4f32) 13499 return 0; 13500 if ((Subtarget->hasAVX512())) { 13501 return fastEmitInst_rr(X86::VSCALEFSSZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13502 } 13503 return 0; 13504} 13505 13506unsigned fastEmit_X86ISD_SCALEFS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13507 if (RetVT.SimpleTy != MVT::v2f64) 13508 return 0; 13509 if ((Subtarget->hasAVX512())) { 13510 return fastEmitInst_rr(X86::VSCALEFSDZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13511 } 13512 return 0; 13513} 13514 13515unsigned fastEmit_X86ISD_SCALEFS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13516 switch (VT.SimpleTy) { 13517 case MVT::v4f32: return fastEmit_X86ISD_SCALEFS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13518 case MVT::v2f64: return fastEmit_X86ISD_SCALEFS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13519 default: return 0; 13520 } 13521} 13522 13523// FastEmit functions for X86ISD::STRICT_FCMP. 13524 13525unsigned fastEmit_X86ISD_STRICT_FCMP_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13526 if (RetVT.SimpleTy != MVT::i32) 13527 return 0; 13528 if ((Subtarget->hasAVX512())) { 13529 return fastEmitInst_rr(X86::VUCOMISSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13530 } 13531 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 13532 return fastEmitInst_rr(X86::UCOMISSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13533 } 13534 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 13535 return fastEmitInst_rr(X86::VUCOMISSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13536 } 13537 if ((!Subtarget->hasSSE1()) && (Subtarget->hasCMov())) { 13538 return fastEmitInst_rr(X86::UCOM_FpIr32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13539 } 13540 return 0; 13541} 13542 13543unsigned fastEmit_X86ISD_STRICT_FCMP_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13544 if (RetVT.SimpleTy != MVT::i32) 13545 return 0; 13546 if ((Subtarget->hasAVX512())) { 13547 return fastEmitInst_rr(X86::VUCOMISDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13548 } 13549 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 13550 return fastEmitInst_rr(X86::UCOMISDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13551 } 13552 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 13553 return fastEmitInst_rr(X86::VUCOMISDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13554 } 13555 if ((!Subtarget->hasSSE2()) && (Subtarget->hasCMov())) { 13556 return fastEmitInst_rr(X86::UCOM_FpIr64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13557 } 13558 return 0; 13559} 13560 13561unsigned fastEmit_X86ISD_STRICT_FCMP_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13562 if (RetVT.SimpleTy != MVT::i32) 13563 return 0; 13564 if ((Subtarget->hasCMov())) { 13565 return fastEmitInst_rr(X86::UCOM_FpIr80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13566 } 13567 return 0; 13568} 13569 13570unsigned fastEmit_X86ISD_STRICT_FCMP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13571 switch (VT.SimpleTy) { 13572 case MVT::f32: return fastEmit_X86ISD_STRICT_FCMP_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13573 case MVT::f64: return fastEmit_X86ISD_STRICT_FCMP_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13574 case MVT::f80: return fastEmit_X86ISD_STRICT_FCMP_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13575 default: return 0; 13576 } 13577} 13578 13579// FastEmit functions for X86ISD::STRICT_FCMPS. 13580 13581unsigned fastEmit_X86ISD_STRICT_FCMPS_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13582 if (RetVT.SimpleTy != MVT::i32) 13583 return 0; 13584 if ((Subtarget->hasAVX512())) { 13585 return fastEmitInst_rr(X86::VCOMISSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13586 } 13587 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 13588 return fastEmitInst_rr(X86::COMISSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13589 } 13590 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 13591 return fastEmitInst_rr(X86::VCOMISSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13592 } 13593 if ((!Subtarget->hasSSE1()) && (Subtarget->hasCMov())) { 13594 return fastEmitInst_rr(X86::COM_FpIr32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13595 } 13596 return 0; 13597} 13598 13599unsigned fastEmit_X86ISD_STRICT_FCMPS_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13600 if (RetVT.SimpleTy != MVT::i32) 13601 return 0; 13602 if ((Subtarget->hasAVX512())) { 13603 return fastEmitInst_rr(X86::VCOMISDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13604 } 13605 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 13606 return fastEmitInst_rr(X86::COMISDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13607 } 13608 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 13609 return fastEmitInst_rr(X86::VCOMISDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13610 } 13611 if ((!Subtarget->hasSSE2()) && (Subtarget->hasCMov())) { 13612 return fastEmitInst_rr(X86::COM_FpIr64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13613 } 13614 return 0; 13615} 13616 13617unsigned fastEmit_X86ISD_STRICT_FCMPS_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13618 if (RetVT.SimpleTy != MVT::i32) 13619 return 0; 13620 if ((Subtarget->hasCMov())) { 13621 return fastEmitInst_rr(X86::COM_FpIr80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13622 } 13623 return 0; 13624} 13625 13626unsigned fastEmit_X86ISD_STRICT_FCMPS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13627 switch (VT.SimpleTy) { 13628 case MVT::f32: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13629 case MVT::f64: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13630 case MVT::f80: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13631 default: return 0; 13632 } 13633} 13634 13635// FastEmit functions for X86ISD::TESTP. 13636 13637unsigned fastEmit_X86ISD_TESTP_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13638 if (RetVT.SimpleTy != MVT::i32) 13639 return 0; 13640 if ((Subtarget->hasAVX())) { 13641 return fastEmitInst_rr(X86::VTESTPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13642 } 13643 return 0; 13644} 13645 13646unsigned fastEmit_X86ISD_TESTP_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13647 if (RetVT.SimpleTy != MVT::i32) 13648 return 0; 13649 if ((Subtarget->hasAVX())) { 13650 return fastEmitInst_rr(X86::VTESTPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13651 } 13652 return 0; 13653} 13654 13655unsigned fastEmit_X86ISD_TESTP_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13656 if (RetVT.SimpleTy != MVT::i32) 13657 return 0; 13658 if ((Subtarget->hasAVX())) { 13659 return fastEmitInst_rr(X86::VTESTPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13660 } 13661 return 0; 13662} 13663 13664unsigned fastEmit_X86ISD_TESTP_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13665 if (RetVT.SimpleTy != MVT::i32) 13666 return 0; 13667 if ((Subtarget->hasAVX())) { 13668 return fastEmitInst_rr(X86::VTESTPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13669 } 13670 return 0; 13671} 13672 13673unsigned fastEmit_X86ISD_TESTP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13674 switch (VT.SimpleTy) { 13675 case MVT::v4f32: return fastEmit_X86ISD_TESTP_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13676 case MVT::v8f32: return fastEmit_X86ISD_TESTP_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13677 case MVT::v2f64: return fastEmit_X86ISD_TESTP_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13678 case MVT::v4f64: return fastEmit_X86ISD_TESTP_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13679 default: return 0; 13680 } 13681} 13682 13683// FastEmit functions for X86ISD::UCOMI. 13684 13685unsigned fastEmit_X86ISD_UCOMI_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13686 if (RetVT.SimpleTy != MVT::i32) 13687 return 0; 13688 if ((Subtarget->hasAVX512())) { 13689 return fastEmitInst_rr(X86::VUCOMISSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13690 } 13691 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 13692 return fastEmitInst_rr(X86::UCOMISSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13693 } 13694 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 13695 return fastEmitInst_rr(X86::VUCOMISSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13696 } 13697 return 0; 13698} 13699 13700unsigned fastEmit_X86ISD_UCOMI_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13701 if (RetVT.SimpleTy != MVT::i32) 13702 return 0; 13703 if ((Subtarget->hasAVX512())) { 13704 return fastEmitInst_rr(X86::VUCOMISDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13705 } 13706 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 13707 return fastEmitInst_rr(X86::UCOMISDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13708 } 13709 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) { 13710 return fastEmitInst_rr(X86::VUCOMISDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13711 } 13712 return 0; 13713} 13714 13715unsigned fastEmit_X86ISD_UCOMI_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13716 switch (VT.SimpleTy) { 13717 case MVT::v4f32: return fastEmit_X86ISD_UCOMI_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13718 case MVT::v2f64: return fastEmit_X86ISD_UCOMI_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13719 default: return 0; 13720 } 13721} 13722 13723// FastEmit functions for X86ISD::UNPCKH. 13724 13725unsigned fastEmit_X86ISD_UNPCKH_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13726 if (RetVT.SimpleTy != MVT::v16i8) 13727 return 0; 13728 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 13729 return fastEmitInst_rr(X86::VPUNPCKHBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13730 } 13731 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 13732 return fastEmitInst_rr(X86::PUNPCKHBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13733 } 13734 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 13735 return fastEmitInst_rr(X86::VPUNPCKHBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13736 } 13737 return 0; 13738} 13739 13740unsigned fastEmit_X86ISD_UNPCKH_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13741 if (RetVT.SimpleTy != MVT::v32i8) 13742 return 0; 13743 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 13744 return fastEmitInst_rr(X86::VPUNPCKHBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13745 } 13746 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 13747 return fastEmitInst_rr(X86::VPUNPCKHBWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13748 } 13749 return 0; 13750} 13751 13752unsigned fastEmit_X86ISD_UNPCKH_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13753 if (RetVT.SimpleTy != MVT::v64i8) 13754 return 0; 13755 if ((Subtarget->hasBWI())) { 13756 return fastEmitInst_rr(X86::VPUNPCKHBWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13757 } 13758 return 0; 13759} 13760 13761unsigned fastEmit_X86ISD_UNPCKH_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13762 if (RetVT.SimpleTy != MVT::v8i16) 13763 return 0; 13764 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 13765 return fastEmitInst_rr(X86::VPUNPCKHWDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13766 } 13767 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 13768 return fastEmitInst_rr(X86::PUNPCKHWDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13769 } 13770 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 13771 return fastEmitInst_rr(X86::VPUNPCKHWDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13772 } 13773 return 0; 13774} 13775 13776unsigned fastEmit_X86ISD_UNPCKH_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13777 if (RetVT.SimpleTy != MVT::v16i16) 13778 return 0; 13779 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 13780 return fastEmitInst_rr(X86::VPUNPCKHWDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13781 } 13782 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 13783 return fastEmitInst_rr(X86::VPUNPCKHWDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13784 } 13785 return 0; 13786} 13787 13788unsigned fastEmit_X86ISD_UNPCKH_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13789 if (RetVT.SimpleTy != MVT::v32i16) 13790 return 0; 13791 if ((Subtarget->hasBWI())) { 13792 return fastEmitInst_rr(X86::VPUNPCKHWDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13793 } 13794 return 0; 13795} 13796 13797unsigned fastEmit_X86ISD_UNPCKH_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13798 if (RetVT.SimpleTy != MVT::v4i32) 13799 return 0; 13800 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 13801 return fastEmitInst_rr(X86::VPUNPCKHDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13802 } 13803 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 13804 return fastEmitInst_rr(X86::PUNPCKHDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13805 } 13806 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 13807 return fastEmitInst_rr(X86::VPUNPCKHDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13808 } 13809 return 0; 13810} 13811 13812unsigned fastEmit_X86ISD_UNPCKH_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13813 if (RetVT.SimpleTy != MVT::v8i32) 13814 return 0; 13815 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 13816 return fastEmitInst_rr(X86::VUNPCKHPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13817 } 13818 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 13819 return fastEmitInst_rr(X86::VPUNPCKHDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13820 } 13821 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 13822 return fastEmitInst_rr(X86::VPUNPCKHDQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13823 } 13824 return 0; 13825} 13826 13827unsigned fastEmit_X86ISD_UNPCKH_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13828 if (RetVT.SimpleTy != MVT::v16i32) 13829 return 0; 13830 if ((Subtarget->hasAVX512())) { 13831 return fastEmitInst_rr(X86::VPUNPCKHDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13832 } 13833 return 0; 13834} 13835 13836unsigned fastEmit_X86ISD_UNPCKH_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13837 if (RetVT.SimpleTy != MVT::v2i64) 13838 return 0; 13839 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 13840 return fastEmitInst_rr(X86::VPUNPCKHQDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13841 } 13842 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 13843 return fastEmitInst_rr(X86::PUNPCKHQDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13844 } 13845 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 13846 return fastEmitInst_rr(X86::VPUNPCKHQDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13847 } 13848 return 0; 13849} 13850 13851unsigned fastEmit_X86ISD_UNPCKH_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13852 if (RetVT.SimpleTy != MVT::v4i64) 13853 return 0; 13854 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 13855 return fastEmitInst_rr(X86::VUNPCKHPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13856 } 13857 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 13858 return fastEmitInst_rr(X86::VPUNPCKHQDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13859 } 13860 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 13861 return fastEmitInst_rr(X86::VPUNPCKHQDQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13862 } 13863 return 0; 13864} 13865 13866unsigned fastEmit_X86ISD_UNPCKH_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13867 if (RetVT.SimpleTy != MVT::v8i64) 13868 return 0; 13869 if ((Subtarget->hasAVX512())) { 13870 return fastEmitInst_rr(X86::VPUNPCKHQDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13871 } 13872 return 0; 13873} 13874 13875unsigned fastEmit_X86ISD_UNPCKH_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13876 if (RetVT.SimpleTy != MVT::v4f32) 13877 return 0; 13878 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 13879 return fastEmitInst_rr(X86::VUNPCKHPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13880 } 13881 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 13882 return fastEmitInst_rr(X86::UNPCKHPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13883 } 13884 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 13885 return fastEmitInst_rr(X86::VUNPCKHPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13886 } 13887 return 0; 13888} 13889 13890unsigned fastEmit_X86ISD_UNPCKH_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13891 if (RetVT.SimpleTy != MVT::v8f32) 13892 return 0; 13893 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 13894 return fastEmitInst_rr(X86::VUNPCKHPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13895 } 13896 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 13897 return fastEmitInst_rr(X86::VUNPCKHPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13898 } 13899 return 0; 13900} 13901 13902unsigned fastEmit_X86ISD_UNPCKH_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13903 if (RetVT.SimpleTy != MVT::v16f32) 13904 return 0; 13905 if ((Subtarget->hasAVX512())) { 13906 return fastEmitInst_rr(X86::VUNPCKHPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13907 } 13908 return 0; 13909} 13910 13911unsigned fastEmit_X86ISD_UNPCKH_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13912 if (RetVT.SimpleTy != MVT::v2f64) 13913 return 0; 13914 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 13915 return fastEmitInst_rr(X86::VUNPCKHPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13916 } 13917 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 13918 return fastEmitInst_rr(X86::UNPCKHPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13919 } 13920 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 13921 return fastEmitInst_rr(X86::VUNPCKHPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13922 } 13923 return 0; 13924} 13925 13926unsigned fastEmit_X86ISD_UNPCKH_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13927 if (RetVT.SimpleTy != MVT::v4f64) 13928 return 0; 13929 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 13930 return fastEmitInst_rr(X86::VUNPCKHPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13931 } 13932 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 13933 return fastEmitInst_rr(X86::VUNPCKHPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13934 } 13935 return 0; 13936} 13937 13938unsigned fastEmit_X86ISD_UNPCKH_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13939 if (RetVT.SimpleTy != MVT::v8f64) 13940 return 0; 13941 if ((Subtarget->hasAVX512())) { 13942 return fastEmitInst_rr(X86::VUNPCKHPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13943 } 13944 return 0; 13945} 13946 13947unsigned fastEmit_X86ISD_UNPCKH_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13948 switch (VT.SimpleTy) { 13949 case MVT::v16i8: return fastEmit_X86ISD_UNPCKH_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13950 case MVT::v32i8: return fastEmit_X86ISD_UNPCKH_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13951 case MVT::v64i8: return fastEmit_X86ISD_UNPCKH_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13952 case MVT::v8i16: return fastEmit_X86ISD_UNPCKH_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13953 case MVT::v16i16: return fastEmit_X86ISD_UNPCKH_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13954 case MVT::v32i16: return fastEmit_X86ISD_UNPCKH_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13955 case MVT::v4i32: return fastEmit_X86ISD_UNPCKH_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13956 case MVT::v8i32: return fastEmit_X86ISD_UNPCKH_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13957 case MVT::v16i32: return fastEmit_X86ISD_UNPCKH_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13958 case MVT::v2i64: return fastEmit_X86ISD_UNPCKH_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13959 case MVT::v4i64: return fastEmit_X86ISD_UNPCKH_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13960 case MVT::v8i64: return fastEmit_X86ISD_UNPCKH_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13961 case MVT::v4f32: return fastEmit_X86ISD_UNPCKH_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13962 case MVT::v8f32: return fastEmit_X86ISD_UNPCKH_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13963 case MVT::v16f32: return fastEmit_X86ISD_UNPCKH_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13964 case MVT::v2f64: return fastEmit_X86ISD_UNPCKH_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13965 case MVT::v4f64: return fastEmit_X86ISD_UNPCKH_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13966 case MVT::v8f64: return fastEmit_X86ISD_UNPCKH_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 13967 default: return 0; 13968 } 13969} 13970 13971// FastEmit functions for X86ISD::UNPCKL. 13972 13973unsigned fastEmit_X86ISD_UNPCKL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13974 if (RetVT.SimpleTy != MVT::v16i8) 13975 return 0; 13976 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 13977 return fastEmitInst_rr(X86::VPUNPCKLBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13978 } 13979 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 13980 return fastEmitInst_rr(X86::PUNPCKLBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13981 } 13982 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 13983 return fastEmitInst_rr(X86::VPUNPCKLBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13984 } 13985 return 0; 13986} 13987 13988unsigned fastEmit_X86ISD_UNPCKL_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 13989 if (RetVT.SimpleTy != MVT::v32i8) 13990 return 0; 13991 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 13992 return fastEmitInst_rr(X86::VPUNPCKLBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13993 } 13994 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 13995 return fastEmitInst_rr(X86::VPUNPCKLBWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 13996 } 13997 return 0; 13998} 13999 14000unsigned fastEmit_X86ISD_UNPCKL_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14001 if (RetVT.SimpleTy != MVT::v64i8) 14002 return 0; 14003 if ((Subtarget->hasBWI())) { 14004 return fastEmitInst_rr(X86::VPUNPCKLBWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14005 } 14006 return 0; 14007} 14008 14009unsigned fastEmit_X86ISD_UNPCKL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14010 if (RetVT.SimpleTy != MVT::v8i16) 14011 return 0; 14012 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 14013 return fastEmitInst_rr(X86::VPUNPCKLWDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14014 } 14015 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 14016 return fastEmitInst_rr(X86::PUNPCKLWDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14017 } 14018 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 14019 return fastEmitInst_rr(X86::VPUNPCKLWDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14020 } 14021 return 0; 14022} 14023 14024unsigned fastEmit_X86ISD_UNPCKL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14025 if (RetVT.SimpleTy != MVT::v16i16) 14026 return 0; 14027 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 14028 return fastEmitInst_rr(X86::VPUNPCKLWDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14029 } 14030 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 14031 return fastEmitInst_rr(X86::VPUNPCKLWDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14032 } 14033 return 0; 14034} 14035 14036unsigned fastEmit_X86ISD_UNPCKL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14037 if (RetVT.SimpleTy != MVT::v32i16) 14038 return 0; 14039 if ((Subtarget->hasBWI())) { 14040 return fastEmitInst_rr(X86::VPUNPCKLWDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14041 } 14042 return 0; 14043} 14044 14045unsigned fastEmit_X86ISD_UNPCKL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14046 if (RetVT.SimpleTy != MVT::v4i32) 14047 return 0; 14048 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14049 return fastEmitInst_rr(X86::VPUNPCKLDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14050 } 14051 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 14052 return fastEmitInst_rr(X86::PUNPCKLDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14053 } 14054 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 14055 return fastEmitInst_rr(X86::VPUNPCKLDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14056 } 14057 return 0; 14058} 14059 14060unsigned fastEmit_X86ISD_UNPCKL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14061 if (RetVT.SimpleTy != MVT::v8i32) 14062 return 0; 14063 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 14064 return fastEmitInst_rr(X86::VUNPCKLPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14065 } 14066 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14067 return fastEmitInst_rr(X86::VPUNPCKLDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14068 } 14069 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 14070 return fastEmitInst_rr(X86::VPUNPCKLDQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14071 } 14072 return 0; 14073} 14074 14075unsigned fastEmit_X86ISD_UNPCKL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14076 if (RetVT.SimpleTy != MVT::v16i32) 14077 return 0; 14078 if ((Subtarget->hasAVX512())) { 14079 return fastEmitInst_rr(X86::VPUNPCKLDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14080 } 14081 return 0; 14082} 14083 14084unsigned fastEmit_X86ISD_UNPCKL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14085 if (RetVT.SimpleTy != MVT::v2i64) 14086 return 0; 14087 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14088 return fastEmitInst_rr(X86::VPUNPCKLQDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14089 } 14090 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 14091 return fastEmitInst_rr(X86::PUNPCKLQDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14092 } 14093 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 14094 return fastEmitInst_rr(X86::VPUNPCKLQDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14095 } 14096 return 0; 14097} 14098 14099unsigned fastEmit_X86ISD_UNPCKL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14100 if (RetVT.SimpleTy != MVT::v4i64) 14101 return 0; 14102 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) { 14103 return fastEmitInst_rr(X86::VUNPCKLPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14104 } 14105 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14106 return fastEmitInst_rr(X86::VPUNPCKLQDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14107 } 14108 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 14109 return fastEmitInst_rr(X86::VPUNPCKLQDQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14110 } 14111 return 0; 14112} 14113 14114unsigned fastEmit_X86ISD_UNPCKL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14115 if (RetVT.SimpleTy != MVT::v8i64) 14116 return 0; 14117 if ((Subtarget->hasAVX512())) { 14118 return fastEmitInst_rr(X86::VPUNPCKLQDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14119 } 14120 return 0; 14121} 14122 14123unsigned fastEmit_X86ISD_UNPCKL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14124 if (RetVT.SimpleTy != MVT::v4f32) 14125 return 0; 14126 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14127 return fastEmitInst_rr(X86::VUNPCKLPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14128 } 14129 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) { 14130 return fastEmitInst_rr(X86::UNPCKLPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14131 } 14132 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 14133 return fastEmitInst_rr(X86::VUNPCKLPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14134 } 14135 return 0; 14136} 14137 14138unsigned fastEmit_X86ISD_UNPCKL_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14139 if (RetVT.SimpleTy != MVT::v8f32) 14140 return 0; 14141 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14142 return fastEmitInst_rr(X86::VUNPCKLPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14143 } 14144 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 14145 return fastEmitInst_rr(X86::VUNPCKLPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14146 } 14147 return 0; 14148} 14149 14150unsigned fastEmit_X86ISD_UNPCKL_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14151 if (RetVT.SimpleTy != MVT::v16f32) 14152 return 0; 14153 if ((Subtarget->hasAVX512())) { 14154 return fastEmitInst_rr(X86::VUNPCKLPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14155 } 14156 return 0; 14157} 14158 14159unsigned fastEmit_X86ISD_UNPCKL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14160 if (RetVT.SimpleTy != MVT::v2f64) 14161 return 0; 14162 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14163 return fastEmitInst_rr(X86::VUNPCKLPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14164 } 14165 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 14166 return fastEmitInst_rr(X86::UNPCKLPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14167 } 14168 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 14169 return fastEmitInst_rr(X86::VUNPCKLPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14170 } 14171 return 0; 14172} 14173 14174unsigned fastEmit_X86ISD_UNPCKL_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14175 if (RetVT.SimpleTy != MVT::v4f64) 14176 return 0; 14177 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14178 return fastEmitInst_rr(X86::VUNPCKLPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14179 } 14180 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 14181 return fastEmitInst_rr(X86::VUNPCKLPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14182 } 14183 return 0; 14184} 14185 14186unsigned fastEmit_X86ISD_UNPCKL_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14187 if (RetVT.SimpleTy != MVT::v8f64) 14188 return 0; 14189 if ((Subtarget->hasAVX512())) { 14190 return fastEmitInst_rr(X86::VUNPCKLPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14191 } 14192 return 0; 14193} 14194 14195unsigned fastEmit_X86ISD_UNPCKL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14196 switch (VT.SimpleTy) { 14197 case MVT::v16i8: return fastEmit_X86ISD_UNPCKL_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14198 case MVT::v32i8: return fastEmit_X86ISD_UNPCKL_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14199 case MVT::v64i8: return fastEmit_X86ISD_UNPCKL_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14200 case MVT::v8i16: return fastEmit_X86ISD_UNPCKL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14201 case MVT::v16i16: return fastEmit_X86ISD_UNPCKL_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14202 case MVT::v32i16: return fastEmit_X86ISD_UNPCKL_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14203 case MVT::v4i32: return fastEmit_X86ISD_UNPCKL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14204 case MVT::v8i32: return fastEmit_X86ISD_UNPCKL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14205 case MVT::v16i32: return fastEmit_X86ISD_UNPCKL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14206 case MVT::v2i64: return fastEmit_X86ISD_UNPCKL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14207 case MVT::v4i64: return fastEmit_X86ISD_UNPCKL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14208 case MVT::v8i64: return fastEmit_X86ISD_UNPCKL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14209 case MVT::v4f32: return fastEmit_X86ISD_UNPCKL_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14210 case MVT::v8f32: return fastEmit_X86ISD_UNPCKL_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14211 case MVT::v16f32: return fastEmit_X86ISD_UNPCKL_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14212 case MVT::v2f64: return fastEmit_X86ISD_UNPCKL_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14213 case MVT::v4f64: return fastEmit_X86ISD_UNPCKL_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14214 case MVT::v8f64: return fastEmit_X86ISD_UNPCKL_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14215 default: return 0; 14216 } 14217} 14218 14219// FastEmit functions for X86ISD::VP2INTERSECT. 14220 14221unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14222 if (RetVT.SimpleTy != MVT::Untyped) 14223 return 0; 14224 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) { 14225 return fastEmitInst_rr(X86::VP2INTERSECTDZ128rr, &X86::VK4PAIRRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14226 } 14227 return 0; 14228} 14229 14230unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14231 if (RetVT.SimpleTy != MVT::Untyped) 14232 return 0; 14233 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) { 14234 return fastEmitInst_rr(X86::VP2INTERSECTDZ256rr, &X86::VK8PAIRRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14235 } 14236 return 0; 14237} 14238 14239unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14240 if (RetVT.SimpleTy != MVT::Untyped) 14241 return 0; 14242 if ((Subtarget->hasAVX512()) && (Subtarget->hasVP2INTERSECT())) { 14243 return fastEmitInst_rr(X86::VP2INTERSECTDZrr, &X86::VK16PAIRRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14244 } 14245 return 0; 14246} 14247 14248unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14249 if (RetVT.SimpleTy != MVT::Untyped) 14250 return 0; 14251 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) { 14252 return fastEmitInst_rr(X86::VP2INTERSECTQZ128rr, &X86::VK2PAIRRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14253 } 14254 return 0; 14255} 14256 14257unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14258 if (RetVT.SimpleTy != MVT::Untyped) 14259 return 0; 14260 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) { 14261 return fastEmitInst_rr(X86::VP2INTERSECTQZ256rr, &X86::VK4PAIRRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14262 } 14263 return 0; 14264} 14265 14266unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14267 if (RetVT.SimpleTy != MVT::Untyped) 14268 return 0; 14269 if ((Subtarget->hasAVX512()) && (Subtarget->hasVP2INTERSECT())) { 14270 return fastEmitInst_rr(X86::VP2INTERSECTQZrr, &X86::VK8PAIRRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14271 } 14272 return 0; 14273} 14274 14275unsigned fastEmit_X86ISD_VP2INTERSECT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14276 switch (VT.SimpleTy) { 14277 case MVT::v4i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14278 case MVT::v8i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14279 case MVT::v16i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14280 case MVT::v2i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14281 case MVT::v4i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14282 case MVT::v8i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14283 default: return 0; 14284 } 14285} 14286 14287// FastEmit functions for X86ISD::VPERMV. 14288 14289unsigned fastEmit_X86ISD_VPERMV_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14290 if (RetVT.SimpleTy != MVT::v16i8) 14291 return 0; 14292 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) { 14293 return fastEmitInst_rr(X86::VPERMBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14294 } 14295 return 0; 14296} 14297 14298unsigned fastEmit_X86ISD_VPERMV_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14299 if (RetVT.SimpleTy != MVT::v32i8) 14300 return 0; 14301 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) { 14302 return fastEmitInst_rr(X86::VPERMBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14303 } 14304 return 0; 14305} 14306 14307unsigned fastEmit_X86ISD_VPERMV_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14308 if (RetVT.SimpleTy != MVT::v64i8) 14309 return 0; 14310 if ((Subtarget->hasVBMI())) { 14311 return fastEmitInst_rr(X86::VPERMBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14312 } 14313 return 0; 14314} 14315 14316unsigned fastEmit_X86ISD_VPERMV_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14317 if (RetVT.SimpleTy != MVT::v8i16) 14318 return 0; 14319 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 14320 return fastEmitInst_rr(X86::VPERMWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14321 } 14322 return 0; 14323} 14324 14325unsigned fastEmit_X86ISD_VPERMV_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14326 if (RetVT.SimpleTy != MVT::v16i16) 14327 return 0; 14328 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 14329 return fastEmitInst_rr(X86::VPERMWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14330 } 14331 return 0; 14332} 14333 14334unsigned fastEmit_X86ISD_VPERMV_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14335 if (RetVT.SimpleTy != MVT::v32i16) 14336 return 0; 14337 if ((Subtarget->hasBWI())) { 14338 return fastEmitInst_rr(X86::VPERMWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14339 } 14340 return 0; 14341} 14342 14343unsigned fastEmit_X86ISD_VPERMV_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14344 if (RetVT.SimpleTy != MVT::v8i32) 14345 return 0; 14346 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14347 return fastEmitInst_rr(X86::VPERMDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14348 } 14349 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 14350 return fastEmitInst_rr(X86::VPERMDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14351 } 14352 return 0; 14353} 14354 14355unsigned fastEmit_X86ISD_VPERMV_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14356 if (RetVT.SimpleTy != MVT::v16i32) 14357 return 0; 14358 if ((Subtarget->hasAVX512())) { 14359 return fastEmitInst_rr(X86::VPERMDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14360 } 14361 return 0; 14362} 14363 14364unsigned fastEmit_X86ISD_VPERMV_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14365 if (RetVT.SimpleTy != MVT::v4i64) 14366 return 0; 14367 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14368 return fastEmitInst_rr(X86::VPERMQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14369 } 14370 return 0; 14371} 14372 14373unsigned fastEmit_X86ISD_VPERMV_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14374 if (RetVT.SimpleTy != MVT::v8i64) 14375 return 0; 14376 if ((Subtarget->hasAVX512())) { 14377 return fastEmitInst_rr(X86::VPERMQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14378 } 14379 return 0; 14380} 14381 14382unsigned fastEmit_X86ISD_VPERMV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14383 switch (VT.SimpleTy) { 14384 case MVT::v16i8: return fastEmit_X86ISD_VPERMV_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14385 case MVT::v32i8: return fastEmit_X86ISD_VPERMV_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14386 case MVT::v64i8: return fastEmit_X86ISD_VPERMV_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14387 case MVT::v8i16: return fastEmit_X86ISD_VPERMV_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14388 case MVT::v16i16: return fastEmit_X86ISD_VPERMV_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14389 case MVT::v32i16: return fastEmit_X86ISD_VPERMV_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14390 case MVT::v8i32: return fastEmit_X86ISD_VPERMV_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14391 case MVT::v16i32: return fastEmit_X86ISD_VPERMV_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14392 case MVT::v4i64: return fastEmit_X86ISD_VPERMV_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14393 case MVT::v8i64: return fastEmit_X86ISD_VPERMV_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14394 default: return 0; 14395 } 14396} 14397 14398// FastEmit functions for X86ISD::VPMADDUBSW. 14399 14400unsigned fastEmit_X86ISD_VPMADDUBSW_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14401 if (RetVT.SimpleTy != MVT::v8i16) 14402 return 0; 14403 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 14404 return fastEmitInst_rr(X86::VPMADDUBSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14405 } 14406 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) { 14407 return fastEmitInst_rr(X86::PMADDUBSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14408 } 14409 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 14410 return fastEmitInst_rr(X86::VPMADDUBSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14411 } 14412 return 0; 14413} 14414 14415unsigned fastEmit_X86ISD_VPMADDUBSW_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14416 if (RetVT.SimpleTy != MVT::v16i16) 14417 return 0; 14418 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 14419 return fastEmitInst_rr(X86::VPMADDUBSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14420 } 14421 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 14422 return fastEmitInst_rr(X86::VPMADDUBSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14423 } 14424 return 0; 14425} 14426 14427unsigned fastEmit_X86ISD_VPMADDUBSW_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14428 if (RetVT.SimpleTy != MVT::v32i16) 14429 return 0; 14430 if ((Subtarget->hasBWI())) { 14431 return fastEmitInst_rr(X86::VPMADDUBSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14432 } 14433 return 0; 14434} 14435 14436unsigned fastEmit_X86ISD_VPMADDUBSW_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14437 switch (VT.SimpleTy) { 14438 case MVT::v16i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14439 case MVT::v32i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14440 case MVT::v64i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14441 default: return 0; 14442 } 14443} 14444 14445// FastEmit functions for X86ISD::VPMADDWD. 14446 14447unsigned fastEmit_X86ISD_VPMADDWD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14448 if (RetVT.SimpleTy != MVT::v4i32) 14449 return 0; 14450 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 14451 return fastEmitInst_rr(X86::VPMADDWDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14452 } 14453 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 14454 return fastEmitInst_rr(X86::PMADDWDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14455 } 14456 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 14457 return fastEmitInst_rr(X86::VPMADDWDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14458 } 14459 return 0; 14460} 14461 14462unsigned fastEmit_X86ISD_VPMADDWD_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14463 if (RetVT.SimpleTy != MVT::v8i32) 14464 return 0; 14465 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 14466 return fastEmitInst_rr(X86::VPMADDWDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14467 } 14468 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 14469 return fastEmitInst_rr(X86::VPMADDWDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14470 } 14471 return 0; 14472} 14473 14474unsigned fastEmit_X86ISD_VPMADDWD_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14475 if (RetVT.SimpleTy != MVT::v16i32) 14476 return 0; 14477 if ((Subtarget->hasBWI())) { 14478 return fastEmitInst_rr(X86::VPMADDWDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14479 } 14480 return 0; 14481} 14482 14483unsigned fastEmit_X86ISD_VPMADDWD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14484 switch (VT.SimpleTy) { 14485 case MVT::v8i16: return fastEmit_X86ISD_VPMADDWD_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14486 case MVT::v16i16: return fastEmit_X86ISD_VPMADDWD_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14487 case MVT::v32i16: return fastEmit_X86ISD_VPMADDWD_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14488 default: return 0; 14489 } 14490} 14491 14492// FastEmit functions for X86ISD::VPSHA. 14493 14494unsigned fastEmit_X86ISD_VPSHA_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14495 if (RetVT.SimpleTy != MVT::v16i8) 14496 return 0; 14497 if ((Subtarget->hasXOP())) { 14498 return fastEmitInst_rr(X86::VPSHABrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14499 } 14500 return 0; 14501} 14502 14503unsigned fastEmit_X86ISD_VPSHA_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14504 if (RetVT.SimpleTy != MVT::v8i16) 14505 return 0; 14506 if ((Subtarget->hasXOP())) { 14507 return fastEmitInst_rr(X86::VPSHAWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14508 } 14509 return 0; 14510} 14511 14512unsigned fastEmit_X86ISD_VPSHA_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14513 if (RetVT.SimpleTy != MVT::v4i32) 14514 return 0; 14515 if ((Subtarget->hasXOP())) { 14516 return fastEmitInst_rr(X86::VPSHADrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14517 } 14518 return 0; 14519} 14520 14521unsigned fastEmit_X86ISD_VPSHA_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14522 if (RetVT.SimpleTy != MVT::v2i64) 14523 return 0; 14524 if ((Subtarget->hasXOP())) { 14525 return fastEmitInst_rr(X86::VPSHAQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14526 } 14527 return 0; 14528} 14529 14530unsigned fastEmit_X86ISD_VPSHA_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14531 switch (VT.SimpleTy) { 14532 case MVT::v16i8: return fastEmit_X86ISD_VPSHA_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14533 case MVT::v8i16: return fastEmit_X86ISD_VPSHA_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14534 case MVT::v4i32: return fastEmit_X86ISD_VPSHA_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14535 case MVT::v2i64: return fastEmit_X86ISD_VPSHA_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14536 default: return 0; 14537 } 14538} 14539 14540// FastEmit functions for X86ISD::VPSHL. 14541 14542unsigned fastEmit_X86ISD_VPSHL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14543 if (RetVT.SimpleTy != MVT::v16i8) 14544 return 0; 14545 if ((Subtarget->hasXOP())) { 14546 return fastEmitInst_rr(X86::VPSHLBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14547 } 14548 return 0; 14549} 14550 14551unsigned fastEmit_X86ISD_VPSHL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14552 if (RetVT.SimpleTy != MVT::v8i16) 14553 return 0; 14554 if ((Subtarget->hasXOP())) { 14555 return fastEmitInst_rr(X86::VPSHLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14556 } 14557 return 0; 14558} 14559 14560unsigned fastEmit_X86ISD_VPSHL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14561 if (RetVT.SimpleTy != MVT::v4i32) 14562 return 0; 14563 if ((Subtarget->hasXOP())) { 14564 return fastEmitInst_rr(X86::VPSHLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14565 } 14566 return 0; 14567} 14568 14569unsigned fastEmit_X86ISD_VPSHL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14570 if (RetVT.SimpleTy != MVT::v2i64) 14571 return 0; 14572 if ((Subtarget->hasXOP())) { 14573 return fastEmitInst_rr(X86::VPSHLQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14574 } 14575 return 0; 14576} 14577 14578unsigned fastEmit_X86ISD_VPSHL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14579 switch (VT.SimpleTy) { 14580 case MVT::v16i8: return fastEmit_X86ISD_VPSHL_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14581 case MVT::v8i16: return fastEmit_X86ISD_VPSHL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14582 case MVT::v4i32: return fastEmit_X86ISD_VPSHL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14583 case MVT::v2i64: return fastEmit_X86ISD_VPSHL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14584 default: return 0; 14585 } 14586} 14587 14588// FastEmit functions for X86ISD::VPSHUFBITQMB. 14589 14590unsigned fastEmit_X86ISD_VPSHUFBITQMB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14591 if (RetVT.SimpleTy != MVT::v16i1) 14592 return 0; 14593 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) { 14594 return fastEmitInst_rr(X86::VPSHUFBITQMBZ128rr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14595 } 14596 return 0; 14597} 14598 14599unsigned fastEmit_X86ISD_VPSHUFBITQMB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14600 if (RetVT.SimpleTy != MVT::v32i1) 14601 return 0; 14602 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) { 14603 return fastEmitInst_rr(X86::VPSHUFBITQMBZ256rr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14604 } 14605 return 0; 14606} 14607 14608unsigned fastEmit_X86ISD_VPSHUFBITQMB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14609 if (RetVT.SimpleTy != MVT::v64i1) 14610 return 0; 14611 if ((Subtarget->hasBITALG())) { 14612 return fastEmitInst_rr(X86::VPSHUFBITQMBZrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14613 } 14614 return 0; 14615} 14616 14617unsigned fastEmit_X86ISD_VPSHUFBITQMB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14618 switch (VT.SimpleTy) { 14619 case MVT::v16i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14620 case MVT::v32i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14621 case MVT::v64i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14622 default: return 0; 14623 } 14624} 14625 14626// FastEmit functions for X86ISD::VSHL. 14627 14628unsigned fastEmit_X86ISD_VSHL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14629 if (RetVT.SimpleTy != MVT::v8i16) 14630 return 0; 14631 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 14632 return fastEmitInst_rr(X86::VPSLLWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14633 } 14634 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 14635 return fastEmitInst_rr(X86::PSLLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14636 } 14637 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 14638 return fastEmitInst_rr(X86::VPSLLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14639 } 14640 return 0; 14641} 14642 14643unsigned fastEmit_X86ISD_VSHL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14644 if (RetVT.SimpleTy != MVT::v4i32) 14645 return 0; 14646 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14647 return fastEmitInst_rr(X86::VPSLLDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14648 } 14649 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 14650 return fastEmitInst_rr(X86::PSLLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14651 } 14652 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 14653 return fastEmitInst_rr(X86::VPSLLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14654 } 14655 return 0; 14656} 14657 14658unsigned fastEmit_X86ISD_VSHL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14659 if (RetVT.SimpleTy != MVT::v2i64) 14660 return 0; 14661 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14662 return fastEmitInst_rr(X86::VPSLLQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14663 } 14664 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 14665 return fastEmitInst_rr(X86::PSLLQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14666 } 14667 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 14668 return fastEmitInst_rr(X86::VPSLLQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14669 } 14670 return 0; 14671} 14672 14673unsigned fastEmit_X86ISD_VSHL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14674 switch (VT.SimpleTy) { 14675 case MVT::v8i16: return fastEmit_X86ISD_VSHL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14676 case MVT::v4i32: return fastEmit_X86ISD_VSHL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14677 case MVT::v2i64: return fastEmit_X86ISD_VSHL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14678 default: return 0; 14679 } 14680} 14681 14682// FastEmit functions for X86ISD::VSHLV. 14683 14684unsigned fastEmit_X86ISD_VSHLV_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14685 if (RetVT.SimpleTy != MVT::v8i16) 14686 return 0; 14687 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 14688 return fastEmitInst_rr(X86::VPSLLVWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14689 } 14690 return 0; 14691} 14692 14693unsigned fastEmit_X86ISD_VSHLV_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14694 if (RetVT.SimpleTy != MVT::v16i16) 14695 return 0; 14696 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 14697 return fastEmitInst_rr(X86::VPSLLVWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14698 } 14699 return 0; 14700} 14701 14702unsigned fastEmit_X86ISD_VSHLV_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14703 if (RetVT.SimpleTy != MVT::v32i16) 14704 return 0; 14705 if ((Subtarget->hasBWI())) { 14706 return fastEmitInst_rr(X86::VPSLLVWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14707 } 14708 return 0; 14709} 14710 14711unsigned fastEmit_X86ISD_VSHLV_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14712 if (RetVT.SimpleTy != MVT::v4i32) 14713 return 0; 14714 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14715 return fastEmitInst_rr(X86::VPSLLVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14716 } 14717 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 14718 return fastEmitInst_rr(X86::VPSLLVDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14719 } 14720 return 0; 14721} 14722 14723unsigned fastEmit_X86ISD_VSHLV_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14724 if (RetVT.SimpleTy != MVT::v8i32) 14725 return 0; 14726 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14727 return fastEmitInst_rr(X86::VPSLLVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14728 } 14729 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 14730 return fastEmitInst_rr(X86::VPSLLVDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14731 } 14732 return 0; 14733} 14734 14735unsigned fastEmit_X86ISD_VSHLV_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14736 if (RetVT.SimpleTy != MVT::v16i32) 14737 return 0; 14738 if ((Subtarget->hasAVX512())) { 14739 return fastEmitInst_rr(X86::VPSLLVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14740 } 14741 return 0; 14742} 14743 14744unsigned fastEmit_X86ISD_VSHLV_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14745 if (RetVT.SimpleTy != MVT::v2i64) 14746 return 0; 14747 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14748 return fastEmitInst_rr(X86::VPSLLVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14749 } 14750 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 14751 return fastEmitInst_rr(X86::VPSLLVQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14752 } 14753 return 0; 14754} 14755 14756unsigned fastEmit_X86ISD_VSHLV_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14757 if (RetVT.SimpleTy != MVT::v4i64) 14758 return 0; 14759 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14760 return fastEmitInst_rr(X86::VPSLLVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14761 } 14762 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 14763 return fastEmitInst_rr(X86::VPSLLVQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14764 } 14765 return 0; 14766} 14767 14768unsigned fastEmit_X86ISD_VSHLV_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14769 if (RetVT.SimpleTy != MVT::v8i64) 14770 return 0; 14771 if ((Subtarget->hasAVX512())) { 14772 return fastEmitInst_rr(X86::VPSLLVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14773 } 14774 return 0; 14775} 14776 14777unsigned fastEmit_X86ISD_VSHLV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14778 switch (VT.SimpleTy) { 14779 case MVT::v8i16: return fastEmit_X86ISD_VSHLV_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14780 case MVT::v16i16: return fastEmit_X86ISD_VSHLV_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14781 case MVT::v32i16: return fastEmit_X86ISD_VSHLV_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14782 case MVT::v4i32: return fastEmit_X86ISD_VSHLV_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14783 case MVT::v8i32: return fastEmit_X86ISD_VSHLV_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14784 case MVT::v16i32: return fastEmit_X86ISD_VSHLV_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14785 case MVT::v2i64: return fastEmit_X86ISD_VSHLV_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14786 case MVT::v4i64: return fastEmit_X86ISD_VSHLV_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14787 case MVT::v8i64: return fastEmit_X86ISD_VSHLV_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14788 default: return 0; 14789 } 14790} 14791 14792// FastEmit functions for X86ISD::VSRA. 14793 14794unsigned fastEmit_X86ISD_VSRA_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14795 if (RetVT.SimpleTy != MVT::v8i16) 14796 return 0; 14797 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 14798 return fastEmitInst_rr(X86::VPSRAWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14799 } 14800 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 14801 return fastEmitInst_rr(X86::PSRAWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14802 } 14803 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 14804 return fastEmitInst_rr(X86::VPSRAWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14805 } 14806 return 0; 14807} 14808 14809unsigned fastEmit_X86ISD_VSRA_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14810 if (RetVT.SimpleTy != MVT::v4i32) 14811 return 0; 14812 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14813 return fastEmitInst_rr(X86::VPSRADZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14814 } 14815 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 14816 return fastEmitInst_rr(X86::PSRADrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14817 } 14818 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 14819 return fastEmitInst_rr(X86::VPSRADrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14820 } 14821 return 0; 14822} 14823 14824unsigned fastEmit_X86ISD_VSRA_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14825 if (RetVT.SimpleTy != MVT::v2i64) 14826 return 0; 14827 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14828 return fastEmitInst_rr(X86::VPSRAQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14829 } 14830 return 0; 14831} 14832 14833unsigned fastEmit_X86ISD_VSRA_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14834 switch (VT.SimpleTy) { 14835 case MVT::v8i16: return fastEmit_X86ISD_VSRA_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14836 case MVT::v4i32: return fastEmit_X86ISD_VSRA_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14837 case MVT::v2i64: return fastEmit_X86ISD_VSRA_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14838 default: return 0; 14839 } 14840} 14841 14842// FastEmit functions for X86ISD::VSRAV. 14843 14844unsigned fastEmit_X86ISD_VSRAV_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14845 if (RetVT.SimpleTy != MVT::v8i16) 14846 return 0; 14847 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 14848 return fastEmitInst_rr(X86::VPSRAVWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14849 } 14850 return 0; 14851} 14852 14853unsigned fastEmit_X86ISD_VSRAV_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14854 if (RetVT.SimpleTy != MVT::v16i16) 14855 return 0; 14856 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 14857 return fastEmitInst_rr(X86::VPSRAVWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14858 } 14859 return 0; 14860} 14861 14862unsigned fastEmit_X86ISD_VSRAV_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14863 if (RetVT.SimpleTy != MVT::v32i16) 14864 return 0; 14865 if ((Subtarget->hasBWI())) { 14866 return fastEmitInst_rr(X86::VPSRAVWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14867 } 14868 return 0; 14869} 14870 14871unsigned fastEmit_X86ISD_VSRAV_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14872 if (RetVT.SimpleTy != MVT::v4i32) 14873 return 0; 14874 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14875 return fastEmitInst_rr(X86::VPSRAVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14876 } 14877 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 14878 return fastEmitInst_rr(X86::VPSRAVDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14879 } 14880 return 0; 14881} 14882 14883unsigned fastEmit_X86ISD_VSRAV_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14884 if (RetVT.SimpleTy != MVT::v8i32) 14885 return 0; 14886 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14887 return fastEmitInst_rr(X86::VPSRAVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14888 } 14889 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 14890 return fastEmitInst_rr(X86::VPSRAVDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14891 } 14892 return 0; 14893} 14894 14895unsigned fastEmit_X86ISD_VSRAV_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14896 if (RetVT.SimpleTy != MVT::v16i32) 14897 return 0; 14898 if ((Subtarget->hasAVX512())) { 14899 return fastEmitInst_rr(X86::VPSRAVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14900 } 14901 return 0; 14902} 14903 14904unsigned fastEmit_X86ISD_VSRAV_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14905 if (RetVT.SimpleTy != MVT::v2i64) 14906 return 0; 14907 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14908 return fastEmitInst_rr(X86::VPSRAVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14909 } 14910 return 0; 14911} 14912 14913unsigned fastEmit_X86ISD_VSRAV_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14914 if (RetVT.SimpleTy != MVT::v4i64) 14915 return 0; 14916 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14917 return fastEmitInst_rr(X86::VPSRAVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14918 } 14919 return 0; 14920} 14921 14922unsigned fastEmit_X86ISD_VSRAV_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14923 if (RetVT.SimpleTy != MVT::v8i64) 14924 return 0; 14925 if ((Subtarget->hasAVX512())) { 14926 return fastEmitInst_rr(X86::VPSRAVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14927 } 14928 return 0; 14929} 14930 14931unsigned fastEmit_X86ISD_VSRAV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14932 switch (VT.SimpleTy) { 14933 case MVT::v8i16: return fastEmit_X86ISD_VSRAV_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14934 case MVT::v16i16: return fastEmit_X86ISD_VSRAV_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14935 case MVT::v32i16: return fastEmit_X86ISD_VSRAV_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14936 case MVT::v4i32: return fastEmit_X86ISD_VSRAV_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14937 case MVT::v8i32: return fastEmit_X86ISD_VSRAV_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14938 case MVT::v16i32: return fastEmit_X86ISD_VSRAV_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14939 case MVT::v2i64: return fastEmit_X86ISD_VSRAV_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14940 case MVT::v4i64: return fastEmit_X86ISD_VSRAV_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14941 case MVT::v8i64: return fastEmit_X86ISD_VSRAV_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14942 default: return 0; 14943 } 14944} 14945 14946// FastEmit functions for X86ISD::VSRL. 14947 14948unsigned fastEmit_X86ISD_VSRL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14949 if (RetVT.SimpleTy != MVT::v8i16) 14950 return 0; 14951 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 14952 return fastEmitInst_rr(X86::VPSRLWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14953 } 14954 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 14955 return fastEmitInst_rr(X86::PSRLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14956 } 14957 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) { 14958 return fastEmitInst_rr(X86::VPSRLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14959 } 14960 return 0; 14961} 14962 14963unsigned fastEmit_X86ISD_VSRL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14964 if (RetVT.SimpleTy != MVT::v4i32) 14965 return 0; 14966 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14967 return fastEmitInst_rr(X86::VPSRLDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14968 } 14969 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 14970 return fastEmitInst_rr(X86::PSRLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14971 } 14972 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 14973 return fastEmitInst_rr(X86::VPSRLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14974 } 14975 return 0; 14976} 14977 14978unsigned fastEmit_X86ISD_VSRL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14979 if (RetVT.SimpleTy != MVT::v2i64) 14980 return 0; 14981 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 14982 return fastEmitInst_rr(X86::VPSRLQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14983 } 14984 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 14985 return fastEmitInst_rr(X86::PSRLQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14986 } 14987 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) { 14988 return fastEmitInst_rr(X86::VPSRLQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 14989 } 14990 return 0; 14991} 14992 14993unsigned fastEmit_X86ISD_VSRL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 14994 switch (VT.SimpleTy) { 14995 case MVT::v8i16: return fastEmit_X86ISD_VSRL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14996 case MVT::v4i32: return fastEmit_X86ISD_VSRL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14997 case MVT::v2i64: return fastEmit_X86ISD_VSRL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 14998 default: return 0; 14999 } 15000} 15001 15002// FastEmit functions for X86ISD::VSRLV. 15003 15004unsigned fastEmit_X86ISD_VSRLV_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 15005 if (RetVT.SimpleTy != MVT::v8i16) 15006 return 0; 15007 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 15008 return fastEmitInst_rr(X86::VPSRLVWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 15009 } 15010 return 0; 15011} 15012 15013unsigned fastEmit_X86ISD_VSRLV_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 15014 if (RetVT.SimpleTy != MVT::v16i16) 15015 return 0; 15016 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) { 15017 return fastEmitInst_rr(X86::VPSRLVWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 15018 } 15019 return 0; 15020} 15021 15022unsigned fastEmit_X86ISD_VSRLV_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 15023 if (RetVT.SimpleTy != MVT::v32i16) 15024 return 0; 15025 if ((Subtarget->hasBWI())) { 15026 return fastEmitInst_rr(X86::VPSRLVWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 15027 } 15028 return 0; 15029} 15030 15031unsigned fastEmit_X86ISD_VSRLV_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 15032 if (RetVT.SimpleTy != MVT::v4i32) 15033 return 0; 15034 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 15035 return fastEmitInst_rr(X86::VPSRLVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 15036 } 15037 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 15038 return fastEmitInst_rr(X86::VPSRLVDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 15039 } 15040 return 0; 15041} 15042 15043unsigned fastEmit_X86ISD_VSRLV_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 15044 if (RetVT.SimpleTy != MVT::v8i32) 15045 return 0; 15046 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 15047 return fastEmitInst_rr(X86::VPSRLVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 15048 } 15049 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 15050 return fastEmitInst_rr(X86::VPSRLVDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 15051 } 15052 return 0; 15053} 15054 15055unsigned fastEmit_X86ISD_VSRLV_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 15056 if (RetVT.SimpleTy != MVT::v16i32) 15057 return 0; 15058 if ((Subtarget->hasAVX512())) { 15059 return fastEmitInst_rr(X86::VPSRLVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 15060 } 15061 return 0; 15062} 15063 15064unsigned fastEmit_X86ISD_VSRLV_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 15065 if (RetVT.SimpleTy != MVT::v2i64) 15066 return 0; 15067 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 15068 return fastEmitInst_rr(X86::VPSRLVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 15069 } 15070 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 15071 return fastEmitInst_rr(X86::VPSRLVQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 15072 } 15073 return 0; 15074} 15075 15076unsigned fastEmit_X86ISD_VSRLV_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 15077 if (RetVT.SimpleTy != MVT::v4i64) 15078 return 0; 15079 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) { 15080 return fastEmitInst_rr(X86::VPSRLVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill); 15081 } 15082 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) { 15083 return fastEmitInst_rr(X86::VPSRLVQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 15084 } 15085 return 0; 15086} 15087 15088unsigned fastEmit_X86ISD_VSRLV_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 15089 if (RetVT.SimpleTy != MVT::v8i64) 15090 return 0; 15091 if ((Subtarget->hasAVX512())) { 15092 return fastEmitInst_rr(X86::VPSRLVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill); 15093 } 15094 return 0; 15095} 15096 15097unsigned fastEmit_X86ISD_VSRLV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) { 15098 switch (VT.SimpleTy) { 15099 case MVT::v8i16: return fastEmit_X86ISD_VSRLV_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15100 case MVT::v16i16: return fastEmit_X86ISD_VSRLV_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15101 case MVT::v32i16: return fastEmit_X86ISD_VSRLV_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15102 case MVT::v4i32: return fastEmit_X86ISD_VSRLV_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15103 case MVT::v8i32: return fastEmit_X86ISD_VSRLV_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15104 case MVT::v16i32: return fastEmit_X86ISD_VSRLV_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15105 case MVT::v2i64: return fastEmit_X86ISD_VSRLV_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15106 case MVT::v4i64: return fastEmit_X86ISD_VSRLV_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15107 case MVT::v8i64: return fastEmit_X86ISD_VSRLV_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15108 default: return 0; 15109 } 15110} 15111 15112// Top-level FastEmit function. 15113 15114unsigned fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) override { 15115 switch (Opcode) { 15116 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15117 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15118 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15119 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15120 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15121 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15122 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15123 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15124 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15125 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15126 case ISD::ROTL: return fastEmit_ISD_ROTL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15127 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15128 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15129 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15130 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15131 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15132 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15133 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15134 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15135 case ISD::STRICT_FADD: return fastEmit_ISD_STRICT_FADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15136 case ISD::STRICT_FDIV: return fastEmit_ISD_STRICT_FDIV_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15137 case ISD::STRICT_FMUL: return fastEmit_ISD_STRICT_FMUL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15138 case ISD::STRICT_FSUB: return fastEmit_ISD_STRICT_FSUB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15139 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15140 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15141 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15142 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15143 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15144 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15145 case X86ISD::ADDSUB: return fastEmit_X86ISD_ADDSUB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15146 case X86ISD::ANDNP: return fastEmit_X86ISD_ANDNP_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15147 case X86ISD::AVG: return fastEmit_X86ISD_AVG_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15148 case X86ISD::BEXTR: return fastEmit_X86ISD_BEXTR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15149 case X86ISD::BT: return fastEmit_X86ISD_BT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15150 case X86ISD::BZHI: return fastEmit_X86ISD_BZHI_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15151 case X86ISD::CMP: return fastEmit_X86ISD_CMP_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15152 case X86ISD::COMI: return fastEmit_X86ISD_COMI_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15153 case X86ISD::CVTNE2PS2BF16: return fastEmit_X86ISD_CVTNE2PS2BF16_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15154 case X86ISD::FADDS: return fastEmit_X86ISD_FADDS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15155 case X86ISD::FAND: return fastEmit_X86ISD_FAND_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15156 case X86ISD::FANDN: return fastEmit_X86ISD_FANDN_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15157 case X86ISD::FDIVS: return fastEmit_X86ISD_FDIVS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15158 case X86ISD::FGETEXPS: return fastEmit_X86ISD_FGETEXPS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15159 case X86ISD::FGETEXPS_SAE: return fastEmit_X86ISD_FGETEXPS_SAE_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15160 case X86ISD::FHADD: return fastEmit_X86ISD_FHADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15161 case X86ISD::FHSUB: return fastEmit_X86ISD_FHSUB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15162 case X86ISD::FMAX: return fastEmit_X86ISD_FMAX_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15163 case X86ISD::FMAXC: return fastEmit_X86ISD_FMAXC_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15164 case X86ISD::FMAXS: return fastEmit_X86ISD_FMAXS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15165 case X86ISD::FMAXS_SAE: return fastEmit_X86ISD_FMAXS_SAE_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15166 case X86ISD::FMAX_SAE: return fastEmit_X86ISD_FMAX_SAE_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15167 case X86ISD::FMIN: return fastEmit_X86ISD_FMIN_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15168 case X86ISD::FMINC: return fastEmit_X86ISD_FMINC_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15169 case X86ISD::FMINS: return fastEmit_X86ISD_FMINS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15170 case X86ISD::FMINS_SAE: return fastEmit_X86ISD_FMINS_SAE_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15171 case X86ISD::FMIN_SAE: return fastEmit_X86ISD_FMIN_SAE_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15172 case X86ISD::FMULS: return fastEmit_X86ISD_FMULS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15173 case X86ISD::FOR: return fastEmit_X86ISD_FOR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15174 case X86ISD::FSQRTS: return fastEmit_X86ISD_FSQRTS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15175 case X86ISD::FSUBS: return fastEmit_X86ISD_FSUBS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15176 case X86ISD::FXOR: return fastEmit_X86ISD_FXOR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15177 case X86ISD::GF2P8MULB: return fastEmit_X86ISD_GF2P8MULB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15178 case X86ISD::HADD: return fastEmit_X86ISD_HADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15179 case X86ISD::HSUB: return fastEmit_X86ISD_HSUB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15180 case X86ISD::KADD: return fastEmit_X86ISD_KADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15181 case X86ISD::KORTEST: return fastEmit_X86ISD_KORTEST_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15182 case X86ISD::KTEST: return fastEmit_X86ISD_KTEST_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15183 case X86ISD::MOVHLPS: return fastEmit_X86ISD_MOVHLPS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15184 case X86ISD::MOVLHPS: return fastEmit_X86ISD_MOVLHPS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15185 case X86ISD::MOVSD: return fastEmit_X86ISD_MOVSD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15186 case X86ISD::MOVSS: return fastEmit_X86ISD_MOVSS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15187 case X86ISD::MULHRS: return fastEmit_X86ISD_MULHRS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15188 case X86ISD::MULTISHIFT: return fastEmit_X86ISD_MULTISHIFT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15189 case X86ISD::PACKSS: return fastEmit_X86ISD_PACKSS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15190 case X86ISD::PACKUS: return fastEmit_X86ISD_PACKUS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15191 case X86ISD::PCMPEQ: return fastEmit_X86ISD_PCMPEQ_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15192 case X86ISD::PCMPGT: return fastEmit_X86ISD_PCMPGT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15193 case X86ISD::PMULDQ: return fastEmit_X86ISD_PMULDQ_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15194 case X86ISD::PMULUDQ: return fastEmit_X86ISD_PMULUDQ_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15195 case X86ISD::PSADBW: return fastEmit_X86ISD_PSADBW_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15196 case X86ISD::PSHUFB: return fastEmit_X86ISD_PSHUFB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15197 case X86ISD::PTEST: return fastEmit_X86ISD_PTEST_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15198 case X86ISD::RCP14S: return fastEmit_X86ISD_RCP14S_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15199 case X86ISD::RCP28S: return fastEmit_X86ISD_RCP28S_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15200 case X86ISD::RCP28S_SAE: return fastEmit_X86ISD_RCP28S_SAE_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15201 case X86ISD::RSQRT14S: return fastEmit_X86ISD_RSQRT14S_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15202 case X86ISD::RSQRT28S: return fastEmit_X86ISD_RSQRT28S_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15203 case X86ISD::RSQRT28S_SAE: return fastEmit_X86ISD_RSQRT28S_SAE_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15204 case X86ISD::SCALEF: return fastEmit_X86ISD_SCALEF_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15205 case X86ISD::SCALEFS: return fastEmit_X86ISD_SCALEFS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15206 case X86ISD::STRICT_FCMP: return fastEmit_X86ISD_STRICT_FCMP_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15207 case X86ISD::STRICT_FCMPS: return fastEmit_X86ISD_STRICT_FCMPS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15208 case X86ISD::TESTP: return fastEmit_X86ISD_TESTP_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15209 case X86ISD::UCOMI: return fastEmit_X86ISD_UCOMI_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15210 case X86ISD::UNPCKH: return fastEmit_X86ISD_UNPCKH_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15211 case X86ISD::UNPCKL: return fastEmit_X86ISD_UNPCKL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15212 case X86ISD::VP2INTERSECT: return fastEmit_X86ISD_VP2INTERSECT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15213 case X86ISD::VPERMV: return fastEmit_X86ISD_VPERMV_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15214 case X86ISD::VPMADDUBSW: return fastEmit_X86ISD_VPMADDUBSW_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15215 case X86ISD::VPMADDWD: return fastEmit_X86ISD_VPMADDWD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15216 case X86ISD::VPSHA: return fastEmit_X86ISD_VPSHA_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15217 case X86ISD::VPSHL: return fastEmit_X86ISD_VPSHL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15218 case X86ISD::VPSHUFBITQMB: return fastEmit_X86ISD_VPSHUFBITQMB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15219 case X86ISD::VSHL: return fastEmit_X86ISD_VSHL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15220 case X86ISD::VSHLV: return fastEmit_X86ISD_VSHLV_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15221 case X86ISD::VSRA: return fastEmit_X86ISD_VSRA_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15222 case X86ISD::VSRAV: return fastEmit_X86ISD_VSRAV_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15223 case X86ISD::VSRL: return fastEmit_X86ISD_VSRL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15224 case X86ISD::VSRLV: return fastEmit_X86ISD_VSRLV_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill); 15225 default: return 0; 15226 } 15227} 15228 15229// FastEmit functions for ISD::ADD. 15230 15231unsigned fastEmit_ISD_ADD_MVT_i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15232 if (RetVT.SimpleTy != MVT::i8) 15233 return 0; 15234 return fastEmitInst_ri(X86::ADD8ri, &X86::GR8RegClass, Op0, Op0IsKill, imm1); 15235} 15236 15237unsigned fastEmit_ISD_ADD_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15238 if (RetVT.SimpleTy != MVT::i16) 15239 return 0; 15240 return fastEmitInst_ri(X86::ADD16ri, &X86::GR16RegClass, Op0, Op0IsKill, imm1); 15241} 15242 15243unsigned fastEmit_ISD_ADD_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15244 if (RetVT.SimpleTy != MVT::i32) 15245 return 0; 15246 return fastEmitInst_ri(X86::ADD32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 15247} 15248 15249unsigned fastEmit_ISD_ADD_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15250 switch (VT.SimpleTy) { 15251 case MVT::i8: return fastEmit_ISD_ADD_MVT_i8_ri(RetVT, Op0, Op0IsKill, imm1); 15252 case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1); 15253 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1); 15254 default: return 0; 15255 } 15256} 15257 15258// FastEmit functions for ISD::AND. 15259 15260unsigned fastEmit_ISD_AND_MVT_i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15261 if (RetVT.SimpleTy != MVT::i8) 15262 return 0; 15263 return fastEmitInst_ri(X86::AND8ri, &X86::GR8RegClass, Op0, Op0IsKill, imm1); 15264} 15265 15266unsigned fastEmit_ISD_AND_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15267 if (RetVT.SimpleTy != MVT::i16) 15268 return 0; 15269 return fastEmitInst_ri(X86::AND16ri, &X86::GR16RegClass, Op0, Op0IsKill, imm1); 15270} 15271 15272unsigned fastEmit_ISD_AND_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15273 if (RetVT.SimpleTy != MVT::i32) 15274 return 0; 15275 return fastEmitInst_ri(X86::AND32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 15276} 15277 15278unsigned fastEmit_ISD_AND_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15279 switch (VT.SimpleTy) { 15280 case MVT::i8: return fastEmit_ISD_AND_MVT_i8_ri(RetVT, Op0, Op0IsKill, imm1); 15281 case MVT::i16: return fastEmit_ISD_AND_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1); 15282 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1); 15283 default: return 0; 15284 } 15285} 15286 15287// FastEmit functions for ISD::EXTRACT_VECTOR_ELT. 15288 15289unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15290 if (RetVT.SimpleTy != MVT::i32) 15291 return 0; 15292 if ((Subtarget->hasDQI())) { 15293 return fastEmitInst_ri(X86::VPEXTRDZrr, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 15294 } 15295 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 15296 return fastEmitInst_ri(X86::PEXTRDrr, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 15297 } 15298 if ((Subtarget->hasAVX()) && (!Subtarget->hasDQI())) { 15299 return fastEmitInst_ri(X86::VPEXTRDrr, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 15300 } 15301 return 0; 15302} 15303 15304unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15305 if (RetVT.SimpleTy != MVT::i64) 15306 return 0; 15307 if ((Subtarget->hasDQI())) { 15308 return fastEmitInst_ri(X86::VPEXTRQZrr, &X86::GR64RegClass, Op0, Op0IsKill, imm1); 15309 } 15310 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 15311 return fastEmitInst_ri(X86::PEXTRQrr, &X86::GR64RegClass, Op0, Op0IsKill, imm1); 15312 } 15313 if ((Subtarget->hasAVX()) && (!Subtarget->hasDQI())) { 15314 return fastEmitInst_ri(X86::VPEXTRQrr, &X86::GR64RegClass, Op0, Op0IsKill, imm1); 15315 } 15316 return 0; 15317} 15318 15319unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15320 switch (VT.SimpleTy) { 15321 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri(RetVT, Op0, Op0IsKill, imm1); 15322 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri(RetVT, Op0, Op0IsKill, imm1); 15323 default: return 0; 15324 } 15325} 15326 15327// FastEmit functions for ISD::MUL. 15328 15329unsigned fastEmit_ISD_MUL_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15330 if (RetVT.SimpleTy != MVT::i16) 15331 return 0; 15332 return fastEmitInst_ri(X86::IMUL16rri, &X86::GR16RegClass, Op0, Op0IsKill, imm1); 15333} 15334 15335unsigned fastEmit_ISD_MUL_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15336 if (RetVT.SimpleTy != MVT::i32) 15337 return 0; 15338 return fastEmitInst_ri(X86::IMUL32rri, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 15339} 15340 15341unsigned fastEmit_ISD_MUL_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15342 switch (VT.SimpleTy) { 15343 case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1); 15344 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1); 15345 default: return 0; 15346 } 15347} 15348 15349// FastEmit functions for ISD::OR. 15350 15351unsigned fastEmit_ISD_OR_MVT_i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15352 if (RetVT.SimpleTy != MVT::i8) 15353 return 0; 15354 return fastEmitInst_ri(X86::OR8ri, &X86::GR8RegClass, Op0, Op0IsKill, imm1); 15355} 15356 15357unsigned fastEmit_ISD_OR_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15358 if (RetVT.SimpleTy != MVT::i16) 15359 return 0; 15360 return fastEmitInst_ri(X86::OR16ri, &X86::GR16RegClass, Op0, Op0IsKill, imm1); 15361} 15362 15363unsigned fastEmit_ISD_OR_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15364 if (RetVT.SimpleTy != MVT::i32) 15365 return 0; 15366 return fastEmitInst_ri(X86::OR32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 15367} 15368 15369unsigned fastEmit_ISD_OR_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15370 switch (VT.SimpleTy) { 15371 case MVT::i8: return fastEmit_ISD_OR_MVT_i8_ri(RetVT, Op0, Op0IsKill, imm1); 15372 case MVT::i16: return fastEmit_ISD_OR_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1); 15373 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1); 15374 default: return 0; 15375 } 15376} 15377 15378// FastEmit functions for ISD::ROTL. 15379 15380unsigned fastEmit_ISD_ROTL_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15381 if (RetVT.SimpleTy != MVT::i32) 15382 return 0; 15383 if ((Subtarget->hasFastSHLDRotate())) { 15384 return fastEmitInst_ri(X86::SHLDROT32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 15385 } 15386 return 0; 15387} 15388 15389unsigned fastEmit_ISD_ROTL_MVT_i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15390 if (RetVT.SimpleTy != MVT::i64) 15391 return 0; 15392 if ((Subtarget->hasFastSHLDRotate())) { 15393 return fastEmitInst_ri(X86::SHLDROT64ri, &X86::GR64RegClass, Op0, Op0IsKill, imm1); 15394 } 15395 return 0; 15396} 15397 15398unsigned fastEmit_ISD_ROTL_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15399 switch (VT.SimpleTy) { 15400 case MVT::i32: return fastEmit_ISD_ROTL_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1); 15401 case MVT::i64: return fastEmit_ISD_ROTL_MVT_i64_ri(RetVT, Op0, Op0IsKill, imm1); 15402 default: return 0; 15403 } 15404} 15405 15406// FastEmit functions for ISD::ROTR. 15407 15408unsigned fastEmit_ISD_ROTR_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15409 if (RetVT.SimpleTy != MVT::i32) 15410 return 0; 15411 if ((Subtarget->hasBMI2())) { 15412 return fastEmitInst_ri(X86::RORX32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 15413 } 15414 if ((Subtarget->hasFastSHLDRotate())) { 15415 return fastEmitInst_ri(X86::SHRDROT32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 15416 } 15417 return 0; 15418} 15419 15420unsigned fastEmit_ISD_ROTR_MVT_i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15421 if (RetVT.SimpleTy != MVT::i64) 15422 return 0; 15423 if ((Subtarget->hasBMI2())) { 15424 return fastEmitInst_ri(X86::RORX64ri, &X86::GR64RegClass, Op0, Op0IsKill, imm1); 15425 } 15426 if ((Subtarget->hasFastSHLDRotate())) { 15427 return fastEmitInst_ri(X86::SHRDROT64ri, &X86::GR64RegClass, Op0, Op0IsKill, imm1); 15428 } 15429 return 0; 15430} 15431 15432unsigned fastEmit_ISD_ROTR_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15433 switch (VT.SimpleTy) { 15434 case MVT::i32: return fastEmit_ISD_ROTR_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1); 15435 case MVT::i64: return fastEmit_ISD_ROTR_MVT_i64_ri(RetVT, Op0, Op0IsKill, imm1); 15436 default: return 0; 15437 } 15438} 15439 15440// FastEmit functions for ISD::SHL. 15441 15442unsigned fastEmit_ISD_SHL_MVT_i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15443 if (RetVT.SimpleTy != MVT::i8) 15444 return 0; 15445 return fastEmitInst_ri(X86::SHL8ri, &X86::GR8RegClass, Op0, Op0IsKill, imm1); 15446} 15447 15448unsigned fastEmit_ISD_SHL_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15449 if (RetVT.SimpleTy != MVT::i16) 15450 return 0; 15451 return fastEmitInst_ri(X86::SHL16ri, &X86::GR16RegClass, Op0, Op0IsKill, imm1); 15452} 15453 15454unsigned fastEmit_ISD_SHL_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15455 if (RetVT.SimpleTy != MVT::i32) 15456 return 0; 15457 return fastEmitInst_ri(X86::SHL32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 15458} 15459 15460unsigned fastEmit_ISD_SHL_MVT_i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15461 if (RetVT.SimpleTy != MVT::i64) 15462 return 0; 15463 return fastEmitInst_ri(X86::SHL64ri, &X86::GR64RegClass, Op0, Op0IsKill, imm1); 15464} 15465 15466unsigned fastEmit_ISD_SHL_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15467 switch (VT.SimpleTy) { 15468 case MVT::i8: return fastEmit_ISD_SHL_MVT_i8_ri(RetVT, Op0, Op0IsKill, imm1); 15469 case MVT::i16: return fastEmit_ISD_SHL_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1); 15470 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1); 15471 case MVT::i64: return fastEmit_ISD_SHL_MVT_i64_ri(RetVT, Op0, Op0IsKill, imm1); 15472 default: return 0; 15473 } 15474} 15475 15476// FastEmit functions for ISD::SRA. 15477 15478unsigned fastEmit_ISD_SRA_MVT_i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15479 if (RetVT.SimpleTy != MVT::i8) 15480 return 0; 15481 return fastEmitInst_ri(X86::SAR8ri, &X86::GR8RegClass, Op0, Op0IsKill, imm1); 15482} 15483 15484unsigned fastEmit_ISD_SRA_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15485 if (RetVT.SimpleTy != MVT::i16) 15486 return 0; 15487 return fastEmitInst_ri(X86::SAR16ri, &X86::GR16RegClass, Op0, Op0IsKill, imm1); 15488} 15489 15490unsigned fastEmit_ISD_SRA_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15491 if (RetVT.SimpleTy != MVT::i32) 15492 return 0; 15493 return fastEmitInst_ri(X86::SAR32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 15494} 15495 15496unsigned fastEmit_ISD_SRA_MVT_i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15497 if (RetVT.SimpleTy != MVT::i64) 15498 return 0; 15499 return fastEmitInst_ri(X86::SAR64ri, &X86::GR64RegClass, Op0, Op0IsKill, imm1); 15500} 15501 15502unsigned fastEmit_ISD_SRA_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15503 switch (VT.SimpleTy) { 15504 case MVT::i8: return fastEmit_ISD_SRA_MVT_i8_ri(RetVT, Op0, Op0IsKill, imm1); 15505 case MVT::i16: return fastEmit_ISD_SRA_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1); 15506 case MVT::i32: return fastEmit_ISD_SRA_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1); 15507 case MVT::i64: return fastEmit_ISD_SRA_MVT_i64_ri(RetVT, Op0, Op0IsKill, imm1); 15508 default: return 0; 15509 } 15510} 15511 15512// FastEmit functions for ISD::SRL. 15513 15514unsigned fastEmit_ISD_SRL_MVT_i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15515 if (RetVT.SimpleTy != MVT::i8) 15516 return 0; 15517 return fastEmitInst_ri(X86::SHR8ri, &X86::GR8RegClass, Op0, Op0IsKill, imm1); 15518} 15519 15520unsigned fastEmit_ISD_SRL_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15521 if (RetVT.SimpleTy != MVT::i16) 15522 return 0; 15523 return fastEmitInst_ri(X86::SHR16ri, &X86::GR16RegClass, Op0, Op0IsKill, imm1); 15524} 15525 15526unsigned fastEmit_ISD_SRL_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15527 if (RetVT.SimpleTy != MVT::i32) 15528 return 0; 15529 return fastEmitInst_ri(X86::SHR32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 15530} 15531 15532unsigned fastEmit_ISD_SRL_MVT_i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15533 if (RetVT.SimpleTy != MVT::i64) 15534 return 0; 15535 return fastEmitInst_ri(X86::SHR64ri, &X86::GR64RegClass, Op0, Op0IsKill, imm1); 15536} 15537 15538unsigned fastEmit_ISD_SRL_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15539 switch (VT.SimpleTy) { 15540 case MVT::i8: return fastEmit_ISD_SRL_MVT_i8_ri(RetVT, Op0, Op0IsKill, imm1); 15541 case MVT::i16: return fastEmit_ISD_SRL_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1); 15542 case MVT::i32: return fastEmit_ISD_SRL_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1); 15543 case MVT::i64: return fastEmit_ISD_SRL_MVT_i64_ri(RetVT, Op0, Op0IsKill, imm1); 15544 default: return 0; 15545 } 15546} 15547 15548// FastEmit functions for ISD::SUB. 15549 15550unsigned fastEmit_ISD_SUB_MVT_i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15551 if (RetVT.SimpleTy != MVT::i8) 15552 return 0; 15553 return fastEmitInst_ri(X86::SUB8ri, &X86::GR8RegClass, Op0, Op0IsKill, imm1); 15554} 15555 15556unsigned fastEmit_ISD_SUB_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15557 if (RetVT.SimpleTy != MVT::i16) 15558 return 0; 15559 return fastEmitInst_ri(X86::SUB16ri, &X86::GR16RegClass, Op0, Op0IsKill, imm1); 15560} 15561 15562unsigned fastEmit_ISD_SUB_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15563 if (RetVT.SimpleTy != MVT::i32) 15564 return 0; 15565 return fastEmitInst_ri(X86::SUB32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 15566} 15567 15568unsigned fastEmit_ISD_SUB_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15569 switch (VT.SimpleTy) { 15570 case MVT::i8: return fastEmit_ISD_SUB_MVT_i8_ri(RetVT, Op0, Op0IsKill, imm1); 15571 case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1); 15572 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1); 15573 default: return 0; 15574 } 15575} 15576 15577// FastEmit functions for ISD::XOR. 15578 15579unsigned fastEmit_ISD_XOR_MVT_i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15580 if (RetVT.SimpleTy != MVT::i8) 15581 return 0; 15582 return fastEmitInst_ri(X86::XOR8ri, &X86::GR8RegClass, Op0, Op0IsKill, imm1); 15583} 15584 15585unsigned fastEmit_ISD_XOR_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15586 if (RetVT.SimpleTy != MVT::i16) 15587 return 0; 15588 return fastEmitInst_ri(X86::XOR16ri, &X86::GR16RegClass, Op0, Op0IsKill, imm1); 15589} 15590 15591unsigned fastEmit_ISD_XOR_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15592 if (RetVT.SimpleTy != MVT::i32) 15593 return 0; 15594 return fastEmitInst_ri(X86::XOR32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 15595} 15596 15597unsigned fastEmit_ISD_XOR_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15598 switch (VT.SimpleTy) { 15599 case MVT::i8: return fastEmit_ISD_XOR_MVT_i8_ri(RetVT, Op0, Op0IsKill, imm1); 15600 case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1); 15601 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1); 15602 default: return 0; 15603 } 15604} 15605 15606// FastEmit functions for X86ISD::BEXTR. 15607 15608unsigned fastEmit_X86ISD_BEXTR_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15609 if (RetVT.SimpleTy != MVT::i32) 15610 return 0; 15611 if ((Subtarget->hasTBM())) { 15612 return fastEmitInst_ri(X86::BEXTRI32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 15613 } 15614 return 0; 15615} 15616 15617unsigned fastEmit_X86ISD_BEXTR_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15618 switch (VT.SimpleTy) { 15619 case MVT::i32: return fastEmit_X86ISD_BEXTR_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1); 15620 default: return 0; 15621 } 15622} 15623 15624// FastEmit functions for X86ISD::BT. 15625 15626unsigned fastEmit_X86ISD_BT_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15627 if (RetVT.SimpleTy != MVT::i32) 15628 return 0; 15629 return fastEmitInst_ri(X86::BT16ri8, &X86::GR16RegClass, Op0, Op0IsKill, imm1); 15630} 15631 15632unsigned fastEmit_X86ISD_BT_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15633 if (RetVT.SimpleTy != MVT::i32) 15634 return 0; 15635 return fastEmitInst_ri(X86::BT32ri8, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 15636} 15637 15638unsigned fastEmit_X86ISD_BT_MVT_i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15639 if (RetVT.SimpleTy != MVT::i32) 15640 return 0; 15641 return fastEmitInst_ri(X86::BT64ri8, &X86::GR64RegClass, Op0, Op0IsKill, imm1); 15642} 15643 15644unsigned fastEmit_X86ISD_BT_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15645 switch (VT.SimpleTy) { 15646 case MVT::i16: return fastEmit_X86ISD_BT_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1); 15647 case MVT::i32: return fastEmit_X86ISD_BT_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1); 15648 case MVT::i64: return fastEmit_X86ISD_BT_MVT_i64_ri(RetVT, Op0, Op0IsKill, imm1); 15649 default: return 0; 15650 } 15651} 15652 15653// FastEmit functions for X86ISD::PEXTRB. 15654 15655unsigned fastEmit_X86ISD_PEXTRB_MVT_v16i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15656 if (RetVT.SimpleTy != MVT::i32) 15657 return 0; 15658 if ((Subtarget->hasBWI())) { 15659 return fastEmitInst_ri(X86::VPEXTRBZrr, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 15660 } 15661 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) { 15662 return fastEmitInst_ri(X86::PEXTRBrr, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 15663 } 15664 if ((Subtarget->hasAVX()) && (!Subtarget->hasBWI())) { 15665 return fastEmitInst_ri(X86::VPEXTRBrr, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 15666 } 15667 return 0; 15668} 15669 15670unsigned fastEmit_X86ISD_PEXTRB_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15671 switch (VT.SimpleTy) { 15672 case MVT::v16i8: return fastEmit_X86ISD_PEXTRB_MVT_v16i8_ri(RetVT, Op0, Op0IsKill, imm1); 15673 default: return 0; 15674 } 15675} 15676 15677// FastEmit functions for X86ISD::PEXTRW. 15678 15679unsigned fastEmit_X86ISD_PEXTRW_MVT_v8i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15680 if (RetVT.SimpleTy != MVT::i32) 15681 return 0; 15682 if ((Subtarget->hasBWI())) { 15683 return fastEmitInst_ri(X86::VPEXTRWZrr, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 15684 } 15685 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) { 15686 return fastEmitInst_ri(X86::PEXTRWrr, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 15687 } 15688 if ((Subtarget->hasAVX()) && (!Subtarget->hasBWI())) { 15689 return fastEmitInst_ri(X86::VPEXTRWrr, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 15690 } 15691 return 0; 15692} 15693 15694unsigned fastEmit_X86ISD_PEXTRW_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15695 switch (VT.SimpleTy) { 15696 case MVT::v8i16: return fastEmit_X86ISD_PEXTRW_MVT_v8i16_ri(RetVT, Op0, Op0IsKill, imm1); 15697 default: return 0; 15698 } 15699} 15700 15701// Top-level FastEmit function. 15702 15703unsigned fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) override { 15704 if (VT == MVT::i64 && Predicate_i64immSExt32(imm1)) 15705 if (unsigned Reg = fastEmit_ri_Predicate_i64immSExt32(VT, RetVT, Opcode, Op0, Op0IsKill, imm1)) 15706 return Reg; 15707 15708 if (VT == MVT::i16 && Predicate_i16immSExt8(imm1)) 15709 if (unsigned Reg = fastEmit_ri_Predicate_i16immSExt8(VT, RetVT, Opcode, Op0, Op0IsKill, imm1)) 15710 return Reg; 15711 15712 if (VT == MVT::i32 && Predicate_i32immSExt8(imm1)) 15713 if (unsigned Reg = fastEmit_ri_Predicate_i32immSExt8(VT, RetVT, Opcode, Op0, Op0IsKill, imm1)) 15714 return Reg; 15715 15716 if (VT == MVT::i64 && Predicate_i64immSExt8(imm1)) 15717 if (unsigned Reg = fastEmit_ri_Predicate_i64immSExt8(VT, RetVT, Opcode, Op0, Op0IsKill, imm1)) 15718 return Reg; 15719 15720 switch (Opcode) { 15721 case ISD::ADD: return fastEmit_ISD_ADD_ri(VT, RetVT, Op0, Op0IsKill, imm1); 15722 case ISD::AND: return fastEmit_ISD_AND_ri(VT, RetVT, Op0, Op0IsKill, imm1); 15723 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(VT, RetVT, Op0, Op0IsKill, imm1); 15724 case ISD::MUL: return fastEmit_ISD_MUL_ri(VT, RetVT, Op0, Op0IsKill, imm1); 15725 case ISD::OR: return fastEmit_ISD_OR_ri(VT, RetVT, Op0, Op0IsKill, imm1); 15726 case ISD::ROTL: return fastEmit_ISD_ROTL_ri(VT, RetVT, Op0, Op0IsKill, imm1); 15727 case ISD::ROTR: return fastEmit_ISD_ROTR_ri(VT, RetVT, Op0, Op0IsKill, imm1); 15728 case ISD::SHL: return fastEmit_ISD_SHL_ri(VT, RetVT, Op0, Op0IsKill, imm1); 15729 case ISD::SRA: return fastEmit_ISD_SRA_ri(VT, RetVT, Op0, Op0IsKill, imm1); 15730 case ISD::SRL: return fastEmit_ISD_SRL_ri(VT, RetVT, Op0, Op0IsKill, imm1); 15731 case ISD::SUB: return fastEmit_ISD_SUB_ri(VT, RetVT, Op0, Op0IsKill, imm1); 15732 case ISD::XOR: return fastEmit_ISD_XOR_ri(VT, RetVT, Op0, Op0IsKill, imm1); 15733 case X86ISD::BEXTR: return fastEmit_X86ISD_BEXTR_ri(VT, RetVT, Op0, Op0IsKill, imm1); 15734 case X86ISD::BT: return fastEmit_X86ISD_BT_ri(VT, RetVT, Op0, Op0IsKill, imm1); 15735 case X86ISD::PEXTRB: return fastEmit_X86ISD_PEXTRB_ri(VT, RetVT, Op0, Op0IsKill, imm1); 15736 case X86ISD::PEXTRW: return fastEmit_X86ISD_PEXTRW_ri(VT, RetVT, Op0, Op0IsKill, imm1); 15737 default: return 0; 15738 } 15739} 15740 15741// FastEmit functions for ISD::ADD. 15742 15743unsigned fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15744 if (RetVT.SimpleTy != MVT::i64) 15745 return 0; 15746 return fastEmitInst_ri(X86::ADD64ri32, &X86::GR64RegClass, Op0, Op0IsKill, imm1); 15747} 15748 15749unsigned fastEmit_ISD_ADD_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15750 switch (VT.SimpleTy) { 15751 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, Op0IsKill, imm1); 15752 default: return 0; 15753 } 15754} 15755 15756// FastEmit functions for ISD::AND. 15757 15758unsigned fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15759 if (RetVT.SimpleTy != MVT::i64) 15760 return 0; 15761 return fastEmitInst_ri(X86::AND64ri32, &X86::GR64RegClass, Op0, Op0IsKill, imm1); 15762} 15763 15764unsigned fastEmit_ISD_AND_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15765 switch (VT.SimpleTy) { 15766 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, Op0IsKill, imm1); 15767 default: return 0; 15768 } 15769} 15770 15771// FastEmit functions for ISD::MUL. 15772 15773unsigned fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15774 if (RetVT.SimpleTy != MVT::i64) 15775 return 0; 15776 return fastEmitInst_ri(X86::IMUL64rri32, &X86::GR64RegClass, Op0, Op0IsKill, imm1); 15777} 15778 15779unsigned fastEmit_ISD_MUL_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15780 switch (VT.SimpleTy) { 15781 case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, Op0IsKill, imm1); 15782 default: return 0; 15783 } 15784} 15785 15786// FastEmit functions for ISD::OR. 15787 15788unsigned fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15789 if (RetVT.SimpleTy != MVT::i64) 15790 return 0; 15791 return fastEmitInst_ri(X86::OR64ri32, &X86::GR64RegClass, Op0, Op0IsKill, imm1); 15792} 15793 15794unsigned fastEmit_ISD_OR_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15795 switch (VT.SimpleTy) { 15796 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, Op0IsKill, imm1); 15797 default: return 0; 15798 } 15799} 15800 15801// FastEmit functions for ISD::SUB. 15802 15803unsigned fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15804 if (RetVT.SimpleTy != MVT::i64) 15805 return 0; 15806 return fastEmitInst_ri(X86::SUB64ri32, &X86::GR64RegClass, Op0, Op0IsKill, imm1); 15807} 15808 15809unsigned fastEmit_ISD_SUB_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15810 switch (VT.SimpleTy) { 15811 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, Op0IsKill, imm1); 15812 default: return 0; 15813 } 15814} 15815 15816// FastEmit functions for ISD::XOR. 15817 15818unsigned fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15819 if (RetVT.SimpleTy != MVT::i64) 15820 return 0; 15821 return fastEmitInst_ri(X86::XOR64ri32, &X86::GR64RegClass, Op0, Op0IsKill, imm1); 15822} 15823 15824unsigned fastEmit_ISD_XOR_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15825 switch (VT.SimpleTy) { 15826 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, Op0IsKill, imm1); 15827 default: return 0; 15828 } 15829} 15830 15831// FastEmit functions for X86ISD::BEXTR. 15832 15833unsigned fastEmit_X86ISD_BEXTR_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15834 if (RetVT.SimpleTy != MVT::i64) 15835 return 0; 15836 if ((Subtarget->hasTBM())) { 15837 return fastEmitInst_ri(X86::BEXTRI64ri, &X86::GR64RegClass, Op0, Op0IsKill, imm1); 15838 } 15839 return 0; 15840} 15841 15842unsigned fastEmit_X86ISD_BEXTR_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15843 switch (VT.SimpleTy) { 15844 case MVT::i64: return fastEmit_X86ISD_BEXTR_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, Op0IsKill, imm1); 15845 default: return 0; 15846 } 15847} 15848 15849// Top-level FastEmit function. 15850 15851unsigned fastEmit_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15852 switch (Opcode) { 15853 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_i64immSExt32(VT, RetVT, Op0, Op0IsKill, imm1); 15854 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_i64immSExt32(VT, RetVT, Op0, Op0IsKill, imm1); 15855 case ISD::MUL: return fastEmit_ISD_MUL_ri_Predicate_i64immSExt32(VT, RetVT, Op0, Op0IsKill, imm1); 15856 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_i64immSExt32(VT, RetVT, Op0, Op0IsKill, imm1); 15857 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_i64immSExt32(VT, RetVT, Op0, Op0IsKill, imm1); 15858 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_i64immSExt32(VT, RetVT, Op0, Op0IsKill, imm1); 15859 case X86ISD::BEXTR: return fastEmit_X86ISD_BEXTR_ri_Predicate_i64immSExt32(VT, RetVT, Op0, Op0IsKill, imm1); 15860 default: return 0; 15861 } 15862} 15863 15864// FastEmit functions for ISD::ADD. 15865 15866unsigned fastEmit_ISD_ADD_MVT_i16_ri_Predicate_i16immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15867 if (RetVT.SimpleTy != MVT::i16) 15868 return 0; 15869 return fastEmitInst_ri(X86::ADD16ri8, &X86::GR16RegClass, Op0, Op0IsKill, imm1); 15870} 15871 15872unsigned fastEmit_ISD_ADD_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15873 switch (VT.SimpleTy) { 15874 case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_ri_Predicate_i16immSExt8(RetVT, Op0, Op0IsKill, imm1); 15875 default: return 0; 15876 } 15877} 15878 15879// FastEmit functions for ISD::AND. 15880 15881unsigned fastEmit_ISD_AND_MVT_i16_ri_Predicate_i16immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15882 if (RetVT.SimpleTy != MVT::i16) 15883 return 0; 15884 return fastEmitInst_ri(X86::AND16ri8, &X86::GR16RegClass, Op0, Op0IsKill, imm1); 15885} 15886 15887unsigned fastEmit_ISD_AND_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15888 switch (VT.SimpleTy) { 15889 case MVT::i16: return fastEmit_ISD_AND_MVT_i16_ri_Predicate_i16immSExt8(RetVT, Op0, Op0IsKill, imm1); 15890 default: return 0; 15891 } 15892} 15893 15894// FastEmit functions for ISD::MUL. 15895 15896unsigned fastEmit_ISD_MUL_MVT_i16_ri_Predicate_i16immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15897 if (RetVT.SimpleTy != MVT::i16) 15898 return 0; 15899 return fastEmitInst_ri(X86::IMUL16rri8, &X86::GR16RegClass, Op0, Op0IsKill, imm1); 15900} 15901 15902unsigned fastEmit_ISD_MUL_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15903 switch (VT.SimpleTy) { 15904 case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_ri_Predicate_i16immSExt8(RetVT, Op0, Op0IsKill, imm1); 15905 default: return 0; 15906 } 15907} 15908 15909// FastEmit functions for ISD::OR. 15910 15911unsigned fastEmit_ISD_OR_MVT_i16_ri_Predicate_i16immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15912 if (RetVT.SimpleTy != MVT::i16) 15913 return 0; 15914 return fastEmitInst_ri(X86::OR16ri8, &X86::GR16RegClass, Op0, Op0IsKill, imm1); 15915} 15916 15917unsigned fastEmit_ISD_OR_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15918 switch (VT.SimpleTy) { 15919 case MVT::i16: return fastEmit_ISD_OR_MVT_i16_ri_Predicate_i16immSExt8(RetVT, Op0, Op0IsKill, imm1); 15920 default: return 0; 15921 } 15922} 15923 15924// FastEmit functions for ISD::SUB. 15925 15926unsigned fastEmit_ISD_SUB_MVT_i16_ri_Predicate_i16immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15927 if (RetVT.SimpleTy != MVT::i16) 15928 return 0; 15929 return fastEmitInst_ri(X86::SUB16ri8, &X86::GR16RegClass, Op0, Op0IsKill, imm1); 15930} 15931 15932unsigned fastEmit_ISD_SUB_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15933 switch (VT.SimpleTy) { 15934 case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_ri_Predicate_i16immSExt8(RetVT, Op0, Op0IsKill, imm1); 15935 default: return 0; 15936 } 15937} 15938 15939// FastEmit functions for ISD::XOR. 15940 15941unsigned fastEmit_ISD_XOR_MVT_i16_ri_Predicate_i16immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15942 if (RetVT.SimpleTy != MVT::i16) 15943 return 0; 15944 return fastEmitInst_ri(X86::XOR16ri8, &X86::GR16RegClass, Op0, Op0IsKill, imm1); 15945} 15946 15947unsigned fastEmit_ISD_XOR_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15948 switch (VT.SimpleTy) { 15949 case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_ri_Predicate_i16immSExt8(RetVT, Op0, Op0IsKill, imm1); 15950 default: return 0; 15951 } 15952} 15953 15954// Top-level FastEmit function. 15955 15956unsigned fastEmit_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15957 switch (Opcode) { 15958 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_i16immSExt8(VT, RetVT, Op0, Op0IsKill, imm1); 15959 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_i16immSExt8(VT, RetVT, Op0, Op0IsKill, imm1); 15960 case ISD::MUL: return fastEmit_ISD_MUL_ri_Predicate_i16immSExt8(VT, RetVT, Op0, Op0IsKill, imm1); 15961 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_i16immSExt8(VT, RetVT, Op0, Op0IsKill, imm1); 15962 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_i16immSExt8(VT, RetVT, Op0, Op0IsKill, imm1); 15963 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_i16immSExt8(VT, RetVT, Op0, Op0IsKill, imm1); 15964 default: return 0; 15965 } 15966} 15967 15968// FastEmit functions for ISD::ADD. 15969 15970unsigned fastEmit_ISD_ADD_MVT_i32_ri_Predicate_i32immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15971 if (RetVT.SimpleTy != MVT::i32) 15972 return 0; 15973 return fastEmitInst_ri(X86::ADD32ri8, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 15974} 15975 15976unsigned fastEmit_ISD_ADD_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15977 switch (VT.SimpleTy) { 15978 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_i32immSExt8(RetVT, Op0, Op0IsKill, imm1); 15979 default: return 0; 15980 } 15981} 15982 15983// FastEmit functions for ISD::AND. 15984 15985unsigned fastEmit_ISD_AND_MVT_i32_ri_Predicate_i32immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15986 if (RetVT.SimpleTy != MVT::i32) 15987 return 0; 15988 return fastEmitInst_ri(X86::AND32ri8, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 15989} 15990 15991unsigned fastEmit_ISD_AND_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 15992 switch (VT.SimpleTy) { 15993 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri_Predicate_i32immSExt8(RetVT, Op0, Op0IsKill, imm1); 15994 default: return 0; 15995 } 15996} 15997 15998// FastEmit functions for ISD::MUL. 15999 16000unsigned fastEmit_ISD_MUL_MVT_i32_ri_Predicate_i32immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 16001 if (RetVT.SimpleTy != MVT::i32) 16002 return 0; 16003 return fastEmitInst_ri(X86::IMUL32rri8, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 16004} 16005 16006unsigned fastEmit_ISD_MUL_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 16007 switch (VT.SimpleTy) { 16008 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_ri_Predicate_i32immSExt8(RetVT, Op0, Op0IsKill, imm1); 16009 default: return 0; 16010 } 16011} 16012 16013// FastEmit functions for ISD::OR. 16014 16015unsigned fastEmit_ISD_OR_MVT_i32_ri_Predicate_i32immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 16016 if (RetVT.SimpleTy != MVT::i32) 16017 return 0; 16018 return fastEmitInst_ri(X86::OR32ri8, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 16019} 16020 16021unsigned fastEmit_ISD_OR_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 16022 switch (VT.SimpleTy) { 16023 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri_Predicate_i32immSExt8(RetVT, Op0, Op0IsKill, imm1); 16024 default: return 0; 16025 } 16026} 16027 16028// FastEmit functions for ISD::SUB. 16029 16030unsigned fastEmit_ISD_SUB_MVT_i32_ri_Predicate_i32immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 16031 if (RetVT.SimpleTy != MVT::i32) 16032 return 0; 16033 return fastEmitInst_ri(X86::SUB32ri8, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 16034} 16035 16036unsigned fastEmit_ISD_SUB_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 16037 switch (VT.SimpleTy) { 16038 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri_Predicate_i32immSExt8(RetVT, Op0, Op0IsKill, imm1); 16039 default: return 0; 16040 } 16041} 16042 16043// FastEmit functions for ISD::XOR. 16044 16045unsigned fastEmit_ISD_XOR_MVT_i32_ri_Predicate_i32immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 16046 if (RetVT.SimpleTy != MVT::i32) 16047 return 0; 16048 return fastEmitInst_ri(X86::XOR32ri8, &X86::GR32RegClass, Op0, Op0IsKill, imm1); 16049} 16050 16051unsigned fastEmit_ISD_XOR_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 16052 switch (VT.SimpleTy) { 16053 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri_Predicate_i32immSExt8(RetVT, Op0, Op0IsKill, imm1); 16054 default: return 0; 16055 } 16056} 16057 16058// Top-level FastEmit function. 16059 16060unsigned fastEmit_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 16061 switch (Opcode) { 16062 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_i32immSExt8(VT, RetVT, Op0, Op0IsKill, imm1); 16063 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_i32immSExt8(VT, RetVT, Op0, Op0IsKill, imm1); 16064 case ISD::MUL: return fastEmit_ISD_MUL_ri_Predicate_i32immSExt8(VT, RetVT, Op0, Op0IsKill, imm1); 16065 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_i32immSExt8(VT, RetVT, Op0, Op0IsKill, imm1); 16066 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_i32immSExt8(VT, RetVT, Op0, Op0IsKill, imm1); 16067 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_i32immSExt8(VT, RetVT, Op0, Op0IsKill, imm1); 16068 default: return 0; 16069 } 16070} 16071 16072// FastEmit functions for ISD::ADD. 16073 16074unsigned fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 16075 if (RetVT.SimpleTy != MVT::i64) 16076 return 0; 16077 return fastEmitInst_ri(X86::ADD64ri8, &X86::GR64RegClass, Op0, Op0IsKill, imm1); 16078} 16079 16080unsigned fastEmit_ISD_ADD_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 16081 switch (VT.SimpleTy) { 16082 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt8(RetVT, Op0, Op0IsKill, imm1); 16083 default: return 0; 16084 } 16085} 16086 16087// FastEmit functions for ISD::AND. 16088 16089unsigned fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 16090 if (RetVT.SimpleTy != MVT::i64) 16091 return 0; 16092 return fastEmitInst_ri(X86::AND64ri8, &X86::GR64RegClass, Op0, Op0IsKill, imm1); 16093} 16094 16095unsigned fastEmit_ISD_AND_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 16096 switch (VT.SimpleTy) { 16097 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt8(RetVT, Op0, Op0IsKill, imm1); 16098 default: return 0; 16099 } 16100} 16101 16102// FastEmit functions for ISD::MUL. 16103 16104unsigned fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 16105 if (RetVT.SimpleTy != MVT::i64) 16106 return 0; 16107 return fastEmitInst_ri(X86::IMUL64rri8, &X86::GR64RegClass, Op0, Op0IsKill, imm1); 16108} 16109 16110unsigned fastEmit_ISD_MUL_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 16111 switch (VT.SimpleTy) { 16112 case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt8(RetVT, Op0, Op0IsKill, imm1); 16113 default: return 0; 16114 } 16115} 16116 16117// FastEmit functions for ISD::OR. 16118 16119unsigned fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 16120 if (RetVT.SimpleTy != MVT::i64) 16121 return 0; 16122 return fastEmitInst_ri(X86::OR64ri8, &X86::GR64RegClass, Op0, Op0IsKill, imm1); 16123} 16124 16125unsigned fastEmit_ISD_OR_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 16126 switch (VT.SimpleTy) { 16127 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt8(RetVT, Op0, Op0IsKill, imm1); 16128 default: return 0; 16129 } 16130} 16131 16132// FastEmit functions for ISD::SUB. 16133 16134unsigned fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 16135 if (RetVT.SimpleTy != MVT::i64) 16136 return 0; 16137 return fastEmitInst_ri(X86::SUB64ri8, &X86::GR64RegClass, Op0, Op0IsKill, imm1); 16138} 16139 16140unsigned fastEmit_ISD_SUB_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 16141 switch (VT.SimpleTy) { 16142 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt8(RetVT, Op0, Op0IsKill, imm1); 16143 default: return 0; 16144 } 16145} 16146 16147// FastEmit functions for ISD::XOR. 16148 16149unsigned fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 16150 if (RetVT.SimpleTy != MVT::i64) 16151 return 0; 16152 return fastEmitInst_ri(X86::XOR64ri8, &X86::GR64RegClass, Op0, Op0IsKill, imm1); 16153} 16154 16155unsigned fastEmit_ISD_XOR_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 16156 switch (VT.SimpleTy) { 16157 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt8(RetVT, Op0, Op0IsKill, imm1); 16158 default: return 0; 16159 } 16160} 16161 16162// Top-level FastEmit function. 16163 16164unsigned fastEmit_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) { 16165 switch (Opcode) { 16166 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_i64immSExt8(VT, RetVT, Op0, Op0IsKill, imm1); 16167 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_i64immSExt8(VT, RetVT, Op0, Op0IsKill, imm1); 16168 case ISD::MUL: return fastEmit_ISD_MUL_ri_Predicate_i64immSExt8(VT, RetVT, Op0, Op0IsKill, imm1); 16169 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_i64immSExt8(VT, RetVT, Op0, Op0IsKill, imm1); 16170 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_i64immSExt8(VT, RetVT, Op0, Op0IsKill, imm1); 16171 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_i64immSExt8(VT, RetVT, Op0, Op0IsKill, imm1); 16172 default: return 0; 16173 } 16174} 16175 16176// FastEmit functions for ISD::Constant. 16177 16178unsigned fastEmit_ISD_Constant_MVT_i8_i(MVT RetVT, uint64_t imm0) { 16179 if (RetVT.SimpleTy != MVT::i8) 16180 return 0; 16181 return fastEmitInst_i(X86::MOV8ri, &X86::GR8RegClass, imm0); 16182} 16183 16184unsigned fastEmit_ISD_Constant_MVT_i16_i(MVT RetVT, uint64_t imm0) { 16185 if (RetVT.SimpleTy != MVT::i16) 16186 return 0; 16187 return fastEmitInst_i(X86::MOV16ri, &X86::GR16RegClass, imm0); 16188} 16189 16190unsigned fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) { 16191 switch (VT.SimpleTy) { 16192 case MVT::i8: return fastEmit_ISD_Constant_MVT_i8_i(RetVT, imm0); 16193 case MVT::i16: return fastEmit_ISD_Constant_MVT_i16_i(RetVT, imm0); 16194 default: return 0; 16195 } 16196} 16197 16198// Top-level FastEmit function. 16199 16200unsigned fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override { 16201 switch (Opcode) { 16202 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0); 16203 default: return 0; 16204 } 16205} 16206 16207