Lines Matching refs:MVT
76 MVT PtrVT = MVT::getIntegerVT(8 * TM.getPointerSize()); in X86TargetLowering()
133 addRegisterClass(MVT::i8, &X86::GR8RegClass); in X86TargetLowering()
134 addRegisterClass(MVT::i16, &X86::GR16RegClass); in X86TargetLowering()
135 addRegisterClass(MVT::i32, &X86::GR32RegClass); in X86TargetLowering()
137 addRegisterClass(MVT::i64, &X86::GR64RegClass); in X86TargetLowering()
139 for (MVT VT : MVT::integer_valuetypes()) in X86TargetLowering()
140 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); in X86TargetLowering()
143 setTruncStoreAction(MVT::i64, MVT::i32, Expand); in X86TargetLowering()
144 setTruncStoreAction(MVT::i64, MVT::i16, Expand); in X86TargetLowering()
145 setTruncStoreAction(MVT::i64, MVT::i8 , Expand); in X86TargetLowering()
146 setTruncStoreAction(MVT::i32, MVT::i16, Expand); in X86TargetLowering()
147 setTruncStoreAction(MVT::i32, MVT::i8 , Expand); in X86TargetLowering()
148 setTruncStoreAction(MVT::i16, MVT::i8, Expand); in X86TargetLowering()
150 setTruncStoreAction(MVT::f64, MVT::f32, Expand); in X86TargetLowering()
153 setCondCodeAction(ISD::SETOEQ, MVT::f32, Expand); in X86TargetLowering()
154 setCondCodeAction(ISD::SETOEQ, MVT::f64, Expand); in X86TargetLowering()
155 setCondCodeAction(ISD::SETOEQ, MVT::f80, Expand); in X86TargetLowering()
156 setCondCodeAction(ISD::SETUNE, MVT::f32, Expand); in X86TargetLowering()
157 setCondCodeAction(ISD::SETUNE, MVT::f64, Expand); in X86TargetLowering()
158 setCondCodeAction(ISD::SETUNE, MVT::f80, Expand); in X86TargetLowering()
162 setOperationAction(ISD::UINT_TO_FP , MVT::i1 , Promote); in X86TargetLowering()
163 setOperationAction(ISD::UINT_TO_FP , MVT::i8 , Promote); in X86TargetLowering()
164 setOperationAction(ISD::UINT_TO_FP , MVT::i16 , Promote); in X86TargetLowering()
169 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Custom); in X86TargetLowering()
171 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote); in X86TargetLowering()
172 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Custom); in X86TargetLowering()
176 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Custom); in X86TargetLowering()
179 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Custom); in X86TargetLowering()
184 setOperationAction(ISD::SINT_TO_FP , MVT::i1 , Promote); in X86TargetLowering()
185 setOperationAction(ISD::SINT_TO_FP , MVT::i8 , Promote); in X86TargetLowering()
190 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote); in X86TargetLowering()
192 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); in X86TargetLowering()
194 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Custom); in X86TargetLowering()
195 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); in X86TargetLowering()
198 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote); in X86TargetLowering()
199 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Promote); in X86TargetLowering()
204 setOperationAction(ISD::FP_TO_SINT , MVT::i1 , Promote); in X86TargetLowering()
205 setOperationAction(ISD::FP_TO_SINT , MVT::i8 , Promote); in X86TargetLowering()
210 setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Custom); in X86TargetLowering()
211 setOperationAction(ISD::SINT_TO_FP , MVT::i64 , Custom); in X86TargetLowering()
214 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote); in X86TargetLowering()
216 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); in X86TargetLowering()
218 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Custom); in X86TargetLowering()
219 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); in X86TargetLowering()
222 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote); in X86TargetLowering()
223 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Expand); in X86TargetLowering()
224 setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Expand); in X86TargetLowering()
229 setOperationAction(ISD::FP_TO_UINT , MVT::i1 , Promote); in X86TargetLowering()
230 setOperationAction(ISD::FP_TO_UINT , MVT::i8 , Promote); in X86TargetLowering()
231 setOperationAction(ISD::FP_TO_UINT , MVT::i16 , Promote); in X86TargetLowering()
236 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Custom); in X86TargetLowering()
237 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Custom); in X86TargetLowering()
239 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote); in X86TargetLowering()
240 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Expand); in X86TargetLowering()
248 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Expand); in X86TargetLowering()
253 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Custom); in X86TargetLowering()
255 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Custom); in X86TargetLowering()
260 setOperationAction(ISD::BITCAST , MVT::f32 , Expand); in X86TargetLowering()
261 setOperationAction(ISD::BITCAST , MVT::i32 , Expand); in X86TargetLowering()
263 setOperationAction(ISD::BITCAST , MVT::f64 , Expand); in X86TargetLowering()
265 setOperationAction(ISD::BITCAST , MVT::i64 , Expand); in X86TargetLowering()
279 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) { in X86TargetLowering()
294 setOperationAction(ISD::BR_JT , MVT::Other, Expand); in X86TargetLowering()
295 setOperationAction(ISD::BRCOND , MVT::Other, Custom); in X86TargetLowering()
296 setOperationAction(ISD::BR_CC , MVT::f32, Expand); in X86TargetLowering()
297 setOperationAction(ISD::BR_CC , MVT::f64, Expand); in X86TargetLowering()
298 setOperationAction(ISD::BR_CC , MVT::f80, Expand); in X86TargetLowering()
299 setOperationAction(ISD::BR_CC , MVT::f128, Expand); in X86TargetLowering()
300 setOperationAction(ISD::BR_CC , MVT::i8, Expand); in X86TargetLowering()
301 setOperationAction(ISD::BR_CC , MVT::i16, Expand); in X86TargetLowering()
302 setOperationAction(ISD::BR_CC , MVT::i32, Expand); in X86TargetLowering()
303 setOperationAction(ISD::BR_CC , MVT::i64, Expand); in X86TargetLowering()
304 setOperationAction(ISD::SELECT_CC , MVT::f32, Expand); in X86TargetLowering()
305 setOperationAction(ISD::SELECT_CC , MVT::f64, Expand); in X86TargetLowering()
306 setOperationAction(ISD::SELECT_CC , MVT::f80, Expand); in X86TargetLowering()
307 setOperationAction(ISD::SELECT_CC , MVT::f128, Expand); in X86TargetLowering()
308 setOperationAction(ISD::SELECT_CC , MVT::i8, Expand); in X86TargetLowering()
309 setOperationAction(ISD::SELECT_CC , MVT::i16, Expand); in X86TargetLowering()
310 setOperationAction(ISD::SELECT_CC , MVT::i32, Expand); in X86TargetLowering()
311 setOperationAction(ISD::SELECT_CC , MVT::i64, Expand); in X86TargetLowering()
313 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal); in X86TargetLowering()
314 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Legal); in X86TargetLowering()
315 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal); in X86TargetLowering()
316 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand); in X86TargetLowering()
317 setOperationAction(ISD::FP_ROUND_INREG , MVT::f32 , Expand); in X86TargetLowering()
324 setOperationAction(ISD::FREM , MVT::f32 , Promote); in X86TargetLowering()
326 setOperationAction(ISD::FREM , MVT::f32 , Expand); in X86TargetLowering()
329 setOperationAction(ISD::FREM , MVT::f64 , Expand); in X86TargetLowering()
330 setOperationAction(ISD::FREM , MVT::f80 , Expand); in X86TargetLowering()
331 setOperationAction(ISD::FLT_ROUNDS_ , MVT::i32 , Custom); in X86TargetLowering()
335 setOperationAction(ISD::CTTZ , MVT::i8 , Promote); in X86TargetLowering()
336 AddPromotedToType (ISD::CTTZ , MVT::i8 , MVT::i32); in X86TargetLowering()
337 setOperationAction(ISD::CTTZ_ZERO_UNDEF , MVT::i8 , Promote); in X86TargetLowering()
338 AddPromotedToType (ISD::CTTZ_ZERO_UNDEF , MVT::i8 , MVT::i32); in X86TargetLowering()
340 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i16 , Expand); in X86TargetLowering()
341 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32 , Expand); in X86TargetLowering()
343 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Expand); in X86TargetLowering()
345 setOperationAction(ISD::CTTZ , MVT::i16 , Custom); in X86TargetLowering()
346 setOperationAction(ISD::CTTZ , MVT::i32 , Custom); in X86TargetLowering()
348 setOperationAction(ISD::CTTZ , MVT::i64 , Custom); in X86TargetLowering()
354 setOperationAction(ISD::CTLZ , MVT::i8 , Promote); in X86TargetLowering()
355 AddPromotedToType (ISD::CTLZ , MVT::i8 , MVT::i32); in X86TargetLowering()
356 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i8 , Promote); in X86TargetLowering()
357 AddPromotedToType (ISD::CTLZ_ZERO_UNDEF, MVT::i8 , MVT::i32); in X86TargetLowering()
358 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i16 , Expand); in X86TargetLowering()
359 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32 , Expand); in X86TargetLowering()
361 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Expand); in X86TargetLowering()
363 setOperationAction(ISD::CTLZ , MVT::i8 , Custom); in X86TargetLowering()
364 setOperationAction(ISD::CTLZ , MVT::i16 , Custom); in X86TargetLowering()
365 setOperationAction(ISD::CTLZ , MVT::i32 , Custom); in X86TargetLowering()
366 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i8 , Custom); in X86TargetLowering()
367 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i16 , Custom); in X86TargetLowering()
368 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32 , Custom); in X86TargetLowering()
370 setOperationAction(ISD::CTLZ , MVT::i64 , Custom); in X86TargetLowering()
371 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Custom); in X86TargetLowering()
379 setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand); in X86TargetLowering()
380 setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand); in X86TargetLowering()
384 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand); in X86TargetLowering()
385 setOperationAction(ISD::FP16_TO_FP, MVT::f80, Expand); in X86TargetLowering()
386 setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand); in X86TargetLowering()
387 setOperationAction(ISD::FP_TO_FP16, MVT::f80, Expand); in X86TargetLowering()
389 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand); in X86TargetLowering()
390 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand); in X86TargetLowering()
391 setLoadExtAction(ISD::EXTLOAD, MVT::f80, MVT::f16, Expand); in X86TargetLowering()
392 setTruncStoreAction(MVT::f32, MVT::f16, Expand); in X86TargetLowering()
393 setTruncStoreAction(MVT::f64, MVT::f16, Expand); in X86TargetLowering()
394 setTruncStoreAction(MVT::f80, MVT::f16, Expand); in X86TargetLowering()
397 setOperationAction(ISD::CTPOP , MVT::i8 , Promote); in X86TargetLowering()
399 setOperationAction(ISD::CTPOP , MVT::i8 , Expand); in X86TargetLowering()
400 setOperationAction(ISD::CTPOP , MVT::i16 , Expand); in X86TargetLowering()
401 setOperationAction(ISD::CTPOP , MVT::i32 , Expand); in X86TargetLowering()
403 setOperationAction(ISD::CTPOP , MVT::i64 , Expand); in X86TargetLowering()
406 setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom); in X86TargetLowering()
409 setOperationAction(ISD::BSWAP , MVT::i16 , Expand); in X86TargetLowering()
412 setOperationAction(ISD::SELECT , MVT::i1 , Promote); in X86TargetLowering()
414 setOperationAction(ISD::SELECT , MVT::i8 , Custom); in X86TargetLowering()
415 setOperationAction(ISD::SELECT , MVT::i16 , Custom); in X86TargetLowering()
416 setOperationAction(ISD::SELECT , MVT::i32 , Custom); in X86TargetLowering()
417 setOperationAction(ISD::SELECT , MVT::f32 , Custom); in X86TargetLowering()
418 setOperationAction(ISD::SELECT , MVT::f64 , Custom); in X86TargetLowering()
419 setOperationAction(ISD::SELECT , MVT::f80 , Custom); in X86TargetLowering()
420 setOperationAction(ISD::SELECT , MVT::f128 , Custom); in X86TargetLowering()
421 setOperationAction(ISD::SETCC , MVT::i8 , Custom); in X86TargetLowering()
422 setOperationAction(ISD::SETCC , MVT::i16 , Custom); in X86TargetLowering()
423 setOperationAction(ISD::SETCC , MVT::i32 , Custom); in X86TargetLowering()
424 setOperationAction(ISD::SETCC , MVT::f32 , Custom); in X86TargetLowering()
425 setOperationAction(ISD::SETCC , MVT::f64 , Custom); in X86TargetLowering()
426 setOperationAction(ISD::SETCC , MVT::f80 , Custom); in X86TargetLowering()
427 setOperationAction(ISD::SETCC , MVT::f128 , Custom); in X86TargetLowering()
428 setOperationAction(ISD::SETCCE , MVT::i8 , Custom); in X86TargetLowering()
429 setOperationAction(ISD::SETCCE , MVT::i16 , Custom); in X86TargetLowering()
430 setOperationAction(ISD::SETCCE , MVT::i32 , Custom); in X86TargetLowering()
432 setOperationAction(ISD::SELECT , MVT::i64 , Custom); in X86TargetLowering()
433 setOperationAction(ISD::SETCC , MVT::i64 , Custom); in X86TargetLowering()
434 setOperationAction(ISD::SETCCE , MVT::i64 , Custom); in X86TargetLowering()
436 setOperationAction(ISD::EH_RETURN , MVT::Other, Custom); in X86TargetLowering()
443 setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom); in X86TargetLowering()
444 setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom); in X86TargetLowering()
447 setOperationAction(ISD::ConstantPool , MVT::i32 , Custom); in X86TargetLowering()
448 setOperationAction(ISD::JumpTable , MVT::i32 , Custom); in X86TargetLowering()
449 setOperationAction(ISD::GlobalAddress , MVT::i32 , Custom); in X86TargetLowering()
450 setOperationAction(ISD::GlobalTLSAddress, MVT::i32 , Custom); in X86TargetLowering()
452 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); in X86TargetLowering()
453 setOperationAction(ISD::ExternalSymbol , MVT::i32 , Custom); in X86TargetLowering()
454 setOperationAction(ISD::BlockAddress , MVT::i32 , Custom); in X86TargetLowering()
456 setOperationAction(ISD::ConstantPool , MVT::i64 , Custom); in X86TargetLowering()
457 setOperationAction(ISD::JumpTable , MVT::i64 , Custom); in X86TargetLowering()
458 setOperationAction(ISD::GlobalAddress , MVT::i64 , Custom); in X86TargetLowering()
459 setOperationAction(ISD::ExternalSymbol, MVT::i64 , Custom); in X86TargetLowering()
460 setOperationAction(ISD::BlockAddress , MVT::i64 , Custom); in X86TargetLowering()
463 setOperationAction(ISD::SHL_PARTS , MVT::i32 , Custom); in X86TargetLowering()
464 setOperationAction(ISD::SRA_PARTS , MVT::i32 , Custom); in X86TargetLowering()
465 setOperationAction(ISD::SRL_PARTS , MVT::i32 , Custom); in X86TargetLowering()
467 setOperationAction(ISD::SHL_PARTS , MVT::i64 , Custom); in X86TargetLowering()
468 setOperationAction(ISD::SRA_PARTS , MVT::i64 , Custom); in X86TargetLowering()
469 setOperationAction(ISD::SRL_PARTS , MVT::i64 , Custom); in X86TargetLowering()
473 setOperationAction(ISD::PREFETCH , MVT::Other, Legal); in X86TargetLowering()
475 setOperationAction(ISD::ATOMIC_FENCE , MVT::Other, Custom); in X86TargetLowering()
478 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) { in X86TargetLowering()
485 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i128, Custom); in X86TargetLowering()
491 setOperationAction(ISD::EH_LABEL, MVT::Other, Expand); in X86TargetLowering()
494 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom); in X86TargetLowering()
495 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i64, Custom); in X86TargetLowering()
497 setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom); in X86TargetLowering()
498 setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom); in X86TargetLowering()
500 setOperationAction(ISD::TRAP, MVT::Other, Legal); in X86TargetLowering()
501 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal); in X86TargetLowering()
504 setOperationAction(ISD::VASTART , MVT::Other, Custom); in X86TargetLowering()
505 setOperationAction(ISD::VAEND , MVT::Other, Expand); in X86TargetLowering()
507 setOperationAction(ISD::VAARG , MVT::Other, Custom); in X86TargetLowering()
508 setOperationAction(ISD::VACOPY , MVT::Other, Custom); in X86TargetLowering()
511 setOperationAction(ISD::VAARG , MVT::Other, Expand); in X86TargetLowering()
512 setOperationAction(ISD::VACOPY , MVT::Other, Expand); in X86TargetLowering()
515 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); in X86TargetLowering()
516 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); in X86TargetLowering()
521 setOperationAction(ISD::GC_TRANSITION_START, MVT::Other, Custom); in X86TargetLowering()
522 setOperationAction(ISD::GC_TRANSITION_END, MVT::Other, Custom); in X86TargetLowering()
527 addRegisterClass(MVT::f32, &X86::FR32RegClass); in X86TargetLowering()
528 addRegisterClass(MVT::f64, &X86::FR64RegClass); in X86TargetLowering()
531 setOperationAction(ISD::FABS , MVT::f64, Custom); in X86TargetLowering()
532 setOperationAction(ISD::FABS , MVT::f32, Custom); in X86TargetLowering()
535 setOperationAction(ISD::FNEG , MVT::f64, Custom); in X86TargetLowering()
536 setOperationAction(ISD::FNEG , MVT::f32, Custom); in X86TargetLowering()
539 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom); in X86TargetLowering()
540 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); in X86TargetLowering()
543 setOperationAction(ISD::FGETSIGN, MVT::i64, Custom); in X86TargetLowering()
544 setOperationAction(ISD::FGETSIGN, MVT::i32, Custom); in X86TargetLowering()
547 setOperationAction(ISD::FSIN , MVT::f64, Expand); in X86TargetLowering()
548 setOperationAction(ISD::FCOS , MVT::f64, Expand); in X86TargetLowering()
549 setOperationAction(ISD::FSINCOS, MVT::f64, Expand); in X86TargetLowering()
550 setOperationAction(ISD::FSIN , MVT::f32, Expand); in X86TargetLowering()
551 setOperationAction(ISD::FCOS , MVT::f32, Expand); in X86TargetLowering()
552 setOperationAction(ISD::FSINCOS, MVT::f32, Expand); in X86TargetLowering()
561 addRegisterClass(MVT::f32, &X86::FR32RegClass); in X86TargetLowering()
562 addRegisterClass(MVT::f64, &X86::RFP64RegClass); in X86TargetLowering()
565 setOperationAction(ISD::FABS , MVT::f32, Custom); in X86TargetLowering()
568 setOperationAction(ISD::FNEG , MVT::f32, Custom); in X86TargetLowering()
570 setOperationAction(ISD::UNDEF, MVT::f64, Expand); in X86TargetLowering()
573 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); in X86TargetLowering()
574 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); in X86TargetLowering()
577 setOperationAction(ISD::FSIN , MVT::f32, Expand); in X86TargetLowering()
578 setOperationAction(ISD::FCOS , MVT::f32, Expand); in X86TargetLowering()
579 setOperationAction(ISD::FSINCOS, MVT::f32, Expand); in X86TargetLowering()
589 setOperationAction(ISD::FSIN , MVT::f64, Expand); in X86TargetLowering()
590 setOperationAction(ISD::FCOS , MVT::f64, Expand); in X86TargetLowering()
591 setOperationAction(ISD::FSINCOS, MVT::f64, Expand); in X86TargetLowering()
596 addRegisterClass(MVT::f64, &X86::RFP64RegClass); in X86TargetLowering()
597 addRegisterClass(MVT::f32, &X86::RFP32RegClass); in X86TargetLowering()
599 setOperationAction(ISD::UNDEF, MVT::f64, Expand); in X86TargetLowering()
600 setOperationAction(ISD::UNDEF, MVT::f32, Expand); in X86TargetLowering()
601 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); in X86TargetLowering()
602 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); in X86TargetLowering()
605 setOperationAction(ISD::FSIN , MVT::f64, Expand); in X86TargetLowering()
606 setOperationAction(ISD::FSIN , MVT::f32, Expand); in X86TargetLowering()
607 setOperationAction(ISD::FCOS , MVT::f64, Expand); in X86TargetLowering()
608 setOperationAction(ISD::FCOS , MVT::f32, Expand); in X86TargetLowering()
609 setOperationAction(ISD::FSINCOS, MVT::f64, Expand); in X86TargetLowering()
610 setOperationAction(ISD::FSINCOS, MVT::f32, Expand); in X86TargetLowering()
623 setOperationAction(ISD::FMA, MVT::f64, Expand); in X86TargetLowering()
624 setOperationAction(ISD::FMA, MVT::f32, Expand); in X86TargetLowering()
629 addRegisterClass(MVT::f128, &X86::FR128RegClass); in X86TargetLowering()
630 ValueTypeActions.setTypeAction(MVT::f128, TypeSoftenFloat); in X86TargetLowering()
631 setOperationAction(ISD::FABS , MVT::f128, Custom); in X86TargetLowering()
632 setOperationAction(ISD::FNEG , MVT::f128, Custom); in X86TargetLowering()
633 setOperationAction(ISD::FCOPYSIGN, MVT::f128, Custom); in X86TargetLowering()
636 addRegisterClass(MVT::f80, &X86::RFP80RegClass); in X86TargetLowering()
637 setOperationAction(ISD::UNDEF, MVT::f80, Expand); in X86TargetLowering()
638 setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand); in X86TargetLowering()
655 setOperationAction(ISD::FSIN , MVT::f80, Expand); in X86TargetLowering()
656 setOperationAction(ISD::FCOS , MVT::f80, Expand); in X86TargetLowering()
657 setOperationAction(ISD::FSINCOS, MVT::f80, Expand); in X86TargetLowering()
660 setOperationAction(ISD::FFLOOR, MVT::f80, Expand); in X86TargetLowering()
661 setOperationAction(ISD::FCEIL, MVT::f80, Expand); in X86TargetLowering()
662 setOperationAction(ISD::FTRUNC, MVT::f80, Expand); in X86TargetLowering()
663 setOperationAction(ISD::FRINT, MVT::f80, Expand); in X86TargetLowering()
664 setOperationAction(ISD::FNEARBYINT, MVT::f80, Expand); in X86TargetLowering()
665 setOperationAction(ISD::FMA, MVT::f80, Expand); in X86TargetLowering()
669 setOperationAction(ISD::FPOW , MVT::f32 , Expand); in X86TargetLowering()
670 setOperationAction(ISD::FPOW , MVT::f64 , Expand); in X86TargetLowering()
671 setOperationAction(ISD::FPOW , MVT::f80 , Expand); in X86TargetLowering()
673 setOperationAction(ISD::FLOG, MVT::f80, Expand); in X86TargetLowering()
674 setOperationAction(ISD::FLOG2, MVT::f80, Expand); in X86TargetLowering()
675 setOperationAction(ISD::FLOG10, MVT::f80, Expand); in X86TargetLowering()
676 setOperationAction(ISD::FEXP, MVT::f80, Expand); in X86TargetLowering()
677 setOperationAction(ISD::FEXP2, MVT::f80, Expand); in X86TargetLowering()
678 setOperationAction(ISD::FMINNUM, MVT::f80, Expand); in X86TargetLowering()
679 setOperationAction(ISD::FMAXNUM, MVT::f80, Expand); in X86TargetLowering()
684 for (MVT VT : MVT::vector_valuetypes()) { in X86TargetLowering()
753 for (MVT InnerVT : MVT::vector_valuetypes()) { in X86TargetLowering()
763 if (VT.getVectorElementType() == MVT::i1) in X86TargetLowering()
768 if (VT.getVectorElementType() == MVT::f16) in X86TargetLowering()
776 addRegisterClass(MVT::x86mmx, &X86::VR64RegClass); in X86TargetLowering()
782 for (MVT MMXTy : {MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v1i64}) { in X86TargetLowering()
791 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v1i64, Expand); in X86TargetLowering()
794 addRegisterClass(MVT::v4f32, &X86::VR128RegClass); in X86TargetLowering()
796 setOperationAction(ISD::FADD, MVT::v4f32, Legal); in X86TargetLowering()
797 setOperationAction(ISD::FSUB, MVT::v4f32, Legal); in X86TargetLowering()
798 setOperationAction(ISD::FMUL, MVT::v4f32, Legal); in X86TargetLowering()
799 setOperationAction(ISD::FDIV, MVT::v4f32, Legal); in X86TargetLowering()
800 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); in X86TargetLowering()
801 setOperationAction(ISD::FNEG, MVT::v4f32, Custom); in X86TargetLowering()
802 setOperationAction(ISD::FABS, MVT::v4f32, Custom); in X86TargetLowering()
803 setOperationAction(ISD::LOAD, MVT::v4f32, Legal); in X86TargetLowering()
804 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); in X86TargetLowering()
805 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom); in X86TargetLowering()
806 setOperationAction(ISD::VSELECT, MVT::v4f32, Custom); in X86TargetLowering()
807 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); in X86TargetLowering()
808 setOperationAction(ISD::SELECT, MVT::v4f32, Custom); in X86TargetLowering()
809 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom); in X86TargetLowering()
813 addRegisterClass(MVT::v2f64, &X86::VR128RegClass); in X86TargetLowering()
817 addRegisterClass(MVT::v16i8, &X86::VR128RegClass); in X86TargetLowering()
818 addRegisterClass(MVT::v8i16, &X86::VR128RegClass); in X86TargetLowering()
819 addRegisterClass(MVT::v4i32, &X86::VR128RegClass); in X86TargetLowering()
820 addRegisterClass(MVT::v2i64, &X86::VR128RegClass); in X86TargetLowering()
822 setOperationAction(ISD::ADD, MVT::v16i8, Legal); in X86TargetLowering()
823 setOperationAction(ISD::ADD, MVT::v8i16, Legal); in X86TargetLowering()
824 setOperationAction(ISD::ADD, MVT::v4i32, Legal); in X86TargetLowering()
825 setOperationAction(ISD::ADD, MVT::v2i64, Legal); in X86TargetLowering()
826 setOperationAction(ISD::MUL, MVT::v16i8, Custom); in X86TargetLowering()
827 setOperationAction(ISD::MUL, MVT::v4i32, Custom); in X86TargetLowering()
828 setOperationAction(ISD::MUL, MVT::v2i64, Custom); in X86TargetLowering()
829 setOperationAction(ISD::UMUL_LOHI, MVT::v4i32, Custom); in X86TargetLowering()
830 setOperationAction(ISD::SMUL_LOHI, MVT::v4i32, Custom); in X86TargetLowering()
831 setOperationAction(ISD::MULHU, MVT::v8i16, Legal); in X86TargetLowering()
832 setOperationAction(ISD::MULHS, MVT::v8i16, Legal); in X86TargetLowering()
833 setOperationAction(ISD::SUB, MVT::v16i8, Legal); in X86TargetLowering()
834 setOperationAction(ISD::SUB, MVT::v8i16, Legal); in X86TargetLowering()
835 setOperationAction(ISD::SUB, MVT::v4i32, Legal); in X86TargetLowering()
836 setOperationAction(ISD::SUB, MVT::v2i64, Legal); in X86TargetLowering()
837 setOperationAction(ISD::MUL, MVT::v8i16, Legal); in X86TargetLowering()
838 setOperationAction(ISD::FADD, MVT::v2f64, Legal); in X86TargetLowering()
839 setOperationAction(ISD::FSUB, MVT::v2f64, Legal); in X86TargetLowering()
840 setOperationAction(ISD::FMUL, MVT::v2f64, Legal); in X86TargetLowering()
841 setOperationAction(ISD::FDIV, MVT::v2f64, Legal); in X86TargetLowering()
842 setOperationAction(ISD::FSQRT, MVT::v2f64, Legal); in X86TargetLowering()
843 setOperationAction(ISD::FNEG, MVT::v2f64, Custom); in X86TargetLowering()
844 setOperationAction(ISD::FABS, MVT::v2f64, Custom); in X86TargetLowering()
846 setOperationAction(ISD::SMAX, MVT::v8i16, Legal); in X86TargetLowering()
847 setOperationAction(ISD::UMAX, MVT::v16i8, Legal); in X86TargetLowering()
848 setOperationAction(ISD::SMIN, MVT::v8i16, Legal); in X86TargetLowering()
849 setOperationAction(ISD::UMIN, MVT::v16i8, Legal); in X86TargetLowering()
851 setOperationAction(ISD::SETCC, MVT::v2i64, Custom); in X86TargetLowering()
852 setOperationAction(ISD::SETCC, MVT::v16i8, Custom); in X86TargetLowering()
853 setOperationAction(ISD::SETCC, MVT::v8i16, Custom); in X86TargetLowering()
854 setOperationAction(ISD::SETCC, MVT::v4i32, Custom); in X86TargetLowering()
856 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Custom); in X86TargetLowering()
857 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Custom); in X86TargetLowering()
858 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); in X86TargetLowering()
859 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); in X86TargetLowering()
860 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); in X86TargetLowering()
862 setOperationAction(ISD::CTPOP, MVT::v16i8, Custom); in X86TargetLowering()
863 setOperationAction(ISD::CTPOP, MVT::v8i16, Custom); in X86TargetLowering()
864 setOperationAction(ISD::CTPOP, MVT::v4i32, Custom); in X86TargetLowering()
865 setOperationAction(ISD::CTPOP, MVT::v2i64, Custom); in X86TargetLowering()
867 setOperationAction(ISD::CTTZ, MVT::v16i8, Custom); in X86TargetLowering()
868 setOperationAction(ISD::CTTZ, MVT::v8i16, Custom); in X86TargetLowering()
869 setOperationAction(ISD::CTTZ, MVT::v4i32, Custom); in X86TargetLowering()
871 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v16i8, Custom); in X86TargetLowering()
872 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v8i16, Custom); in X86TargetLowering()
873 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v4i32, Custom); in X86TargetLowering()
877 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) { in X86TargetLowering()
888 for (MVT VT : MVT::integer_vector_valuetypes()) { in X86TargetLowering()
889 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i8, Custom); in X86TargetLowering()
890 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i16, Custom); in X86TargetLowering()
891 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v8i8, Custom); in X86TargetLowering()
892 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i8, Custom); in X86TargetLowering()
893 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i16, Custom); in X86TargetLowering()
894 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i32, Custom); in X86TargetLowering()
895 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i8, Custom); in X86TargetLowering()
896 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i16, Custom); in X86TargetLowering()
897 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v8i8, Custom); in X86TargetLowering()
900 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Custom); in X86TargetLowering()
901 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom); in X86TargetLowering()
902 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Custom); in X86TargetLowering()
903 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Custom); in X86TargetLowering()
904 setOperationAction(ISD::VSELECT, MVT::v2f64, Custom); in X86TargetLowering()
905 setOperationAction(ISD::VSELECT, MVT::v2i64, Custom); in X86TargetLowering()
906 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f64, Custom); in X86TargetLowering()
907 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Custom); in X86TargetLowering()
910 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Custom); in X86TargetLowering()
911 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Custom); in X86TargetLowering()
915 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) { in X86TargetLowering()
917 AddPromotedToType (ISD::AND, VT, MVT::v2i64); in X86TargetLowering()
919 AddPromotedToType (ISD::OR, VT, MVT::v2i64); in X86TargetLowering()
921 AddPromotedToType (ISD::XOR, VT, MVT::v2i64); in X86TargetLowering()
923 AddPromotedToType (ISD::LOAD, VT, MVT::v2i64); in X86TargetLowering()
925 AddPromotedToType (ISD::SELECT, VT, MVT::v2i64); in X86TargetLowering()
929 setOperationAction(ISD::LOAD, MVT::v2f64, Legal); in X86TargetLowering()
930 setOperationAction(ISD::LOAD, MVT::v2i64, Legal); in X86TargetLowering()
931 setOperationAction(ISD::SELECT, MVT::v2f64, Custom); in X86TargetLowering()
932 setOperationAction(ISD::SELECT, MVT::v2i64, Custom); in X86TargetLowering()
934 setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal); in X86TargetLowering()
935 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal); in X86TargetLowering()
937 setOperationAction(ISD::SINT_TO_FP, MVT::v2i32, Custom); in X86TargetLowering()
939 setOperationAction(ISD::UINT_TO_FP, MVT::v4i8, Custom); in X86TargetLowering()
940 setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom); in X86TargetLowering()
944 setOperationAction(ISD::UINT_TO_FP, MVT::v2f32, Custom); in X86TargetLowering()
946 setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Custom); in X86TargetLowering()
947 setOperationAction(ISD::FP_ROUND, MVT::v2f32, Custom); in X86TargetLowering()
949 for (MVT VT : MVT::fp_vector_valuetypes()) in X86TargetLowering()
950 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2f32, Legal); in X86TargetLowering()
952 setOperationAction(ISD::BITCAST, MVT::v2i32, Custom); in X86TargetLowering()
953 setOperationAction(ISD::BITCAST, MVT::v4i16, Custom); in X86TargetLowering()
954 setOperationAction(ISD::BITCAST, MVT::v8i8, Custom); in X86TargetLowering()
958 for (MVT RoundedTy : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) { in X86TargetLowering()
966 setOperationAction(ISD::SMAX, MVT::v16i8, Legal); in X86TargetLowering()
967 setOperationAction(ISD::SMAX, MVT::v4i32, Legal); in X86TargetLowering()
968 setOperationAction(ISD::UMAX, MVT::v8i16, Legal); in X86TargetLowering()
969 setOperationAction(ISD::UMAX, MVT::v4i32, Legal); in X86TargetLowering()
970 setOperationAction(ISD::SMIN, MVT::v16i8, Legal); in X86TargetLowering()
971 setOperationAction(ISD::SMIN, MVT::v4i32, Legal); in X86TargetLowering()
972 setOperationAction(ISD::UMIN, MVT::v8i16, Legal); in X86TargetLowering()
973 setOperationAction(ISD::UMIN, MVT::v4i32, Legal); in X86TargetLowering()
976 setOperationAction(ISD::MUL, MVT::v4i32, Legal); in X86TargetLowering()
980 setOperationAction(ISD::VSELECT, MVT::v16i8, Legal); in X86TargetLowering()
984 for (MVT VT : MVT::integer_vector_valuetypes()) { in X86TargetLowering()
985 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i8, Custom); in X86TargetLowering()
986 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i16, Custom); in X86TargetLowering()
987 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i32, Custom); in X86TargetLowering()
991 setLoadExtAction(ISD::SEXTLOAD, MVT::v8i16, MVT::v8i8, Legal); in X86TargetLowering()
992 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i32, MVT::v4i8, Legal); in X86TargetLowering()
993 setLoadExtAction(ISD::SEXTLOAD, MVT::v2i64, MVT::v2i8, Legal); in X86TargetLowering()
994 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i32, MVT::v4i16, Legal); in X86TargetLowering()
995 setLoadExtAction(ISD::SEXTLOAD, MVT::v2i64, MVT::v2i16, Legal); in X86TargetLowering()
996 setLoadExtAction(ISD::SEXTLOAD, MVT::v2i64, MVT::v2i32, Legal); in X86TargetLowering()
998 setLoadExtAction(ISD::ZEXTLOAD, MVT::v8i16, MVT::v8i8, Legal); in X86TargetLowering()
999 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i32, MVT::v4i8, Legal); in X86TargetLowering()
1000 setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i64, MVT::v2i8, Legal); in X86TargetLowering()
1001 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i32, MVT::v4i16, Legal); in X86TargetLowering()
1002 setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i64, MVT::v2i16, Legal); in X86TargetLowering()
1003 setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i64, MVT::v2i32, Legal); in X86TargetLowering()
1009 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom); in X86TargetLowering()
1010 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); in X86TargetLowering()
1011 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); in X86TargetLowering()
1012 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); in X86TargetLowering()
1014 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i8, Custom); in X86TargetLowering()
1015 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i16, Custom); in X86TargetLowering()
1016 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Custom); in X86TargetLowering()
1017 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); in X86TargetLowering()
1022 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Custom); in X86TargetLowering()
1023 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Custom); in X86TargetLowering()
1028 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v2i64, Custom); in X86TargetLowering()
1029 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v4i32, Custom); in X86TargetLowering()
1030 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v8i16, Custom); in X86TargetLowering()
1032 setOperationAction(ISD::SRL, MVT::v8i16, Custom); in X86TargetLowering()
1033 setOperationAction(ISD::SRL, MVT::v16i8, Custom); in X86TargetLowering()
1035 setOperationAction(ISD::SHL, MVT::v8i16, Custom); in X86TargetLowering()
1036 setOperationAction(ISD::SHL, MVT::v16i8, Custom); in X86TargetLowering()
1038 setOperationAction(ISD::SRA, MVT::v8i16, Custom); in X86TargetLowering()
1039 setOperationAction(ISD::SRA, MVT::v16i8, Custom); in X86TargetLowering()
1043 setOperationAction(ISD::SRL, MVT::v2i64, Custom); in X86TargetLowering()
1044 setOperationAction(ISD::SRL, MVT::v4i32, Custom); in X86TargetLowering()
1046 setOperationAction(ISD::SHL, MVT::v2i64, Custom); in X86TargetLowering()
1047 setOperationAction(ISD::SHL, MVT::v4i32, Custom); in X86TargetLowering()
1049 setOperationAction(ISD::SRA, MVT::v2i64, Custom); in X86TargetLowering()
1050 setOperationAction(ISD::SRA, MVT::v4i32, Custom); in X86TargetLowering()
1054 setOperationAction(ISD::ROTL, MVT::v16i8, Custom); in X86TargetLowering()
1055 setOperationAction(ISD::ROTL, MVT::v8i16, Custom); in X86TargetLowering()
1056 setOperationAction(ISD::ROTL, MVT::v4i32, Custom); in X86TargetLowering()
1057 setOperationAction(ISD::ROTL, MVT::v2i64, Custom); in X86TargetLowering()
1058 setOperationAction(ISD::ROTL, MVT::v32i8, Custom); in X86TargetLowering()
1059 setOperationAction(ISD::ROTL, MVT::v16i16, Custom); in X86TargetLowering()
1060 setOperationAction(ISD::ROTL, MVT::v8i32, Custom); in X86TargetLowering()
1061 setOperationAction(ISD::ROTL, MVT::v4i64, Custom); in X86TargetLowering()
1065 addRegisterClass(MVT::v32i8, &X86::VR256RegClass); in X86TargetLowering()
1066 addRegisterClass(MVT::v16i16, &X86::VR256RegClass); in X86TargetLowering()
1067 addRegisterClass(MVT::v8i32, &X86::VR256RegClass); in X86TargetLowering()
1068 addRegisterClass(MVT::v8f32, &X86::VR256RegClass); in X86TargetLowering()
1069 addRegisterClass(MVT::v4i64, &X86::VR256RegClass); in X86TargetLowering()
1070 addRegisterClass(MVT::v4f64, &X86::VR256RegClass); in X86TargetLowering()
1072 setOperationAction(ISD::LOAD, MVT::v8f32, Legal); in X86TargetLowering()
1073 setOperationAction(ISD::LOAD, MVT::v4f64, Legal); in X86TargetLowering()
1074 setOperationAction(ISD::LOAD, MVT::v4i64, Legal); in X86TargetLowering()
1076 setOperationAction(ISD::FADD, MVT::v8f32, Legal); in X86TargetLowering()
1077 setOperationAction(ISD::FSUB, MVT::v8f32, Legal); in X86TargetLowering()
1078 setOperationAction(ISD::FMUL, MVT::v8f32, Legal); in X86TargetLowering()
1079 setOperationAction(ISD::FDIV, MVT::v8f32, Legal); in X86TargetLowering()
1080 setOperationAction(ISD::FSQRT, MVT::v8f32, Legal); in X86TargetLowering()
1081 setOperationAction(ISD::FFLOOR, MVT::v8f32, Legal); in X86TargetLowering()
1082 setOperationAction(ISD::FCEIL, MVT::v8f32, Legal); in X86TargetLowering()
1083 setOperationAction(ISD::FTRUNC, MVT::v8f32, Legal); in X86TargetLowering()
1084 setOperationAction(ISD::FRINT, MVT::v8f32, Legal); in X86TargetLowering()
1085 setOperationAction(ISD::FNEARBYINT, MVT::v8f32, Legal); in X86TargetLowering()
1086 setOperationAction(ISD::FNEG, MVT::v8f32, Custom); in X86TargetLowering()
1087 setOperationAction(ISD::FABS, MVT::v8f32, Custom); in X86TargetLowering()
1089 setOperationAction(ISD::FADD, MVT::v4f64, Legal); in X86TargetLowering()
1090 setOperationAction(ISD::FSUB, MVT::v4f64, Legal); in X86TargetLowering()
1091 setOperationAction(ISD::FMUL, MVT::v4f64, Legal); in X86TargetLowering()
1092 setOperationAction(ISD::FDIV, MVT::v4f64, Legal); in X86TargetLowering()
1093 setOperationAction(ISD::FSQRT, MVT::v4f64, Legal); in X86TargetLowering()
1094 setOperationAction(ISD::FFLOOR, MVT::v4f64, Legal); in X86TargetLowering()
1095 setOperationAction(ISD::FCEIL, MVT::v4f64, Legal); in X86TargetLowering()
1096 setOperationAction(ISD::FTRUNC, MVT::v4f64, Legal); in X86TargetLowering()
1097 setOperationAction(ISD::FRINT, MVT::v4f64, Legal); in X86TargetLowering()
1098 setOperationAction(ISD::FNEARBYINT, MVT::v4f64, Legal); in X86TargetLowering()
1099 setOperationAction(ISD::FNEG, MVT::v4f64, Custom); in X86TargetLowering()
1100 setOperationAction(ISD::FABS, MVT::v4f64, Custom); in X86TargetLowering()
1104 setOperationAction(ISD::FP_TO_SINT, MVT::v8i16, Promote); in X86TargetLowering()
1105 setOperationAction(ISD::FP_TO_UINT, MVT::v8i16, Promote); in X86TargetLowering()
1106 setOperationAction(ISD::FP_TO_SINT, MVT::v8i32, Legal); in X86TargetLowering()
1108 setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Promote); in X86TargetLowering()
1109 setOperationAction(ISD::SINT_TO_FP, MVT::v8i32, Legal); in X86TargetLowering()
1110 setOperationAction(ISD::FP_ROUND, MVT::v4f32, Legal); in X86TargetLowering()
1112 setOperationAction(ISD::UINT_TO_FP, MVT::v8i8, Custom); in X86TargetLowering()
1113 setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Custom); in X86TargetLowering()
1115 for (MVT VT : MVT::fp_vector_valuetypes()) in X86TargetLowering()
1116 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4f32, Legal); in X86TargetLowering()
1118 setOperationAction(ISD::SRL, MVT::v16i16, Custom); in X86TargetLowering()
1119 setOperationAction(ISD::SRL, MVT::v32i8, Custom); in X86TargetLowering()
1121 setOperationAction(ISD::SHL, MVT::v16i16, Custom); in X86TargetLowering()
1122 setOperationAction(ISD::SHL, MVT::v32i8, Custom); in X86TargetLowering()
1124 setOperationAction(ISD::SRA, MVT::v16i16, Custom); in X86TargetLowering()
1125 setOperationAction(ISD::SRA, MVT::v32i8, Custom); in X86TargetLowering()
1127 setOperationAction(ISD::SETCC, MVT::v32i8, Custom); in X86TargetLowering()
1128 setOperationAction(ISD::SETCC, MVT::v16i16, Custom); in X86TargetLowering()
1129 setOperationAction(ISD::SETCC, MVT::v8i32, Custom); in X86TargetLowering()
1130 setOperationAction(ISD::SETCC, MVT::v4i64, Custom); in X86TargetLowering()
1132 setOperationAction(ISD::SELECT, MVT::v4f64, Custom); in X86TargetLowering()
1133 setOperationAction(ISD::SELECT, MVT::v4i64, Custom); in X86TargetLowering()
1134 setOperationAction(ISD::SELECT, MVT::v8f32, Custom); in X86TargetLowering()
1136 setOperationAction(ISD::SIGN_EXTEND, MVT::v4i64, Custom); in X86TargetLowering()
1137 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i32, Custom); in X86TargetLowering()
1138 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i16, Custom); in X86TargetLowering()
1139 setOperationAction(ISD::ZERO_EXTEND, MVT::v4i64, Custom); in X86TargetLowering()
1140 setOperationAction(ISD::ZERO_EXTEND, MVT::v8i32, Custom); in X86TargetLowering()
1141 setOperationAction(ISD::ZERO_EXTEND, MVT::v16i16, Custom); in X86TargetLowering()
1142 setOperationAction(ISD::ANY_EXTEND, MVT::v4i64, Custom); in X86TargetLowering()
1143 setOperationAction(ISD::ANY_EXTEND, MVT::v8i32, Custom); in X86TargetLowering()
1144 setOperationAction(ISD::ANY_EXTEND, MVT::v16i16, Custom); in X86TargetLowering()
1145 setOperationAction(ISD::TRUNCATE, MVT::v16i8, Custom); in X86TargetLowering()
1146 setOperationAction(ISD::TRUNCATE, MVT::v8i16, Custom); in X86TargetLowering()
1147 setOperationAction(ISD::TRUNCATE, MVT::v4i32, Custom); in X86TargetLowering()
1149 setOperationAction(ISD::CTPOP, MVT::v32i8, Custom); in X86TargetLowering()
1150 setOperationAction(ISD::CTPOP, MVT::v16i16, Custom); in X86TargetLowering()
1151 setOperationAction(ISD::CTPOP, MVT::v8i32, Custom); in X86TargetLowering()
1152 setOperationAction(ISD::CTPOP, MVT::v4i64, Custom); in X86TargetLowering()
1154 setOperationAction(ISD::CTTZ, MVT::v32i8, Custom); in X86TargetLowering()
1155 setOperationAction(ISD::CTTZ, MVT::v16i16, Custom); in X86TargetLowering()
1156 setOperationAction(ISD::CTTZ, MVT::v8i32, Custom); in X86TargetLowering()
1157 setOperationAction(ISD::CTTZ, MVT::v4i64, Custom); in X86TargetLowering()
1158 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v32i8, Custom); in X86TargetLowering()
1159 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v16i16, Custom); in X86TargetLowering()
1160 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v8i32, Custom); in X86TargetLowering()
1161 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v4i64, Custom); in X86TargetLowering()
1164 setOperationAction(ISD::FMA, MVT::v8f32, Legal); in X86TargetLowering()
1165 setOperationAction(ISD::FMA, MVT::v4f64, Legal); in X86TargetLowering()
1166 setOperationAction(ISD::FMA, MVT::v4f32, Legal); in X86TargetLowering()
1167 setOperationAction(ISD::FMA, MVT::v2f64, Legal); in X86TargetLowering()
1168 setOperationAction(ISD::FMA, MVT::f32, Legal); in X86TargetLowering()
1169 setOperationAction(ISD::FMA, MVT::f64, Legal); in X86TargetLowering()
1173 setOperationAction(ISD::ADD, MVT::v4i64, Legal); in X86TargetLowering()
1174 setOperationAction(ISD::ADD, MVT::v8i32, Legal); in X86TargetLowering()
1175 setOperationAction(ISD::ADD, MVT::v16i16, Legal); in X86TargetLowering()
1176 setOperationAction(ISD::ADD, MVT::v32i8, Legal); in X86TargetLowering()
1178 setOperationAction(ISD::SUB, MVT::v4i64, Legal); in X86TargetLowering()
1179 setOperationAction(ISD::SUB, MVT::v8i32, Legal); in X86TargetLowering()
1180 setOperationAction(ISD::SUB, MVT::v16i16, Legal); in X86TargetLowering()
1181 setOperationAction(ISD::SUB, MVT::v32i8, Legal); in X86TargetLowering()
1183 setOperationAction(ISD::MUL, MVT::v4i64, Custom); in X86TargetLowering()
1184 setOperationAction(ISD::MUL, MVT::v8i32, Legal); in X86TargetLowering()
1185 setOperationAction(ISD::MUL, MVT::v16i16, Legal); in X86TargetLowering()
1186 setOperationAction(ISD::MUL, MVT::v32i8, Custom); in X86TargetLowering()
1188 setOperationAction(ISD::UMUL_LOHI, MVT::v8i32, Custom); in X86TargetLowering()
1189 setOperationAction(ISD::SMUL_LOHI, MVT::v8i32, Custom); in X86TargetLowering()
1190 setOperationAction(ISD::MULHU, MVT::v16i16, Legal); in X86TargetLowering()
1191 setOperationAction(ISD::MULHS, MVT::v16i16, Legal); in X86TargetLowering()
1193 setOperationAction(ISD::SMAX, MVT::v32i8, Legal); in X86TargetLowering()
1194 setOperationAction(ISD::SMAX, MVT::v16i16, Legal); in X86TargetLowering()
1195 setOperationAction(ISD::SMAX, MVT::v8i32, Legal); in X86TargetLowering()
1196 setOperationAction(ISD::UMAX, MVT::v32i8, Legal); in X86TargetLowering()
1197 setOperationAction(ISD::UMAX, MVT::v16i16, Legal); in X86TargetLowering()
1198 setOperationAction(ISD::UMAX, MVT::v8i32, Legal); in X86TargetLowering()
1199 setOperationAction(ISD::SMIN, MVT::v32i8, Legal); in X86TargetLowering()
1200 setOperationAction(ISD::SMIN, MVT::v16i16, Legal); in X86TargetLowering()
1201 setOperationAction(ISD::SMIN, MVT::v8i32, Legal); in X86TargetLowering()
1202 setOperationAction(ISD::UMIN, MVT::v32i8, Legal); in X86TargetLowering()
1203 setOperationAction(ISD::UMIN, MVT::v16i16, Legal); in X86TargetLowering()
1204 setOperationAction(ISD::UMIN, MVT::v8i32, Legal); in X86TargetLowering()
1208 setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Custom); in X86TargetLowering()
1211 setLoadExtAction(ISD::SEXTLOAD, MVT::v16i16, MVT::v16i8, Legal); in X86TargetLowering()
1212 setLoadExtAction(ISD::SEXTLOAD, MVT::v8i32, MVT::v8i8, Legal); in X86TargetLowering()
1213 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i64, MVT::v4i8, Legal); in X86TargetLowering()
1214 setLoadExtAction(ISD::SEXTLOAD, MVT::v8i32, MVT::v8i16, Legal); in X86TargetLowering()
1215 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i64, MVT::v4i16, Legal); in X86TargetLowering()
1216 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i64, MVT::v4i32, Legal); in X86TargetLowering()
1218 setLoadExtAction(ISD::ZEXTLOAD, MVT::v16i16, MVT::v16i8, Legal); in X86TargetLowering()
1219 setLoadExtAction(ISD::ZEXTLOAD, MVT::v8i32, MVT::v8i8, Legal); in X86TargetLowering()
1220 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i64, MVT::v4i8, Legal); in X86TargetLowering()
1221 setLoadExtAction(ISD::ZEXTLOAD, MVT::v8i32, MVT::v8i16, Legal); in X86TargetLowering()
1222 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i64, MVT::v4i16, Legal); in X86TargetLowering()
1223 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i64, MVT::v4i32, Legal); in X86TargetLowering()
1225 setOperationAction(ISD::ADD, MVT::v4i64, Custom); in X86TargetLowering()
1226 setOperationAction(ISD::ADD, MVT::v8i32, Custom); in X86TargetLowering()
1227 setOperationAction(ISD::ADD, MVT::v16i16, Custom); in X86TargetLowering()
1228 setOperationAction(ISD::ADD, MVT::v32i8, Custom); in X86TargetLowering()
1230 setOperationAction(ISD::SUB, MVT::v4i64, Custom); in X86TargetLowering()
1231 setOperationAction(ISD::SUB, MVT::v8i32, Custom); in X86TargetLowering()
1232 setOperationAction(ISD::SUB, MVT::v16i16, Custom); in X86TargetLowering()
1233 setOperationAction(ISD::SUB, MVT::v32i8, Custom); in X86TargetLowering()
1235 setOperationAction(ISD::MUL, MVT::v4i64, Custom); in X86TargetLowering()
1236 setOperationAction(ISD::MUL, MVT::v8i32, Custom); in X86TargetLowering()
1237 setOperationAction(ISD::MUL, MVT::v16i16, Custom); in X86TargetLowering()
1238 setOperationAction(ISD::MUL, MVT::v32i8, Custom); in X86TargetLowering()
1240 setOperationAction(ISD::SMAX, MVT::v32i8, Custom); in X86TargetLowering()
1241 setOperationAction(ISD::SMAX, MVT::v16i16, Custom); in X86TargetLowering()
1242 setOperationAction(ISD::SMAX, MVT::v8i32, Custom); in X86TargetLowering()
1243 setOperationAction(ISD::UMAX, MVT::v32i8, Custom); in X86TargetLowering()
1244 setOperationAction(ISD::UMAX, MVT::v16i16, Custom); in X86TargetLowering()
1245 setOperationAction(ISD::UMAX, MVT::v8i32, Custom); in X86TargetLowering()
1246 setOperationAction(ISD::SMIN, MVT::v32i8, Custom); in X86TargetLowering()
1247 setOperationAction(ISD::SMIN, MVT::v16i16, Custom); in X86TargetLowering()
1248 setOperationAction(ISD::SMIN, MVT::v8i32, Custom); in X86TargetLowering()
1249 setOperationAction(ISD::UMIN, MVT::v32i8, Custom); in X86TargetLowering()
1250 setOperationAction(ISD::UMIN, MVT::v16i16, Custom); in X86TargetLowering()
1251 setOperationAction(ISD::UMIN, MVT::v8i32, Custom); in X86TargetLowering()
1256 setOperationAction(ISD::SRL, MVT::v4i64, Custom); in X86TargetLowering()
1257 setOperationAction(ISD::SRL, MVT::v8i32, Custom); in X86TargetLowering()
1259 setOperationAction(ISD::SHL, MVT::v4i64, Custom); in X86TargetLowering()
1260 setOperationAction(ISD::SHL, MVT::v8i32, Custom); in X86TargetLowering()
1262 setOperationAction(ISD::SRA, MVT::v4i64, Custom); in X86TargetLowering()
1263 setOperationAction(ISD::SRA, MVT::v8i32, Custom); in X86TargetLowering()
1266 for (MVT VT : MVT::vector_valuetypes()) { in X86TargetLowering()
1291 setOperationAction(ISD::VSELECT, MVT::v32i8, Legal); in X86TargetLowering()
1294 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32 }) { in X86TargetLowering()
1296 AddPromotedToType (ISD::AND, VT, MVT::v4i64); in X86TargetLowering()
1298 AddPromotedToType (ISD::OR, VT, MVT::v4i64); in X86TargetLowering()
1300 AddPromotedToType (ISD::XOR, VT, MVT::v4i64); in X86TargetLowering()
1302 AddPromotedToType (ISD::LOAD, VT, MVT::v4i64); in X86TargetLowering()
1304 AddPromotedToType (ISD::SELECT, VT, MVT::v4i64); in X86TargetLowering()
1309 addRegisterClass(MVT::v16i32, &X86::VR512RegClass); in X86TargetLowering()
1310 addRegisterClass(MVT::v16f32, &X86::VR512RegClass); in X86TargetLowering()
1311 addRegisterClass(MVT::v8i64, &X86::VR512RegClass); in X86TargetLowering()
1312 addRegisterClass(MVT::v8f64, &X86::VR512RegClass); in X86TargetLowering()
1314 addRegisterClass(MVT::i1, &X86::VK1RegClass); in X86TargetLowering()
1315 addRegisterClass(MVT::v8i1, &X86::VK8RegClass); in X86TargetLowering()
1316 addRegisterClass(MVT::v16i1, &X86::VK16RegClass); in X86TargetLowering()
1318 for (MVT VT : MVT::fp_vector_valuetypes()) in X86TargetLowering()
1319 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v8f32, Legal); in X86TargetLowering()
1321 setLoadExtAction(ISD::ZEXTLOAD, MVT::v16i32, MVT::v16i8, Legal); in X86TargetLowering()
1322 setLoadExtAction(ISD::SEXTLOAD, MVT::v16i32, MVT::v16i8, Legal); in X86TargetLowering()
1323 setLoadExtAction(ISD::ZEXTLOAD, MVT::v16i32, MVT::v16i16, Legal); in X86TargetLowering()
1324 setLoadExtAction(ISD::SEXTLOAD, MVT::v16i32, MVT::v16i16, Legal); in X86TargetLowering()
1325 setLoadExtAction(ISD::ZEXTLOAD, MVT::v32i16, MVT::v32i8, Legal); in X86TargetLowering()
1326 setLoadExtAction(ISD::SEXTLOAD, MVT::v32i16, MVT::v32i8, Legal); in X86TargetLowering()
1327 setLoadExtAction(ISD::ZEXTLOAD, MVT::v8i64, MVT::v8i8, Legal); in X86TargetLowering()
1328 setLoadExtAction(ISD::SEXTLOAD, MVT::v8i64, MVT::v8i8, Legal); in X86TargetLowering()
1329 setLoadExtAction(ISD::ZEXTLOAD, MVT::v8i64, MVT::v8i16, Legal); in X86TargetLowering()
1330 setLoadExtAction(ISD::SEXTLOAD, MVT::v8i64, MVT::v8i16, Legal); in X86TargetLowering()
1331 setLoadExtAction(ISD::ZEXTLOAD, MVT::v8i64, MVT::v8i32, Legal); in X86TargetLowering()
1332 setLoadExtAction(ISD::SEXTLOAD, MVT::v8i64, MVT::v8i32, Legal); in X86TargetLowering()
1334 setOperationAction(ISD::BR_CC, MVT::i1, Expand); in X86TargetLowering()
1335 setOperationAction(ISD::SETCC, MVT::i1, Custom); in X86TargetLowering()
1336 setOperationAction(ISD::SELECT_CC, MVT::i1, Expand); in X86TargetLowering()
1337 setOperationAction(ISD::XOR, MVT::i1, Legal); in X86TargetLowering()
1338 setOperationAction(ISD::OR, MVT::i1, Legal); in X86TargetLowering()
1339 setOperationAction(ISD::AND, MVT::i1, Legal); in X86TargetLowering()
1340 setOperationAction(ISD::SUB, MVT::i1, Custom); in X86TargetLowering()
1341 setOperationAction(ISD::ADD, MVT::i1, Custom); in X86TargetLowering()
1342 setOperationAction(ISD::MUL, MVT::i1, Custom); in X86TargetLowering()
1343 setOperationAction(ISD::LOAD, MVT::v16f32, Legal); in X86TargetLowering()
1344 setOperationAction(ISD::LOAD, MVT::v8f64, Legal); in X86TargetLowering()
1345 setOperationAction(ISD::LOAD, MVT::v8i64, Legal); in X86TargetLowering()
1346 setOperationAction(ISD::LOAD, MVT::v16i32, Legal); in X86TargetLowering()
1347 setOperationAction(ISD::LOAD, MVT::v16i1, Legal); in X86TargetLowering()
1349 setOperationAction(ISD::FADD, MVT::v16f32, Legal); in X86TargetLowering()
1350 setOperationAction(ISD::FSUB, MVT::v16f32, Legal); in X86TargetLowering()
1351 setOperationAction(ISD::FMUL, MVT::v16f32, Legal); in X86TargetLowering()
1352 setOperationAction(ISD::FDIV, MVT::v16f32, Legal); in X86TargetLowering()
1353 setOperationAction(ISD::FSQRT, MVT::v16f32, Legal); in X86TargetLowering()
1354 setOperationAction(ISD::FNEG, MVT::v16f32, Custom); in X86TargetLowering()
1355 setOperationAction(ISD::FABS, MVT::v16f32, Custom); in X86TargetLowering()
1357 setOperationAction(ISD::FADD, MVT::v8f64, Legal); in X86TargetLowering()
1358 setOperationAction(ISD::FSUB, MVT::v8f64, Legal); in X86TargetLowering()
1359 setOperationAction(ISD::FMUL, MVT::v8f64, Legal); in X86TargetLowering()
1360 setOperationAction(ISD::FDIV, MVT::v8f64, Legal); in X86TargetLowering()
1361 setOperationAction(ISD::FSQRT, MVT::v8f64, Legal); in X86TargetLowering()
1362 setOperationAction(ISD::FNEG, MVT::v8f64, Custom); in X86TargetLowering()
1363 setOperationAction(ISD::FABS, MVT::v8f64, Custom); in X86TargetLowering()
1364 setOperationAction(ISD::FMA, MVT::v8f64, Legal); in X86TargetLowering()
1365 setOperationAction(ISD::FMA, MVT::v16f32, Legal); in X86TargetLowering()
1367 setOperationAction(ISD::FP_TO_SINT, MVT::v16i32, Legal); in X86TargetLowering()
1368 setOperationAction(ISD::FP_TO_UINT, MVT::v16i32, Legal); in X86TargetLowering()
1369 setOperationAction(ISD::FP_TO_UINT, MVT::v8i32, Legal); in X86TargetLowering()
1370 setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Legal); in X86TargetLowering()
1371 setOperationAction(ISD::SINT_TO_FP, MVT::v16i32, Legal); in X86TargetLowering()
1372 setOperationAction(ISD::SINT_TO_FP, MVT::v8i1, Custom); in X86TargetLowering()
1373 setOperationAction(ISD::SINT_TO_FP, MVT::v16i1, Custom); in X86TargetLowering()
1374 setOperationAction(ISD::SINT_TO_FP, MVT::v16i8, Promote); in X86TargetLowering()
1375 setOperationAction(ISD::SINT_TO_FP, MVT::v16i16, Promote); in X86TargetLowering()
1376 setOperationAction(ISD::UINT_TO_FP, MVT::v16i32, Legal); in X86TargetLowering()
1377 setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Legal); in X86TargetLowering()
1378 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Legal); in X86TargetLowering()
1379 setOperationAction(ISD::UINT_TO_FP, MVT::v16i8, Custom); in X86TargetLowering()
1380 setOperationAction(ISD::UINT_TO_FP, MVT::v16i16, Custom); in X86TargetLowering()
1381 setOperationAction(ISD::FP_ROUND, MVT::v8f32, Legal); in X86TargetLowering()
1382 setOperationAction(ISD::FP_EXTEND, MVT::v8f32, Legal); in X86TargetLowering()
1384 setTruncStoreAction(MVT::v8i64, MVT::v8i8, Legal); in X86TargetLowering()
1385 setTruncStoreAction(MVT::v8i64, MVT::v8i16, Legal); in X86TargetLowering()
1386 setTruncStoreAction(MVT::v8i64, MVT::v8i32, Legal); in X86TargetLowering()
1387 setTruncStoreAction(MVT::v16i32, MVT::v16i8, Legal); in X86TargetLowering()
1388 setTruncStoreAction(MVT::v16i32, MVT::v16i16, Legal); in X86TargetLowering()
1390 setTruncStoreAction(MVT::v4i64, MVT::v4i8, Legal); in X86TargetLowering()
1391 setTruncStoreAction(MVT::v4i64, MVT::v4i16, Legal); in X86TargetLowering()
1392 setTruncStoreAction(MVT::v4i64, MVT::v4i32, Legal); in X86TargetLowering()
1393 setTruncStoreAction(MVT::v8i32, MVT::v8i8, Legal); in X86TargetLowering()
1394 setTruncStoreAction(MVT::v8i32, MVT::v8i16, Legal); in X86TargetLowering()
1396 setTruncStoreAction(MVT::v2i64, MVT::v2i8, Legal); in X86TargetLowering()
1397 setTruncStoreAction(MVT::v2i64, MVT::v2i16, Legal); in X86TargetLowering()
1398 setTruncStoreAction(MVT::v2i64, MVT::v2i32, Legal); in X86TargetLowering()
1399 setTruncStoreAction(MVT::v4i32, MVT::v4i8, Legal); in X86TargetLowering()
1400 setTruncStoreAction(MVT::v4i32, MVT::v4i16, Legal); in X86TargetLowering()
1402 setOperationAction(ISD::MLOAD, MVT::v8i32, Custom); in X86TargetLowering()
1403 setOperationAction(ISD::MLOAD, MVT::v8f32, Custom); in X86TargetLowering()
1404 setOperationAction(ISD::MSTORE, MVT::v8i32, Custom); in X86TargetLowering()
1405 setOperationAction(ISD::MSTORE, MVT::v8f32, Custom); in X86TargetLowering()
1407 setOperationAction(ISD::TRUNCATE, MVT::i1, Custom); in X86TargetLowering()
1408 setOperationAction(ISD::TRUNCATE, MVT::v16i8, Custom); in X86TargetLowering()
1409 setOperationAction(ISD::TRUNCATE, MVT::v8i32, Custom); in X86TargetLowering()
1410 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v8i1, Custom); in X86TargetLowering()
1411 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v16i1, Custom); in X86TargetLowering()
1413 setOperationAction(ISD::TRUNCATE, MVT::v2i1, Custom); in X86TargetLowering()
1414 setOperationAction(ISD::TRUNCATE, MVT::v4i1, Custom); in X86TargetLowering()
1416 setOperationAction(ISD::SINT_TO_FP, MVT::v8i64, Legal); in X86TargetLowering()
1417 setOperationAction(ISD::UINT_TO_FP, MVT::v8i64, Legal); in X86TargetLowering()
1418 setOperationAction(ISD::FP_TO_SINT, MVT::v8i64, Legal); in X86TargetLowering()
1419 setOperationAction(ISD::FP_TO_UINT, MVT::v8i64, Legal); in X86TargetLowering()
1421 setOperationAction(ISD::SINT_TO_FP, MVT::v4i64, Legal); in X86TargetLowering()
1422 setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Legal); in X86TargetLowering()
1423 setOperationAction(ISD::UINT_TO_FP, MVT::v4i64, Legal); in X86TargetLowering()
1424 setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Legal); in X86TargetLowering()
1425 setOperationAction(ISD::FP_TO_SINT, MVT::v4i64, Legal); in X86TargetLowering()
1426 setOperationAction(ISD::FP_TO_SINT, MVT::v2i64, Legal); in X86TargetLowering()
1427 setOperationAction(ISD::FP_TO_UINT, MVT::v4i64, Legal); in X86TargetLowering()
1428 setOperationAction(ISD::FP_TO_UINT, MVT::v2i64, Legal); in X86TargetLowering()
1432 setOperationAction(ISD::SINT_TO_FP, MVT::v8i32, Legal); in X86TargetLowering()
1433 setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Legal); in X86TargetLowering()
1434 setOperationAction(ISD::FP_TO_SINT, MVT::v8i32, Legal); in X86TargetLowering()
1435 setOperationAction(ISD::FP_TO_UINT, MVT::v8i32, Legal); in X86TargetLowering()
1436 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal); in X86TargetLowering()
1437 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Legal); in X86TargetLowering()
1438 setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal); in X86TargetLowering()
1439 setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Legal); in X86TargetLowering()
1441 setOperationAction(ISD::TRUNCATE, MVT::v8i1, Custom); in X86TargetLowering()
1442 setOperationAction(ISD::TRUNCATE, MVT::v16i1, Custom); in X86TargetLowering()
1443 setOperationAction(ISD::TRUNCATE, MVT::v16i16, Custom); in X86TargetLowering()
1444 setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom); in X86TargetLowering()
1445 setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom); in X86TargetLowering()
1446 setOperationAction(ISD::ANY_EXTEND, MVT::v16i32, Custom); in X86TargetLowering()
1447 setOperationAction(ISD::ANY_EXTEND, MVT::v8i64, Custom); in X86TargetLowering()
1448 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom); in X86TargetLowering()
1449 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64, Custom); in X86TargetLowering()
1450 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i8, Custom); in X86TargetLowering()
1451 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i16, Custom); in X86TargetLowering()
1452 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i16, Custom); in X86TargetLowering()
1454 setOperationAction(ISD::SIGN_EXTEND, MVT::v4i32, Custom); in X86TargetLowering()
1455 setOperationAction(ISD::SIGN_EXTEND, MVT::v2i64, Custom); in X86TargetLowering()
1457 setOperationAction(ISD::FFLOOR, MVT::v16f32, Legal); in X86TargetLowering()
1458 setOperationAction(ISD::FFLOOR, MVT::v8f64, Legal); in X86TargetLowering()
1459 setOperationAction(ISD::FCEIL, MVT::v16f32, Legal); in X86TargetLowering()
1460 setOperationAction(ISD::FCEIL, MVT::v8f64, Legal); in X86TargetLowering()
1461 setOperationAction(ISD::FTRUNC, MVT::v16f32, Legal); in X86TargetLowering()
1462 setOperationAction(ISD::FTRUNC, MVT::v8f64, Legal); in X86TargetLowering()
1463 setOperationAction(ISD::FRINT, MVT::v16f32, Legal); in X86TargetLowering()
1464 setOperationAction(ISD::FRINT, MVT::v8f64, Legal); in X86TargetLowering()
1465 setOperationAction(ISD::FNEARBYINT, MVT::v16f32, Legal); in X86TargetLowering()
1466 setOperationAction(ISD::FNEARBYINT, MVT::v8f64, Legal); in X86TargetLowering()
1468 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8f64, Custom); in X86TargetLowering()
1469 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i64, Custom); in X86TargetLowering()
1470 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16f32, Custom); in X86TargetLowering()
1471 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16i32, Custom); in X86TargetLowering()
1472 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16i1, Custom); in X86TargetLowering()
1474 setOperationAction(ISD::SETCC, MVT::v16i1, Custom); in X86TargetLowering()
1475 setOperationAction(ISD::SETCC, MVT::v8i1, Custom); in X86TargetLowering()
1477 setOperationAction(ISD::MUL, MVT::v8i64, Custom); in X86TargetLowering()
1479 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i1, Custom); in X86TargetLowering()
1480 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i1, Custom); in X86TargetLowering()
1481 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v16i1, Custom); in X86TargetLowering()
1482 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i1, Custom); in X86TargetLowering()
1483 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i1, Custom); in X86TargetLowering()
1484 setOperationAction(ISD::BUILD_VECTOR, MVT::v8i1, Custom); in X86TargetLowering()
1485 setOperationAction(ISD::BUILD_VECTOR, MVT::v16i1, Custom); in X86TargetLowering()
1486 setOperationAction(ISD::SELECT, MVT::v8f64, Custom); in X86TargetLowering()
1487 setOperationAction(ISD::SELECT, MVT::v8i64, Custom); in X86TargetLowering()
1488 setOperationAction(ISD::SELECT, MVT::v16f32, Custom); in X86TargetLowering()
1489 setOperationAction(ISD::SELECT, MVT::v16i1, Custom); in X86TargetLowering()
1490 setOperationAction(ISD::SELECT, MVT::v8i1, Custom); in X86TargetLowering()
1492 setOperationAction(ISD::SMAX, MVT::v16i32, Legal); in X86TargetLowering()
1493 setOperationAction(ISD::SMAX, MVT::v8i64, Legal); in X86TargetLowering()
1494 setOperationAction(ISD::UMAX, MVT::v16i32, Legal); in X86TargetLowering()
1495 setOperationAction(ISD::UMAX, MVT::v8i64, Legal); in X86TargetLowering()
1496 setOperationAction(ISD::SMIN, MVT::v16i32, Legal); in X86TargetLowering()
1497 setOperationAction(ISD::SMIN, MVT::v8i64, Legal); in X86TargetLowering()
1498 setOperationAction(ISD::UMIN, MVT::v16i32, Legal); in X86TargetLowering()
1499 setOperationAction(ISD::UMIN, MVT::v8i64, Legal); in X86TargetLowering()
1501 setOperationAction(ISD::ADD, MVT::v8i64, Legal); in X86TargetLowering()
1502 setOperationAction(ISD::ADD, MVT::v16i32, Legal); in X86TargetLowering()
1504 setOperationAction(ISD::SUB, MVT::v8i64, Legal); in X86TargetLowering()
1505 setOperationAction(ISD::SUB, MVT::v16i32, Legal); in X86TargetLowering()
1507 setOperationAction(ISD::MUL, MVT::v16i32, Legal); in X86TargetLowering()
1509 setOperationAction(ISD::SRL, MVT::v8i64, Custom); in X86TargetLowering()
1510 setOperationAction(ISD::SRL, MVT::v16i32, Custom); in X86TargetLowering()
1512 setOperationAction(ISD::SHL, MVT::v8i64, Custom); in X86TargetLowering()
1513 setOperationAction(ISD::SHL, MVT::v16i32, Custom); in X86TargetLowering()
1515 setOperationAction(ISD::SRA, MVT::v8i64, Custom); in X86TargetLowering()
1516 setOperationAction(ISD::SRA, MVT::v16i32, Custom); in X86TargetLowering()
1518 setOperationAction(ISD::AND, MVT::v8i64, Legal); in X86TargetLowering()
1519 setOperationAction(ISD::OR, MVT::v8i64, Legal); in X86TargetLowering()
1520 setOperationAction(ISD::XOR, MVT::v8i64, Legal); in X86TargetLowering()
1521 setOperationAction(ISD::AND, MVT::v16i32, Legal); in X86TargetLowering()
1522 setOperationAction(ISD::OR, MVT::v16i32, Legal); in X86TargetLowering()
1523 setOperationAction(ISD::XOR, MVT::v16i32, Legal); in X86TargetLowering()
1526 setOperationAction(ISD::CTLZ, MVT::v8i64, Legal); in X86TargetLowering()
1527 setOperationAction(ISD::CTLZ, MVT::v16i32, Legal); in X86TargetLowering()
1528 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::v8i64, Legal); in X86TargetLowering()
1529 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::v16i32, Legal); in X86TargetLowering()
1531 setOperationAction(ISD::CTLZ, MVT::v8i16, Custom); in X86TargetLowering()
1532 setOperationAction(ISD::CTLZ, MVT::v16i8, Custom); in X86TargetLowering()
1533 setOperationAction(ISD::CTLZ, MVT::v16i16, Custom); in X86TargetLowering()
1534 setOperationAction(ISD::CTLZ, MVT::v32i8, Custom); in X86TargetLowering()
1535 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::v8i16, Custom); in X86TargetLowering()
1536 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::v16i8, Custom); in X86TargetLowering()
1537 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::v16i16, Custom); in X86TargetLowering()
1538 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::v32i8, Custom); in X86TargetLowering()
1540 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v8i64, Custom); in X86TargetLowering()
1541 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v16i32, Custom); in X86TargetLowering()
1544 setOperationAction(ISD::CTLZ, MVT::v4i64, Legal); in X86TargetLowering()
1545 setOperationAction(ISD::CTLZ, MVT::v8i32, Legal); in X86TargetLowering()
1546 setOperationAction(ISD::CTLZ, MVT::v2i64, Legal); in X86TargetLowering()
1547 setOperationAction(ISD::CTLZ, MVT::v4i32, Legal); in X86TargetLowering()
1548 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::v4i64, Legal); in X86TargetLowering()
1549 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::v8i32, Legal); in X86TargetLowering()
1550 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::v2i64, Legal); in X86TargetLowering()
1551 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::v4i32, Legal); in X86TargetLowering()
1553 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v4i64, Custom); in X86TargetLowering()
1554 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v8i32, Custom); in X86TargetLowering()
1555 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v2i64, Custom); in X86TargetLowering()
1556 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v4i32, Custom); in X86TargetLowering()
1558 setOperationAction(ISD::CTLZ, MVT::v4i64, Custom); in X86TargetLowering()
1559 setOperationAction(ISD::CTLZ, MVT::v8i32, Custom); in X86TargetLowering()
1560 setOperationAction(ISD::CTLZ, MVT::v2i64, Custom); in X86TargetLowering()
1561 setOperationAction(ISD::CTLZ, MVT::v4i32, Custom); in X86TargetLowering()
1562 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::v4i64, Custom); in X86TargetLowering()
1563 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::v8i32, Custom); in X86TargetLowering()
1564 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::v2i64, Custom); in X86TargetLowering()
1565 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::v4i32, Custom); in X86TargetLowering()
1570 setOperationAction(ISD::MUL, MVT::v2i64, Legal); in X86TargetLowering()
1571 setOperationAction(ISD::MUL, MVT::v4i64, Legal); in X86TargetLowering()
1572 setOperationAction(ISD::MUL, MVT::v8i64, Legal); in X86TargetLowering()
1575 for (MVT VT : MVT::vector_valuetypes()) { in X86TargetLowering()
1591 if (VT.getVectorElementType() == MVT::i1) in X86TargetLowering()
1612 for (auto VT : { MVT::v64i8, MVT::v32i16, MVT::v16i32 }) { in X86TargetLowering()
1614 AddPromotedToType (ISD::SELECT, VT, MVT::v8i64); in X86TargetLowering()
1619 addRegisterClass(MVT::v32i16, &X86::VR512RegClass); in X86TargetLowering()
1620 addRegisterClass(MVT::v64i8, &X86::VR512RegClass); in X86TargetLowering()
1622 addRegisterClass(MVT::v32i1, &X86::VK32RegClass); in X86TargetLowering()
1623 addRegisterClass(MVT::v64i1, &X86::VK64RegClass); in X86TargetLowering()
1625 setOperationAction(ISD::LOAD, MVT::v32i16, Legal); in X86TargetLowering()
1626 setOperationAction(ISD::LOAD, MVT::v64i8, Legal); in X86TargetLowering()
1627 setOperationAction(ISD::SETCC, MVT::v32i1, Custom); in X86TargetLowering()
1628 setOperationAction(ISD::SETCC, MVT::v64i1, Custom); in X86TargetLowering()
1629 setOperationAction(ISD::ADD, MVT::v32i16, Legal); in X86TargetLowering()
1630 setOperationAction(ISD::ADD, MVT::v64i8, Legal); in X86TargetLowering()
1631 setOperationAction(ISD::SUB, MVT::v32i16, Legal); in X86TargetLowering()
1632 setOperationAction(ISD::SUB, MVT::v64i8, Legal); in X86TargetLowering()
1633 setOperationAction(ISD::MUL, MVT::v32i16, Legal); in X86TargetLowering()
1634 setOperationAction(ISD::MULHS, MVT::v32i16, Legal); in X86TargetLowering()
1635 setOperationAction(ISD::MULHU, MVT::v32i16, Legal); in X86TargetLowering()
1636 setOperationAction(ISD::CONCAT_VECTORS, MVT::v32i1, Custom); in X86TargetLowering()
1637 setOperationAction(ISD::CONCAT_VECTORS, MVT::v64i1, Custom); in X86TargetLowering()
1638 setOperationAction(ISD::CONCAT_VECTORS, MVT::v32i16, Custom); in X86TargetLowering()
1639 setOperationAction(ISD::CONCAT_VECTORS, MVT::v64i8, Custom); in X86TargetLowering()
1640 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v32i1, Custom); in X86TargetLowering()
1641 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v64i1, Custom); in X86TargetLowering()
1642 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v32i16, Custom); in X86TargetLowering()
1643 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v64i8, Custom); in X86TargetLowering()
1644 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v32i16, Custom); in X86TargetLowering()
1645 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v64i8, Custom); in X86TargetLowering()
1646 setOperationAction(ISD::SELECT, MVT::v32i1, Custom); in X86TargetLowering()
1647 setOperationAction(ISD::SELECT, MVT::v64i1, Custom); in X86TargetLowering()
1648 setOperationAction(ISD::SIGN_EXTEND, MVT::v32i8, Custom); in X86TargetLowering()
1649 setOperationAction(ISD::ZERO_EXTEND, MVT::v32i8, Custom); in X86TargetLowering()
1650 setOperationAction(ISD::SIGN_EXTEND, MVT::v32i16, Custom); in X86TargetLowering()
1651 setOperationAction(ISD::ZERO_EXTEND, MVT::v32i16, Custom); in X86TargetLowering()
1652 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v32i16, Custom); in X86TargetLowering()
1653 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v64i8, Custom); in X86TargetLowering()
1654 setOperationAction(ISD::SIGN_EXTEND, MVT::v64i8, Custom); in X86TargetLowering()
1655 setOperationAction(ISD::ZERO_EXTEND, MVT::v64i8, Custom); in X86TargetLowering()
1656 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v32i1, Custom); in X86TargetLowering()
1657 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v64i1, Custom); in X86TargetLowering()
1658 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v32i16, Custom); in X86TargetLowering()
1659 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v64i8, Custom); in X86TargetLowering()
1660 setOperationAction(ISD::VSELECT, MVT::v32i16, Legal); in X86TargetLowering()
1661 setOperationAction(ISD::VSELECT, MVT::v64i8, Legal); in X86TargetLowering()
1662 setOperationAction(ISD::TRUNCATE, MVT::v32i1, Custom); in X86TargetLowering()
1663 setOperationAction(ISD::TRUNCATE, MVT::v64i1, Custom); in X86TargetLowering()
1664 setOperationAction(ISD::TRUNCATE, MVT::v32i8, Custom); in X86TargetLowering()
1665 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v32i1, Custom); in X86TargetLowering()
1666 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v64i1, Custom); in X86TargetLowering()
1668 setOperationAction(ISD::SMAX, MVT::v64i8, Legal); in X86TargetLowering()
1669 setOperationAction(ISD::SMAX, MVT::v32i16, Legal); in X86TargetLowering()
1670 setOperationAction(ISD::UMAX, MVT::v64i8, Legal); in X86TargetLowering()
1671 setOperationAction(ISD::UMAX, MVT::v32i16, Legal); in X86TargetLowering()
1672 setOperationAction(ISD::SMIN, MVT::v64i8, Legal); in X86TargetLowering()
1673 setOperationAction(ISD::SMIN, MVT::v32i16, Legal); in X86TargetLowering()
1674 setOperationAction(ISD::UMIN, MVT::v64i8, Legal); in X86TargetLowering()
1675 setOperationAction(ISD::UMIN, MVT::v32i16, Legal); in X86TargetLowering()
1677 setTruncStoreAction(MVT::v32i16, MVT::v32i8, Legal); in X86TargetLowering()
1678 setTruncStoreAction(MVT::v16i16, MVT::v16i8, Legal); in X86TargetLowering()
1680 setTruncStoreAction(MVT::v8i16, MVT::v8i8, Legal); in X86TargetLowering()
1683 setOperationAction(ISD::CTLZ, MVT::v32i16, Custom); in X86TargetLowering()
1684 setOperationAction(ISD::CTLZ, MVT::v64i8, Custom); in X86TargetLowering()
1685 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::v32i16, Custom); in X86TargetLowering()
1686 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::v64i8, Custom); in X86TargetLowering()
1689 for (auto VT : { MVT::v64i8, MVT::v32i16 }) { in X86TargetLowering()
1694 AddPromotedToType (ISD::AND, VT, MVT::v8i64); in X86TargetLowering()
1696 AddPromotedToType (ISD::OR, VT, MVT::v8i64); in X86TargetLowering()
1698 AddPromotedToType (ISD::XOR, VT, MVT::v8i64); in X86TargetLowering()
1703 addRegisterClass(MVT::v4i1, &X86::VK4RegClass); in X86TargetLowering()
1704 addRegisterClass(MVT::v2i1, &X86::VK2RegClass); in X86TargetLowering()
1706 setOperationAction(ISD::SETCC, MVT::v4i1, Custom); in X86TargetLowering()
1707 setOperationAction(ISD::SETCC, MVT::v2i1, Custom); in X86TargetLowering()
1708 setOperationAction(ISD::CONCAT_VECTORS, MVT::v4i1, Custom); in X86TargetLowering()
1709 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i1, Custom); in X86TargetLowering()
1710 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v8i1, Custom); in X86TargetLowering()
1711 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v4i1, Custom); in X86TargetLowering()
1712 setOperationAction(ISD::SELECT, MVT::v4i1, Custom); in X86TargetLowering()
1713 setOperationAction(ISD::SELECT, MVT::v2i1, Custom); in X86TargetLowering()
1714 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i1, Custom); in X86TargetLowering()
1715 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i1, Custom); in X86TargetLowering()
1716 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i1, Custom); in X86TargetLowering()
1717 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4i1, Custom); in X86TargetLowering()
1719 setOperationAction(ISD::AND, MVT::v8i32, Legal); in X86TargetLowering()
1720 setOperationAction(ISD::OR, MVT::v8i32, Legal); in X86TargetLowering()
1721 setOperationAction(ISD::XOR, MVT::v8i32, Legal); in X86TargetLowering()
1722 setOperationAction(ISD::AND, MVT::v4i32, Legal); in X86TargetLowering()
1723 setOperationAction(ISD::OR, MVT::v4i32, Legal); in X86TargetLowering()
1724 setOperationAction(ISD::XOR, MVT::v4i32, Legal); in X86TargetLowering()
1725 setOperationAction(ISD::SRA, MVT::v2i64, Custom); in X86TargetLowering()
1726 setOperationAction(ISD::SRA, MVT::v4i64, Custom); in X86TargetLowering()
1728 setOperationAction(ISD::SMAX, MVT::v2i64, Legal); in X86TargetLowering()
1729 setOperationAction(ISD::SMAX, MVT::v4i64, Legal); in X86TargetLowering()
1730 setOperationAction(ISD::UMAX, MVT::v2i64, Legal); in X86TargetLowering()
1731 setOperationAction(ISD::UMAX, MVT::v4i64, Legal); in X86TargetLowering()
1732 setOperationAction(ISD::SMIN, MVT::v2i64, Legal); in X86TargetLowering()
1733 setOperationAction(ISD::SMIN, MVT::v4i64, Legal); in X86TargetLowering()
1734 setOperationAction(ISD::UMIN, MVT::v2i64, Legal); in X86TargetLowering()
1735 setOperationAction(ISD::UMIN, MVT::v4i64, Legal); in X86TargetLowering()
1739 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); in X86TargetLowering()
1740 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom); in X86TargetLowering()
1741 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); in X86TargetLowering()
1743 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom); in X86TargetLowering()
1744 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom); in X86TargetLowering()
1753 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) { in X86TargetLowering()
1754 if (VT == MVT::i64 && !Subtarget->is64Bit()) in X86TargetLowering()
1779 setOperationAction(ISD::FSINCOS, MVT::f64, Custom); in X86TargetLowering()
1780 setOperationAction(ISD::FSINCOS, MVT::f32, Custom); in X86TargetLowering()
1785 setOperationAction(ISD::SDIV, MVT::i128, Custom); in X86TargetLowering()
1786 setOperationAction(ISD::UDIV, MVT::i128, Custom); in X86TargetLowering()
1787 setOperationAction(ISD::SREM, MVT::i128, Custom); in X86TargetLowering()
1788 setOperationAction(ISD::UREM, MVT::i128, Custom); in X86TargetLowering()
1789 setOperationAction(ISD::SDIVREM, MVT::i128, Custom); in X86TargetLowering()
1790 setOperationAction(ISD::UDIVREM, MVT::i128, Custom); in X86TargetLowering()
1857 VT.getVectorElementType().getSimpleVT() != MVT::i1) in getPreferredVectorAction()
1866 return Subtarget->hasAVX512() ? MVT::i1: MVT::i8; in getSetCCResultType()
1869 MVT VVT = VT.getSimpleVT(); in getSetCCResultType()
1871 const MVT EltVT = VVT.getVectorElementType(); in getSetCCResultType()
1874 if (EltVT == MVT::i32 || EltVT == MVT::i64 || in getSetCCResultType()
1875 EltVT == MVT::f32 || EltVT == MVT::f64) in getSetCCResultType()
1877 case 8: return MVT::v8i1; in getSetCCResultType()
1878 case 16: return MVT::v16i1; in getSetCCResultType()
1881 if (EltVT == MVT::i8 || EltVT == MVT::i16) in getSetCCResultType()
1883 case 32: return MVT::v32i1; in getSetCCResultType()
1884 case 64: return MVT::v64i1; in getSetCCResultType()
1890 if (EltVT == MVT::i32 || EltVT == MVT::i64 || in getSetCCResultType()
1891 EltVT == MVT::f32 || EltVT == MVT::f64) in getSetCCResultType()
1893 case 2: return MVT::v2i1; in getSetCCResultType()
1894 case 4: return MVT::v4i1; in getSetCCResultType()
1895 case 8: return MVT::v8i1; in getSetCCResultType()
1898 if (EltVT == MVT::i8 || EltVT == MVT::i16) in getSetCCResultType()
1900 case 8: return MVT::v8i1; in getSetCCResultType()
1901 case 16: return MVT::v16i1; in getSetCCResultType()
1902 case 32: return MVT::v32i1; in getSetCCResultType()
1982 return MVT::v8i32; in getOptimalMemOpType()
1984 return MVT::v8f32; in getOptimalMemOpType()
1987 return MVT::v4i32; in getOptimalMemOpType()
1989 return MVT::v4f32; in getOptimalMemOpType()
1995 return MVT::f64; in getOptimalMemOpType()
2002 return MVT::i64; in getOptimalMemOpType()
2003 return MVT::i32; in getOptimalMemOpType()
2006 bool X86TargetLowering::isSafeMemOpType(MVT VT) const { in isSafeMemOpType()
2007 if (VT == MVT::f32) in isSafeMemOpType()
2009 else if (VT == MVT::f64) in isSafeMemOpType()
2094 MVT VT) const { in findRepresentativeClass()
2100 case MVT::i8: case MVT::i16: case MVT::i32: case MVT::i64: in findRepresentativeClass()
2103 case MVT::x86mmx: in findRepresentativeClass()
2106 case MVT::f32: case MVT::f64: in findRepresentativeClass()
2107 case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64: in findRepresentativeClass()
2108 case MVT::v4f32: case MVT::v2f64: in findRepresentativeClass()
2109 case MVT::v32i8: case MVT::v8i32: case MVT::v4i64: case MVT::v8f32: in findRepresentativeClass()
2110 case MVT::v4f64: in findRepresentativeClass()
2210 MVT::i16)); in LowerReturn()
2225 if (ValVT.isVector() && ValVT.getVectorElementType() == MVT::i1) in LowerReturn()
2238 if ((ValVT == MVT::f32 || ValVT == MVT::f64 || in LowerReturn()
2246 if (ValVT == MVT::f64 && in LowerReturn()
2257 ValToCopy = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f80, ValToCopy); in LowerReturn()
2266 if (ValVT == MVT::x86mmx) { in LowerReturn()
2268 ValToCopy = DAG.getBitcast(MVT::i64, ValToCopy); in LowerReturn()
2269 ValToCopy = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, in LowerReturn()
2274 ValToCopy = DAG.getBitcast(MVT::v4f32, ValToCopy); in LowerReturn()
2317 return DAG.getNode(opcode, dl, MVT::Other, RetOps); in LowerReturn()
2331 if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue) in isUsedByReturnOnly()
2347 UI->getOperand(UI->getNumOperands()-1).getValueType() != MVT::Glue) in isUsedByReturnOnly()
2362 MVT ReturnMVT; in getTypeForExtArgOrReturn()
2364 if (Subtarget->is64Bit() && VT == MVT::i1 && ExtendKind == ISD::ZERO_EXTEND) in getTypeForExtArgOrReturn()
2365 ReturnMVT = MVT::i8; in getTypeForExtArgOrReturn()
2367 ReturnMVT = MVT::i32; in getTypeForExtArgOrReturn()
2396 if ((CopyVT == MVT::f32 || CopyVT == MVT::f64 || CopyVT == MVT::f128) && in LowerCallResult()
2406 CopyVT = MVT::f80; in LowerCallResult()
2419 if (VA.isExtInLoc() && VA.getValVT().getScalarType() == MVT::i1) in LowerCallResult()
2480 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32); in CreateCopyOfByValArgument()
2550 VA.getValVT().getScalarType() == MVT::i1; in LowerMemArgument()
2672 (Ins.size() == 2 && ((Is64Bit && Ins[1].VT == MVT::i64) || in LowerFormalArguments()
2673 (!Is64Bit && Ins[1].VT == MVT::i32))); in LowerFormalArguments()
2702 if (RegVT == MVT::i32) in LowerFormalArguments()
2704 else if (Is64Bit && RegVT == MVT::i64) in LowerFormalArguments()
2706 else if (RegVT == MVT::f32) in LowerFormalArguments()
2708 else if (RegVT == MVT::f64) in LowerFormalArguments()
2710 else if (RegVT == MVT::f128) in LowerFormalArguments()
2718 else if (RegVT == MVT::x86mmx) in LowerFormalArguments()
2720 else if (RegVT == MVT::i1) in LowerFormalArguments()
2722 else if (RegVT == MVT::v8i1) in LowerFormalArguments()
2724 else if (RegVT == MVT::v16i1) in LowerFormalArguments()
2726 else if (RegVT == MVT::v32i1) in LowerFormalArguments()
2728 else if (RegVT == MVT::v64i1) in LowerFormalArguments()
2750 if (RegVT.isVector() && VA.getValVT().getScalarType() != MVT::i1) in LowerFormalArguments()
2776 MVT PtrTy = getPointerTy(DAG.getDataLayout()); in LowerFormalArguments()
2781 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Copy, Chain); in LowerFormalArguments()
2825 DAG.getCopyFromReg(Chain, dl, GPR, MVT::i64)); in LowerFormalArguments()
2829 ALVal = DAG.getCopyFromReg(Chain, dl, AL, MVT::i8); in LowerFormalArguments()
2833 DAG.getCopyFromReg(Chain, dl, XMMReg, MVT::v4f32)); in LowerFormalArguments()
2886 MVT::Other, SaveXMMOps)); in LowerFormalArguments()
2890 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); in LowerFormalArguments()
2895 MVT VecVT = MVT::Other; in LowerFormalArguments()
2900 VecVT = MVT::v16f32; in LowerFormalArguments()
2902 VecVT = MVT::v8f32; in LowerFormalArguments()
2904 VecVT = MVT::v4f32; in LowerFormalArguments()
2907 SmallVector<MVT, 2> RegParmTypes; in LowerFormalArguments()
2908 MVT IntVT = Is64Bit ? MVT::i64 : MVT::i32; in LowerFormalArguments()
2910 if (VecVT != MVT::Other) in LowerFormalArguments()
2921 Forwards.push_back(ForwardedRegister(ALVReg, X86::AL, MVT::i8)); in LowerFormalArguments()
3038 static SDValue getMOVL(SelectionDAG &DAG, SDLoc dl, MVT VT, SDValue V1, in getMOVL()
3203 Arg.getValueType().getVectorElementType() == MVT::i1) in LowerCall()
3207 Arg = DAG.getBitcast(MVT::i64, Arg); in LowerCall()
3208 Arg = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Arg); in LowerCall()
3209 Arg = getMOVL(DAG, dl, MVT::v2i64, DAG.getUNDEF(MVT::v2i64), Arg); in LowerCall()
3255 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); in LowerCall()
3303 MVT::i8))); in LowerCall()
3367 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains2); in LowerCall()
3464 Callee->getValueType(0) == MVT::i32) { in LowerCall()
3466 Callee = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Callee); in LowerCall()
3470 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerCall()
3484 Ops.push_back(DAG.getConstant(FPDiff, dl, MVT::i32)); in LowerCall()
3927 static SDValue getTargetShuffleNode(unsigned Opc, SDLoc dl, MVT VT, in getTargetShuffleNode()
3938 DAG.getConstant(TargetMask, dl, MVT::i8)); in getTargetShuffleNode()
3942 static SDValue getTargetShuffleNode(unsigned Opc, SDLoc dl, MVT VT, in getTargetShuffleNode()
4250 MVT VT = N->getSimpleValueType(0); in isVEXTRACTIndex()
4268 MVT VT = N->getSimpleValueType(0); in isVINSERTIndex()
4299 MVT VecVT = N->getOperand(0).getSimpleValueType(); in getExtractVEXTRACTImmediate()
4300 MVT ElVT = VecVT.getVectorElementType(); in getExtractVEXTRACTImmediate()
4314 MVT VecVT = N->getSimpleValueType(0); in getInsertVINSERTImmediate()
4315 MVT ElVT = VecVT.getVectorElementType(); in getInsertVINSERTImmediate()
4353 static SDValue getConstVector(ArrayRef<int> Values, MVT VT, in getConstVector()
4360 MVT ConstVecVT = VT; in getConstVector()
4362 bool In64BitMode = DAG.getTargetLoweringInfo().isTypeLegal(MVT::i64); in getConstVector()
4363 if (!In64BitMode && VT.getVectorElementType() == MVT::i64) { in getConstVector()
4364 ConstVecVT = MVT::getVectorVT(MVT::i32, NumElts * 2); in getConstVector()
4368 MVT EltVT = ConstVecVT.getVectorElementType(); in getConstVector()
4385 static SDValue getZeroVector(MVT VT, const X86Subtarget *Subtarget, in getZeroVector()
4394 SDValue Cst = DAG.getConstant(0, dl, MVT::i32); in getZeroVector()
4395 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst); in getZeroVector()
4397 SDValue Cst = DAG.getConstantFP(+0.0, dl, MVT::f32); in getZeroVector()
4398 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4f32, Cst, Cst, Cst, Cst); in getZeroVector()
4402 SDValue Cst = DAG.getConstant(0, dl, MVT::i32); in getZeroVector()
4404 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8i32, Ops); in getZeroVector()
4408 SDValue Cst = DAG.getConstantFP(+0.0, dl, MVT::f32); in getZeroVector()
4410 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8f32, Ops); in getZeroVector()
4413 SDValue Cst = DAG.getConstant(0, dl, MVT::i32); in getZeroVector()
4416 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v16i32, Ops); in getZeroVector()
4417 } else if (VT.getVectorElementType() == MVT::i1) { in getZeroVector()
4423 SDValue Cst = DAG.getConstant(0, dl, MVT::i1); in getZeroVector()
4532 MVT ScalarType = ResultVT.getVectorElementType().getSimpleVT(); in Insert128BitVector()
4538 SDValue Mask = DAG.getConstant(MaskVal, dl, MVT::i8); in Insert128BitVector()
4552 MVT CastVT = Subtarget.hasAVX2() ? MVT::v8i32 : MVT::v8f32; in Insert128BitVector()
4554 SDValue Mask = DAG.getConstant(0x0f, dl, MVT::i8); in Insert128BitVector()
4584 MVT OpVT = Op.getSimpleValueType(); in Insert1BitVector()
4585 MVT SubVecVT = SubVec.getSimpleValueType(); in Insert1BitVector()
4606 DAG.getConstant(IdxVal, dl, MVT::i8)); in Insert1BitVector()
4612 DAG.getConstant(ShiftLeft, dl, MVT::i8)); in Insert1BitVector()
4614 DAG.getConstant(ShiftRight, dl, MVT::i8)) : WideSubVec; in Insert1BitVector()
4619 SDValue ShiftBits = DAG.getConstant(SubVecNumElems, dl, MVT::i8); in Insert1BitVector()
4630 DAG.getConstant(IdxVal, dl, MVT::i8)); in Insert1BitVector()
4631 SDValue ShiftBits = DAG.getConstant(SubVecNumElems, dl, MVT::i8); in Insert1BitVector()
4670 SDValue Cst = DAG.getConstant(~0U, dl, MVT::i32); in getOnesVector()
4675 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v16i32, Ops); in getOnesVector()
4679 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8i32, Ops); in getOnesVector()
4681 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst); in getOnesVector()
4682 Vec = Concat128BitVectors(Vec, Vec, MVT::v8i32, 8, DAG, dl); in getOnesVector()
4685 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst); in getOnesVector()
4693 static SDValue getUnpackl(SelectionDAG &DAG, SDLoc dl, MVT VT, SDValue V1, in getUnpackl()
4705 static SDValue getUnpackh(SelectionDAG &DAG, SDLoc dl, MVT VT, SDValue V1, in getUnpackh()
4724 MVT VT = V2.getSimpleValueType(); in getShuffleVectorZeroOrUndef()
4741 static bool getTargetShuffleMask(SDNode *N, MVT VT, in getTargetShuffleMask()
4802 MVT VT = MaskNode.getSimpleValueType(); in getTargetShuffleMask()
5048 MVT ShufVT = V.getSimpleValueType(); in getShuffleScalarElt()
5106 V = getZeroVector(MVT::v16i8, Subtarget, DAG, dl); in LowerBuildVectorv16i8()
5108 V = DAG.getUNDEF(MVT::v16i8); in LowerBuildVectorv16i8()
5112 MVT::v16i8, V, Op.getOperand(i), in LowerBuildVectorv16i8()
5125 V = getZeroVector(MVT::v8i16, Subtarget, DAG, dl); in LowerBuildVectorv16i8()
5127 V = DAG.getUNDEF(MVT::v8i16); in LowerBuildVectorv16i8()
5136 MVT::i16, Op.getOperand(i-1)); in LowerBuildVectorv16i8()
5139 ThisElt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Op.getOperand(i)); in LowerBuildVectorv16i8()
5140 ThisElt = DAG.getNode(ISD::SHL, dl, MVT::i16, in LowerBuildVectorv16i8()
5141 ThisElt, DAG.getConstant(8, dl, MVT::i8)); in LowerBuildVectorv16i8()
5143 ThisElt = DAG.getNode(ISD::OR, dl, MVT::i16, ThisElt, LastElt); in LowerBuildVectorv16i8()
5148 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, V, ThisElt, in LowerBuildVectorv16i8()
5153 return DAG.getBitcast(MVT::v16i8, V); in LowerBuildVectorv16i8()
5173 V = getZeroVector(MVT::v8i16, Subtarget, DAG, dl); in LowerBuildVectorv8i16()
5175 V = DAG.getUNDEF(MVT::v8i16); in LowerBuildVectorv8i16()
5179 MVT::v8i16, V, Op.getOperand(i), in LowerBuildVectorv8i16()
5212 MVT VT = Elt.getOperand(0).getSimpleValueType(); in LowerBuildVectorv4x32()
5223 MVT VT = V1.getSimpleValueType(); in LowerBuildVectorv4x32()
5277 if (V1.getSimpleValueType() != MVT::v4f32) in LowerBuildVectorv4x32()
5278 V1 = DAG.getNode(ISD::BITCAST, SDLoc(V1), MVT::v4f32, V1); in LowerBuildVectorv4x32()
5279 if (V2.getSimpleValueType() != MVT::v4f32) in LowerBuildVectorv4x32()
5280 V2 = DAG.getNode(ISD::BITCAST, SDLoc(V2), MVT::v4f32, V2); in LowerBuildVectorv4x32()
5288 SDValue Result = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2, in LowerBuildVectorv4x32()
5298 MVT ShVT = MVT::v2i64; in getVShift()
5301 MVT ScalarShiftTy = TLI.getScalarShiftAmountTy(DAG.getDataLayout(), VT); in getVShift()
5308 LowerAsSplatVectorLoad(SDValue SrcOp, MVT VT, SDLoc dl, SelectionDAG &DAG) { in LowerAsSplatVectorLoad()
5318 if (PVT != MVT::i32 && PVT != MVT::f32) in LowerAsSplatVectorLoad()
5453 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, in EltsFromConsecutiveLoads()
5468 DAG.getTargetLoweringInfo().isTypeLegal(MVT::v2i64)) { in EltsFromConsecutiveLoads()
5469 SDVTList Tys = DAG.getVTList(MVT::v2i64, MVT::Other); in EltsFromConsecutiveLoads()
5472 DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, DL, Tys, Ops, MVT::i64, in EltsFromConsecutiveLoads()
5482 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, in EltsFromConsecutiveLoads()
5509 MVT VT = Op.getSimpleValueType(); in LowerVectorBroadcast()
5679 MVT ShuffleVecVT = ShuffleVec.getSimpleValueType(); in getUnderlyingExtractedFromVec()
5692 MVT VT = Op.getSimpleValueType(); in buildFromShuffleMostly()
5774 MVT VT = in ConvertI1VectorToInteger()
5775 MVT::getIntegerVT(std::max((int)Op.getValueType().getSizeInBits(), 8)); in ConvertI1VectorToInteger()
5782 MVT VT = Op.getSimpleValueType(); in LowerBUILD_VECTORvXi1()
5783 assert((VT.getVectorElementType() == MVT::i1) && in LowerBUILD_VECTORvXi1()
5788 SDValue Cst = DAG.getTargetConstant(0, dl, MVT::i1); in LowerBUILD_VECTORvXi1()
5794 SDValue Cst = DAG.getTargetConstant(1, dl, MVT::i1); in LowerBUILD_VECTORvXi1()
5803 SDValue ExtVec = DAG.getBitcast(MVT::v8i1, Imm); in LowerBUILD_VECTORvXi1()
5840 MVT ImmVT = MVT::getIntegerVT(std::max((int)VT.getSizeInBits(), 8)); in LowerBUILD_VECTORvXi1()
5850 SDValue ExtVec = DAG.getBitcast(MVT::v8i1, Imm); in LowerBUILD_VECTORvXi1()
6034 MVT VT = BV->getSimpleValueType(0); in LowerToAddSub()
6035 if ((!Subtarget->hasSSE3() || (VT != MVT::v4f32 && VT != MVT::v2f64)) && in LowerToAddSub()
6036 (!Subtarget->hasAVX() || (VT != MVT::v8f32 && VT != MVT::v4f64))) in LowerToAddSub()
6044 assert((VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v4f32 || in LowerToAddSub()
6045 VT == MVT::v2f64) && "build_vector with an invalid type found!"); in LowerToAddSub()
6137 MVT VT = BV->getSimpleValueType(0); in LowerToHorizontalOp()
6159 if ((VT == MVT::v4f32 || VT == MVT::v2f64) && Subtarget->hasSSE3()) { in LowerToHorizontalOp()
6166 } else if ((VT == MVT::v4i32 || VT == MVT::v8i16) && Subtarget->hasSSSE3()) { in LowerToHorizontalOp()
6178 if ((VT == MVT::v8f32 || VT == MVT::v4f64)) { in LowerToHorizontalOp()
6197 } else if (VT == MVT::v8i32 || VT == MVT::v16i16) { in LowerToHorizontalOp()
6240 if ((VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v8i32 || in LowerToHorizontalOp()
6241 VT == MVT::v16i16) && Subtarget->hasAVX()) { in LowerToHorizontalOp()
6274 MVT VT = Op.getSimpleValueType(); in LowerBUILD_VECTOR()
6275 MVT ExtVT = VT.getVectorElementType(); in LowerBUILD_VECTOR()
6279 if (VT.getVectorElementType() == MVT::i1 && Subtarget->hasAVX512()) in LowerBUILD_VECTOR()
6286 if (VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32) in LowerBUILD_VECTOR()
6296 if (VT == MVT::v4i32 || (VT == MVT::v8i32 && Subtarget->hasInt256())) in LowerBUILD_VECTOR()
6349 if (ExtVT == MVT::i64 && !Subtarget->is64Bit() && in LowerBUILD_VECTOR()
6353 assert(VT == MVT::v2i64 && "Expected an SSE value type!"); in LowerBUILD_VECTOR()
6354 MVT VecVT = MVT::v4i32; in LowerBUILD_VECTOR()
6358 Item = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Item); in LowerBUILD_VECTOR()
6373 if (ExtVT == MVT::i32 || ExtVT == MVT::f32 || ExtVT == MVT::f64 || in LowerBUILD_VECTOR()
6374 (ExtVT == MVT::i64 && Subtarget->is64Bit())) { in LowerBUILD_VECTOR()
6389 if (ExtVT == MVT::i16 || ExtVT == MVT::i8) { in LowerBUILD_VECTOR()
6390 Item = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Item); in LowerBUILD_VECTOR()
6393 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v8i32, Item); in LowerBUILD_VECTOR()
6398 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Item); in LowerBUILD_VECTOR()
6399 SDValue ZeroVec = getZeroVector(MVT::v8i32, Subtarget, DAG, dl); in LowerBUILD_VECTOR()
6404 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Item); in LowerBUILD_VECTOR()
6616 MVT ResVT = Op.getSimpleValueType(); in LowerAVXCONCAT_VECTORS()
6628 MVT HalfVT = MVT::getVectorVT(ResVT.getVectorElementType(), in LowerAVXCONCAT_VECTORS()
6642 MVT ResVT = Op.getSimpleValueType(); in LowerCONCAT_VECTORSvXi1()
6668 MVT HalfVT = MVT::getVectorVT(ResVT.getVectorElementType(), in LowerCONCAT_VECTORSvXi1()
6718 MVT VT = Op.getSimpleValueType(); in LowerCONCAT_VECTORS()
6719 if (VT.getVectorElementType() == MVT::i1) in LowerCONCAT_VECTORS()
6778 static bool is128BitLaneCrossingShuffleMask(MVT VT, ArrayRef<int> Mask) { in is128BitLaneCrossingShuffleMask()
6799 is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask, in is128BitLaneRepeatedShuffleMask()
6879 return DAG.getConstant(Imm, DL, MVT::i8); in getV4X86ShuffleImm8ForMask()
6928 static SDValue lowerVectorShuffleWithUNPCK(SDLoc DL, MVT VT, ArrayRef<int> Mask, in lowerVectorShuffleWithUNPCK()
6965 static SDValue lowerVectorShuffleAsBitMask(SDLoc DL, MVT VT, SDValue V1, in lowerVectorShuffleAsBitMask()
6968 MVT EltVT = VT.getVectorElementType(); in lowerVectorShuffleAsBitMask()
6970 MVT IntEltVT = MVT::getIntegerVT(NumEltBits); in lowerVectorShuffleAsBitMask()
7008 static SDValue lowerVectorShuffleAsBitBlend(SDLoc DL, MVT VT, SDValue V1, in lowerVectorShuffleAsBitBlend()
7012 MVT EltVT = VT.getVectorElementType(); in lowerVectorShuffleAsBitBlend()
7027 MVT MaskVT = MVT::getVectorVT(MVT::i64, VT.getSizeInBits() / 64); in lowerVectorShuffleAsBitBlend()
7040 static SDValue lowerVectorShuffleAsBlend(SDLoc DL, MVT VT, SDValue V1, in lowerVectorShuffleAsBlend()
7096 case MVT::v2f64: in lowerVectorShuffleAsBlend()
7097 case MVT::v4f32: in lowerVectorShuffleAsBlend()
7098 case MVT::v4f64: in lowerVectorShuffleAsBlend()
7099 case MVT::v8f32: in lowerVectorShuffleAsBlend()
7101 DAG.getConstant(BlendMask, DL, MVT::i8)); in lowerVectorShuffleAsBlend()
7103 case MVT::v4i64: in lowerVectorShuffleAsBlend()
7104 case MVT::v8i32: in lowerVectorShuffleAsBlend()
7107 case MVT::v2i64: in lowerVectorShuffleAsBlend()
7108 case MVT::v4i32: in lowerVectorShuffleAsBlend()
7115 MVT BlendVT = VT.getSizeInBits() > 128 ? MVT::v8i32 : MVT::v4i32; in lowerVectorShuffleAsBlend()
7120 DAG.getConstant(BlendMask, DL, MVT::i8))); in lowerVectorShuffleAsBlend()
7123 case MVT::v8i16: { in lowerVectorShuffleAsBlend()
7128 V1 = DAG.getBitcast(MVT::v8i16, V1); in lowerVectorShuffleAsBlend()
7129 V2 = DAG.getBitcast(MVT::v8i16, V2); in lowerVectorShuffleAsBlend()
7131 DAG.getNode(X86ISD::BLENDI, DL, MVT::v8i16, V1, V2, in lowerVectorShuffleAsBlend()
7132 DAG.getConstant(BlendMask, DL, MVT::i8))); in lowerVectorShuffleAsBlend()
7135 case MVT::v16i16: { in lowerVectorShuffleAsBlend()
7138 if (is128BitLaneRepeatedShuffleMask(MVT::v16i16, Mask, RepeatedMask)) { in lowerVectorShuffleAsBlend()
7145 return DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2, in lowerVectorShuffleAsBlend()
7146 DAG.getConstant(BlendMask, DL, MVT::i8)); in lowerVectorShuffleAsBlend()
7150 case MVT::v16i8: in lowerVectorShuffleAsBlend()
7151 case MVT::v32i8: { in lowerVectorShuffleAsBlend()
7164 MVT BlendVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8); in lowerVectorShuffleAsBlend()
7181 Mask[i] < 0 ? DAG.getUNDEF(MVT::i8) in lowerVectorShuffleAsBlend()
7183 MVT::i8)); in lowerVectorShuffleAsBlend()
7203 static SDValue lowerVectorShuffleAsBlendAndPermute(SDLoc DL, MVT VT, SDValue V1, in lowerVectorShuffleAsBlendAndPermute()
7237 static SDValue lowerVectorShuffleAsDecomposedShuffleBlend(SDLoc DL, MVT VT, in lowerVectorShuffleAsDecomposedShuffleBlend()
7287 static SDValue lowerVectorShuffleAsByteRotate(SDLoc DL, MVT VT, SDValue V1, in lowerVectorShuffleAsByteRotate()
7370 MVT AlignVT = MVT::getVectorVT(MVT::i8, 16 * NumLanes); in lowerVectorShuffleAsByteRotate()
7376 DAG.getConstant(Rotation * Scale, DL, MVT::i8))); in lowerVectorShuffleAsByteRotate()
7389 Lo = DAG.getBitcast(MVT::v2i64, Lo); in lowerVectorShuffleAsByteRotate()
7390 Hi = DAG.getBitcast(MVT::v2i64, Hi); in lowerVectorShuffleAsByteRotate()
7392 SDValue LoShift = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v2i64, Lo, in lowerVectorShuffleAsByteRotate()
7393 DAG.getConstant(LoByteShift, DL, MVT::i8)); in lowerVectorShuffleAsByteRotate()
7394 SDValue HiShift = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v2i64, Hi, in lowerVectorShuffleAsByteRotate()
7395 DAG.getConstant(HiByteShift, DL, MVT::i8)); in lowerVectorShuffleAsByteRotate()
7397 DAG.getNode(ISD::OR, DL, MVT::v2i64, LoShift, HiShift)); in lowerVectorShuffleAsByteRotate()
7423 static SDValue lowerVectorShuffleAsShift(SDLoc DL, MVT VT, SDValue V1, in lowerVectorShuffleAsShift()
7461 MVT ShiftSVT = MVT::getIntegerVT(VT.getScalarSizeInBits() * Scale); in lowerVectorShuffleAsShift()
7462 MVT ShiftVT = MVT::getVectorVT(ShiftSVT, Size / Scale); in lowerVectorShuffleAsShift()
7468 DAG.getConstant(ShiftAmt, DL, MVT::i8)); in lowerVectorShuffleAsShift()
7491 static SDValue lowerVectorShuffleWithSSE4A(SDLoc DL, MVT VT, SDValue V1, in lowerVectorShuffleWithSSE4A()
7546 DAG.getConstant(BitLen, DL, MVT::i8), in lowerVectorShuffleWithSSE4A()
7547 DAG.getConstant(BitIdx, DL, MVT::i8)); in lowerVectorShuffleWithSSE4A()
7607 DAG.getConstant(BitLen, DL, MVT::i8), in lowerVectorShuffleWithSSE4A()
7608 DAG.getConstant(BitIdx, DL, MVT::i8)); in lowerVectorShuffleWithSSE4A()
7631 SDLoc DL, MVT VT, int Scale, int Offset, bool AnyExt, SDValue InputV, in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
7670 MVT ExtVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits * Scale), in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
7684 VT, DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
7685 DAG.getBitcast(MVT::v4i32, InputV), in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
7691 InputV = DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
7692 DAG.getBitcast(MVT::v4i32, InputV), in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
7697 VT, DAG.getNode(OddEvenOp, DL, MVT::v8i16, in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
7698 DAG.getBitcast(MVT::v8i16, InputV), in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
7709 SDValue Lo = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
7711 DAG.getConstant(EltBits, DL, MVT::i8), in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
7712 DAG.getConstant(LoIdx, DL, MVT::i8))); in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
7719 SDValue Hi = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
7721 DAG.getConstant(EltBits, DL, MVT::i8), in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
7722 DAG.getConstant(HiIdx, DL, MVT::i8))); in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
7724 DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2i64, Lo, Hi)); in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
7736 (i % Scale == 0 && SafeOffset(Idx)) ? Idx : 0x80, DL, MVT::i8); in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
7738 InputV = DAG.getBitcast(MVT::v16i8, InputV); in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
7740 DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, InputV, in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
7742 MVT::v16i8, PSHUFBMask))); in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
7764 MVT InputVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits), NumElements); in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
7789 SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerVectorShuffleAsZeroOrAnyExtend()
7896 V = DAG.getBitcast(MVT::v2i64, V); in lowerVectorShuffleAsZeroOrAnyExtend()
7897 V = DAG.getNode(X86ISD::VZEXT_MOVL, DL, MVT::v2i64, V); in lowerVectorShuffleAsZeroOrAnyExtend()
7910 MVT VT = V.getSimpleValueType(); in getScalarValueForVectorElement()
7911 MVT EltVT = VT.getVectorElementType(); in getScalarValueForVectorElement()
7916 MVT NewVT = V.getSimpleValueType(); in getScalarValueForVectorElement()
7948 SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerVectorShuffleAsElementInsertion()
7951 MVT ExtVT = VT; in lowerVectorShuffleAsElementInsertion()
7952 MVT EltVT = VT.getVectorElementType(); in lowerVectorShuffleAsElementInsertion()
7974 if (EltVT == MVT::i8 || EltVT == MVT::i16) { in lowerVectorShuffleAsElementInsertion()
7981 ExtVT = MVT::v4i32; in lowerVectorShuffleAsElementInsertion()
7982 V2S = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, V2S); in lowerVectorShuffleAsElementInsertion()
7985 } else if (Mask[V2Index] != (int)Mask.size() || EltVT == MVT::i8 || in lowerVectorShuffleAsElementInsertion()
7986 EltVT == MVT::i16) { in lowerVectorShuffleAsElementInsertion()
8010 assert((EltVT == MVT::f32 || EltVT == MVT::f64) && in lowerVectorShuffleAsElementInsertion()
8012 return DAG.getNode(EltVT == MVT::f32 ? X86ISD::MOVSS : X86ISD::MOVSD, DL, in lowerVectorShuffleAsElementInsertion()
8034 V2 = DAG.getBitcast(MVT::v2i64, V2); in lowerVectorShuffleAsElementInsertion()
8036 X86ISD::VSHLDQ, DL, MVT::v2i64, V2, in lowerVectorShuffleAsElementInsertion()
8050 static SDValue lowerVectorShuffleAsTruncBroadcast(SDLoc DL, MVT VT, SDValue V0, in lowerVectorShuffleAsTruncBroadcast()
8105 static SDValue lowerVectorShuffleAsBroadcast(SDLoc DL, MVT VT, SDValue V, in lowerVectorShuffleAsBroadcast()
8209 assert(Op.getSimpleValueType() == MVT::v4f32 && "Bad shuffle type!"); in lowerVectorShuffleAsInsertPS()
8210 assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"); in lowerVectorShuffleAsInsertPS()
8211 assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"); in lowerVectorShuffleAsInsertPS()
8267 V1 = DAG.getUNDEF(MVT::v4f32); in lowerVectorShuffleAsInsertPS()
8274 return DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2, in lowerVectorShuffleAsInsertPS()
8275 DAG.getConstant(InsertPSMask, DL, MVT::i8)); in lowerVectorShuffleAsInsertPS()
8287 static SDValue lowerVectorShuffleAsPermuteAndUnpack(SDLoc DL, MVT VT, in lowerVectorShuffleAsPermuteAndUnpack()
8307 auto TryUnpack = [&](MVT UnpackVT, int Scale) { in lowerVectorShuffleAsPermuteAndUnpack()
8357 MVT UnpackVT = MVT::getVectorVT(MVT::getIntegerVT(ScalarSize), NumElements); in lowerVectorShuffleAsPermuteAndUnpack()
8405 assert(Op.getSimpleValueType() == MVT::v2f64 && "Bad shuffle type!"); in lowerV2F64VectorShuffle()
8406 assert(V1.getSimpleValueType() == MVT::v2f64 && "Bad operand type!"); in lowerV2F64VectorShuffle()
8407 assert(V2.getSimpleValueType() == MVT::v2f64 && "Bad operand type!"); in lowerV2F64VectorShuffle()
8416 return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, V1); in lowerV2F64VectorShuffle()
8425 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v2f64, V1, in lowerV2F64VectorShuffle()
8426 DAG.getConstant(SHUFPDMask, DL, MVT::i8)); in lowerV2F64VectorShuffle()
8429 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v2f64, V1, V1, in lowerV2F64VectorShuffle()
8430 DAG.getConstant(SHUFPDMask, DL, MVT::i8)); in lowerV2F64VectorShuffle()
8438 DL, MVT::v2f64, V1, V2, Mask, Subtarget, DAG)) in lowerV2F64VectorShuffle()
8445 DL, MVT::v2f64, V2, V1, InverseMask, Subtarget, DAG)) in lowerV2F64VectorShuffle()
8458 DL, MVT::v2f64, V2, in lowerV2F64VectorShuffle()
8459 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, V1S)); in lowerV2F64VectorShuffle()
8462 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v2f64, V1, V2, Mask, in lowerV2F64VectorShuffle()
8468 lowerVectorShuffleWithUNPCK(DL, MVT::v2f64, Mask, V1, V2, DAG)) in lowerV2F64VectorShuffle()
8472 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v2f64, V1, V2, in lowerV2F64VectorShuffle()
8473 DAG.getConstant(SHUFPDMask, DL, MVT::i8)); in lowerV2F64VectorShuffle()
8486 assert(Op.getSimpleValueType() == MVT::v2i64 && "Bad shuffle type!"); in lowerV2I64VectorShuffle()
8487 assert(V1.getSimpleValueType() == MVT::v2i64 && "Bad operand type!"); in lowerV2I64VectorShuffle()
8488 assert(V2.getSimpleValueType() == MVT::v2i64 && "Bad operand type!"); in lowerV2I64VectorShuffle()
8495 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v2i64, V1, in lowerV2I64VectorShuffle()
8502 V1 = DAG.getBitcast(MVT::v4i32, V1); in lowerV2I64VectorShuffle()
8507 MVT::v2i64, in lowerV2I64VectorShuffle()
8508 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1, in lowerV2I64VectorShuffle()
8528 return DAG.getBitcast(MVT::v2i64, in lowerV2I64VectorShuffle()
8529 DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, in lowerV2I64VectorShuffle()
8537 lowerVectorShuffleAsShift(DL, MVT::v2i64, V1, V2, Mask, DAG)) in lowerV2I64VectorShuffle()
8543 DL, MVT::v2i64, V1, V2, Mask, Subtarget, DAG)) in lowerV2I64VectorShuffle()
8549 DL, MVT::v2i64, V2, V1, InverseMask, Subtarget, DAG)) in lowerV2I64VectorShuffle()
8556 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v2i64, V1, V2, Mask, in lowerV2I64VectorShuffle()
8562 lowerVectorShuffleWithUNPCK(DL, MVT::v2i64, Mask, V1, V2, DAG)) in lowerV2I64VectorShuffle()
8569 DL, MVT::v2i64, V1, V2, Mask, Subtarget, DAG)) in lowerV2I64VectorShuffle()
8575 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v2i64, V1, V2, in lowerV2I64VectorShuffle()
8582 V1 = DAG.getBitcast(MVT::v2f64, V1); in lowerV2I64VectorShuffle()
8583 V2 = DAG.getBitcast(MVT::v2f64, V2); in lowerV2I64VectorShuffle()
8584 return DAG.getBitcast(MVT::v2i64, in lowerV2I64VectorShuffle()
8585 DAG.getVectorShuffle(MVT::v2f64, DL, V1, V2, Mask)); in lowerV2I64VectorShuffle()
8611 static SDValue lowerVectorShuffleWithSHUFPS(SDLoc DL, MVT VT, in lowerVectorShuffleWithSHUFPS()
8705 assert(Op.getSimpleValueType() == MVT::v4f32 && "Bad shuffle type!"); in lowerV4F32VectorShuffle()
8706 assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"); in lowerV4F32VectorShuffle()
8707 assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"); in lowerV4F32VectorShuffle()
8717 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v4f32, V1, in lowerV4F32VectorShuffle()
8724 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v4f32, V1); in lowerV4F32VectorShuffle()
8726 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v4f32, V1); in lowerV4F32VectorShuffle()
8732 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f32, V1, in lowerV4F32VectorShuffle()
8738 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, V1, V1, in lowerV4F32VectorShuffle()
8748 if (SDValue V = lowerVectorShuffleAsElementInsertion(DL, MVT::v4f32, V1, V2, in lowerV4F32VectorShuffle()
8753 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4f32, V1, V2, Mask, in lowerV4F32VectorShuffle()
8763 DL, MVT::v4f32, V1, V2, Mask, DAG)) in lowerV4F32VectorShuffle()
8769 lowerVectorShuffleWithUNPCK(DL, MVT::v4f32, Mask, V1, V2, DAG)) in lowerV4F32VectorShuffle()
8773 return lowerVectorShuffleWithSHUFPS(DL, MVT::v4f32, Mask, V1, V2, DAG); in lowerV4F32VectorShuffle()
8784 assert(Op.getSimpleValueType() == MVT::v4i32 && "Bad shuffle type!"); in lowerV4I32VectorShuffle()
8785 assert(V1.getSimpleValueType() == MVT::v4i32 && "Bad operand type!"); in lowerV4I32VectorShuffle()
8786 assert(V2.getSimpleValueType() == MVT::v4i32 && "Bad operand type!"); in lowerV4I32VectorShuffle()
8794 if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(DL, MVT::v4i32, V1, V2, in lowerV4I32VectorShuffle()
8803 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v4i32, V1, in lowerV4I32VectorShuffle()
8819 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1, in lowerV4I32VectorShuffle()
8825 lowerVectorShuffleAsShift(DL, MVT::v4i32, V1, V2, Mask, DAG)) in lowerV4I32VectorShuffle()
8830 if (SDValue V = lowerVectorShuffleAsElementInsertion(DL, MVT::v4i32, V1, V2, in lowerV4I32VectorShuffle()
8838 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4i32, V1, V2, Mask, in lowerV4I32VectorShuffle()
8843 lowerVectorShuffleAsBitMask(DL, MVT::v4i32, V1, V2, Mask, DAG)) in lowerV4I32VectorShuffle()
8848 lowerVectorShuffleWithUNPCK(DL, MVT::v4i32, Mask, V1, V2, DAG)) in lowerV4I32VectorShuffle()
8855 DL, MVT::v4i32, V1, V2, Mask, Subtarget, DAG)) in lowerV4I32VectorShuffle()
8861 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v4i32, V1, V2, in lowerV4I32VectorShuffle()
8865 if (SDValue Unpack = lowerVectorShuffleAsPermuteAndUnpack(DL, MVT::v4i32, V1, in lowerV4I32VectorShuffle()
8875 MVT::v4i32, in lowerV4I32VectorShuffle()
8876 DAG.getVectorShuffle(MVT::v4f32, DL, DAG.getBitcast(MVT::v4f32, V1), in lowerV4I32VectorShuffle()
8877 DAG.getBitcast(MVT::v4f32, V2), Mask)); in lowerV4I32VectorShuffle()
8897 SDLoc DL, MVT VT, SDValue V, MutableArrayRef<int> Mask, in lowerV8I16GeneralSingleInputVectorShuffle()
8899 assert(VT.getVectorElementType() == MVT::i16 && "Bad input type!"); in lowerV8I16GeneralSingleInputVectorShuffle()
8900 MVT PSHUFDVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() / 2); in lowerV8I16GeneralSingleInputVectorShuffle()
9032 MVT::v8i16, V, in lowerV8I16GeneralSingleInputVectorShuffle()
9331 static SDValue lowerVectorShuffleAsPSHUFB(SDLoc DL, MVT VT, SDValue V1, in lowerVectorShuffleAsPSHUFB()
9345 V1Mask[i] = V2Mask[i] = DAG.getUNDEF(MVT::i8); in lowerVectorShuffleAsPSHUFB()
9355 V1Mask[i] = DAG.getConstant(V1Idx, DL, MVT::i8); in lowerVectorShuffleAsPSHUFB()
9356 V2Mask[i] = DAG.getConstant(V2Idx, DL, MVT::i8); in lowerVectorShuffleAsPSHUFB()
9363 V1 = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, in lowerVectorShuffleAsPSHUFB()
9364 DAG.getBitcast(MVT::v16i8, V1), in lowerVectorShuffleAsPSHUFB()
9365 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v16i8, V1Mask)); in lowerVectorShuffleAsPSHUFB()
9367 V2 = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, in lowerVectorShuffleAsPSHUFB()
9368 DAG.getBitcast(MVT::v16i8, V2), in lowerVectorShuffleAsPSHUFB()
9369 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v16i8, V2Mask)); in lowerVectorShuffleAsPSHUFB()
9374 V = DAG.getNode(ISD::OR, DL, MVT::v16i8, V1, V2); in lowerVectorShuffleAsPSHUFB()
9398 assert(Op.getSimpleValueType() == MVT::v8i16 && "Bad shuffle type!"); in lowerV8I16VectorShuffle()
9399 assert(V1.getSimpleValueType() == MVT::v8i16 && "Bad operand type!"); in lowerV8I16VectorShuffle()
9400 assert(V2.getSimpleValueType() == MVT::v8i16 && "Bad operand type!"); in lowerV8I16VectorShuffle()
9412 DL, MVT::v8i16, V1, V2, OrigMask, Subtarget, DAG)) in lowerV8I16VectorShuffle()
9423 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v8i16, V1, in lowerV8I16VectorShuffle()
9429 lowerVectorShuffleAsShift(DL, MVT::v8i16, V1, V1, Mask, DAG)) in lowerV8I16VectorShuffle()
9434 lowerVectorShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG)) in lowerV8I16VectorShuffle()
9438 if (SDValue Rotate = lowerVectorShuffleAsByteRotate(DL, MVT::v8i16, V1, V1, in lowerV8I16VectorShuffle()
9442 return lowerV8I16GeneralSingleInputVectorShuffle(DL, MVT::v8i16, V1, Mask, in lowerV8I16VectorShuffle()
9452 lowerVectorShuffleAsShift(DL, MVT::v8i16, V1, V2, Mask, DAG)) in lowerV8I16VectorShuffle()
9457 if (SDValue V = lowerVectorShuffleWithSSE4A(DL, MVT::v8i16, V1, V2, Mask, DAG)) in lowerV8I16VectorShuffle()
9462 if (SDValue V = lowerVectorShuffleAsElementInsertion(DL, MVT::v8i16, V1, V2, in lowerV8I16VectorShuffle()
9470 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8i16, V1, V2, Mask, in lowerV8I16VectorShuffle()
9475 lowerVectorShuffleAsBitMask(DL, MVT::v8i16, V1, V2, Mask, DAG)) in lowerV8I16VectorShuffle()
9480 lowerVectorShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG)) in lowerV8I16VectorShuffle()
9485 DL, MVT::v8i16, V1, V2, Mask, Subtarget, DAG)) in lowerV8I16VectorShuffle()
9489 lowerVectorShuffleAsBitBlend(DL, MVT::v8i16, V1, V2, Mask, DAG)) in lowerV8I16VectorShuffle()
9492 if (SDValue Unpack = lowerVectorShuffleAsPermuteAndUnpack(DL, MVT::v8i16, V1, in lowerV8I16VectorShuffle()
9500 return lowerVectorShuffleAsPSHUFB(DL, MVT::v8i16, V1, V2, Mask, DAG, in lowerV8I16VectorShuffle()
9506 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v8i16, V1, V2, in lowerV8I16VectorShuffle()
9589 assert(Op.getSimpleValueType() == MVT::v16i8 && "Bad shuffle type!"); in lowerV16I8VectorShuffle()
9590 assert(V1.getSimpleValueType() == MVT::v16i8 && "Bad operand type!"); in lowerV16I8VectorShuffle()
9591 assert(V2.getSimpleValueType() == MVT::v16i8 && "Bad operand type!"); in lowerV16I8VectorShuffle()
9598 lowerVectorShuffleAsShift(DL, MVT::v16i8, V1, V2, Mask, DAG)) in lowerV16I8VectorShuffle()
9603 DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG)) in lowerV16I8VectorShuffle()
9608 DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG)) in lowerV16I8VectorShuffle()
9613 if (SDValue V = lowerVectorShuffleWithSSE4A(DL, MVT::v16i8, V1, V2, Mask, DAG)) in lowerV16I8VectorShuffle()
9622 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v16i8, V1, in lowerV16I8VectorShuffle()
9689 MVT::v16i8, in lowerV16I8VectorShuffle()
9690 DAG.getVectorShuffle(MVT::v8i16, DL, DAG.getBitcast(MVT::v8i16, V1), in lowerV16I8VectorShuffle()
9691 DAG.getUNDEF(MVT::v8i16), PreDupI16Shuffle)); in lowerV16I8VectorShuffle()
9695 MVT::v16i8, V1, V1); in lowerV16I8VectorShuffle()
9709 MVT::v16i8, in lowerV16I8VectorShuffle()
9710 DAG.getVectorShuffle(MVT::v8i16, DL, DAG.getBitcast(MVT::v8i16, V1), in lowerV16I8VectorShuffle()
9711 DAG.getUNDEF(MVT::v8i16), PostDupI16Shuffle)); in lowerV16I8VectorShuffle()
9718 lowerVectorShuffleAsBitMask(DL, MVT::v16i8, V1, V2, Mask, DAG)) in lowerV16I8VectorShuffle()
9723 lowerVectorShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG)) in lowerV16I8VectorShuffle()
9743 SDValue PSHUFB = lowerVectorShuffleAsPSHUFB(DL, MVT::v16i8, V1, V2, Mask, in lowerV16I8VectorShuffle()
9751 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v16i8, V1, V2, in lowerV16I8VectorShuffle()
9764 DL, MVT::v16i8, V1, V2, Mask, DAG)) in lowerV16I8VectorShuffle()
9773 if (SDValue V = lowerVectorShuffleAsElementInsertion(DL, MVT::v16i8, V1, V2, in lowerV16I8VectorShuffle()
9778 lowerVectorShuffleAsBitBlend(DL, MVT::v16i8, V1, V2, Mask, DAG)) in lowerV16I8VectorShuffle()
9799 MVT MaskVTs[] = { MVT::v8i16, MVT::v4i32, MVT::v2i64 }; in lowerV16I8VectorShuffle()
9801 MVT::v16i8, DAG.getConstant(0xFF, DL, MaskVTs[NumEvenDrops - 1])); in lowerV16I8VectorShuffle()
9802 V1 = DAG.getNode(ISD::AND, DL, MVT::v16i8, V1, ByteClearMask); in lowerV16I8VectorShuffle()
9804 V2 = DAG.getNode(ISD::AND, DL, MVT::v16i8, V2, ByteClearMask); in lowerV16I8VectorShuffle()
9807 V1 = DAG.getBitcast(MVT::v8i16, V1); in lowerV16I8VectorShuffle()
9808 V2 = IsSingleInput ? V1 : DAG.getBitcast(MVT::v8i16, V2); in lowerV16I8VectorShuffle()
9809 SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1, V2); in lowerV16I8VectorShuffle()
9811 Result = DAG.getBitcast(MVT::v8i16, Result); in lowerV16I8VectorShuffle()
9812 Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, Result, Result); in lowerV16I8VectorShuffle()
9820 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v16i8, V1, V2, in lowerV16I8VectorShuffle()
9834 SDValue Zero = getZeroVector(MVT::v8i16, Subtarget, DAG, DL); in lowerV16I8VectorShuffle()
9845 VLoHalf = DAG.getBitcast(MVT::v8i16, V); in lowerV16I8VectorShuffle()
9846 VLoHalf = DAG.getNode(ISD::AND, DL, MVT::v8i16, VLoHalf, in lowerV16I8VectorShuffle()
9847 DAG.getConstant(0x00FF, DL, MVT::v8i16)); in lowerV16I8VectorShuffle()
9850 VHiHalf = DAG.getUNDEF(MVT::v8i16); in lowerV16I8VectorShuffle()
9863 MVT::v8i16, DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, V, Zero)); in lowerV16I8VectorShuffle()
9865 MVT::v8i16, DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i8, V, Zero)); in lowerV16I8VectorShuffle()
9868 SDValue LoV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, LoBlendMask); in lowerV16I8VectorShuffle()
9869 SDValue HiV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, HiBlendMask); in lowerV16I8VectorShuffle()
9871 return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, LoV, HiV); in lowerV16I8VectorShuffle()
9879 MVT VT, const X86Subtarget *Subtarget, in lower128BitVectorShuffle()
9882 case MVT::v2i64: in lower128BitVectorShuffle()
9884 case MVT::v2f64: in lower128BitVectorShuffle()
9886 case MVT::v4i32: in lower128BitVectorShuffle()
9888 case MVT::v4f32: in lower128BitVectorShuffle()
9890 case MVT::v8i16: in lower128BitVectorShuffle()
9892 case MVT::v16i8: in lower128BitVectorShuffle()
9960 static SDValue splitAndLowerVectorShuffle(SDLoc DL, MVT VT, SDValue V1, in splitAndLowerVectorShuffle()
9973 MVT ScalarVT = VT.getVectorElementType(); in splitAndLowerVectorShuffle()
9974 MVT SplitVT = MVT::getVectorVT(ScalarVT, NumElements / 2); in splitAndLowerVectorShuffle()
9982 MVT OrigVT = V.getSimpleValueType(); in splitAndLowerVectorShuffle()
9985 MVT OrigScalarVT = OrigVT.getVectorElementType(); in splitAndLowerVectorShuffle()
9986 MVT OrigSplitVT = MVT::getVectorVT(OrigScalarVT, OrigNumElements / 2); in splitAndLowerVectorShuffle()
10091 static SDValue lowerVectorShuffleAsSplitOrBlend(SDLoc DL, MVT VT, SDValue V1, in lowerVectorShuffleAsSplitOrBlend()
10150 static SDValue lowerVectorShuffleAsLanePermuteAndBlend(SDLoc DL, MVT VT, in lowerVectorShuffleAsLanePermuteAndBlend()
10184 V1, DAG.getConstant(PERMMask, DL, MVT::i8)); in lowerVectorShuffleAsLanePermuteAndBlend()
10195 static SDValue lowerV2X128VectorShuffle(SDLoc DL, MVT VT, SDValue V1, in lowerV2X128VectorShuffle()
10218 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), in lowerV2X128VectorShuffle()
10273 DAG.getConstant(PermMask, DL, MVT::i8)); in lowerV2X128VectorShuffle()
10289 SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerVectorShuffleByMerging128BitLanes()
10330 MVT LaneVT = MVT::getVectorVT(VT.isFloatingPoint() ? MVT::f64 : MVT::i64, in lowerVectorShuffleByMerging128BitLanes()
10374 static SDValue lowerVectorShuffleWithSHUFPD(SDLoc DL, MVT VT, in lowerVectorShuffleWithSHUFPD()
10398 DAG.getConstant(Immediate, DL, MVT::i8)); in lowerVectorShuffleWithSHUFPD()
10401 DAG.getConstant(Immediate, DL, MVT::i8)); in lowerVectorShuffleWithSHUFPD()
10413 assert(V1.getSimpleValueType() == MVT::v4f64 && "Bad operand type!"); in lowerV4F64VectorShuffle()
10414 assert(V2.getSimpleValueType() == MVT::v4f64 && "Bad operand type!"); in lowerV4F64VectorShuffle()
10421 return lowerV2X128VectorShuffle(DL, MVT::v4f64, V1, V2, Mask, Subtarget, in lowerV4F64VectorShuffle()
10426 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v4f64, V1, in lowerV4F64VectorShuffle()
10432 return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v4f64, V1); in lowerV4F64VectorShuffle()
10434 if (!is128BitLaneCrossingShuffleMask(MVT::v4f64, Mask)) { in lowerV4F64VectorShuffle()
10439 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f64, V1, in lowerV4F64VectorShuffle()
10440 DAG.getConstant(VPERMILPMask, DL, MVT::i8)); in lowerV4F64VectorShuffle()
10445 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4f64, V1, in lowerV4F64VectorShuffle()
10449 return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v4f64, V1, V2, Mask, in lowerV4F64VectorShuffle()
10455 lowerVectorShuffleWithUNPCK(DL, MVT::v4f64, Mask, V1, V2, DAG)) in lowerV4F64VectorShuffle()
10458 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4f64, V1, V2, Mask, in lowerV4F64VectorShuffle()
10464 lowerVectorShuffleWithSHUFPD(DL, MVT::v4f64, Mask, V1, V2, DAG)) in lowerV4F64VectorShuffle()
10474 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG)) in lowerV4F64VectorShuffle()
10480 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v4f64, V1, V2, in lowerV4F64VectorShuffle()
10484 return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v4f64, V1, V2, Mask, DAG); in lowerV4F64VectorShuffle()
10495 assert(V1.getSimpleValueType() == MVT::v4i64 && "Bad operand type!"); in lowerV4I64VectorShuffle()
10496 assert(V2.getSimpleValueType() == MVT::v4i64 && "Bad operand type!"); in lowerV4I64VectorShuffle()
10504 return lowerV2X128VectorShuffle(DL, MVT::v4i64, V1, V2, Mask, Subtarget, in lowerV4I64VectorShuffle()
10507 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4i64, V1, V2, Mask, in lowerV4I64VectorShuffle()
10512 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v4i64, V1, in lowerV4I64VectorShuffle()
10519 if (is128BitLaneRepeatedShuffleMask(MVT::v4i64, Mask, RepeatedMask)) { in lowerV4I64VectorShuffle()
10528 MVT::v4i64, in lowerV4I64VectorShuffle()
10529 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32, in lowerV4I64VectorShuffle()
10530 DAG.getBitcast(MVT::v8i32, V1), in lowerV4I64VectorShuffle()
10538 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4i64, V1, in lowerV4I64VectorShuffle()
10543 lowerVectorShuffleAsShift(DL, MVT::v4i64, V1, V2, Mask, DAG)) in lowerV4I64VectorShuffle()
10548 lowerVectorShuffleWithUNPCK(DL, MVT::v4i64, Mask, V1, V2, DAG)) in lowerV4I64VectorShuffle()
10558 DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG)) in lowerV4I64VectorShuffle()
10562 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v4i64, V1, V2, in lowerV4I64VectorShuffle()
10574 assert(V1.getSimpleValueType() == MVT::v8f32 && "Bad operand type!"); in lowerV8F32VectorShuffle()
10575 assert(V2.getSimpleValueType() == MVT::v8f32 && "Bad operand type!"); in lowerV8F32VectorShuffle()
10580 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8f32, V1, V2, Mask, in lowerV8F32VectorShuffle()
10585 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v8f32, V1, in lowerV8F32VectorShuffle()
10592 if (is128BitLaneRepeatedShuffleMask(MVT::v8f32, Mask, RepeatedMask)) { in lowerV8F32VectorShuffle()
10598 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v8f32, V1); in lowerV8F32VectorShuffle()
10600 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v8f32, V1); in lowerV8F32VectorShuffle()
10603 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f32, V1, in lowerV8F32VectorShuffle()
10608 lowerVectorShuffleWithUNPCK(DL, MVT::v8f32, Mask, V1, V2, DAG)) in lowerV8F32VectorShuffle()
10617 return lowerVectorShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask, V1, V2, DAG); in lowerV8F32VectorShuffle()
10625 VPermMask[i] = Mask[i] < 0 ? DAG.getUNDEF(MVT::i32) in lowerV8F32VectorShuffle()
10626 : DAG.getConstant(Mask[i], DL, MVT::i32); in lowerV8F32VectorShuffle()
10627 if (!is128BitLaneCrossingShuffleMask(MVT::v8f32, Mask)) in lowerV8F32VectorShuffle()
10629 X86ISD::VPERMILPV, DL, MVT::v8f32, V1, in lowerV8F32VectorShuffle()
10630 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v8i32, VPermMask)); in lowerV8F32VectorShuffle()
10634 X86ISD::VPERMV, DL, MVT::v8f32, in lowerV8F32VectorShuffle()
10635 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v8i32, VPermMask), V1); in lowerV8F32VectorShuffle()
10638 return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v8f32, V1, V2, Mask, in lowerV8F32VectorShuffle()
10645 DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG)) in lowerV8F32VectorShuffle()
10651 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v8f32, V1, V2, in lowerV8F32VectorShuffle()
10655 return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask, DAG); in lowerV8F32VectorShuffle()
10666 assert(V1.getSimpleValueType() == MVT::v8i32 && "Bad operand type!"); in lowerV8I32VectorShuffle()
10667 assert(V2.getSimpleValueType() == MVT::v8i32 && "Bad operand type!"); in lowerV8I32VectorShuffle()
10676 if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(DL, MVT::v8i32, V1, V2, in lowerV8I32VectorShuffle()
10680 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8i32, V1, V2, Mask, in lowerV8I32VectorShuffle()
10685 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v8i32, V1, in lowerV8I32VectorShuffle()
10693 if (is128BitLaneRepeatedShuffleMask(MVT::v8i32, Mask, RepeatedMask)) { in lowerV8I32VectorShuffle()
10696 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32, V1, in lowerV8I32VectorShuffle()
10701 lowerVectorShuffleWithUNPCK(DL, MVT::v8i32, Mask, V1, V2, DAG)) in lowerV8I32VectorShuffle()
10707 lowerVectorShuffleAsShift(DL, MVT::v8i32, V1, V2, Mask, DAG)) in lowerV8I32VectorShuffle()
10711 DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG)) in lowerV8I32VectorShuffle()
10719 VPermMask[i] = Mask[i] < 0 ? DAG.getUNDEF(MVT::i32) in lowerV8I32VectorShuffle()
10720 : DAG.getConstant(Mask[i], DL, MVT::i32); in lowerV8I32VectorShuffle()
10722 X86ISD::VPERMV, DL, MVT::v8i32, in lowerV8I32VectorShuffle()
10723 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v8i32, VPermMask), V1); in lowerV8I32VectorShuffle()
10729 DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG)) in lowerV8I32VectorShuffle()
10733 return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v8i32, V1, V2, in lowerV8I32VectorShuffle()
10745 assert(V1.getSimpleValueType() == MVT::v16i16 && "Bad operand type!"); in lowerV16I16VectorShuffle()
10746 assert(V2.getSimpleValueType() == MVT::v16i16 && "Bad operand type!"); in lowerV16I16VectorShuffle()
10755 if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(DL, MVT::v16i16, V1, V2, in lowerV16I16VectorShuffle()
10760 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v16i16, V1, in lowerV16I16VectorShuffle()
10764 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v16i16, V1, V2, Mask, in lowerV16I16VectorShuffle()
10770 lowerVectorShuffleWithUNPCK(DL, MVT::v16i16, Mask, V1, V2, DAG)) in lowerV16I16VectorShuffle()
10775 lowerVectorShuffleAsShift(DL, MVT::v16i16, V1, V2, Mask, DAG)) in lowerV16I16VectorShuffle()
10780 DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG)) in lowerV16I16VectorShuffle()
10786 if (is128BitLaneCrossingShuffleMask(MVT::v16i16, Mask)) in lowerV16I16VectorShuffle()
10787 return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v16i16, V1, V2, in lowerV16I16VectorShuffle()
10791 if (is128BitLaneRepeatedShuffleMask(MVT::v16i16, Mask, RepeatedMask)) { in lowerV16I16VectorShuffle()
10796 DL, MVT::v16i16, V1, RepeatedMask, Subtarget, DAG); in lowerV16I16VectorShuffle()
10802 PSHUFBMask[2 * i] = PSHUFBMask[2 * i + 1] = DAG.getUNDEF(MVT::i8); in lowerV16I16VectorShuffle()
10808 PSHUFBMask[2 * i] = DAG.getConstant(2 * M, DL, MVT::i8); in lowerV16I16VectorShuffle()
10809 PSHUFBMask[2 * i + 1] = DAG.getConstant(2 * M + 1, DL, MVT::i8); in lowerV16I16VectorShuffle()
10811 return DAG.getBitcast(MVT::v16i16, in lowerV16I16VectorShuffle()
10812 DAG.getNode(X86ISD::PSHUFB, DL, MVT::v32i8, in lowerV16I16VectorShuffle()
10813 DAG.getBitcast(MVT::v32i8, V1), in lowerV16I16VectorShuffle()
10815 MVT::v32i8, PSHUFBMask))); in lowerV16I16VectorShuffle()
10821 DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG)) in lowerV16I16VectorShuffle()
10825 return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v16i16, V1, V2, Mask, DAG); in lowerV16I16VectorShuffle()
10836 assert(V1.getSimpleValueType() == MVT::v32i8 && "Bad operand type!"); in lowerV32I8VectorShuffle()
10837 assert(V2.getSimpleValueType() == MVT::v32i8 && "Bad operand type!"); in lowerV32I8VectorShuffle()
10846 if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(DL, MVT::v32i8, V1, V2, in lowerV32I8VectorShuffle()
10851 if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v32i8, V1, in lowerV32I8VectorShuffle()
10855 if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v32i8, V1, V2, Mask, in lowerV32I8VectorShuffle()
10861 lowerVectorShuffleWithUNPCK(DL, MVT::v32i8, Mask, V1, V2, DAG)) in lowerV32I8VectorShuffle()
10866 lowerVectorShuffleAsShift(DL, MVT::v32i8, V1, V2, Mask, DAG)) in lowerV32I8VectorShuffle()
10871 DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG)) in lowerV32I8VectorShuffle()
10877 if (is128BitLaneCrossingShuffleMask(MVT::v32i8, Mask)) in lowerV32I8VectorShuffle()
10878 return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v32i8, V1, V2, in lowerV32I8VectorShuffle()
10885 ? DAG.getUNDEF(MVT::i8) in lowerV32I8VectorShuffle()
10887 MVT::i8); in lowerV32I8VectorShuffle()
10890 X86ISD::PSHUFB, DL, MVT::v32i8, V1, in lowerV32I8VectorShuffle()
10891 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v32i8, PSHUFBMask)); in lowerV32I8VectorShuffle()
10897 DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG)) in lowerV32I8VectorShuffle()
10901 return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v32i8, V1, V2, Mask, DAG); in lowerV32I8VectorShuffle()
10910 MVT VT, const X86Subtarget *Subtarget, in lower256BitVectorShuffle()
10940 MVT FpVT = MVT::getVectorVT(MVT::getFloatingPointVT(ElementBits), in lower256BitVectorShuffle()
10948 case MVT::v4f64: in lower256BitVectorShuffle()
10950 case MVT::v4i64: in lower256BitVectorShuffle()
10952 case MVT::v8f32: in lower256BitVectorShuffle()
10954 case MVT::v8i32: in lower256BitVectorShuffle()
10956 case MVT::v16i16: in lower256BitVectorShuffle()
10958 case MVT::v32i8: in lower256BitVectorShuffle()
10967 static SDValue lowerV4X128VectorShuffle(SDLoc DL, MVT VT, in lowerV4X128VectorShuffle()
10998 DAG.getConstant(PermMask, DL, MVT::i8)); in lowerV4X128VectorShuffle()
11001 static SDValue lowerVectorShuffleWithPERMV(SDLoc DL, MVT VT, in lowerVectorShuffleWithPERMV()
11007 MVT MaskEltVT = MVT::getIntegerVT(VT.getScalarSizeInBits()); in lowerVectorShuffleWithPERMV()
11008 MVT MaskVecVT = MVT::getVectorVT(MaskEltVT, VT.getVectorNumElements()); in lowerVectorShuffleWithPERMV()
11022 assert(V1.getSimpleValueType() == MVT::v8f64 && "Bad operand type!"); in lowerV8F64VectorShuffle()
11023 assert(V2.getSimpleValueType() == MVT::v8f64 && "Bad operand type!"); in lowerV8F64VectorShuffle()
11029 lowerV4X128VectorShuffle(DL, MVT::v8f64, Mask, V1, V2, DAG)) in lowerV8F64VectorShuffle()
11033 lowerVectorShuffleWithUNPCK(DL, MVT::v8f64, Mask, V1, V2, DAG)) in lowerV8F64VectorShuffle()
11036 return lowerVectorShuffleWithPERMV(DL, MVT::v8f64, Mask, V1, V2, DAG); in lowerV8F64VectorShuffle()
11044 assert(V1.getSimpleValueType() == MVT::v16f32 && "Bad operand type!"); in lowerV16F32VectorShuffle()
11045 assert(V2.getSimpleValueType() == MVT::v16f32 && "Bad operand type!"); in lowerV16F32VectorShuffle()
11051 lowerVectorShuffleWithUNPCK(DL, MVT::v16f32, Mask, V1, V2, DAG)) in lowerV16F32VectorShuffle()
11054 return lowerVectorShuffleWithPERMV(DL, MVT::v16f32, Mask, V1, V2, DAG); in lowerV16F32VectorShuffle()
11062 assert(V1.getSimpleValueType() == MVT::v8i64 && "Bad operand type!"); in lowerV8I64VectorShuffle()
11063 assert(V2.getSimpleValueType() == MVT::v8i64 && "Bad operand type!"); in lowerV8I64VectorShuffle()
11069 lowerV4X128VectorShuffle(DL, MVT::v8i64, Mask, V1, V2, DAG)) in lowerV8I64VectorShuffle()
11073 lowerVectorShuffleWithUNPCK(DL, MVT::v8i64, Mask, V1, V2, DAG)) in lowerV8I64VectorShuffle()
11076 return lowerVectorShuffleWithPERMV(DL, MVT::v8i64, Mask, V1, V2, DAG); in lowerV8I64VectorShuffle()
11084 assert(V1.getSimpleValueType() == MVT::v16i32 && "Bad operand type!"); in lowerV16I32VectorShuffle()
11085 assert(V2.getSimpleValueType() == MVT::v16i32 && "Bad operand type!"); in lowerV16I32VectorShuffle()
11091 lowerVectorShuffleWithUNPCK(DL, MVT::v16i32, Mask, V1, V2, DAG)) in lowerV16I32VectorShuffle()
11094 return lowerVectorShuffleWithPERMV(DL, MVT::v16i32, Mask, V1, V2, DAG); in lowerV16I32VectorShuffle()
11102 assert(V1.getSimpleValueType() == MVT::v32i16 && "Bad operand type!"); in lowerV32I16VectorShuffle()
11103 assert(V2.getSimpleValueType() == MVT::v32i16 && "Bad operand type!"); in lowerV32I16VectorShuffle()
11109 return lowerVectorShuffleWithPERMV(DL, MVT::v32i16, Mask, V1, V2, DAG); in lowerV32I16VectorShuffle()
11117 assert(V1.getSimpleValueType() == MVT::v64i8 && "Bad operand type!"); in lowerV64I8VectorShuffle()
11118 assert(V2.getSimpleValueType() == MVT::v64i8 && "Bad operand type!"); in lowerV64I8VectorShuffle()
11125 return splitAndLowerVectorShuffle(DL, MVT::v64i8, V1, V2, Mask, DAG); in lowerV64I8VectorShuffle()
11134 MVT VT, const X86Subtarget *Subtarget, in lower512BitVectorShuffle()
11152 case MVT::v8f64: in lower512BitVectorShuffle()
11154 case MVT::v16f32: in lower512BitVectorShuffle()
11156 case MVT::v8i64: in lower512BitVectorShuffle()
11158 case MVT::v16i32: in lower512BitVectorShuffle()
11160 case MVT::v32i16: in lower512BitVectorShuffle()
11164 case MVT::v64i8: in lower512BitVectorShuffle()
11182 MVT VT, const X86Subtarget *Subtarget, in lower1BitVectorShuffle()
11189 MVT ExtVT; in lower1BitVectorShuffle()
11193 case MVT::v2i1: in lower1BitVectorShuffle()
11194 ExtVT = MVT::v2i64; in lower1BitVectorShuffle()
11196 case MVT::v4i1: in lower1BitVectorShuffle()
11197 ExtVT = MVT::v4i32; in lower1BitVectorShuffle()
11199 case MVT::v8i1: in lower1BitVectorShuffle()
11200 ExtVT = MVT::v8i64; // Take 512-bit type, more shuffles on KNL in lower1BitVectorShuffle()
11202 case MVT::v16i1: in lower1BitVectorShuffle()
11203 ExtVT = MVT::v16i32; in lower1BitVectorShuffle()
11205 case MVT::v32i1: in lower1BitVectorShuffle()
11206 ExtVT = MVT::v32i16; in lower1BitVectorShuffle()
11208 case MVT::v64i1: in lower1BitVectorShuffle()
11209 ExtVT = MVT::v64i8; in lower1BitVectorShuffle()
11244 MVT VT = Op.getSimpleValueType(); in lowerVectorShuffle()
11247 bool Is1BitVector = (VT.getVectorElementType() == MVT::i1); in lowerVectorShuffle()
11290 MVT NewEltVT = VT.isFloatingPoint() in lowerVectorShuffle()
11291 ? MVT::getFloatingPointVT(VT.getScalarSizeInBits() * 2) in lowerVectorShuffle()
11292 : MVT::getIntegerVT(VT.getScalarSizeInBits() * 2); in lowerVectorShuffle()
11293 MVT NewVT = MVT::getVectorVT(NewEltVT, VT.getVectorNumElements() / 2); in lowerVectorShuffle()
11425 MVT VT = Op.getSimpleValueType(); in lowerVSELECTtoVectorShuffle()
11468 case MVT::v32i8: in LowerVSELECT()
11475 case MVT::v8i16: in LowerVSELECT()
11476 case MVT::v16i16: in LowerVSELECT()
11488 MVT VT = Op.getSimpleValueType(); in LowerEXTRACT_VECTOR_ELT_SSE4()
11495 SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
11497 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract, in LowerEXTRACT_VECTOR_ELT_SSE4()
11506 ISD::TRUNCATE, dl, MVT::i16, in LowerEXTRACT_VECTOR_ELT_SSE4()
11507 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
11508 DAG.getBitcast(MVT::v4i32, Op.getOperand(0)), in LowerEXTRACT_VECTOR_ELT_SSE4()
11510 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
11512 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract, in LowerEXTRACT_VECTOR_ELT_SSE4()
11517 if (VT == MVT::f32) { in LowerEXTRACT_VECTOR_ELT_SSE4()
11529 User->getValueType(0) != MVT::i32)) in LowerEXTRACT_VECTOR_ELT_SSE4()
11531 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
11532 DAG.getBitcast(MVT::v4i32, Op.getOperand(0)), in LowerEXTRACT_VECTOR_ELT_SSE4()
11534 return DAG.getBitcast(MVT::f32, Extract); in LowerEXTRACT_VECTOR_ELT_SSE4()
11537 if (VT == MVT::i32 || VT == MVT::i64) { in LowerEXTRACT_VECTOR_ELT_SSE4()
11551 MVT VecVT = Vec.getSimpleValueType(); in ExtractBitFromMaskVector()
11553 MVT EltVT = Op.getSimpleValueType(); in ExtractBitFromMaskVector()
11555 assert((EltVT == MVT::i1) && "Unexpected operands in ExtractBitFromMaskVector"); in ExtractBitFromMaskVector()
11562 MVT ExtVT = (VecVT == MVT::v8i1 ? MVT::v8i64 : MVT::v16i32); in ExtractBitFromMaskVector()
11572 rc = getRegClassFor(MVT::v16i1); in ExtractBitFromMaskVector()
11575 DAG.getConstant(MaxSift - IdxVal, dl, MVT::i8)); in ExtractBitFromMaskVector()
11577 DAG.getConstant(MaxSift, dl, MVT::i8)); in ExtractBitFromMaskVector()
11578 return DAG.getNode(X86ISD::VEXTRACT, dl, MVT::i1, Vec, in ExtractBitFromMaskVector()
11587 MVT VecVT = Vec.getSimpleValueType(); in LowerEXTRACT_VECTOR_ELT()
11590 if (Op.getSimpleValueType() == MVT::i1) in LowerEXTRACT_VECTOR_ELT()
11598 MVT MaskEltVT = in LowerEXTRACT_VECTOR_ELT()
11599 MVT::getIntegerVT(VecVT.getVectorElementType().getSizeInBits()); in LowerEXTRACT_VECTOR_ELT()
11600 MVT MaskVT = MVT::getVectorVT(MaskEltVT, VecVT.getSizeInBits() / in LowerEXTRACT_VECTOR_ELT()
11622 MVT EltVT = VecVT.getVectorElementType(); in LowerEXTRACT_VECTOR_ELT()
11631 DAG.getConstant(IdxVal, dl, MVT::i32)); in LowerEXTRACT_VECTOR_ELT()
11640 MVT VT = Op.getSimpleValueType(); in LowerEXTRACT_VECTOR_ELT()
11645 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, in LowerEXTRACT_VECTOR_ELT()
11646 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT()
11647 DAG.getBitcast(MVT::v4i32, Vec), in LowerEXTRACT_VECTOR_ELT()
11650 MVT EltVT = MVT::i32; in LowerEXTRACT_VECTOR_ELT()
11665 MVT VVT = Op.getOperand(0).getSimpleValueType(); in LowerEXTRACT_VECTOR_ELT()
11683 MVT VVT = Op.getOperand(0).getSimpleValueType(); in LowerEXTRACT_VECTOR_ELT()
11701 MVT VecVT = Vec.getSimpleValueType(); in InsertBitToMaskVector()
11706 MVT ExtVecVT = (VecVT == MVT::v8i1 ? MVT::v8i64 : MVT::v16i32); in InsertBitToMaskVector()
11707 MVT ExtEltVT = (VecVT == MVT::v8i1 ? MVT::i64 : MVT::i32); in InsertBitToMaskVector()
11718 DAG.getConstant(IdxVal, dl, MVT::i8)); in InsertBitToMaskVector()
11726 MVT VT = Op.getSimpleValueType(); in LowerINSERT_VECTOR_ELT()
11727 MVT EltVT = VT.getVectorElementType(); in LowerINSERT_VECTOR_ELT()
11729 if (EltVT == MVT::i1) in LowerINSERT_VECTOR_ELT()
11750 if ((Subtarget->hasAVX() && (EltVT == MVT::f64 || EltVT == MVT::f32)) || in LowerINSERT_VECTOR_ELT()
11751 (Subtarget->hasAVX2() && EltVT == MVT::i32)) { in LowerINSERT_VECTOR_ELT()
11768 DAG.getConstant(IdxIn128, dl, MVT::i32)); in LowerINSERT_VECTOR_ELT()
11778 if (VT == MVT::v8i16) { in LowerINSERT_VECTOR_ELT()
11781 assert(VT == MVT::v16i8); in LowerINSERT_VECTOR_ELT()
11787 if (N1.getValueType() != MVT::i32) in LowerINSERT_VECTOR_ELT()
11788 N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1); in LowerINSERT_VECTOR_ELT()
11789 if (N2.getValueType() != MVT::i32) in LowerINSERT_VECTOR_ELT()
11794 if (EltVT == MVT::f32) { in LowerINSERT_VECTOR_ELT()
11814 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1); in LowerINSERT_VECTOR_ELT()
11819 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1); in LowerINSERT_VECTOR_ELT()
11823 if (EltVT == MVT::i32 || EltVT == MVT::i64) { in LowerINSERT_VECTOR_ELT()
11829 if (EltVT == MVT::i8) in LowerINSERT_VECTOR_ELT()
11835 if (N1.getValueType() != MVT::i32) in LowerINSERT_VECTOR_ELT()
11836 N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1); in LowerINSERT_VECTOR_ELT()
11837 if (N2.getValueType() != MVT::i32) in LowerINSERT_VECTOR_ELT()
11846 MVT OpVT = Op.getSimpleValueType(); in LowerSCALAR_TO_VECTOR()
11853 MVT VT128 = MVT::getVectorVT(OpVT.getVectorElementType(), in LowerSCALAR_TO_VECTOR()
11862 if (OpVT == MVT::v1i64 && in LowerSCALAR_TO_VECTOR()
11863 Op.getOperand(0).getValueType() == MVT::i64) in LowerSCALAR_TO_VECTOR()
11864 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i64, Op.getOperand(0)); in LowerSCALAR_TO_VECTOR()
11866 SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op.getOperand(0)); in LowerSCALAR_TO_VECTOR()
11869 OpVT, DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, AnyExt)); in LowerSCALAR_TO_VECTOR()
11881 MVT ResVT = Op.getSimpleValueType(); in LowerEXTRACT_SUBVECTOR()
11882 MVT InVT = In.getSimpleValueType(); in LowerEXTRACT_SUBVECTOR()
11915 MVT OpVT = Op.getSimpleValueType(); in LowerINSERT_SUBVECTOR()
11916 MVT SubVecVT = SubVec.getSimpleValueType(); in LowerINSERT_SUBVECTOR()
11954 if (OpVT.getVectorElementType() == MVT::i1) in LowerINSERT_SUBVECTOR()
12163 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in GetTLSADDR()
12369 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerGlobalTLSAddress()
12426 MachinePointerInfo(), MVT::i32, false, false, in LowerGlobalTLSAddress()
12461 MVT VT = Op.getSimpleValueType(); in LowerShiftParts()
12471 SDValue SafeShAmt = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt, in LowerShiftParts()
12472 DAG.getConstant(VTBits - 1, dl, MVT::i8)); in LowerShiftParts()
12474 DAG.getConstant(VTBits - 1, dl, MVT::i8)) in LowerShiftParts()
12489 SDValue AndNode = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt, in LowerShiftParts()
12490 DAG.getConstant(VTBits, dl, MVT::i8)); in LowerShiftParts()
12491 SDValue Cond = DAG.getNode(X86ISD::CMP, dl, MVT::i32, in LowerShiftParts()
12492 AndNode, DAG.getConstant(0, dl, MVT::i8)); in LowerShiftParts()
12495 SDValue CC = DAG.getConstant(X86::COND_NE, dl, MVT::i8); in LowerShiftParts()
12514 MVT SrcVT = Src.getSimpleValueType(); in LowerSINT_TO_FP()
12515 MVT VT = Op.getSimpleValueType(); in LowerSINT_TO_FP()
12519 if (SrcVT == MVT::v2i32 && VT == MVT::v2f64) { in LowerSINT_TO_FP()
12521 DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src, in LowerSINT_TO_FP()
12524 if (SrcVT.getVectorElementType() == MVT::i1) { in LowerSINT_TO_FP()
12525 MVT IntegerVT = MVT::getVectorVT(MVT::i32, SrcVT.getVectorNumElements()); in LowerSINT_TO_FP()
12532 assert(SrcVT <= MVT::i64 && SrcVT >= MVT::i16 && in LowerSINT_TO_FP()
12537 if (SrcVT == MVT::i32 && isScalarFPTypeInSSEReg(Op.getValueType())) in LowerSINT_TO_FP()
12539 if (SrcVT == MVT::i64 && isScalarFPTypeInSSEReg(Op.getValueType()) && in LowerSINT_TO_FP()
12564 Tys = DAG.getVTList(MVT::f64, MVT::Other, MVT::Glue); in BuildFILD()
12566 Tys = DAG.getVTList(Op.getValueType(), MVT::Other); in BuildFILD()
12598 Tys = DAG.getVTList(MVT::Other); in BuildFILD()
12653 SDValue XR1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, in LowerUINT_TO_FP_i64()
12656 DAG.getLoad(MVT::v4i32, dl, DAG.getEntryNode(), CPIdx0, in LowerUINT_TO_FP_i64()
12660 getUnpackl(DAG, dl, MVT::v4i32, DAG.getBitcast(MVT::v4i32, XR1), CLod0); in LowerUINT_TO_FP_i64()
12663 DAG.getLoad(MVT::v2f64, dl, CLod0.getValue(1), CPIdx1, in LowerUINT_TO_FP_i64()
12666 SDValue XR2F = DAG.getBitcast(MVT::v2f64, Unpck1); in LowerUINT_TO_FP_i64()
12668 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, XR2F, CLod1); in LowerUINT_TO_FP_i64()
12673 Result = DAG.getNode(X86ISD::FHADD, dl, MVT::v2f64, Sub, Sub); in LowerUINT_TO_FP_i64()
12675 SDValue S2F = DAG.getBitcast(MVT::v4i32, Sub); in LowerUINT_TO_FP_i64()
12676 SDValue Shuffle = getTargetShuffleNode(X86ISD::PSHUFD, dl, MVT::v4i32, in LowerUINT_TO_FP_i64()
12678 Result = DAG.getNode(ISD::FADD, dl, MVT::v2f64, in LowerUINT_TO_FP_i64()
12679 DAG.getBitcast(MVT::v2f64, Shuffle), Sub); in LowerUINT_TO_FP_i64()
12682 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Result, in LowerUINT_TO_FP_i64()
12692 MVT::f64); in LowerUINT_TO_FP_i32()
12695 SDValue Load = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, in LowerUINT_TO_FP_i32()
12701 Load = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, in LowerUINT_TO_FP_i32()
12702 DAG.getBitcast(MVT::v2f64, Load), in LowerUINT_TO_FP_i32()
12707 ISD::OR, dl, MVT::v2i64, in LowerUINT_TO_FP_i32()
12708 DAG.getBitcast(MVT::v2i64, in LowerUINT_TO_FP_i32()
12709 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f64, Load)), in LowerUINT_TO_FP_i32()
12710 DAG.getBitcast(MVT::v2i64, in LowerUINT_TO_FP_i32()
12711 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f64, Bias))); in LowerUINT_TO_FP_i32()
12713 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, in LowerUINT_TO_FP_i32()
12714 DAG.getBitcast(MVT::v2f64, Or), DAG.getIntPtrConstant(0, dl)); in LowerUINT_TO_FP_i32()
12718 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Or, Bias); in LowerUINT_TO_FP_i32()
12721 MVT DestVT = Op.getSimpleValueType(); in LowerUINT_TO_FP_i32()
12723 if (DestVT.bitsLT(MVT::f64)) in LowerUINT_TO_FP_i32()
12726 if (DestVT.bitsGT(MVT::f64)) in LowerUINT_TO_FP_i32()
12758 MVT VecIntVT = V.getSimpleValueType(); in lowerUINT_TO_FP_vXi32()
12759 bool Is128 = VecIntVT == MVT::v4i32; in lowerUINT_TO_FP_vXi32()
12760 MVT VecFloatVT = Is128 ? MVT::v4f32 : MVT::v8f32; in lowerUINT_TO_FP_vXi32()
12767 assert((VecIntVT == MVT::v4i32 || VecIntVT == MVT::v8i32) && in lowerUINT_TO_FP_vXi32()
12779 SDValue CstLow = DAG.getConstant(0x4b000000, DL, MVT::i32); in lowerUINT_TO_FP_vXi32()
12785 SDValue CstHigh = DAG.getConstant(0x53000000, DL, MVT::i32); in lowerUINT_TO_FP_vXi32()
12792 SDValue CstShift = DAG.getConstant(16, DL, MVT::i32); in lowerUINT_TO_FP_vXi32()
12801 MVT VecI16VT = Is128 ? MVT::v8i16 : MVT::v16i16; in lowerUINT_TO_FP_vXi32()
12808 VecCstLowBitcast, DAG.getConstant(0xaa, DL, MVT::i32)); in lowerUINT_TO_FP_vXi32()
12816 VecCstHighBitcast, DAG.getConstant(0xaa, DL, MVT::i32)); in lowerUINT_TO_FP_vXi32()
12818 SDValue CstMask = DAG.getConstant(0xffff, DL, MVT::i32); in lowerUINT_TO_FP_vXi32()
12831 APFloat(APFloat::IEEEsingle, APInt(32, 0xD3000080)), DL, MVT::f32); in lowerUINT_TO_FP_vXi32()
12850 MVT SVT = N0.getSimpleValueType(); in lowerUINT_TO_FP_vec()
12856 case MVT::v4i8: in lowerUINT_TO_FP_vec()
12857 case MVT::v4i16: in lowerUINT_TO_FP_vec()
12858 case MVT::v8i8: in lowerUINT_TO_FP_vec()
12859 case MVT::v8i16: { in lowerUINT_TO_FP_vec()
12860 MVT NVT = MVT::getVectorVT(MVT::i32, SVT.getVectorNumElements()); in lowerUINT_TO_FP_vec()
12864 case MVT::v4i32: in lowerUINT_TO_FP_vec()
12865 case MVT::v8i32: in lowerUINT_TO_FP_vec()
12867 case MVT::v16i8: in lowerUINT_TO_FP_vec()
12868 case MVT::v16i16: in lowerUINT_TO_FP_vec()
12871 DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v16i32, N0)); in lowerUINT_TO_FP_vec()
12890 MVT SrcVT = N0.getSimpleValueType(); in LowerUINT_TO_FP()
12891 MVT DstVT = Op.getSimpleValueType(); in LowerUINT_TO_FP()
12894 (SrcVT == MVT::i32 || (SrcVT == MVT::i64 && Subtarget->is64Bit()))) { in LowerUINT_TO_FP()
12900 if (SrcVT == MVT::i64 && DstVT == MVT::f64 && X86ScalarSSEf64) in LowerUINT_TO_FP()
12902 if (SrcVT == MVT::i32 && X86ScalarSSEf64) in LowerUINT_TO_FP()
12904 if (Subtarget->is64Bit() && SrcVT == MVT::i64 && DstVT == MVT::f32) in LowerUINT_TO_FP()
12908 SDValue StackSlot = DAG.CreateStackTemporary(MVT::i64); in LowerUINT_TO_FP()
12909 if (SrcVT == MVT::i32) { in LowerUINT_TO_FP()
12915 SDValue Store2 = DAG.getStore(Store1, dl, DAG.getConstant(0, dl, MVT::i32), in LowerUINT_TO_FP()
12918 SDValue Fild = BuildFILD(Op, MVT::i64, Store2, StackSlot, DAG); in LowerUINT_TO_FP()
12922 assert(SrcVT == MVT::i64 && "Unexpected type in UINT_TO_FP"); in LowerUINT_TO_FP()
12936 SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other); in LowerUINT_TO_FP()
12937 SDValue Ops[] = { Store, StackSlot, DAG.getValueType(MVT::i64) }; in LowerUINT_TO_FP()
12939 MVT::i64, MMO); in LowerUINT_TO_FP()
12945 dl, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i64), in LowerUINT_TO_FP()
12946 Op.getOperand(0), DAG.getConstant(0, dl, MVT::i64), ISD::SETLT); in LowerUINT_TO_FP()
12962 ISD::EXTLOAD, dl, MVT::f80, DAG.getEntryNode(), FudgePtr, in LowerUINT_TO_FP()
12963 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32, in LowerUINT_TO_FP()
12967 SDValue Add = DAG.getNode(ISD::FADD, dl, MVT::f80, Fild, Fudge); in LowerUINT_TO_FP()
12990 if (TheVT != MVT::f32 && TheVT != MVT::f64 && TheVT != MVT::f80) { in FP_TO_INTHelper()
13000 DstTy == MVT::i64 && in FP_TO_INTHelper()
13004 if (!IsSigned && DstTy != MVT::i64 && !Subtarget->hasAVX512()) { in FP_TO_INTHelper()
13007 assert(DstTy == MVT::i32 && "Unexpected FP_TO_UINT"); in FP_TO_INTHelper()
13008 DstTy = MVT::i64; in FP_TO_INTHelper()
13011 assert(DstTy.getSimpleVT() <= MVT::i64 && in FP_TO_INTHelper()
13012 DstTy.getSimpleVT() >= MVT::i16 && in FP_TO_INTHelper()
13016 if (DstTy == MVT::i32 && in FP_TO_INTHelper()
13020 DstTy == MVT::i64 && in FP_TO_INTHelper()
13034 case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break; in FP_TO_INTHelper()
13035 case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break; in FP_TO_INTHelper()
13036 case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break; in FP_TO_INTHelper()
13063 if (TheVT == MVT::f64) in FP_TO_INTHelper()
13067 else if (TheVT == MVT::f80) in FP_TO_INTHelper()
13080 Adjust = DAG.getSelect(DL, MVT::i32, Cmp, in FP_TO_INTHelper()
13081 DAG.getConstant(0, DL, MVT::i32), in FP_TO_INTHelper()
13082 DAG.getConstant(0x80000000, DL, MVT::i32)); in FP_TO_INTHelper()
13093 assert(DstTy == MVT::i64 && "Invalid FP_TO_SINT to lower!"); in FP_TO_INTHelper()
13097 SDVTList Tys = DAG.getVTList(Op.getOperand(0).getValueType(), MVT::Other); in FP_TO_INTHelper()
13121 SDValue FIST = DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::Other), in FP_TO_INTHelper()
13124 SDValue Low32 = DAG.getLoad(MVT::i32, DL, FIST, StackSlot, in FP_TO_INTHelper()
13130 SDValue High32 = DAG.getLoad(MVT::i32, DL, FIST, HighAddr, in FP_TO_INTHelper()
13133 High32 = DAG.getNode(ISD::XOR, DL, MVT::i32, High32, Adjust); in FP_TO_INTHelper()
13138 Low32 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Low32); in FP_TO_INTHelper()
13139 High32 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, High32); in FP_TO_INTHelper()
13140 High32 = DAG.getNode(ISD::SHL, DL, MVT::i64, High32, in FP_TO_INTHelper()
13141 DAG.getConstant(32, DL, MVT::i8)); in FP_TO_INTHelper()
13142 SDValue Result = DAG.getNode(ISD::OR, DL, MVT::i64, High32, Low32); in FP_TO_INTHelper()
13149 ? DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, ResultOps) in FP_TO_INTHelper()
13155 SDValue FIST = DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::Other), in FP_TO_INTHelper()
13163 MVT VT = Op->getSimpleValueType(0); in LowerAVXExtend()
13165 MVT InVT = In.getSimpleValueType(); in LowerAVXExtend()
13168 if (VT.is512BitVector() || InVT.getVectorElementType() == MVT::i1) in LowerAVXExtend()
13184 if (((VT != MVT::v16i16) || (InVT != MVT::v16i8)) && in LowerAVXExtend()
13185 ((VT != MVT::v8i32) || (InVT != MVT::v8i16)) && in LowerAVXExtend()
13186 ((VT != MVT::v4i64) || (InVT != MVT::v4i32))) in LowerAVXExtend()
13198 MVT HVT = MVT::getVectorVT(VT.getVectorElementType(), in LowerAVXExtend()
13209 MVT VT = Op->getSimpleValueType(0); in LowerZERO_EXTEND_AVX512()
13211 MVT InVT = In.getSimpleValueType(); in LowerZERO_EXTEND_AVX512()
13217 if (VT.is512BitVector() && InVT.getVectorElementType() != MVT::i1) in LowerZERO_EXTEND_AVX512()
13220 assert(InVT.getVectorElementType() == MVT::i1); in LowerZERO_EXTEND_AVX512()
13221 MVT ExtVT = NumElts == 8 ? MVT::v8i64 : MVT::v16i32; in LowerZERO_EXTEND_AVX512()
13245 MVT VT = Op.getSimpleValueType(); in LowerZERO_EXTEND()
13247 MVT SVT = In.getSimpleValueType(); in LowerZERO_EXTEND()
13249 if (VT.is512BitVector() || SVT.getVectorElementType() == MVT::i1) in LowerZERO_EXTEND()
13263 MVT VT = Op.getSimpleValueType(); in LowerTRUNCATE()
13265 MVT InVT = In.getSimpleValueType(); in LowerTRUNCATE()
13267 if (VT == MVT::i1) { in LowerTRUNCATE()
13272 In = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, In); in LowerTRUNCATE()
13279 if (VT.getVectorElementType() == MVT::i1) { in LowerTRUNCATE()
13296 if (VT.getVectorElementType() == MVT::i1) { in LowerTRUNCATE()
13297 assert(VT.getVectorElementType() == MVT::i1 && "Unexpected vector type"); in LowerTRUNCATE()
13301 MVT ExtVT = (NumElts == 16)? MVT::v16i32 : MVT::v8i64; in LowerTRUNCATE()
13315 if (InVT == MVT::v16i16 && !Subtarget->hasBWI()) // v16i16 -> v16i8 in LowerTRUNCATE()
13317 DAG.getNode(X86ISD::VSEXT, DL, MVT::v16i32, In)); in LowerTRUNCATE()
13320 if ((VT == MVT::v4i32) && (InVT == MVT::v4i64)) { in LowerTRUNCATE()
13324 In = DAG.getBitcast(MVT::v8i32, In); in LowerTRUNCATE()
13325 In = DAG.getVectorShuffle(MVT::v8i32, DL, In, DAG.getUNDEF(MVT::v8i32), in LowerTRUNCATE()
13331 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In, in LowerTRUNCATE()
13333 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In, in LowerTRUNCATE()
13335 OpLo = DAG.getBitcast(MVT::v4i32, OpLo); in LowerTRUNCATE()
13336 OpHi = DAG.getBitcast(MVT::v4i32, OpHi); in LowerTRUNCATE()
13341 if ((VT == MVT::v8i16) && (InVT == MVT::v8i32)) { in LowerTRUNCATE()
13344 In = DAG.getBitcast(MVT::v32i8, In); in LowerTRUNCATE()
13348 pshufbMask.push_back(DAG.getConstant(0x0, DL, MVT::i8)); in LowerTRUNCATE()
13349 pshufbMask.push_back(DAG.getConstant(0x1, DL, MVT::i8)); in LowerTRUNCATE()
13350 pshufbMask.push_back(DAG.getConstant(0x4, DL, MVT::i8)); in LowerTRUNCATE()
13351 pshufbMask.push_back(DAG.getConstant(0x5, DL, MVT::i8)); in LowerTRUNCATE()
13352 pshufbMask.push_back(DAG.getConstant(0x8, DL, MVT::i8)); in LowerTRUNCATE()
13353 pshufbMask.push_back(DAG.getConstant(0x9, DL, MVT::i8)); in LowerTRUNCATE()
13354 pshufbMask.push_back(DAG.getConstant(0xc, DL, MVT::i8)); in LowerTRUNCATE()
13355 pshufbMask.push_back(DAG.getConstant(0xd, DL, MVT::i8)); in LowerTRUNCATE()
13357 pshufbMask.push_back(DAG.getConstant(0x80, DL, MVT::i8)); in LowerTRUNCATE()
13359 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v32i8, pshufbMask); in LowerTRUNCATE()
13360 In = DAG.getNode(X86ISD::PSHUFB, DL, MVT::v32i8, In, BV); in LowerTRUNCATE()
13361 In = DAG.getBitcast(MVT::v4i64, In); in LowerTRUNCATE()
13364 In = DAG.getVectorShuffle(MVT::v4i64, DL, In, DAG.getUNDEF(MVT::v4i64), in LowerTRUNCATE()
13366 In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In, in LowerTRUNCATE()
13371 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In, in LowerTRUNCATE()
13374 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In, in LowerTRUNCATE()
13377 OpLo = DAG.getBitcast(MVT::v16i8, OpLo); in LowerTRUNCATE()
13378 OpHi = DAG.getBitcast(MVT::v16i8, OpHi); in LowerTRUNCATE()
13384 SDValue Undef = DAG.getUNDEF(MVT::v16i8); in LowerTRUNCATE()
13385 OpLo = DAG.getVectorShuffle(MVT::v16i8, DL, OpLo, Undef, ShufMask1); in LowerTRUNCATE()
13386 OpHi = DAG.getVectorShuffle(MVT::v16i8, DL, OpHi, Undef, ShufMask1); in LowerTRUNCATE()
13388 OpLo = DAG.getBitcast(MVT::v4i32, OpLo); in LowerTRUNCATE()
13389 OpHi = DAG.getBitcast(MVT::v4i32, OpHi); in LowerTRUNCATE()
13393 SDValue res = DAG.getVectorShuffle(MVT::v4i32, DL, OpLo, OpHi, ShufMask2); in LowerTRUNCATE()
13394 return DAG.getBitcast(MVT::v8i16, res); in LowerTRUNCATE()
13404 MVT NVT = MVT::getVectorVT(VT.getVectorElementType(), NumElems * 2); in LowerTRUNCATE()
13458 MVT VT = Op.getSimpleValueType(); in LowerFP_EXTEND()
13460 MVT SVT = In.getSimpleValueType(); in LowerFP_EXTEND()
13462 assert(SVT == MVT::v2f32 && "Only customize MVT::v2f32 type legalization!"); in LowerFP_EXTEND()
13465 DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f32, in LowerFP_EXTEND()
13485 MVT VT = Op.getSimpleValueType(); in LowerFABSorFNEG()
13487 bool IsF128 = (VT == MVT::f128); in LowerFABSorFNEG()
13493 MVT LogicVT; in LowerFABSorFNEG()
13494 MVT EltVT; in LowerFABSorFNEG()
13503 LogicVT = MVT::f128; in LowerFABSorFNEG()
13511 LogicVT = (VT == MVT::f64) ? MVT::v2f64 : MVT::v4f32; in LowerFABSorFNEG()
13513 NumElts = (VT == MVT::f64) ? 2 : 4; in LowerFABSorFNEG()
13554 MVT VT = Op.getSimpleValueType(); in LowerFCOPYSIGN()
13555 MVT SrcVT = Op1.getSimpleValueType(); in LowerFCOPYSIGN()
13556 bool IsF128 = (VT == MVT::f128); in LowerFCOPYSIGN()
13571 assert((VT == MVT::f64 || VT == MVT::f32 || IsF128) && in LowerFCOPYSIGN()
13575 VT == MVT::f64 ? APFloat::IEEEdouble : in LowerFCOPYSIGN()
13580 VT == MVT::f64 ? 2 : (IsF128 ? 1 : 4), in LowerFCOPYSIGN()
13593 MVT LogicVT = (VT == MVT::f64) ? MVT::v2f64 : (IsF128 ? MVT::f128 : MVT::v4f32); in LowerFCOPYSIGN()
13640 MVT VT = Op.getSimpleValueType(); in LowerFGETSIGN()
13665 EVT VT = MVT::Other; in LowerVectorAllZeroTest()
13721 MVT TestVT = VT.is128BitVector() ? MVT::v2i64 : MVT::v4i64; in LowerVectorAllZeroTest()
13736 return DAG.getNode(X86ISD::PTEST, DL, MVT::i32, in LowerVectorAllZeroTest()
13763 if (Op.getValueType() == MVT::i1) { in EmitTest()
13764 SDValue ExtOp = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i8, Op); in EmitTest()
13765 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, ExtOp, in EmitTest()
13766 DAG.getConstant(0, dl, MVT::i8)); in EmitTest()
13808 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, in EmitTest()
13982 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op, in EmitTest()
13985 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32); in EmitTest()
14000 assert(!(isa<ConstantSDNode>(Op1) && Op0.getValueType() == MVT::i1) && in EmitCmp()
14003 if ((Op0.getValueType() == MVT::i8 || Op0.getValueType() == MVT::i16 || in EmitCmp()
14004 Op0.getValueType() == MVT::i32 || Op0.getValueType() == MVT::i64)) { in EmitCmp()
14009 if (Op0.getValueType() != MVT::i32 && Op0.getValueType() != MVT::i64 && in EmitCmp()
14014 Op0 = DAG.getNode(ExtendOp, dl, MVT::i32, Op0); in EmitCmp()
14015 Op1 = DAG.getNode(ExtendOp, dl, MVT::i32, Op1); in EmitCmp()
14018 SDVTList VTs = DAG.getVTList(Op0.getValueType(), MVT::i32); in EmitCmp()
14023 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op0, Op1); in EmitCmp()
14042 SDValue TruncFPSW = DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Cmp); in ConvertCmpIfNecessary()
14043 SDValue FNStSW = DAG.getNode(X86ISD::FNSTSW16r, dl, MVT::i16, TruncFPSW); in ConvertCmpIfNecessary()
14044 SDValue Srl = DAG.getNode(ISD::SRL, dl, MVT::i16, FNStSW, in ConvertCmpIfNecessary()
14045 DAG.getConstant(8, dl, MVT::i8)); in ConvertCmpIfNecessary()
14046 SDValue TruncSrl = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Srl); in ConvertCmpIfNecessary()
14050 return DAG.getNode(X86ISD::SAHF, dl, MVT::i32, TruncSrl); in ConvertCmpIfNecessary()
14069 if (VT == MVT::f32 && Subtarget->hasSSE1()) in getRsqrtEstimate()
14071 else if ((VT == MVT::v4f32 && Subtarget->hasSSE1()) || in getRsqrtEstimate()
14072 (VT == MVT::v8f32 && Subtarget->hasAVX())) in getRsqrtEstimate()
14101 if (VT == MVT::f32 && Subtarget->hasSSE1()) in getRecipEstimate()
14103 else if ((VT == MVT::v4f32 && Subtarget->hasSSE1()) || in getRecipEstimate()
14104 (VT == MVT::v8f32 && Subtarget->hasAVX())) in getRecipEstimate()
14179 if (LHS.getValueType() == MVT::i8 || in LowerToBT()
14180 LHS.getValueType() == MVT::i16) in LowerToBT()
14181 LHS = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, LHS); in LowerToBT()
14188 SDValue BT = DAG.getNode(X86ISD::BT, dl, MVT::i32, LHS, RHS); in LowerToBT()
14190 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in LowerToBT()
14191 DAG.getConstant(Cond, dl, MVT::i8), BT); in LowerToBT()
14245 MVT VT = Op.getSimpleValueType(); in Lower256IntVSETCC()
14265 MVT EltVT = VT.getVectorElementType(); in Lower256IntVSETCC()
14266 MVT NewVT = MVT::getVectorVT(EltVT, NumElems/2); in Lower256IntVSETCC()
14276 MVT VT = Op.getSimpleValueType(); in LowerBoolVSETCC_AVX512()
14279 assert(Op0.getSimpleValueType().getVectorElementType() == MVT::i1 && in LowerBoolVSETCC_AVX512()
14320 MVT VT = Op.getSimpleValueType(); in LowerIntVSETCC_AVX512()
14324 Op.getSimpleValueType().getVectorElementType() == MVT::i1 && in LowerIntVSETCC_AVX512()
14352 DAG.getConstant(SSECC, dl, MVT::i8)); in LowerIntVSETCC_AVX512()
14364 MVT VT = Op1.getSimpleValueType(); in ChangeVSETULTtoVSETULE()
14365 MVT EVT = VT.getVectorElementType(); in ChangeVSETULTtoVSETULE()
14390 MVT VT = Op.getSimpleValueType(); in LowerVSETCC()
14397 MVT EltVT = Op0.getSimpleValueType().getVectorElementType(); in LowerVSETCC()
14398 assert(EltVT == MVT::f32 || EltVT == MVT::f64); in LowerVSETCC()
14403 if (Subtarget->hasAVX512() && VT.getVectorElementType() == MVT::i1) { in LowerVSETCC()
14419 DAG.getConstant(CC0, dl, MVT::i8)); in LowerVSETCC()
14421 DAG.getConstant(CC1, dl, MVT::i8)); in LowerVSETCC()
14426 DAG.getConstant(SSECC, dl, MVT::i8)); in LowerVSETCC()
14429 MVT VTOp0 = Op0.getSimpleValueType(); in LowerVSETCC()
14462 MVT OpVT = Op1.getSimpleValueType(); in LowerVSETCC()
14463 if (OpVT.getVectorElementType() == MVT::i1) in LowerVSETCC()
14466 bool MaskResult = (VT.getVectorElementType() == MVT::i1); in LowerVSETCC()
14485 if ((VT == MVT::v16i8 || VT == MVT::v8i16 || in LowerVSETCC()
14486 VT == MVT::v4i32 || VT == MVT::v2i64) && Subtarget->hasXOP()) { in LowerVSETCC()
14508 DAG.getConstant(CmpMode, dl, MVT::i8)); in LowerVSETCC()
14536 MVT VET = VT.getVectorElementType(); in LowerVSETCC()
14538 (Subtarget->hasSSE41() && (VET >= MVT::i8 && VET <= MVT::i32)) in LowerVSETCC()
14539 || (Subtarget->hasSSE2() && (VET == MVT::i8)); in LowerVSETCC()
14551 bool hasSubus = Subtarget->hasSSE2() && (VET == MVT::i8 || VET == MVT::i16); in LowerVSETCC()
14590 if (VT == MVT::v2i64) { in LowerVSETCC()
14595 Op0 = DAG.getBitcast(MVT::v4i32, Op0); in LowerVSETCC()
14596 Op1 = DAG.getBitcast(MVT::v4i32, Op1); in LowerVSETCC()
14603 SB = DAG.getConstant(0x80000000U, dl, MVT::v4i32); in LowerVSETCC()
14605 SDValue Sign = DAG.getConstant(0x80000000U, dl, MVT::i32); in LowerVSETCC()
14606 SDValue Zero = DAG.getConstant(0x00000000U, dl, MVT::i32); in LowerVSETCC()
14607 SB = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, in LowerVSETCC()
14610 Op0 = DAG.getNode(ISD::XOR, dl, MVT::v4i32, Op0, SB); in LowerVSETCC()
14611 Op1 = DAG.getNode(ISD::XOR, dl, MVT::v4i32, Op1, SB); in LowerVSETCC()
14614 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1); in LowerVSETCC()
14615 SDValue EQ = DAG.getNode(X86ISD::PCMPEQ, dl, MVT::v4i32, Op0, Op1); in LowerVSETCC()
14620 SDValue EQHi = DAG.getVectorShuffle(MVT::v4i32, dl, EQ, EQ, MaskHi); in LowerVSETCC()
14621 SDValue GTLo = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskLo); in LowerVSETCC()
14622 SDValue GTHi = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi); in LowerVSETCC()
14624 SDValue Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, EQHi, GTLo); in LowerVSETCC()
14625 Result = DAG.getNode(ISD::OR, dl, MVT::v4i32, Result, GTHi); in LowerVSETCC()
14628 Result = DAG.getNOT(dl, Result, MVT::v4i32); in LowerVSETCC()
14639 Op0 = DAG.getBitcast(MVT::v4i32, Op0); in LowerVSETCC()
14640 Op1 = DAG.getBitcast(MVT::v4i32, Op1); in LowerVSETCC()
14643 SDValue Result = DAG.getNode(Opc, dl, MVT::v4i32, Op0, Op1); in LowerVSETCC()
14647 SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Result, Result, Mask); in LowerVSETCC()
14648 Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, Result, Shuf); in LowerVSETCC()
14651 Result = DAG.getNOT(dl, Result, MVT::v4i32); in LowerVSETCC()
14660 MVT EltVT = VT.getVectorElementType(); in LowerVSETCC()
14685 MVT VT = Op.getSimpleValueType(); in LowerSETCC()
14689 assert(((!Subtarget->hasAVX512() && VT == MVT::i8) || (VT == MVT::i1)) in LowerSETCC()
14704 if (VT == MVT::i1) in LowerSETCC()
14705 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, NewSetCC); in LowerSETCC()
14724 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in LowerSETCC()
14725 DAG.getConstant(CCode, dl, MVT::i8), in LowerSETCC()
14727 if (VT == MVT::i1) in LowerSETCC()
14728 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, SetCC); in LowerSETCC()
14732 if ((Op0.getValueType() == MVT::i1) && isOneConstant(Op1) && in LowerSETCC()
14736 return DAG.getSetCC(dl, VT, Op0, DAG.getConstant(0, dl, MVT::i1), NewCC); in LowerSETCC()
14746 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in LowerSETCC()
14747 DAG.getConstant(X86CC, dl, MVT::i8), EFLAGS); in LowerSETCC()
14748 if (VT == MVT::i1) in LowerSETCC()
14749 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, SetCC); in LowerSETCC()
14764 SDVTList VTs = DAG.getVTList(LHS.getValueType(), MVT::i32); in LowerSETCCE()
14767 DAG.getConstant(CC, DL, MVT::i8), Cmp.getValue(1)); in LowerSETCCE()
14812 MVT VT = Op1.getSimpleValueType(); in LowerSELECT()
14819 ((Subtarget->hasSSE2() && (VT == MVT::f32 || VT == MVT::f64)) || in LowerSELECT()
14820 (Subtarget->hasSSE1() && VT == MVT::f32)) && in LowerSELECT()
14828 SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, MVT::i1, CondOp0, CondOp1, in LowerSELECT()
14829 DAG.getConstant(SSECC, DL, MVT::i8)); in LowerSELECT()
14834 DAG.getConstant(SSECC, DL, MVT::i8)); in LowerSELECT()
14855 MVT VecVT = VT == MVT::f32 ? MVT::v4f32 : MVT::v2f64; in LowerSELECT()
14860 MVT VCmpVT = VT == MVT::f32 ? MVT::v4i32 : MVT::v2i64; in LowerSELECT()
14874 if (VT.isVector() && VT.getVectorElementType() == MVT::i1) { in LowerSELECT()
14891 SDValue ExtVec = DAG.getBitcast(MVT::v8i1, newSelect); in LowerSELECT()
14897 if (VT == MVT::v4i1 || VT == MVT::v2i1) { in LowerSELECT()
14899 Op1 = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v8i1, in LowerSELECT()
14900 DAG.getUNDEF(MVT::v8i1), Op1, zeroConst); in LowerSELECT()
14901 Op2 = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v8i1, in LowerSELECT()
14902 DAG.getUNDEF(MVT::v8i1), Op2, zeroConst); in LowerSELECT()
14903 SDValue newSelect = DAG.getNode(ISD::SELECT, DL, MVT::v8i1, in LowerSELECT()
14935 SDVTList VTs = DAG.getVTList(CmpOp0.getValueType(), MVT::i32); in LowerSELECT()
14941 DAG.getConstant(X86::COND_B, DL, MVT::i8), in LowerSELECT()
14946 Cmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32, in LowerSELECT()
14952 DAG.getConstant(X86::COND_B, DL, MVT::i8), Cmp); in LowerSELECT()
14978 MVT VT = Op.getSimpleValueType(); in LowerSELECT()
14993 Cond.getOperand(0).getValueType() != MVT::i8)) { in LowerSELECT()
15010 MVT::i32); in LowerSELECT()
15012 VTs = DAG.getVTList(LHS.getValueType(), MVT::i32); in LowerSELECT()
15021 CC = DAG.getConstant(X86Cond, DL, MVT::i8); in LowerSELECT()
15042 CC = DAG.getConstant(X86::COND_NE, DL, MVT::i8); in LowerSELECT()
15058 DAG.getConstant(X86::COND_B, DL, MVT::i8), in LowerSELECT()
15069 if (Op.getValueType() == MVT::i8 && in LowerSELECT()
15075 SDVTList VTs = DAG.getVTList(T1.getValueType(), MVT::Glue); in LowerSELECT()
15083 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue); in LowerSELECT()
15091 MVT VT = Op->getSimpleValueType(0); in LowerSIGN_EXTEND_AVX512()
15093 MVT InVT = In.getSimpleValueType(); in LowerSIGN_EXTEND_AVX512()
15094 MVT VTElt = VT.getVectorElementType(); in LowerSIGN_EXTEND_AVX512()
15095 MVT InVTElt = InVT.getVectorElementType(); in LowerSIGN_EXTEND_AVX512()
15099 if ((InVTElt == MVT::i1) && in LowerSIGN_EXTEND_AVX512()
15118 if (VT.is512BitVector() && InVT.getVectorElementType() != MVT::i1) { in LowerSIGN_EXTEND_AVX512()
15124 assert (InVT.getVectorElementType() == MVT::i1 && "Unexpected vector type"); in LowerSIGN_EXTEND_AVX512()
15125 MVT ExtVT = NumElts == 8 ? MVT::v8i64 : MVT::v16i32; in LowerSIGN_EXTEND_AVX512()
15142 MVT VT = Op->getSimpleValueType(0); in LowerSIGN_EXTEND_VECTOR_INREG()
15143 MVT InVT = In.getSimpleValueType(); in LowerSIGN_EXTEND_VECTOR_INREG()
15146 MVT InSVT = InVT.getVectorElementType(); in LowerSIGN_EXTEND_VECTOR_INREG()
15149 if (VT != MVT::v2i64 && VT != MVT::v4i32 && VT != MVT::v8i16) in LowerSIGN_EXTEND_VECTOR_INREG()
15151 if (InSVT != MVT::i32 && InSVT != MVT::i16 && InSVT != MVT::i8) in LowerSIGN_EXTEND_VECTOR_INREG()
15162 MVT CurrVT = InVT; in LowerSIGN_EXTEND_VECTOR_INREG()
15166 while (CurrVT != VT && CurrVT.getVectorElementType() != MVT::i32) { in LowerSIGN_EXTEND_VECTOR_INREG()
15168 MVT CurrSVT = MVT::getIntegerVT(CurrVT.getScalarSizeInBits() * 2); in LowerSIGN_EXTEND_VECTOR_INREG()
15169 CurrVT = MVT::getVectorVT(CurrSVT, CurrVT.getVectorNumElements() / 2); in LowerSIGN_EXTEND_VECTOR_INREG()
15178 DAG.getConstant(SignExtShift, dl, MVT::i8)); in LowerSIGN_EXTEND_VECTOR_INREG()
15184 if (VT == MVT::v2i64 && CurrVT == MVT::v4i32) { in LowerSIGN_EXTEND_VECTOR_INREG()
15186 DAG.getConstant(31, dl, MVT::i8)); in LowerSIGN_EXTEND_VECTOR_INREG()
15196 MVT VT = Op->getSimpleValueType(0); in LowerSIGN_EXTEND()
15198 MVT InVT = In.getSimpleValueType(); in LowerSIGN_EXTEND()
15201 if (VT.is512BitVector() || InVT.getVectorElementType() == MVT::i1) in LowerSIGN_EXTEND()
15204 if ((VT != MVT::v4i64 || InVT != MVT::v4i32) && in LowerSIGN_EXTEND()
15205 (VT != MVT::v8i32 || InVT != MVT::v8i16) && in LowerSIGN_EXTEND()
15206 (VT != MVT::v16i16 || InVT != MVT::v16i8)) in LowerSIGN_EXTEND()
15236 MVT HalfVT = MVT::getVectorVT(VT.getVectorElementType(), in LowerSIGN_EXTEND()
15254 MVT RegVT = Op.getSimpleValueType(); in LowerExtendedLoad()
15328 MVT SclrLoadTy = MVT::i8; in LowerExtendedLoad()
15329 for (MVT Tp : MVT::integer_valuetypes()) { in LowerExtendedLoad()
15336 if (TLI.isTypeLegal(MVT::f64) && SclrLoadTy.getSizeInBits() < 64 && in LowerExtendedLoad()
15338 SclrLoadTy = MVT::f64; in LowerExtendedLoad()
15393 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); in LowerExtendedLoad()
15529 Cond.getOperand(0).getValueType() != MVT::i8)) { in LowerBRCOND()
15561 MVT::i32); in LowerBRCOND()
15563 VTs = DAG.getVTList(LHS.getValueType(), MVT::i32); in LowerBRCOND()
15572 CC = DAG.getConstant(X86Cond, dl, MVT::i8); in LowerBRCOND()
15603 CC = DAG.getConstant(CCode, dl, MVT::i8); in LowerBRCOND()
15621 CC = DAG.getConstant(CCode, dl, MVT::i8); in LowerBRCOND()
15634 CC = DAG.getConstant(CCode, dl, MVT::i8); in LowerBRCOND()
15657 SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32, in LowerBRCOND()
15660 CC = DAG.getConstant(X86::COND_NE, dl, MVT::i8); in LowerBRCOND()
15663 CC = DAG.getConstant(X86::COND_P, dl, MVT::i8); in LowerBRCOND()
15687 SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32, in LowerBRCOND()
15690 CC = DAG.getConstant(X86::COND_NE, dl, MVT::i8); in LowerBRCOND()
15693 CC = DAG.getConstant(X86::COND_NP, dl, MVT::i8); in LowerBRCOND()
15720 CC = DAG.getConstant(X86Cond, dl, MVT::i8); in LowerBRCOND()
15754 MVT SPTy = getPointerTy(DAG.getDataLayout()); in LowerDYNAMIC_STACKALLOC()
15801 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerDYNAMIC_STACKALLOC()
15853 DL, MVT::i32), in LowerVASTART()
15861 MVT::i32), in LowerVASTART()
15880 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); in LowerVASTART()
15907 if (ArgVT == MVT::f80) { in LowerVAARG()
15926 SDValue InstOps[] = {Chain, SrcPtr, DAG.getConstant(ArgSize, dl, MVT::i32), in LowerVAARG()
15927 DAG.getConstant(ArgMode, dl, MVT::i8), in LowerVAARG()
15928 DAG.getConstant(Align, dl, MVT::i32)}; in LowerVAARG()
15929 SDVTList VTs = DAG.getVTList(getPointerTy(DAG.getDataLayout()), MVT::Other); in LowerVAARG()
15931 VTs, InstOps, MVT::i64, in LowerVAARG()
15972 static SDValue getTargetVShiftByConstNode(unsigned Opc, SDLoc dl, MVT VT, in getTargetVShiftByConstNode()
15975 MVT ElementType = VT.getVectorElementType(); in getTargetVShiftByConstNode()
16044 DAG.getConstant(ShiftAmt, dl, MVT::i8)); in getTargetVShiftByConstNode()
16049 static SDValue getTargetVShiftNode(unsigned Opc, SDLoc dl, MVT VT, in getTargetVShiftNode()
16052 MVT SVT = ShAmt.getSimpleValueType(); in getTargetVShiftNode()
16053 assert((SVT == MVT::i32 || SVT == MVT::i64) && "Unexpected value type!"); in getTargetVShiftNode()
16071 ShAmt.getOperand(0).getSimpleValueType() == MVT::i16) { in getTargetVShiftNode()
16074 Op0 = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(Op0), MVT::v8i16, Op0); in getTargetVShiftNode()
16081 if (SVT == MVT::i32) { in getTargetVShiftNode()
16087 MVT BVT = SVT == MVT::i32 ? MVT::v4i32 : MVT::v2i64; in getTargetVShiftNode()
16093 MVT EltVT = VT.getVectorElementType(); in getTargetVShiftNode()
16094 MVT ShVT = MVT::getVectorVT(EltVT, 128/EltVT.getSizeInBits()); in getTargetVShiftNode()
16102 static SDValue getMaskNode(SDValue Mask, MVT MaskVT, in getMaskNode()
16109 MVT::getIntegerVT(MaskVT.getSizeInBits()), Mask); in getMaskNode()
16112 if (Mask.getSimpleValueType() == MVT::i64 && Subtarget->is32Bit()) { in getMaskNode()
16113 if (MaskVT == MVT::v64i1) { in getMaskNode()
16117 Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Mask, in getMaskNode()
16118 DAG.getConstant(0, dl, MVT::i32)); in getMaskNode()
16119 Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Mask, in getMaskNode()
16120 DAG.getConstant(1, dl, MVT::i32)); in getMaskNode()
16122 Lo = DAG.getBitcast(MVT::v32i1, Lo); in getMaskNode()
16123 Hi = DAG.getBitcast(MVT::v32i1, Hi); in getMaskNode()
16125 return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Lo, Hi); in getMaskNode()
16129 MVT TruncVT = MVT::getIntegerVT(MaskVT.getSizeInBits()); in getMaskNode()
16135 MVT BitcastVT = MVT::getVectorVT(MVT::i1, in getMaskNode()
16152 MVT VT = Op.getSimpleValueType(); in getVectorMaskingNode()
16153 MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements()); in getVectorMaskingNode()
16200 MVT VT = Op.getSimpleValueType(); in getScalarMaskingNode()
16203 SDValue IMask = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Mask); in getScalarMaskingNode()
16245 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); in recoverFramePointer()
16281 MVT VT = Op.getSimpleValueType(); in LowerINTRINSIC_WO_CHAIN()
16292 DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op.getOperand(2))); in LowerINTRINSIC_WO_CHAIN()
16309 DAG.getConstant(X86::STATIC_ROUNDING::CUR_DIRECTION, dl, MVT::i32); in LowerINTRINSIC_WO_CHAIN()
16382 Src2 = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Src2); in LowerINTRINSIC_WO_CHAIN()
16415 Rnd = DAG.getConstant(X86::STATIC_ROUNDING::CUR_DIRECTION, dl, MVT::i32); in LowerINTRINSIC_WO_CHAIN()
16446 Rnd = DAG.getConstant(X86::STATIC_ROUNDING::CUR_DIRECTION, dl, MVT::i32); in LowerINTRINSIC_WO_CHAIN()
16461 Src3 = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Src3); in LowerINTRINSIC_WO_CHAIN()
16467 Src3 = DAG.getTargetConstant(Imm, dl, MVT::i32); in LowerINTRINSIC_WO_CHAIN()
16495 MVT VT = Op.getSimpleValueType(); in LowerINTRINSIC_WO_CHAIN()
16517 MVT VT = Op.getSimpleValueType(); in LowerINTRINSIC_WO_CHAIN()
16551 SDValue Src4 = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op.getOperand(4)); in LowerINTRINSIC_WO_CHAIN()
16553 MVT VT = Op.getSimpleValueType(); in LowerINTRINSIC_WO_CHAIN()
16566 MVT VT = Src1.getSimpleValueType(); in LowerINTRINSIC_WO_CHAIN()
16567 MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements()); in LowerINTRINSIC_WO_CHAIN()
16570 MVT BitcastVT = MVT::getVectorVT(MVT::i1, in LowerINTRINSIC_WO_CHAIN()
16585 SDValue FPclass = DAG.getNode(IntrData->Opc0, dl, MVT::i1, Src1, Imm); in LowerINTRINSIC_WO_CHAIN()
16587 DAG.getTargetConstant(0, dl, MVT::i1), Subtarget, DAG); in LowerINTRINSIC_WO_CHAIN()
16588 return DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i8, FPclassMask); in LowerINTRINSIC_WO_CHAIN()
16601 MVT VT = Op.getOperand(1).getSimpleValueType(); in LowerINTRINSIC_WO_CHAIN()
16602 MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements()); in LowerINTRINSIC_WO_CHAIN()
16604 MVT BitcastVT = MVT::getVectorVT(MVT::i1, in LowerINTRINSIC_WO_CHAIN()
16609 CC = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, CC); in LowerINTRINSIC_WO_CHAIN()
16642 SDValue CC = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op.getOperand(3)); in LowerINTRINSIC_WO_CHAIN()
16650 Cmp = DAG.getNode(IntrData->Opc1, dl, MVT::i1, Src1, Src2, CC, Rnd); in LowerINTRINSIC_WO_CHAIN()
16654 Cmp = DAG.getNode(IntrData->Opc0, dl, MVT::i1, Src1, Src2, CC); in LowerINTRINSIC_WO_CHAIN()
16658 MVT::i1), in LowerINTRINSIC_WO_CHAIN()
16661 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::i8, in LowerINTRINSIC_WO_CHAIN()
16662 DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i8, CmpMask), in LowerINTRINSIC_WO_CHAIN()
16663 DAG.getValueType(MVT::i1)); in LowerINTRINSIC_WO_CHAIN()
16671 SDValue Cond = DAG.getNode(IntrData->Opc0, dl, MVT::i32, LHS, RHS); in LowerINTRINSIC_WO_CHAIN()
16672 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in LowerINTRINSIC_WO_CHAIN()
16673 DAG.getConstant(X86CC, dl, MVT::i8), Cond); in LowerINTRINSIC_WO_CHAIN()
16674 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); in LowerINTRINSIC_WO_CHAIN()
16687 Cond = DAG.getNode(comiOp, dl, MVT::i32, LHS, RHS, Sae); in LowerINTRINSIC_WO_CHAIN()
16689 Cond = DAG.getNode(comiOp, dl, MVT::i32, LHS, RHS); in LowerINTRINSIC_WO_CHAIN()
16690 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in LowerINTRINSIC_WO_CHAIN()
16691 DAG.getConstant(std::get<1>(ComiType), dl, MVT::i8), Cond); in LowerINTRINSIC_WO_CHAIN()
16692 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); in LowerINTRINSIC_WO_CHAIN()
16717 MVT MaskVT = MVT::getVectorVT(MVT::i1, Mask.getSimpleValueType().getSizeInBits()); in LowerINTRINSIC_WO_CHAIN()
16723 MVT VT = Op.getSimpleValueType(); in LowerINTRINSIC_WO_CHAIN()
16724 MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements()); in LowerINTRINSIC_WO_CHAIN()
16730 MVT VT = Op.getSimpleValueType(); in LowerINTRINSIC_WO_CHAIN()
16731 MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getSizeInBits()/2); in LowerINTRINSIC_WO_CHAIN()
16737 MVT::getVectorVT(MVT::i1, VT.getSizeInBits()), in LowerINTRINSIC_WO_CHAIN()
16816 SDValue Test = DAG.getNode(TestOpc, dl, MVT::i32, LHS, RHS); in LowerINTRINSIC_WO_CHAIN()
16817 SDValue CC = DAG.getConstant(X86CC, dl, MVT::i8); in LowerINTRINSIC_WO_CHAIN()
16818 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, CC, Test); in LowerINTRINSIC_WO_CHAIN()
16819 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); in LowerINTRINSIC_WO_CHAIN()
16824 SDValue LHS = DAG.getBitcast(MVT::v16i1, Op.getOperand(1)); in LowerINTRINSIC_WO_CHAIN()
16825 SDValue RHS = DAG.getBitcast(MVT::v16i1, Op.getOperand(2)); in LowerINTRINSIC_WO_CHAIN()
16826 SDValue CC = DAG.getConstant(X86CC, dl, MVT::i8); in LowerINTRINSIC_WO_CHAIN()
16827 SDValue Test = DAG.getNode(X86ISD::KORTEST, dl, MVT::i32, LHS, RHS); in LowerINTRINSIC_WO_CHAIN()
16828 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i1, CC, Test); in LowerINTRINSIC_WO_CHAIN()
16829 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); in LowerINTRINSIC_WO_CHAIN()
16888 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32); in LowerINTRINSIC_WO_CHAIN()
16890 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in LowerINTRINSIC_WO_CHAIN()
16891 DAG.getConstant(X86CC, dl, MVT::i8), in LowerINTRINSIC_WO_CHAIN()
16893 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); in LowerINTRINSIC_WO_CHAIN()
16905 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32); in LowerINTRINSIC_WO_CHAIN()
16955 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl, MVT::i8); in getGatherNode()
16956 MVT MaskVT = MVT::getVectorVT(MVT::i1, in getGatherNode()
16963 MVT BitcastVT = MVT::getVectorVT(MVT::i1, in getGatherNode()
16972 SDVTList VTs = DAG.getVTList(Op.getValueType(), MaskVT, MVT::Other); in getGatherNode()
16973 SDValue Disp = DAG.getTargetConstant(0, dl, MVT::i32); in getGatherNode()
16974 SDValue Segment = DAG.getRegister(0, MVT::i32); in getGatherNode()
16988 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl, MVT::i8); in getScatterNode()
16989 SDValue Disp = DAG.getTargetConstant(0, dl, MVT::i32); in getScatterNode()
16990 SDValue Segment = DAG.getRegister(0, MVT::i32); in getScatterNode()
16991 MVT MaskVT = MVT::getVectorVT(MVT::i1, in getScatterNode()
16998 MVT BitcastVT = MVT::getVectorVT(MVT::i1, in getScatterNode()
17007 SDVTList VTs = DAG.getVTList(MaskVT, MVT::Other); in getScatterNode()
17018 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl, MVT::i8); in getPrefetchNode()
17019 SDValue Disp = DAG.getTargetConstant(0, dl, MVT::i32); in getPrefetchNode()
17020 SDValue Segment = DAG.getRegister(0, MVT::i32); in getPrefetchNode()
17021 MVT MaskVT = in getPrefetchNode()
17022 MVT::getVectorVT(MVT::i1, Index.getSimpleValueType().getVectorNumElements()); in getPrefetchNode()
17031 SDNode *Res = DAG.getMachineNode(Opc, dl, MVT::Other, Ops); in getPrefetchNode()
17041 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); in getReadPerformanceCounter()
17053 LO = DAG.getCopyFromReg(rd, DL, X86::RAX, MVT::i64, rd.getValue(1)); in getReadPerformanceCounter()
17054 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::RDX, MVT::i64, in getReadPerformanceCounter()
17057 LO = DAG.getCopyFromReg(rd, DL, X86::EAX, MVT::i32, rd.getValue(1)); in getReadPerformanceCounter()
17058 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::EDX, MVT::i32, in getReadPerformanceCounter()
17066 SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI, in getReadPerformanceCounter()
17067 DAG.getConstant(32, DL, MVT::i8)); in getReadPerformanceCounter()
17068 Results.push_back(DAG.getNode(ISD::OR, DL, MVT::i64, LO, Tmp)); in getReadPerformanceCounter()
17075 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops); in getReadPerformanceCounter()
17086 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); in getReadTimeStampCounter()
17094 LO = DAG.getCopyFromReg(rd, DL, X86::RAX, MVT::i64, rd.getValue(1)); in getReadTimeStampCounter()
17095 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::RDX, MVT::i64, in getReadTimeStampCounter()
17098 LO = DAG.getCopyFromReg(rd, DL, X86::EAX, MVT::i32, rd.getValue(1)); in getReadTimeStampCounter()
17099 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::EDX, MVT::i32, in getReadTimeStampCounter()
17109 SDValue ecx = DAG.getCopyFromReg(Chain, DL, X86::ECX, MVT::i32, in getReadTimeStampCounter()
17120 SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI, in getReadTimeStampCounter()
17121 DAG.getConstant(32, DL, MVT::i8)); in getReadTimeStampCounter()
17122 Results.push_back(DAG.getNode(ISD::OR, DL, MVT::i64, LO, Tmp)); in getReadTimeStampCounter()
17129 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops); in getReadTimeStampCounter()
17165 MVT ElementType) { in LowerINTRINSIC_TRUNCATE_TO_MEM()
17172 MVT VT = DataToTruncate.getSimpleValueType(); in LowerINTRINSIC_TRUNCATE_TO_MEM()
17173 MVT SVT = MVT::getVectorVT(ElementType, VT.getVectorNumElements()); in LowerINTRINSIC_TRUNCATE_TO_MEM()
17180 MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements()); in LowerINTRINSIC_TRUNCATE_TO_MEM()
17181 MVT BitcastVT = MVT::getVectorVT(MVT::i1, in LowerINTRINSIC_TRUNCATE_TO_MEM()
17215 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::Glue, MVT::Other); in LowerINTRINSIC_W_CHAIN()
17222 DAG.getConstant(X86::COND_B, dl, MVT::i32), in LowerINTRINSIC_W_CHAIN()
17225 DAG.getVTList(Op->getValueType(1), MVT::Glue), in LowerINTRINSIC_W_CHAIN()
17281 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::Other); in LowerINTRINSIC_W_CHAIN()
17283 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in LowerINTRINSIC_W_CHAIN()
17284 DAG.getConstant(X86::COND_NE, dl, MVT::i8), in LowerINTRINSIC_W_CHAIN()
17293 SDVTList CFVTs = DAG.getVTList(Op->getValueType(0), MVT::Other); in LowerINTRINSIC_W_CHAIN()
17294 SDVTList VTs = DAG.getVTList(Op.getOperand(3)->getValueType(0), MVT::Other); in LowerINTRINSIC_W_CHAIN()
17296 DAG.getConstant(-1, dl, MVT::i8)); in LowerINTRINSIC_W_CHAIN()
17302 SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in LowerINTRINSIC_W_CHAIN()
17303 DAG.getConstant(X86::COND_B, dl, MVT::i8), in LowerINTRINSIC_W_CHAIN()
17316 MVT VT = DataToCompress.getSimpleValueType(); in LowerINTRINSIC_W_CHAIN()
17330 return LowerINTRINSIC_TRUNCATE_TO_MEM(Op, DAG, MVT::i8); in LowerINTRINSIC_W_CHAIN()
17332 return LowerINTRINSIC_TRUNCATE_TO_MEM(Op, DAG, MVT::i16); in LowerINTRINSIC_W_CHAIN()
17334 return LowerINTRINSIC_TRUNCATE_TO_MEM(Op, DAG, MVT::i32); in LowerINTRINSIC_W_CHAIN()
17341 MVT VT = Op.getSimpleValueType(); in LowerINTRINSIC_W_CHAIN()
17415 assert(((FrameReg == X86::RBP && VT == MVT::i64) || in LowerFRAMEADDR()
17416 (FrameReg == X86::EBP && VT == MVT::i32)) && in LowerFRAMEADDR()
17491 assert(((FrameReg == X86::RBP && PtrVT == MVT::i64) || in LowerEH_RETURN()
17492 (FrameReg == X86::EBP && PtrVT == MVT::i32)) && in LowerEH_RETURN()
17495 unsigned StoreAddrReg = (PtrVT == MVT::i64) ? X86::RCX : X86::ECX; in LowerEH_RETURN()
17505 return DAG.getNode(X86ISD::EH_RETURN, dl, MVT::Other, Chain, in LowerEH_RETURN()
17513 DAG.getVTList(MVT::i32, MVT::Other), in lowerEH_SJLJ_SETJMP()
17520 return DAG.getNode(X86ISD::EH_SJLJ_LONGJMP, DL, MVT::Other, in lowerEH_SJLJ_LONGJMP()
17554 OutChains[0] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16), in LowerINIT_TRAMPOLINE()
17558 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
17559 DAG.getConstant(2, dl, MVT::i64)); in LowerINIT_TRAMPOLINE()
17567 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
17568 DAG.getConstant(10, dl, MVT::i64)); in LowerINIT_TRAMPOLINE()
17569 OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16), in LowerINIT_TRAMPOLINE()
17573 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
17574 DAG.getConstant(12, dl, MVT::i64)); in LowerINIT_TRAMPOLINE()
17581 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
17582 DAG.getConstant(20, dl, MVT::i64)); in LowerINIT_TRAMPOLINE()
17583 OutChains[4] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16), in LowerINIT_TRAMPOLINE()
17588 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
17589 DAG.getConstant(22, dl, MVT::i64)); in LowerINIT_TRAMPOLINE()
17590 OutChains[5] = DAG.getStore(Root, dl, DAG.getConstant(ModRM, dl, MVT::i8), in LowerINIT_TRAMPOLINE()
17594 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); in LowerINIT_TRAMPOLINE()
17645 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerINIT_TRAMPOLINE()
17646 DAG.getConstant(10, dl, MVT::i32)); in LowerINIT_TRAMPOLINE()
17647 Disp = DAG.getNode(ISD::SUB, dl, MVT::i32, FPtr, Addr); in LowerINIT_TRAMPOLINE()
17653 DAG.getConstant(MOV32ri|N86Reg, dl, MVT::i8), in LowerINIT_TRAMPOLINE()
17657 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerINIT_TRAMPOLINE()
17658 DAG.getConstant(1, dl, MVT::i32)); in LowerINIT_TRAMPOLINE()
17664 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerINIT_TRAMPOLINE()
17665 DAG.getConstant(5, dl, MVT::i32)); in LowerINIT_TRAMPOLINE()
17666 OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(JMP, dl, MVT::i8), in LowerINIT_TRAMPOLINE()
17670 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerINIT_TRAMPOLINE()
17671 DAG.getConstant(6, dl, MVT::i32)); in LowerINIT_TRAMPOLINE()
17676 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); in LowerINIT_TRAMPOLINE()
17704 MVT VT = Op.getSimpleValueType(); in LowerFLT_ROUNDS_()
17718 DAG.getVTList(MVT::Other), in LowerFLT_ROUNDS_()
17719 Ops, MVT::i16, MMO); in LowerFLT_ROUNDS_()
17722 SDValue CWD = DAG.getLoad(MVT::i16, DL, Chain, StackSlot, in LowerFLT_ROUNDS_()
17727 DAG.getNode(ISD::SRL, DL, MVT::i16, in LowerFLT_ROUNDS_()
17728 DAG.getNode(ISD::AND, DL, MVT::i16, in LowerFLT_ROUNDS_()
17729 CWD, DAG.getConstant(0x800, DL, MVT::i16)), in LowerFLT_ROUNDS_()
17730 DAG.getConstant(11, DL, MVT::i8)); in LowerFLT_ROUNDS_()
17732 DAG.getNode(ISD::SRL, DL, MVT::i16, in LowerFLT_ROUNDS_()
17733 DAG.getNode(ISD::AND, DL, MVT::i16, in LowerFLT_ROUNDS_()
17734 CWD, DAG.getConstant(0x400, DL, MVT::i16)), in LowerFLT_ROUNDS_()
17735 DAG.getConstant(9, DL, MVT::i8)); in LowerFLT_ROUNDS_()
17738 DAG.getNode(ISD::AND, DL, MVT::i16, in LowerFLT_ROUNDS_()
17739 DAG.getNode(ISD::ADD, DL, MVT::i16, in LowerFLT_ROUNDS_()
17740 DAG.getNode(ISD::OR, DL, MVT::i16, CWD1, CWD2), in LowerFLT_ROUNDS_()
17741 DAG.getConstant(1, DL, MVT::i16)), in LowerFLT_ROUNDS_()
17742 DAG.getConstant(3, DL, MVT::i16)); in LowerFLT_ROUNDS_()
17758 MVT VT = Op.getSimpleValueType(); in LowerVectorCTLZ_AVX512()
17759 MVT EltVT = VT.getVectorElementType(); in LowerVectorCTLZ_AVX512()
17762 if (EltVT == MVT::i64 || EltVT == MVT::i32) { in LowerVectorCTLZ_AVX512()
17767 MVT NewVT = MVT::getVectorVT(EltVT, 512 / VT.getScalarSizeInBits()); in LowerVectorCTLZ_AVX512()
17778 assert((EltVT == MVT::i8 || EltVT == MVT::i16) && in LowerVectorCTLZ_AVX512()
17785 MVT OutVT = MVT::getVectorVT(EltVT, NumElems/2); in LowerVectorCTLZ_AVX512()
17793 MVT NewVT = MVT::getVectorVT(MVT::i32, NumElems); in LowerVectorCTLZ_AVX512()
17809 MVT VT = Op.getSimpleValueType(); in LowerCTLZ()
17810 MVT OpVT = VT; in LowerCTLZ()
17818 if (VT == MVT::i8) { in LowerCTLZ()
17820 OpVT = MVT::i32; in LowerCTLZ()
17825 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); in LowerCTLZ()
17832 DAG.getConstant(X86::COND_E, dl, MVT::i8), in LowerCTLZ()
17841 if (VT == MVT::i8) in LowerCTLZ()
17842 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op); in LowerCTLZ()
17848 MVT VT = Op.getSimpleValueType(); in LowerCTLZ_ZERO_UNDEF()
17857 if (VT == MVT::i8) { in LowerCTLZ_ZERO_UNDEF()
17859 OpVT = MVT::i32; in LowerCTLZ_ZERO_UNDEF()
17864 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32); in LowerCTLZ_ZERO_UNDEF()
17871 if (VT == MVT::i8) in LowerCTLZ_ZERO_UNDEF()
17872 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op); in LowerCTLZ_ZERO_UNDEF()
17877 MVT VT = Op.getSimpleValueType(); in LowerCTTZ()
17909 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in LowerCTTZ()
17916 DAG.getConstant(X86::COND_E, dl, MVT::i8), in LowerCTTZ()
17925 MVT VT = Op.getSimpleValueType(); in Lower256IntArith()
17943 MVT EltVT = VT.getVectorElementType(); in Lower256IntArith()
17944 MVT NewVT = MVT::getVectorVT(EltVT, NumElems/2); in Lower256IntArith()
17952 if (Op.getValueType() == MVT::i1) in LowerADD()
17962 if (Op.getValueType() == MVT::i1) in LowerSUB()
17981 MVT VT = Op.getSimpleValueType(); in LowerMUL()
17983 if (VT == MVT::i1) in LowerMUL()
17995 if (VT == MVT::v16i8 || VT == MVT::v32i8) { in LowerMUL()
17997 if (VT == MVT::v32i8) { in LowerMUL()
17998 MVT SubVT = MVT::getVectorVT(MVT::i8, VT.getVectorNumElements() / 2); in LowerMUL()
18010 MVT ExVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements()); in LowerMUL()
18018 assert(VT == MVT::v16i8 && in LowerMUL()
18020 MVT ExVT = MVT::v8i16; in LowerMUL()
18067 if (VT == MVT::v4i32) { in LowerMUL()
18077 SDValue Evens = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64, A, B); in LowerMUL()
18079 SDValue Odds = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64, Aodds, Bodds); in LowerMUL()
18090 assert((VT == MVT::v2i64 || VT == MVT::v4i64 || VT == MVT::v8i64) && in LowerMUL()
18110 MVT MulVT = (VT == MVT::v2i64) ? MVT::v4i32 : in LowerMUL()
18111 (VT == MVT::v4i64) ? MVT::v8i32 : MVT::v16i32; in LowerMUL()
18176 static_cast<EVT>(MVT::v2i64).getTypeForEVT(*DAG.getContext()), in LowerWin64_i128OP()
18187 MVT VT = Op0.getSimpleValueType(); in LowerMUL_LOHI()
18190 assert((VT == MVT::v4i32 && Subtarget->hasSSE2()) || in LowerMUL_LOHI()
18191 (VT == MVT::v8i32 && Subtarget->hasInt256())); in LowerMUL_LOHI()
18213 MVT MulVT = VT == MVT::v4i32 ? MVT::v2i64 : MVT::v4i64; in LowerMUL_LOHI()
18226 if (VT == MVT::v8i32) { in LowerMUL_LOHI()
18261 static bool SupportedVectorShiftWithImm(MVT VT, const X86Subtarget *Subtarget, in SupportedVectorShiftWithImm()
18274 (VT != MVT::v2i64 && VT != MVT::v4i64)); in SupportedVectorShiftWithImm()
18281 bool SupportedVectorShiftWithBaseAmnt(MVT VT, const X86Subtarget *Subtarget, in SupportedVectorShiftWithBaseAmnt()
18288 static bool SupportedVectorVarShift(MVT VT, const X86Subtarget *Subtarget, in SupportedVectorVarShift()
18302 bool AShift = LShift && VT != MVT::v2i64 && VT != MVT::v4i64; in SupportedVectorVarShift()
18308 MVT VT = Op.getSimpleValueType(); in LowerScalarImmediateShift()
18317 assert((VT == MVT::v2i64 || VT == MVT::v4i64) && "Unexpected SRA type"); in LowerScalarImmediateShift()
18318 MVT ExVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() * 2); in LowerScalarImmediateShift()
18327 if (VT == MVT::v2i64) in LowerScalarImmediateShift()
18329 if (VT == MVT::v4i64) in LowerScalarImmediateShift()
18339 if (VT == MVT::v2i64) in LowerScalarImmediateShift()
18341 if (VT == MVT::v4i64) in LowerScalarImmediateShift()
18357 if ((VT == MVT::v2i64 || (Subtarget->hasInt256() && VT == MVT::v4i64)) && in LowerScalarImmediateShift()
18361 if (VT == MVT::v16i8 || (Subtarget->hasInt256() && VT == MVT::v32i8)) { in LowerScalarImmediateShift()
18363 MVT ShiftVT = MVT::getVectorVT(MVT::i16, NumElts / 2); in LowerScalarImmediateShift()
18376 if (VT == MVT::v16i8 && Subtarget->hasXOP()) in LowerScalarImmediateShift()
18386 NumElts, DAG.getConstant(uint8_t(-1U << ShiftAmt), dl, MVT::i8)); in LowerScalarImmediateShift()
18397 NumElts, DAG.getConstant(uint8_t(-1U) >> ShiftAmt, dl, MVT::i8)); in LowerScalarImmediateShift()
18406 MVT::i8)); in LowerScalarImmediateShift()
18419 (VT == MVT::v2i64 || (Subtarget->hasInt256() && VT == MVT::v4i64))) { in LowerScalarImmediateShift()
18477 MVT VT = Op.getSimpleValueType(); in LowerScalarVariableShift()
18490 MVT EltVT = VT.getVectorElementType(); in LowerScalarVariableShift()
18526 assert(EltVT.bitsLE(MVT::i64) && "Unexpected element type!"); in LowerScalarVariableShift()
18527 if (EltVT != MVT::i64 && EltVT.bitsGT(MVT::i32)) in LowerScalarVariableShift()
18528 BaseShAmt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, BaseShAmt); in LowerScalarVariableShift()
18529 else if (EltVT.bitsLT(MVT::i32)) in LowerScalarVariableShift()
18530 BaseShAmt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, BaseShAmt); in LowerScalarVariableShift()
18537 if (!Subtarget->is64Bit() && VT == MVT::v2i64 && in LowerScalarVariableShift()
18560 MVT VT = Op.getSimpleValueType(); in LowerShift()
18580 (VT == MVT::v2i64 || VT == MVT::v4i32 || in LowerShift()
18581 VT == MVT::v8i16 || VT == MVT::v16i8)) { in LowerShift()
18594 if (VT == MVT::v2i64 && Op.getOpcode() != ISD::SRA) { in LowerShift()
18606 if ((VT == MVT::v2i64 || (VT == MVT::v4i64 && Subtarget->hasInt256())) && in LowerShift()
18620 (VT == MVT::v8i16 || VT == MVT::v4i32 || in LowerShift()
18621 (Subtarget->hasInt256() && VT == MVT::v16i16)) && in LowerShift()
18624 MVT SVT = VT.getVectorElementType(); in LowerShift()
18650 if (VT == MVT::v4i32 && Op->getOpcode() == ISD::SHL) { in LowerShift()
18655 Op = DAG.getBitcast(MVT::v4f32, Op); in LowerShift()
18672 if ((VT == MVT::v8i16 || VT == MVT::v4i32) && in LowerShift()
18679 SDValue Amt2 = (VT == MVT::v4i32) ? Amt->getOperand(1) : in LowerShift()
18684 if (VT == MVT::v4i32) { in LowerShift()
18716 MVT CastVT = MVT::v4i32; in LowerShift()
18724 CastVT = MVT::v2i64; in LowerShift()
18738 if (VT == MVT::v4i32) { in LowerShift()
18780 if (VT == MVT::v16i8 || in LowerShift()
18781 (VT == MVT::v32i8 && Subtarget->hasInt256() && !Subtarget->hasXOP())) { in LowerShift()
18782 MVT ExtVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements() / 2); in LowerShift()
18785 auto SignBitSelect = [&](MVT SelVT, SDValue Sel, SDValue V0, SDValue V1) { in LowerShift()
18891 if (Subtarget->hasInt256() && VT == MVT::v8i16) { in LowerShift()
18892 MVT ExtVT = MVT::v8i32; in LowerShift()
18901 if (Subtarget->hasInt256() && !Subtarget->hasXOP() && VT == MVT::v16i16) { in LowerShift()
18902 MVT ExtVT = MVT::v8i32; in LowerShift()
18919 if (VT == MVT::v8i16) { in LowerShift()
18926 MVT ExtVT = MVT::getVectorVT(MVT::i8, VT.getVectorNumElements() * 2); in LowerShift()
18983 MVT EltVT = VT.getVectorElementType(); in LowerShift()
18984 MVT NewVT = MVT::getVectorVT(EltVT, NumElems/2); in LowerShift()
19019 MVT VT = Op.getSimpleValueType(); in LowerRotate()
19043 DAG.getConstant(RotateAmt, DL, MVT::i8)); in LowerRotate()
19095 BaseOp = N->getValueType(0) == MVT::i8 ? X86ISD::SMUL8 : X86ISD::SMUL; in LowerXALUO()
19099 if (N->getValueType(0) == MVT::i8) { in LowerXALUO()
19105 MVT::i32); in LowerXALUO()
19109 DAG.getNode(X86ISD::SETCC, DL, MVT::i8, in LowerXALUO()
19110 DAG.getConstant(X86::COND_O, DL, MVT::i32), in LowerXALUO()
19118 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::i32); in LowerXALUO()
19123 DAG.getConstant(Cond, DL, MVT::i32), in LowerXALUO()
19274 return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0)); in LowerATOMIC_FENCE()
19277 SDValue Zero = DAG.getConstant(0, dl, MVT::i32); in LowerATOMIC_FENCE()
19279 DAG.getRegister(X86::ESP, MVT::i32), // Base in LowerATOMIC_FENCE()
19280 DAG.getTargetConstant(1, dl, MVT::i8), // Scale in LowerATOMIC_FENCE()
19281 DAG.getRegister(0, MVT::i32), // Index in LowerATOMIC_FENCE()
19282 DAG.getTargetConstant(0, dl, MVT::i32), // Disp in LowerATOMIC_FENCE()
19283 DAG.getRegister(0, MVT::i32), // Segment. in LowerATOMIC_FENCE()
19287 SDNode *Res = DAG.getMachineNode(X86::OR32mrLocked, dl, MVT::Other, Ops); in LowerATOMIC_FENCE()
19292 return DAG.getNode(X86ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0)); in LowerATOMIC_FENCE()
19297 MVT T = Op.getSimpleValueType(); in LowerCMP_SWAP()
19303 case MVT::i8: Reg = X86::AL; size = 1; break; in LowerCMP_SWAP()
19304 case MVT::i16: Reg = X86::AX; size = 2; break; in LowerCMP_SWAP()
19305 case MVT::i32: Reg = X86::EAX; size = 4; break; in LowerCMP_SWAP()
19306 case MVT::i64: in LowerCMP_SWAP()
19316 DAG.getTargetConstant(size, DL, MVT::i8), in LowerCMP_SWAP()
19318 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerCMP_SWAP()
19326 MVT::i32, cpOut.getValue(2)); in LowerCMP_SWAP()
19328 DAG.getConstant(X86::COND_E, DL, MVT::i8), in LowerCMP_SWAP()
19339 MVT SrcVT = Op.getOperand(0).getSimpleValueType(); in LowerBITCAST()
19340 MVT DstVT = Op.getSimpleValueType(); in LowerBITCAST()
19342 if (SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 || SrcVT == MVT::v8i8) { in LowerBITCAST()
19344 if (DstVT != MVT::f64) in LowerBITCAST()
19351 MVT SVT = SrcVT.getVectorElementType(); in LowerBITCAST()
19365 SDValue ToV2F64 = DAG.getBitcast(MVT::v2f64, BV); in LowerBITCAST()
19366 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, ToV2F64, in LowerBITCAST()
19372 assert((DstVT == MVT::i64 || in LowerBITCAST()
19376 if (SrcVT==MVT::i64 && DstVT.isVector()) in LowerBITCAST()
19378 if (DstVT==MVT::i64 && SrcVT.isVector()) in LowerBITCAST()
19393 static SDValue LowerHorizontalByteSum(SDValue V, MVT VT, in LowerHorizontalByteSum()
19397 MVT ByteVecVT = V.getSimpleValueType(); in LowerHorizontalByteSum()
19398 MVT EltVT = VT.getVectorElementType(); in LowerHorizontalByteSum()
19400 assert(ByteVecVT.getVectorElementType() == MVT::i8 && in LowerHorizontalByteSum()
19402 assert(EltVT != MVT::i8 && in LowerHorizontalByteSum()
19409 if (EltVT == MVT::i64) { in LowerHorizontalByteSum()
19411 MVT SadVecVT = MVT::getVectorVT(MVT::i64, VecSize / 64); in LowerHorizontalByteSum()
19416 if (EltVT == MVT::i32) { in LowerHorizontalByteSum()
19428 MVT SadVecVT = MVT::getVectorVT(MVT::i64, VecSize / 64); in LowerHorizontalByteSum()
19435 MVT ShortVecVT = MVT::getVectorVT(MVT::i16, VecSize / 16); in LowerHorizontalByteSum()
19444 assert(EltVT == MVT::i16 && "Unknown how to handle type"); in LowerHorizontalByteSum()
19461 MVT VT = Op.getSimpleValueType(); in LowerVectorCTPOPInRegLUT()
19462 MVT EltVT = VT.getVectorElementType(); in LowerVectorCTPOPInRegLUT()
19485 MVT ByteVecVT = MVT::getVectorVT(MVT::i8, NumByteElts); in LowerVectorCTPOPInRegLUT()
19489 LUTVec.push_back(DAG.getConstant(LUT[i % 16], DL, MVT::i8)); in LowerVectorCTPOPInRegLUT()
19492 DAG.getConstant(0x0F, DL, MVT::i8)); in LowerVectorCTPOPInRegLUT()
19496 SmallVector<SDValue, 16> Four(NumByteElts, DAG.getConstant(4, DL, MVT::i8)); in LowerVectorCTPOPInRegLUT()
19512 if (EltVT == MVT::i8) in LowerVectorCTPOPInRegLUT()
19521 MVT VT = Op.getSimpleValueType(); in LowerVectorCTPOPBitmath()
19526 MVT EltVT = VT.getVectorElementType(); in LowerVectorCTPOPBitmath()
19537 MVT VT = V.getSimpleValueType(); in LowerVectorCTPOPBitmath()
19545 MVT VT = V.getSimpleValueType(); in LowerVectorCTPOPBitmath()
19557 MVT SrlVT = Len > 8 ? VT : MVT::getVectorVT(MVT::i16, VecSize / 16); in LowerVectorCTPOPBitmath()
19581 if (EltVT == MVT::i8) in LowerVectorCTPOPBitmath()
19585 DAG.getBitcast(MVT::getVectorVT(MVT::i8, VecSize / 8), V), VT, Subtarget, in LowerVectorCTPOPBitmath()
19591 MVT VT = Op.getSimpleValueType(); in LowerVectorCTPOP()
19667 MVT VT = Op.getNode()->getSimpleValueType(0); in LowerADDC_ADDE_SUBC_SUBE()
19673 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in LowerADDC_ADDE_SUBC_SUBE()
19713 bool isF64 = ArgVT == MVT::f64; in LowerFSINCOS()
19747 static SDValue ExtendToType(SDValue InOp, MVT NVT, SelectionDAG &DAG, in ExtendToType()
19750 MVT InVT = InOp.getSimpleValueType(); in ExtendToType()
19809 MVT VT = Src.getSimpleValueType(); in LowerMSCATTER()
19818 MVT MemVT = N->getMemoryVT().getSimpleVT(); in LowerMSCATTER()
19819 MVT IndexVT = Index.getSimpleValueType(); in LowerMSCATTER()
19820 MVT MaskVT = Mask.getSimpleValueType(); in LowerMSCATTER()
19827 assert((MemVT == MVT::v2i32 && VT == MVT::v2i64) && in LowerMSCATTER()
19830 Src = DAG.getVectorShuffle(MVT::v4i32, dl, DAG.getBitcast(MVT::v4i32, Src), in LowerMSCATTER()
19831 DAG.getUNDEF(MVT::v4i32), ShuffleMask); in LowerMSCATTER()
19834 MVT NewIndexVT = MVT::getVectorVT(IndexVT.getScalarType(), 4); in LowerMSCATTER()
19839 assert((MaskVT == MVT::v2i1 || MaskVT == MVT::v2i64) && in LowerMSCATTER()
19841 MVT ExtMaskVT = MVT::getVectorVT(MaskVT.getScalarType(), 4); in LowerMSCATTER()
19843 VT = MVT::v4i32; in LowerMSCATTER()
19852 if (IndexVT == MVT::v8i32) in LowerMSCATTER()
19854 Index = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v8i64, Index); in LowerMSCATTER()
19859 MVT NewIndexVT = MVT::getVectorVT(IndexVT.getScalarType(), NumElts); in LowerMSCATTER()
19862 if (IndexVT.getScalarType() == MVT::i32) in LowerMSCATTER()
19863 Index = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v8i64, Index); in LowerMSCATTER()
19868 MVT ExtMaskVT = MVT::getVectorVT(MaskVT.getScalarType(), NumElts); in LowerMSCATTER()
19873 MVT NewVT = MVT::getVectorVT(VT.getScalarType(), NumElts); in LowerMSCATTER()
19878 MVT BitMaskVT = MVT::getVectorVT(MVT::i1, NumElts); in LowerMSCATTER()
19882 SDVTList VTs = DAG.getVTList(BitMaskVT, MVT::Other); in LowerMSCATTER()
19894 MVT VT = Op.getSimpleValueType(); in LowerMLOAD()
19899 !VT.is512BitVector() && Mask.getValueType() == MVT::v8i1) { in LowerMLOAD()
19903 MVT WideDataVT = MVT::getVectorVT(VT.getScalarType(), NumEltsInWideVec); in LowerMLOAD()
19904 MVT WideMaskVT = MVT::getVectorVT(MVT::i1, NumEltsInWideVec); in LowerMLOAD()
19926 MVT VT = DataToStore.getSimpleValueType(); in LowerMSTORE()
19931 !VT.is512BitVector() && Mask.getValueType() == MVT::v8i1) { in LowerMSTORE()
19935 MVT WideDataVT = MVT::getVectorVT(VT.getScalarType(), NumEltsInWideVec); in LowerMSTORE()
19936 MVT WideMaskVT = MVT::getVectorVT(MVT::i1, NumEltsInWideVec); in LowerMSTORE()
19953 MVT VT = Op.getSimpleValueType(); in LowerMGATHER()
19957 MVT IndexVT = Index.getSimpleValueType(); in LowerMGATHER()
19958 MVT MaskVT = Mask.getSimpleValueType(); in LowerMGATHER()
19969 Index = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v8i64, Index); in LowerMGATHER()
19979 MVT NewIndexVT = MVT::getVectorVT(IndexVT.getScalarType(), NumElts); in LowerMGATHER()
19981 if (IndexVT.getScalarType() == MVT::i32) in LowerMGATHER()
19982 Index = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v8i64, Index); in LowerMGATHER()
19985 MVT MaskBitVT = MVT::getVectorVT(MVT::i1, NumElts); in LowerMGATHER()
19988 MVT ExtMaskVT = MVT::getVectorVT(MaskVT.getScalarType(), NumElts); in LowerMGATHER()
19993 MVT NewVT = MVT::getVectorVT(VT.getScalarType(), NumElts); in LowerMGATHER()
19997 SDValue NewGather = DAG.getMaskedGather(DAG.getVTList(NewVT, MVT::Other), in LowerMGATHER()
20024 SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue); in LowerGC_TRANSITION_START()
20045 SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue); in LowerGC_TRANSITION_END()
20203 assert(VT == MVT::v2f32 && "Unexpected type (!= v2f32) on FMIN/FMAX."); in ReplaceNodeResults()
20205 SDValue LHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, in ReplaceNodeResults()
20207 SDValue RHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, in ReplaceNodeResults()
20209 Results.push_back(DAG.getNode(N->getOpcode(), dl, MVT::v4f32, LHS, RHS)); in ReplaceNodeResults()
20250 if (N->getOperand(0).getValueType() != MVT::v2i32 || in ReplaceNodeResults()
20251 N->getValueType(0) != MVT::v2f32) in ReplaceNodeResults()
20253 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v2i64, in ReplaceNodeResults()
20256 MVT::f64); in ReplaceNodeResults()
20257 SDValue VBias = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v2f64, Bias, Bias); in ReplaceNodeResults()
20258 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64, ZExtIn, in ReplaceNodeResults()
20259 DAG.getBitcast(MVT::v2i64, VBias)); in ReplaceNodeResults()
20260 Or = DAG.getBitcast(MVT::v2f64, Or); in ReplaceNodeResults()
20262 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, Or, VBias); in ReplaceNodeResults()
20263 Results.push_back(DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, Sub)); in ReplaceNodeResults()
20269 SDValue V = DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, N->getOperand(0)); in ReplaceNodeResults()
20276 assert(N->getValueType(0) == MVT::v2f32 && in ReplaceNodeResults()
20306 assert((T == MVT::i64 || T == MVT::i128) && "can only expand cmpxchg pair"); in ReplaceNodeResults()
20307 bool Regs64bit = T == MVT::i128; in ReplaceNodeResults()
20308 MVT HalfT = Regs64bit ? MVT::i64 : MVT::i32; in ReplaceNodeResults()
20334 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue); in ReplaceNodeResults()
20348 MVT::i32, cpOutH.getValue(2)); in ReplaceNodeResults()
20350 DAG.getNode(X86ISD::SETCC, dl, MVT::i8, in ReplaceNodeResults()
20351 DAG.getConstant(X86::COND_E, dl, MVT::i8), EFLAGS); in ReplaceNodeResults()
20380 if (SrcVT != MVT::f64 || in ReplaceNodeResults()
20381 (DstVT != MVT::v2i32 && DstVT != MVT::v4i16 && DstVT != MVT::v8i8)) in ReplaceNodeResults()
20388 MVT::v2f64, N->getOperand(0)); in ReplaceNodeResults()
20765 return VT1 == MVT::i32 && VT2 == MVT::i64 && Subtarget->is64Bit(); in isZExtFree()
20782 case MVT::i8: in isZExtFree()
20783 case MVT::i16: in isZExtFree()
20784 case MVT::i32: in isZExtFree()
20805 case MVT::f32: in isFMAFasterThanFMulAndFAdd()
20806 case MVT::f64: in isFMAFasterThanFMulAndFAdd()
20817 return !(VT1 == MVT::i32 && VT2 == MVT::i16); in isNarrowingProfitable()
20831 if (VT.getSimpleVT().getScalarType() == MVT::i1) in isShuffleMaskLegal()
21043 const TargetRegisterClass *AddrRegClass = getRegClassFor(MVT::i64); in EmitVAARG64WithCustomInserter()
21044 const TargetRegisterClass *OffsetRegClass = getRegClassFor(MVT::i32); in EmitVAARG64WithCustomInserter()
22001 assert(RC->hasType(MVT::i32) && "Invalid destination!"); in emitEHSjLjSetJmp()
22007 MVT PVT = getPointerTy(MF->getDataLayout()); in emitEHSjLjSetJmp()
22008 assert((PVT == MVT::i64 || PVT == MVT::i32) && in emitEHSjLjSetJmp()
22053 PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mr : X86::MOV32mr; in emitEHSjLjSetJmp()
22073 PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mi32 : X86::MOV32mi; in emitEHSjLjSetJmp()
22140 MVT PVT = getPointerTy(MF->getDataLayout()); in emitEHSjLjLongJmp()
22141 assert((PVT == MVT::i64 || PVT == MVT::i32) && in emitEHSjLjLongJmp()
22145 (PVT == MVT::i64) ? &X86::GR64RegClass : &X86::GR32RegClass; in emitEHSjLjLongJmp()
22149 unsigned FP = (PVT == MVT::i64) ? X86::RBP : X86::EBP; in emitEHSjLjLongJmp()
22157 unsigned PtrLoadOpc = (PVT == MVT::i64) ? X86::MOV64rm : X86::MOV32rm; in emitEHSjLjLongJmp()
22158 unsigned IJmpOpc = (PVT == MVT::i64) ? X86::JMP64r : X86::JMP32r; in emitEHSjLjLongJmp()
22653 MVT VT = SVOp->getSimpleValueType(0); in PerformShuffleCombine256()
22687 SDVTList Tys = DAG.getVTList(MVT::v4i64, MVT::Other); in PerformShuffleCombine256()
22701 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, in PerformShuffleCombine256()
22761 MVT VT = Input.getSimpleValueType(); in combineX86ShuffleChain()
22762 MVT RootVT = Root.getSimpleValueType(); in combineX86ShuffleChain()
22800 MVT ShuffleVT; in combineX86ShuffleChain()
22806 ShuffleVT = MVT::v2f64; in combineX86ShuffleChain()
22811 ShuffleVT = MVT::v4f32; in combineX86ShuffleChain()
22830 MVT ShuffleVT = MVT::v4f32; in combineX86ShuffleChain()
22844 MVT ShuffleVT = MVT::v4f32; in combineX86ShuffleChain()
22870 MVT ShuffleVT; in combineX86ShuffleChain()
22873 ShuffleVT = MVT::v8i16; in combineX86ShuffleChain()
22876 ShuffleVT = MVT::v16i8; in combineX86ShuffleChain()
22905 PSHUFBMask.push_back(DAG.getUNDEF(MVT::i8)); in combineX86ShuffleChain()
22911 PSHUFBMask.push_back(DAG.getConstant(M, DL, MVT::i8)); in combineX86ShuffleChain()
22913 MVT ByteVT = MVT::getVectorVT(MVT::i8, NumBytes); in combineX86ShuffleChain()
22974 MVT VT = Op.getSimpleValueType(); in combineX86ShufflesRecursively()
23081 MVT VT = N.getSimpleValueType(); in getPSHUFShuffleMask()
23173 if (V.getSimpleValueType().getVectorElementType() != MVT::i8 && in combineRedundantDWordShuffle()
23174 V.getSimpleValueType().getVectorElementType() != MVT::i16) in combineRedundantDWordShuffle()
23306 V = DAG.getNode(V.getOpcode(), DL, MVT::v8i16, V.getOperand(0), in combineRedundantHalfShuffle()
23324 MVT VT = N.getSimpleValueType(); in PerformTargetShuffleCombine()
23384 assert(VT.getVectorElementType() == MVT::i16 && "Bad word shuffle type!"); in PerformTargetShuffleCombine()
23397 MVT DVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() / 2); in PerformTargetShuffleCombine()
23508 assert((VT == MVT::v4f32 || VT == MVT::v2f64 || VT == MVT::v8f32 || in combineShuffleToAddSub()
23509 VT == MVT::v4f64) && in combineShuffleToAddSub()
23729 if (VT == MVT::x86mmx && N0.getOpcode() == ISD::BUILD_VECTOR && in PerformBITCASTCombine()
23730 N0.getValueType() == MVT::v2i32 && in PerformBITCASTCombine()
23733 if (N00.getValueType() == MVT::i32) in PerformBITCASTCombine()
23750 if (((Subtarget->hasSSE1() && VT == MVT::f32) || in PerformBITCASTCombine()
23751 (Subtarget->hasSSE2() && VT == MVT::f64)) && in PerformBITCASTCombine()
23777 N->getValueType(0) == MVT::i32 && in PerformEXTRACT_VECTOR_ELTCombine()
23778 InputVector.getValueType() == MVT::v2i32) { in PerformEXTRACT_VECTOR_ELTCombine()
23782 if (MMXSrc.getValueType() == MVT::x86mmx) in PerformEXTRACT_VECTOR_ELTCombine()
23789 MMXSrc.getValueType() == MVT::i64) { in PerformEXTRACT_VECTOR_ELTCombine()
23792 MMXSrcOp.getValueType() == MVT::v1i64 && in PerformEXTRACT_VECTOR_ELTCombine()
23793 MMXSrcOp.getOperand(0).getValueType() == MVT::x86mmx) in PerformEXTRACT_VECTOR_ELTCombine()
23801 if (VT == MVT::i1 && isa<ConstantSDNode>(N->getOperand(1)) && in PerformEXTRACT_VECTOR_ELTCombine()
23809 return DAG.getConstant(Res, dl, MVT::i1); in PerformEXTRACT_VECTOR_ELTCombine()
23813 if (InputVector.getValueType() != MVT::v4i32) in PerformEXTRACT_VECTOR_ELTCombine()
23830 if (Extract->getValueType(0) != MVT::i32) in PerformEXTRACT_VECTOR_ELTCombine()
23857 if (TLI.isOperationLegal(ISD::SRA, MVT::i64)) { in PerformEXTRACT_VECTOR_ELTCombine()
23858 SDValue Cst = DAG.getBitcast(MVT::v2i64, InputVector); in PerformEXTRACT_VECTOR_ELTCombine()
23861 SDValue BottomHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Cst, in PerformEXTRACT_VECTOR_ELTCombine()
23863 SDValue TopHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Cst, in PerformEXTRACT_VECTOR_ELTCombine()
23867 32, dl, DAG.getTargetLoweringInfo().getShiftAmountTy(MVT::i64, DL)); in PerformEXTRACT_VECTOR_ELTCombine()
23868 Vals[0] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, BottomHalf); in PerformEXTRACT_VECTOR_ELTCombine()
23869 Vals[1] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, in PerformEXTRACT_VECTOR_ELTCombine()
23870 DAG.getNode(ISD::SRA, dl, MVT::i64, BottomHalf, ShAmt)); in PerformEXTRACT_VECTOR_ELTCombine()
23871 Vals[2] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, TopHalf); in PerformEXTRACT_VECTOR_ELTCombine()
23872 Vals[3] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, in PerformEXTRACT_VECTOR_ELTCombine()
23873 DAG.getNode(ISD::SRA, dl, MVT::i64, TopHalf, ShAmt)); in PerformEXTRACT_VECTOR_ELTCombine()
23941 MVT VT = N->getSimpleValueType(0); in transformVSELECTtoBlendVECTOR_SHUFFLE()
23977 VT != MVT::f80 && VT != MVT::f128 && in PerformSELECTCombine()
23978 (TLI.isTypeLegal(VT) || VT == MVT::v2f32) && in PerformSELECTCombine()
23980 (Subtarget->hasSSE1() && VT.getScalarType() == MVT::f32))) { in PerformSELECTCombine()
24119 CondVT.getVectorElementType() == MVT::i1) { in PerformSELECTCombine()
24127 (OpVT.getVectorElementType() == MVT::i8 || in PerformSELECTCombine()
24128 OpVT.getVectorElementType() == MVT::i16) && in PerformSELECTCombine()
24166 DAG.getConstant(ShAmt, DL, MVT::i8)); in PerformSELECTCombine()
24184 if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) { in PerformSELECTCombine()
24186 if (N->getValueType(0) == MVT::i32) Diff = (unsigned)Diff; in PerformSELECTCombine()
24267 ((Subtarget->hasSSE2() && (VT == MVT::v16i8 || VT == MVT::v8i16)) || in PerformSELECTCombine()
24268 (Subtarget->hasAVX2() && (VT == MVT::v32i8 || VT == MVT::v16i16)))) { in PerformSELECTCombine()
24419 if (VT.getVectorElementType() == MVT::i16) in PerformSELECTCombine()
24425 if (VT == MVT::v32i8 && !Subtarget->hasAVX2()) in PerformSELECTCombine()
24681 (FalseOp.getValueType() != MVT::f80 || hasFPCMov(CC))) { in PerformCMOVCombine()
24683 DAG.getConstant(CC, DL, MVT::i8), Flags }; in PerformCMOVCombine()
24704 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8, in PerformCMOVCombine()
24705 DAG.getConstant(CC, DL, MVT::i8), Cond); in PerformCMOVCombine()
24712 DAG.getConstant(ShAmt, DL, MVT::i8)); in PerformCMOVCombine()
24721 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8, in PerformCMOVCombine()
24722 DAG.getConstant(CC, DL, MVT::i8), Cond); in PerformCMOVCombine()
24737 if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) { in PerformCMOVCombine()
24739 if (N->getValueType(0) == MVT::i32) Diff = (unsigned)Diff; in PerformCMOVCombine()
24759 Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8, in PerformCMOVCombine()
24760 DAG.getConstant(CC, DL, MVT::i8), Cond); in PerformCMOVCombine()
24813 DAG.getConstant(CC, DL, MVT::i8), Cond }; in PerformCMOVCombine()
24847 SDValue LOps[] = {FalseOp, TrueOp, DAG.getConstant(CC0, DL, MVT::i8), in PerformCMOVCombine()
24850 SDValue Ops[] = {LCMOV, TrueOp, DAG.getConstant(CC1, DL, MVT::i8), Flags}; in PerformCMOVCombine()
24873 if (VT != MVT::i64 && VT != MVT::i32) in PerformMulCombine()
24910 DAG.getConstant(Log2_64(MulAmt1), DL, MVT::i8)); in PerformMulCombine()
24917 DAG.getConstant(Log2_64(MulAmt2), DL, MVT::i8)); in PerformMulCombine()
24924 assert(MulAmt != 0 && MulAmt != (VT == MVT::i64 ? UINT64_MAX : UINT32_MAX) in PerformMulCombine()
24932 MVT::i8))); in PerformMulCombine()
24939 DL, MVT::i8)), N->getOperand(0)); in PerformMulCombine()
25040 for (MVT SVT : MVT::integer_valuetypes()) { in PerformSRACombine()
25069 if (VT != MVT::v2i64 && VT != MVT::v4i32 && VT != MVT::v8i16 && in performShiftToAllZeros()
25071 (VT != MVT::v4i64 && VT != MVT::v8i32 && VT != MVT::v16i16))) in performShiftToAllZeros()
25138 if (VT == MVT::f32 || VT == MVT::f64) { in CMPEQCombine()
25173 SDValue FSetCC = DAG.getNode(X86ISD::FSETCC, DL, MVT::i1, CMP00, in CMPEQCombine()
25175 DAG.getConstant(x86cc, DL, MVT::i8)); in CMPEQCombine()
25176 if (N->getValueType(0) != MVT::i1) in CMPEQCombine()
25184 MVT::i8)); in CMPEQCombine()
25186 bool is64BitFP = (CMP00.getValueType() == MVT::f64); in CMPEQCombine()
25187 MVT IntVT = is64BitFP ? MVT::i64 : MVT::i32; in CMPEQCombine()
25195 SDValue Vector64 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, in CMPEQCombine()
25197 SDValue Vector32 = DAG.getBitcast(MVT::v4f32, Vector64); in CMPEQCombine()
25198 OnesOrZeroesF = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, in CMPEQCombine()
25200 IntVT = MVT::i32; in CMPEQCombine()
25206 SDValue OneBitOfTruth = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, in CMPEQCombine()
25440 ((Subtarget->hasSSE1() && VT == MVT::i32) || in convertIntLogicToFPLogic()
25441 (Subtarget->hasSSE2() && VT == MVT::i64))) { in convertIntLogicToFPLogic()
25476 if (VT == MVT::i32 || VT == MVT::i64) { in PerformAndCombine()
25502 if (VT != MVT::v2i64 && VT != MVT::v4i64) in PerformAndCombine()
25537 if (VT == MVT::v2i64 || VT == MVT::v4i64) { in PerformOrCombine()
25539 (VT == MVT::v4i64 && !Subtarget->hasInt256())) in PerformOrCombine()
25603 MVT BlendVT = (VT == MVT::v4i64) ? MVT::v32i8 : MVT::v16i8; in PerformOrCombine()
25613 if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64) in PerformOrCombine()
25635 if (ShAmt0.getValueType() != MVT::i8) in PerformOrCombine()
25638 if (ShAmt1.getValueType() != MVT::i8) in PerformOrCombine()
25666 MVT::i8, ShAmt0)); in PerformOrCombine()
25675 MVT::i8, ShAmt0)); in PerformOrCombine()
25704 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, DAG.getVTList(VT, MVT::i32), in performIntegerAbsCombine()
25708 DAG.getConstant(X86::COND_GE, DL, MVT::i8), in performIntegerAbsCombine()
25710 return DAG.getNode(X86ISD::CMOV, DL, DAG.getVTList(VT, MVT::Glue), Ops); in performIntegerAbsCombine()
25721 if (N->getValueType(0) != MVT::i8) in foldXorTruncShiftIntoCmp()
25742 if (ShiftTy != MVT::i16 && ShiftTy != MVT::i32 && ShiftTy != MVT::i64) in foldXorTruncShiftIntoCmp()
25756 SDValue Cond = DAG.getSetCC(DL, MVT::i8, ShiftOp, in foldXorTruncShiftIntoCmp()
25791 if (!((ScalarVT == MVT::i8 || ScalarVT == MVT::i16) && in detectAVGPattern()
25858 if (IsConstVectorInRange(Operands[1], 1, ScalarVT == MVT::i8 ? 256 : 65536) && in detectAVGPattern()
25939 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, in PerformLOADCombine()
26009 assert(Mask.getValueType().getVectorElementType() == MVT::i1); in PerformMLOADCombine()
26012 EVT NewMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, in PerformMLOADCombine()
26101 assert(Mask.getValueType().getVectorElementType() == MVT::i1); in PerformMSTORECombine()
26104 EVT NewMaskVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, in PerformMSTORECombine()
26158 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Ch0, Ch1); in PerformSTORECombine()
26220 MVT StoreType = MVT::i8; in PerformSTORECombine()
26221 for (MVT Tp : MVT::integer_valuetypes()) { in PerformSTORECombine()
26227 if (TLI.isTypeLegal(MVT::f64) && StoreType.getSizeInBits() < 64 && in PerformSTORECombine()
26229 StoreType = MVT::f64; in PerformSTORECombine()
26253 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); in PerformSTORECombine()
26270 (VT == MVT::i64 && F64IsLegal && !Subtarget->is64Bit())) && in PerformSTORECombine()
26310 MVT LdVT = Subtarget->is64Bit() ? MVT::i64 : MVT::f64; in PerformSTORECombine()
26318 NewChain = DAG.getNode(ISD::TokenFactor, LdDL, MVT::Other, Ops); in PerformSTORECombine()
26328 SDValue HiAddr = DAG.getNode(ISD::ADD, LdDL, MVT::i32, LoAddr, in PerformSTORECombine()
26329 DAG.getConstant(4, LdDL, MVT::i32)); in PerformSTORECombine()
26331 SDValue LoLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), LoAddr, in PerformSTORECombine()
26335 SDValue HiLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), HiAddr, in PerformSTORECombine()
26345 NewChain = DAG.getNode(ISD::TokenFactor, LdDL, MVT::Other, Ops); in PerformSTORECombine()
26349 HiAddr = DAG.getNode(ISD::ADD, StDL, MVT::i32, LoAddr, in PerformSTORECombine()
26350 DAG.getConstant(4, StDL, MVT::i32)); in PerformSTORECombine()
26361 return DAG.getNode(ISD::TokenFactor, StDL, MVT::Other, LoSt, HiSt); in PerformSTORECombine()
26370 if (VT == MVT::i64 && F64IsLegal && !Subtarget->is64Bit() && in PerformSTORECombine()
26375 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64, VecSize / 64); in PerformSTORECombine()
26377 SDValue NewExtract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, in PerformSTORECombine()
26417 MVT VT = LHS.getSimpleValueType(); in isHorizontalBinOp()
26521 if (((Subtarget->hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) || in PerformFADDCombine()
26522 (Subtarget->hasFp256() && (VT == MVT::v8f32 || VT == MVT::v4f64))) && in PerformFADDCombine()
26536 if (((Subtarget->hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) || in PerformFSUBCombine()
26537 (Subtarget->hasFp256() && (VT == MVT::v8f32 || VT == MVT::v4f64))) && in PerformFSUBCombine()
26547 assert(Regs.size() > 0 && (Regs[0].getValueType() == MVT::v4i32 || in combineVectorTruncationWithPACKUS()
26548 Regs[0].getValueType() == MVT::v2i64)); in combineVectorTruncationWithPACKUS()
26556 assert((OutSVT == MVT::i8 || OutSVT == MVT::i16) && in combineVectorTruncationWithPACKUS()
26559 DAG.getConstant(OutSVT == MVT::i8 ? 0xFF : 0xFFFF, DL, InSVT); in combineVectorTruncationWithPACKUS()
26566 MVT UnpackedVT, PackedVT; in combineVectorTruncationWithPACKUS()
26567 if (OutSVT == MVT::i8) { in combineVectorTruncationWithPACKUS()
26568 UnpackedVT = MVT::v8i16; in combineVectorTruncationWithPACKUS()
26569 PackedVT = MVT::v16i8; in combineVectorTruncationWithPACKUS()
26571 UnpackedVT = MVT::v4i32; in combineVectorTruncationWithPACKUS()
26572 PackedVT = MVT::v8i16; in combineVectorTruncationWithPACKUS()
26588 if (OutVT == MVT::v8i8) { in combineVectorTruncationWithPACKUS()
26604 assert(Regs.size() > 0 && Regs[0].getValueType() == MVT::v4i32); in combineVectorTruncationWithPACKSS()
26609 SDValue ShAmt = DAG.getConstant(16, DL, MVT::i32); in combineVectorTruncationWithPACKSS()
26611 Reg = getTargetVShiftNode(X86ISD::VSHLI, DL, MVT::v4i32, Reg, ShAmt, DAG); in combineVectorTruncationWithPACKSS()
26612 Reg = getTargetVShiftNode(X86ISD::VSRAI, DL, MVT::v4i32, Reg, ShAmt, DAG); in combineVectorTruncationWithPACKSS()
26616 Regs[i] = DAG.getNode(X86ISD::PACKSS, DL, MVT::v8i16, Regs[i * 2], in combineVectorTruncationWithPACKSS()
26652 if (!((InSVT == MVT::i32 || InSVT == MVT::i64) && in combineVectorTruncation()
26653 (OutSVT == MVT::i8 || OutSVT == MVT::i16) && isPowerOf2_32(NumElems) && in combineVectorTruncation()
26659 ((OutSVT == MVT::i8 && InSVT != MVT::i64) || in combineVectorTruncation()
26660 (InSVT == MVT::i32 && OutSVT == MVT::i16))) in combineVectorTruncation()
26668 if (InSVT == MVT::i32) { in combineVectorTruncation()
26670 SubVec[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In, in combineVectorTruncation()
26674 SubVec[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In, in combineVectorTruncation()
26681 if (Subtarget->hasSSE41() || OutSVT == MVT::i8) in combineVectorTruncation()
26683 else if (InSVT == MVT::i32) in combineVectorTruncation()
26715 if (Arg.getOpcode() == ISD::FMUL && (SVT == MVT::f32 || SVT == MVT::f64) && in PerformFNEGCombine()
26750 MVT IntScalar = MVT::getIntegerVT(VT.getScalarSizeInBits()); in lowerX86FPLogicOp()
26751 MVT IntVT = MVT::getVectorVT(IntScalar, VT.getVectorNumElements()); in lowerX86FPLogicOp()
26822 if (VT != MVT::f32 || !Subtarget->hasSSE1() || Subtarget->useSoftFloat()) in performFMaxNumCombine()
26937 if (VT == MVT::v4i64 && (N0.getOpcode() == ISD::ANY_EXTEND || in PerformSIGN_EXTEND_INREGCombine()
26947 if (N00.getValueType() == MVT::v4i32 && ExtraVT.getSizeInBits() < 128) { in PerformSIGN_EXTEND_INREGCombine()
26948 SDValue Tmp = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::v4i32, in PerformSIGN_EXTEND_INREGCombine()
26950 return DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i64, Tmp); in PerformSIGN_EXTEND_INREGCombine()
26964 if (VT != MVT::i64) in promoteSextBeforeAddNSW()
27022 InVT == MVT::i8 && VT == MVT::i32) { in PerformSExtCombine()
27023 SDVTList NodeTys = DAG.getVTList(MVT::i8, VT); in PerformSExtCombine()
27031 if (InVT == MVT::i1) { in PerformSExtCombine()
27054 (SVT == MVT::i64 || SVT == MVT::i32 || SVT == MVT::i16) && in PerformSExtCombine()
27055 (InSVT == MVT::i32 || InSVT == MVT::i16 || InSVT == MVT::i8)) { in PerformSExtCombine()
27068 (SVT == MVT::i64 || SVT == MVT::i32 || SVT == MVT::i16) && in PerformSExtCombine()
27069 (InSVT == MVT::i32 || InSVT == MVT::i16 || InSVT == MVT::i8)) { in PerformSExtCombine()
27077 (SVT == MVT::i64 || SVT == MVT::i32 || SVT == MVT::i16) && in PerformSExtCombine()
27078 (InSVT == MVT::i32 || InSVT == MVT::i16 || InSVT == MVT::i8)) { in PerformSExtCombine()
27117 if ((ScalarVT != MVT::f32 && ScalarVT != MVT::f64) || !Subtarget->hasAnyFMA()) in PerformFMACombine()
27192 N0.getResNo() == 1 && N0.getValueType() == MVT::i8 && in PerformZExtCombine()
27193 (VT == MVT::i32 || VT == MVT::i64)) { in PerformZExtCombine()
27194 SDVTList NodeTys = DAG.getVTList(MVT::i8, VT); in PerformZExtCombine()
27229 if (VT.getScalarType() == MVT::i1 && in PerformISDSETCCCombine()
27233 (LHS.getOperand(0).getValueType().getScalarType() == MVT::i1); in PerformISDSETCCCombine()
27242 (LHS.getOperand(0).getValueType().getScalarType() == MVT::i1); in PerformISDSETCCCombine()
27281 if (VT == MVT::v2f64) in PerformBLENDICombine()
27284 SDValue NewMask = DAG.getConstant(1, DL, MVT::i8); in PerformBLENDICombine()
27309 MVT VT) { in MaterializeSETB()
27310 if (VT == MVT::i8) in MaterializeSETB()
27312 DAG.getNode(X86ISD::SETCC_CARRY, DL, MVT::i8, in MaterializeSETB()
27313 DAG.getConstant(X86::COND_B, DL, MVT::i8), in MaterializeSETB()
27316 assert (VT == MVT::i1 && "Unexpected type for SECCC node"); in MaterializeSETB()
27317 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, in MaterializeSETB()
27318 DAG.getNode(X86ISD::SETCC_CARRY, DL, MVT::i8, in MaterializeSETB()
27319 DAG.getConstant(X86::COND_B, DL, MVT::i8), in MaterializeSETB()
27356 SDValue Cond = DAG.getConstant(CC, DL, MVT::i8); in PerformSETCCCombine()
27375 SDValue Cond = DAG.getConstant(CC, DL, MVT::i8); in PerformBrCondCombine()
27439 if (InVT.isVector() && (InSVT == MVT::i8 || InSVT == MVT::i16)) { in PerformUINT_TO_FPCombine()
27441 EVT DstVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, in PerformUINT_TO_FPCombine()
27469 if (InVT.isVector() && (InSVT == MVT::i8 || InSVT == MVT::i16)) { in PerformSINT_TO_FPCombine()
27471 EVT DstVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, in PerformSINT_TO_FPCombine()
27484 if (VT == MVT::f16) in PerformSINT_TO_FPCombine()
27489 !Subtarget->is64Bit() && LdVT == MVT::i64) { in PerformSINT_TO_FPCombine()
27516 MVT::i8), in PerformADCCombine()
27552 SDValue NewCmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32, CmpOp0, in OptimizeConditionalInDecrement()
27574 if (((Subtarget->hasSSSE3() && (VT == MVT::v8i16 || VT == MVT::v4i32)) || in PerformAddCombine()
27575 (Subtarget->hasInt256() && (VT == MVT::v16i16 || VT == MVT::v8i32))) && in PerformAddCombine()
27607 if (((Subtarget->hasSSSE3() && (VT == MVT::v8i16 || VT == MVT::v4i32)) || in PerformSubCombine()
27608 (Subtarget->hasInt256() && (VT == MVT::v16i16 || VT == MVT::v8i32))) && in PerformSubCombine()
27620 MVT VT = N->getSimpleValueType(0); in performVZEXTCombine()
27622 MVT OpVT = Op.getSimpleValueType(); in performVZEXTCombine()
27623 MVT OpEltVT = OpVT.getVectorElementType(); in performVZEXTCombine()
27632 MVT InnerVT = V.getSimpleValueType(); in performVZEXTCombine()
27633 MVT InnerEltVT = InnerVT.getVectorElementType(); in performVZEXTCombine()
27662 MVT OrigVT = OrigV.getSimpleValueType(); in performVZEXTCombine()
27666 OrigVT = MVT::getVectorVT(OrigVT.getVectorElementType(), in performVZEXTCombine()
27761 if (VT != MVT::i16) in isTypeDesirableForOp()
27788 if (VT != MVT::i16) in IsDesirableToPromoteOp()
27847 PVT = MVT::i32; in IsDesirableToPromoteOp()
28215 Result = DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op), MVT::i64); in LowerAsmOperandForConstraint()
28241 Result = DAG.getTargetConstant(CST->getSExtValue(), SDLoc(Op), MVT::i64); in LowerAsmOperandForConstraint()
28302 MVT VT) const { in getRegForInlineAsmConstraint()
28314 if (VT == MVT::i32 || VT == MVT::f32) in getRegForInlineAsmConstraint()
28316 if (VT == MVT::i16) in getRegForInlineAsmConstraint()
28318 if (VT == MVT::i8 || VT == MVT::i1) in getRegForInlineAsmConstraint()
28320 if (VT == MVT::i64 || VT == MVT::f64) in getRegForInlineAsmConstraint()
28326 if (VT == MVT::i32 || VT == MVT::f32) in getRegForInlineAsmConstraint()
28328 if (VT == MVT::i16) in getRegForInlineAsmConstraint()
28330 if (VT == MVT::i8 || VT == MVT::i1) in getRegForInlineAsmConstraint()
28332 if (VT == MVT::i64) in getRegForInlineAsmConstraint()
28337 if (VT == MVT::i8 || VT == MVT::i1) in getRegForInlineAsmConstraint()
28339 if (VT == MVT::i16) in getRegForInlineAsmConstraint()
28341 if (VT == MVT::i32 || VT == MVT::f32 || !Subtarget->is64Bit()) in getRegForInlineAsmConstraint()
28345 if (VT == MVT::i8 || VT == MVT::i1) in getRegForInlineAsmConstraint()
28347 if (VT == MVT::i16) in getRegForInlineAsmConstraint()
28349 if (VT == MVT::i32 || !Subtarget->is64Bit()) in getRegForInlineAsmConstraint()
28355 if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT)) in getRegForInlineAsmConstraint()
28357 if (VT == MVT::f64 && !isScalarFPTypeInSSEReg(VT)) in getRegForInlineAsmConstraint()
28372 case MVT::f32: in getRegForInlineAsmConstraint()
28373 case MVT::i32: in getRegForInlineAsmConstraint()
28375 case MVT::f64: in getRegForInlineAsmConstraint()
28376 case MVT::i64: in getRegForInlineAsmConstraint()
28380 case MVT::v16i8: in getRegForInlineAsmConstraint()
28381 case MVT::v8i16: in getRegForInlineAsmConstraint()
28382 case MVT::v4i32: in getRegForInlineAsmConstraint()
28383 case MVT::v2i64: in getRegForInlineAsmConstraint()
28384 case MVT::v4f32: in getRegForInlineAsmConstraint()
28385 case MVT::v2f64: in getRegForInlineAsmConstraint()
28388 case MVT::v32i8: in getRegForInlineAsmConstraint()
28389 case MVT::v16i16: in getRegForInlineAsmConstraint()
28390 case MVT::v8i32: in getRegForInlineAsmConstraint()
28391 case MVT::v4i64: in getRegForInlineAsmConstraint()
28392 case MVT::v8f32: in getRegForInlineAsmConstraint()
28393 case MVT::v4f64: in getRegForInlineAsmConstraint()
28395 case MVT::v8f64: in getRegForInlineAsmConstraint()
28396 case MVT::v16f32: in getRegForInlineAsmConstraint()
28397 case MVT::v16i32: in getRegForInlineAsmConstraint()
28398 case MVT::v8i64: in getRegForInlineAsmConstraint()
28453 if (Res.second->hasType(VT) || VT == MVT::Other) in getRegForInlineAsmConstraint()
28463 MVT::SimpleValueType SimpleTy = Size == 1 || Size == 8 ? MVT::i8 in getRegForInlineAsmConstraint()
28464 : Size == 16 ? MVT::i16 in getRegForInlineAsmConstraint()
28465 : Size == 32 ? MVT::i32 in getRegForInlineAsmConstraint()
28466 : Size == 64 ? MVT::i64 in getRegForInlineAsmConstraint()
28467 : MVT::Other; in getRegForInlineAsmConstraint()
28471 Res.second = SimpleTy == MVT::i8 ? &X86::GR8RegClass in getRegForInlineAsmConstraint()
28472 : SimpleTy == MVT::i16 ? &X86::GR16RegClass in getRegForInlineAsmConstraint()
28473 : SimpleTy == MVT::i32 ? &X86::GR32RegClass in getRegForInlineAsmConstraint()
28492 if (VT == MVT::f32 || VT == MVT::i32) in getRegForInlineAsmConstraint()
28494 else if (VT == MVT::f64 || VT == MVT::i64) in getRegForInlineAsmConstraint()