Lines Matching refs:ISD
157 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); in X86TargetLowering()
170 setCondCodeAction(ISD::SETOEQ, MVT::f32, Expand); in X86TargetLowering()
171 setCondCodeAction(ISD::SETOEQ, MVT::f64, Expand); in X86TargetLowering()
172 setCondCodeAction(ISD::SETOEQ, MVT::f80, Expand); in X86TargetLowering()
173 setCondCodeAction(ISD::SETUNE, MVT::f32, Expand); in X86TargetLowering()
174 setCondCodeAction(ISD::SETUNE, MVT::f64, Expand); in X86TargetLowering()
175 setCondCodeAction(ISD::SETUNE, MVT::f80, Expand); in X86TargetLowering()
179 setOperationAction(ISD::UINT_TO_FP , MVT::i1 , Promote); in X86TargetLowering()
180 setOperationAction(ISD::UINT_TO_FP , MVT::i8 , Promote); in X86TargetLowering()
181 setOperationAction(ISD::UINT_TO_FP , MVT::i16 , Promote); in X86TargetLowering()
184 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote); in X86TargetLowering()
185 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Custom); in X86TargetLowering()
189 setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Custom); in X86TargetLowering()
192 setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Custom); in X86TargetLowering()
197 setOperationAction(ISD::SINT_TO_FP , MVT::i1 , Promote); in X86TargetLowering()
198 setOperationAction(ISD::SINT_TO_FP , MVT::i8 , Promote); in X86TargetLowering()
203 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote); in X86TargetLowering()
205 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); in X86TargetLowering()
207 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Custom); in X86TargetLowering()
208 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom); in X86TargetLowering()
211 setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote); in X86TargetLowering()
212 setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Promote); in X86TargetLowering()
217 setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Custom); in X86TargetLowering()
218 setOperationAction(ISD::SINT_TO_FP , MVT::i64 , Custom); in X86TargetLowering()
222 setOperationAction(ISD::FP_TO_SINT , MVT::i1 , Promote); in X86TargetLowering()
223 setOperationAction(ISD::FP_TO_SINT , MVT::i8 , Promote); in X86TargetLowering()
226 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote); in X86TargetLowering()
228 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); in X86TargetLowering()
230 setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Custom); in X86TargetLowering()
231 setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom); in X86TargetLowering()
236 setOperationAction(ISD::FP_TO_UINT , MVT::i1 , Promote); in X86TargetLowering()
237 setOperationAction(ISD::FP_TO_UINT , MVT::i8 , Promote); in X86TargetLowering()
238 setOperationAction(ISD::FP_TO_UINT , MVT::i16 , Promote); in X86TargetLowering()
241 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Expand); in X86TargetLowering()
242 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote); in X86TargetLowering()
249 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Expand); in X86TargetLowering()
253 setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Custom); in X86TargetLowering()
259 setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Custom); in X86TargetLowering()
264 setOperationAction(ISD::BITCAST , MVT::f32 , Expand); in X86TargetLowering()
265 setOperationAction(ISD::BITCAST , MVT::i32 , Expand); in X86TargetLowering()
267 setOperationAction(ISD::BITCAST , MVT::f64 , Expand); in X86TargetLowering()
269 setOperationAction(ISD::BITCAST , MVT::i64 , Expand); in X86TargetLowering()
285 setOperationAction(ISD::MULHS, VT, Expand); in X86TargetLowering()
286 setOperationAction(ISD::MULHU, VT, Expand); in X86TargetLowering()
287 setOperationAction(ISD::SDIV, VT, Expand); in X86TargetLowering()
288 setOperationAction(ISD::UDIV, VT, Expand); in X86TargetLowering()
289 setOperationAction(ISD::SREM, VT, Expand); in X86TargetLowering()
290 setOperationAction(ISD::UREM, VT, Expand); in X86TargetLowering()
293 setOperationAction(ISD::ADDC, VT, Custom); in X86TargetLowering()
294 setOperationAction(ISD::ADDE, VT, Custom); in X86TargetLowering()
295 setOperationAction(ISD::SUBC, VT, Custom); in X86TargetLowering()
296 setOperationAction(ISD::SUBE, VT, Custom); in X86TargetLowering()
299 setOperationAction(ISD::BR_JT , MVT::Other, Expand); in X86TargetLowering()
300 setOperationAction(ISD::BRCOND , MVT::Other, Custom); in X86TargetLowering()
301 setOperationAction(ISD::BR_CC , MVT::f32, Expand); in X86TargetLowering()
302 setOperationAction(ISD::BR_CC , MVT::f64, Expand); in X86TargetLowering()
303 setOperationAction(ISD::BR_CC , MVT::f80, Expand); in X86TargetLowering()
304 setOperationAction(ISD::BR_CC , MVT::i8, Expand); in X86TargetLowering()
305 setOperationAction(ISD::BR_CC , MVT::i16, Expand); in X86TargetLowering()
306 setOperationAction(ISD::BR_CC , MVT::i32, Expand); in X86TargetLowering()
307 setOperationAction(ISD::BR_CC , MVT::i64, Expand); in X86TargetLowering()
308 setOperationAction(ISD::SELECT_CC , MVT::f32, Expand); in X86TargetLowering()
309 setOperationAction(ISD::SELECT_CC , MVT::f64, Expand); in X86TargetLowering()
310 setOperationAction(ISD::SELECT_CC , MVT::f80, Expand); in X86TargetLowering()
311 setOperationAction(ISD::SELECT_CC , MVT::i8, Expand); in X86TargetLowering()
312 setOperationAction(ISD::SELECT_CC , MVT::i16, Expand); in X86TargetLowering()
313 setOperationAction(ISD::SELECT_CC , MVT::i32, Expand); in X86TargetLowering()
314 setOperationAction(ISD::SELECT_CC , MVT::i64, Expand); in X86TargetLowering()
316 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal); in X86TargetLowering()
317 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Legal); in X86TargetLowering()
318 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal); in X86TargetLowering()
319 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand); in X86TargetLowering()
320 setOperationAction(ISD::FP_ROUND_INREG , MVT::f32 , Expand); in X86TargetLowering()
321 setOperationAction(ISD::FREM , MVT::f32 , Expand); in X86TargetLowering()
322 setOperationAction(ISD::FREM , MVT::f64 , Expand); in X86TargetLowering()
323 setOperationAction(ISD::FREM , MVT::f80 , Expand); in X86TargetLowering()
324 setOperationAction(ISD::FLT_ROUNDS_ , MVT::i32 , Custom); in X86TargetLowering()
328 setOperationAction(ISD::CTTZ , MVT::i8 , Promote); in X86TargetLowering()
329 AddPromotedToType (ISD::CTTZ , MVT::i8 , MVT::i32); in X86TargetLowering()
330 setOperationAction(ISD::CTTZ_ZERO_UNDEF , MVT::i8 , Promote); in X86TargetLowering()
331 AddPromotedToType (ISD::CTTZ_ZERO_UNDEF , MVT::i8 , MVT::i32); in X86TargetLowering()
333 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i16 , Expand); in X86TargetLowering()
334 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32 , Expand); in X86TargetLowering()
336 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Expand); in X86TargetLowering()
338 setOperationAction(ISD::CTTZ , MVT::i16 , Custom); in X86TargetLowering()
339 setOperationAction(ISD::CTTZ , MVT::i32 , Custom); in X86TargetLowering()
341 setOperationAction(ISD::CTTZ , MVT::i64 , Custom); in X86TargetLowering()
347 setOperationAction(ISD::CTLZ , MVT::i8 , Promote); in X86TargetLowering()
348 AddPromotedToType (ISD::CTLZ , MVT::i8 , MVT::i32); in X86TargetLowering()
349 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i8 , Promote); in X86TargetLowering()
350 AddPromotedToType (ISD::CTLZ_ZERO_UNDEF, MVT::i8 , MVT::i32); in X86TargetLowering()
351 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i16 , Expand); in X86TargetLowering()
352 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32 , Expand); in X86TargetLowering()
354 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Expand); in X86TargetLowering()
356 setOperationAction(ISD::CTLZ , MVT::i8 , Custom); in X86TargetLowering()
357 setOperationAction(ISD::CTLZ , MVT::i16 , Custom); in X86TargetLowering()
358 setOperationAction(ISD::CTLZ , MVT::i32 , Custom); in X86TargetLowering()
359 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i8 , Custom); in X86TargetLowering()
360 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i16 , Custom); in X86TargetLowering()
361 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32 , Custom); in X86TargetLowering()
363 setOperationAction(ISD::CTLZ , MVT::i64 , Custom); in X86TargetLowering()
364 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Custom); in X86TargetLowering()
372 setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand); in X86TargetLowering()
373 setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand); in X86TargetLowering()
377 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand); in X86TargetLowering()
378 setOperationAction(ISD::FP16_TO_FP, MVT::f80, Expand); in X86TargetLowering()
379 setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand); in X86TargetLowering()
380 setOperationAction(ISD::FP_TO_FP16, MVT::f80, Expand); in X86TargetLowering()
382 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand); in X86TargetLowering()
383 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand); in X86TargetLowering()
384 setLoadExtAction(ISD::EXTLOAD, MVT::f80, MVT::f16, Expand); in X86TargetLowering()
390 setOperationAction(ISD::CTPOP , MVT::i8 , Promote); in X86TargetLowering()
392 setOperationAction(ISD::CTPOP , MVT::i8 , Expand); in X86TargetLowering()
393 setOperationAction(ISD::CTPOP , MVT::i16 , Expand); in X86TargetLowering()
394 setOperationAction(ISD::CTPOP , MVT::i32 , Expand); in X86TargetLowering()
396 setOperationAction(ISD::CTPOP , MVT::i64 , Expand); in X86TargetLowering()
399 setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom); in X86TargetLowering()
402 setOperationAction(ISD::BSWAP , MVT::i16 , Expand); in X86TargetLowering()
405 setOperationAction(ISD::SELECT , MVT::i1 , Promote); in X86TargetLowering()
407 setOperationAction(ISD::SELECT , MVT::i8 , Custom); in X86TargetLowering()
408 setOperationAction(ISD::SELECT , MVT::i16 , Custom); in X86TargetLowering()
409 setOperationAction(ISD::SELECT , MVT::i32 , Custom); in X86TargetLowering()
410 setOperationAction(ISD::SELECT , MVT::f32 , Custom); in X86TargetLowering()
411 setOperationAction(ISD::SELECT , MVT::f64 , Custom); in X86TargetLowering()
412 setOperationAction(ISD::SELECT , MVT::f80 , Custom); in X86TargetLowering()
413 setOperationAction(ISD::SETCC , MVT::i8 , Custom); in X86TargetLowering()
414 setOperationAction(ISD::SETCC , MVT::i16 , Custom); in X86TargetLowering()
415 setOperationAction(ISD::SETCC , MVT::i32 , Custom); in X86TargetLowering()
416 setOperationAction(ISD::SETCC , MVT::f32 , Custom); in X86TargetLowering()
417 setOperationAction(ISD::SETCC , MVT::f64 , Custom); in X86TargetLowering()
418 setOperationAction(ISD::SETCC , MVT::f80 , Custom); in X86TargetLowering()
420 setOperationAction(ISD::SELECT , MVT::i64 , Custom); in X86TargetLowering()
421 setOperationAction(ISD::SETCC , MVT::i64 , Custom); in X86TargetLowering()
423 setOperationAction(ISD::EH_RETURN , MVT::Other, Custom); in X86TargetLowering()
430 setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom); in X86TargetLowering()
431 setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom); in X86TargetLowering()
434 setOperationAction(ISD::ConstantPool , MVT::i32 , Custom); in X86TargetLowering()
435 setOperationAction(ISD::JumpTable , MVT::i32 , Custom); in X86TargetLowering()
436 setOperationAction(ISD::GlobalAddress , MVT::i32 , Custom); in X86TargetLowering()
437 setOperationAction(ISD::GlobalTLSAddress, MVT::i32 , Custom); in X86TargetLowering()
439 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); in X86TargetLowering()
440 setOperationAction(ISD::ExternalSymbol , MVT::i32 , Custom); in X86TargetLowering()
441 setOperationAction(ISD::BlockAddress , MVT::i32 , Custom); in X86TargetLowering()
443 setOperationAction(ISD::ConstantPool , MVT::i64 , Custom); in X86TargetLowering()
444 setOperationAction(ISD::JumpTable , MVT::i64 , Custom); in X86TargetLowering()
445 setOperationAction(ISD::GlobalAddress , MVT::i64 , Custom); in X86TargetLowering()
446 setOperationAction(ISD::ExternalSymbol, MVT::i64 , Custom); in X86TargetLowering()
447 setOperationAction(ISD::BlockAddress , MVT::i64 , Custom); in X86TargetLowering()
450 setOperationAction(ISD::SHL_PARTS , MVT::i32 , Custom); in X86TargetLowering()
451 setOperationAction(ISD::SRA_PARTS , MVT::i32 , Custom); in X86TargetLowering()
452 setOperationAction(ISD::SRL_PARTS , MVT::i32 , Custom); in X86TargetLowering()
454 setOperationAction(ISD::SHL_PARTS , MVT::i64 , Custom); in X86TargetLowering()
455 setOperationAction(ISD::SRA_PARTS , MVT::i64 , Custom); in X86TargetLowering()
456 setOperationAction(ISD::SRL_PARTS , MVT::i64 , Custom); in X86TargetLowering()
460 setOperationAction(ISD::PREFETCH , MVT::Other, Legal); in X86TargetLowering()
462 setOperationAction(ISD::ATOMIC_FENCE , MVT::Other, Custom); in X86TargetLowering()
467 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT, Custom); in X86TargetLowering()
468 setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom); in X86TargetLowering()
469 setOperationAction(ISD::ATOMIC_STORE, VT, Custom); in X86TargetLowering()
473 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i128, Custom); in X86TargetLowering()
479 setOperationAction(ISD::EH_LABEL, MVT::Other, Expand); in X86TargetLowering()
489 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom); in X86TargetLowering()
490 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i64, Custom); in X86TargetLowering()
492 setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom); in X86TargetLowering()
493 setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom); in X86TargetLowering()
495 setOperationAction(ISD::TRAP, MVT::Other, Legal); in X86TargetLowering()
496 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal); in X86TargetLowering()
499 setOperationAction(ISD::VASTART , MVT::Other, Custom); in X86TargetLowering()
500 setOperationAction(ISD::VAEND , MVT::Other, Expand); in X86TargetLowering()
503 setOperationAction(ISD::VAARG , MVT::Other, Custom); in X86TargetLowering()
504 setOperationAction(ISD::VACOPY , MVT::Other, Custom); in X86TargetLowering()
507 setOperationAction(ISD::VAARG , MVT::Other, Expand); in X86TargetLowering()
508 setOperationAction(ISD::VACOPY , MVT::Other, Expand); in X86TargetLowering()
511 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); in X86TargetLowering()
512 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); in X86TargetLowering()
514 setOperationAction(ISD::DYNAMIC_STACKALLOC, getPointerTy(), Custom); in X86TargetLowering()
523 setOperationAction(ISD::FABS , MVT::f64, Custom); in X86TargetLowering()
524 setOperationAction(ISD::FABS , MVT::f32, Custom); in X86TargetLowering()
527 setOperationAction(ISD::FNEG , MVT::f64, Custom); in X86TargetLowering()
528 setOperationAction(ISD::FNEG , MVT::f32, Custom); in X86TargetLowering()
531 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom); in X86TargetLowering()
532 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); in X86TargetLowering()
535 setOperationAction(ISD::FGETSIGN, MVT::i64, Custom); in X86TargetLowering()
536 setOperationAction(ISD::FGETSIGN, MVT::i32, Custom); in X86TargetLowering()
539 setOperationAction(ISD::FSIN , MVT::f64, Expand); in X86TargetLowering()
540 setOperationAction(ISD::FCOS , MVT::f64, Expand); in X86TargetLowering()
541 setOperationAction(ISD::FSINCOS, MVT::f64, Expand); in X86TargetLowering()
542 setOperationAction(ISD::FSIN , MVT::f32, Expand); in X86TargetLowering()
543 setOperationAction(ISD::FCOS , MVT::f32, Expand); in X86TargetLowering()
544 setOperationAction(ISD::FSINCOS, MVT::f32, Expand); in X86TargetLowering()
557 setOperationAction(ISD::FABS , MVT::f32, Custom); in X86TargetLowering()
560 setOperationAction(ISD::FNEG , MVT::f32, Custom); in X86TargetLowering()
562 setOperationAction(ISD::UNDEF, MVT::f64, Expand); in X86TargetLowering()
565 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); in X86TargetLowering()
566 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); in X86TargetLowering()
569 setOperationAction(ISD::FSIN , MVT::f32, Expand); in X86TargetLowering()
570 setOperationAction(ISD::FCOS , MVT::f32, Expand); in X86TargetLowering()
571 setOperationAction(ISD::FSINCOS, MVT::f32, Expand); in X86TargetLowering()
581 setOperationAction(ISD::FSIN , MVT::f64, Expand); in X86TargetLowering()
582 setOperationAction(ISD::FCOS , MVT::f64, Expand); in X86TargetLowering()
583 setOperationAction(ISD::FSINCOS, MVT::f64, Expand); in X86TargetLowering()
591 setOperationAction(ISD::UNDEF, MVT::f64, Expand); in X86TargetLowering()
592 setOperationAction(ISD::UNDEF, MVT::f32, Expand); in X86TargetLowering()
593 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); in X86TargetLowering()
594 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); in X86TargetLowering()
597 setOperationAction(ISD::FSIN , MVT::f64, Expand); in X86TargetLowering()
598 setOperationAction(ISD::FSIN , MVT::f32, Expand); in X86TargetLowering()
599 setOperationAction(ISD::FCOS , MVT::f64, Expand); in X86TargetLowering()
600 setOperationAction(ISD::FCOS , MVT::f32, Expand); in X86TargetLowering()
601 setOperationAction(ISD::FSINCOS, MVT::f64, Expand); in X86TargetLowering()
602 setOperationAction(ISD::FSINCOS, MVT::f32, Expand); in X86TargetLowering()
615 setOperationAction(ISD::FMA, MVT::f64, Expand); in X86TargetLowering()
616 setOperationAction(ISD::FMA, MVT::f32, Expand); in X86TargetLowering()
621 setOperationAction(ISD::UNDEF, MVT::f80, Expand); in X86TargetLowering()
622 setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand); in X86TargetLowering()
639 setOperationAction(ISD::FSIN , MVT::f80, Expand); in X86TargetLowering()
640 setOperationAction(ISD::FCOS , MVT::f80, Expand); in X86TargetLowering()
641 setOperationAction(ISD::FSINCOS, MVT::f80, Expand); in X86TargetLowering()
644 setOperationAction(ISD::FFLOOR, MVT::f80, Expand); in X86TargetLowering()
645 setOperationAction(ISD::FCEIL, MVT::f80, Expand); in X86TargetLowering()
646 setOperationAction(ISD::FTRUNC, MVT::f80, Expand); in X86TargetLowering()
647 setOperationAction(ISD::FRINT, MVT::f80, Expand); in X86TargetLowering()
648 setOperationAction(ISD::FNEARBYINT, MVT::f80, Expand); in X86TargetLowering()
649 setOperationAction(ISD::FMA, MVT::f80, Expand); in X86TargetLowering()
653 setOperationAction(ISD::FPOW , MVT::f32 , Expand); in X86TargetLowering()
654 setOperationAction(ISD::FPOW , MVT::f64 , Expand); in X86TargetLowering()
655 setOperationAction(ISD::FPOW , MVT::f80 , Expand); in X86TargetLowering()
657 setOperationAction(ISD::FLOG, MVT::f80, Expand); in X86TargetLowering()
658 setOperationAction(ISD::FLOG2, MVT::f80, Expand); in X86TargetLowering()
659 setOperationAction(ISD::FLOG10, MVT::f80, Expand); in X86TargetLowering()
660 setOperationAction(ISD::FEXP, MVT::f80, Expand); in X86TargetLowering()
661 setOperationAction(ISD::FEXP2, MVT::f80, Expand); in X86TargetLowering()
662 setOperationAction(ISD::FMINNUM, MVT::f80, Expand); in X86TargetLowering()
663 setOperationAction(ISD::FMAXNUM, MVT::f80, Expand); in X86TargetLowering()
669 setOperationAction(ISD::ADD , VT, Expand); in X86TargetLowering()
670 setOperationAction(ISD::SUB , VT, Expand); in X86TargetLowering()
671 setOperationAction(ISD::FADD, VT, Expand); in X86TargetLowering()
672 setOperationAction(ISD::FNEG, VT, Expand); in X86TargetLowering()
673 setOperationAction(ISD::FSUB, VT, Expand); in X86TargetLowering()
674 setOperationAction(ISD::MUL , VT, Expand); in X86TargetLowering()
675 setOperationAction(ISD::FMUL, VT, Expand); in X86TargetLowering()
676 setOperationAction(ISD::SDIV, VT, Expand); in X86TargetLowering()
677 setOperationAction(ISD::UDIV, VT, Expand); in X86TargetLowering()
678 setOperationAction(ISD::FDIV, VT, Expand); in X86TargetLowering()
679 setOperationAction(ISD::SREM, VT, Expand); in X86TargetLowering()
680 setOperationAction(ISD::UREM, VT, Expand); in X86TargetLowering()
681 setOperationAction(ISD::LOAD, VT, Expand); in X86TargetLowering()
682 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Expand); in X86TargetLowering()
683 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT,Expand); in X86TargetLowering()
684 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand); in X86TargetLowering()
685 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT,Expand); in X86TargetLowering()
686 setOperationAction(ISD::INSERT_SUBVECTOR, VT,Expand); in X86TargetLowering()
687 setOperationAction(ISD::FABS, VT, Expand); in X86TargetLowering()
688 setOperationAction(ISD::FSIN, VT, Expand); in X86TargetLowering()
689 setOperationAction(ISD::FSINCOS, VT, Expand); in X86TargetLowering()
690 setOperationAction(ISD::FCOS, VT, Expand); in X86TargetLowering()
691 setOperationAction(ISD::FSINCOS, VT, Expand); in X86TargetLowering()
692 setOperationAction(ISD::FREM, VT, Expand); in X86TargetLowering()
693 setOperationAction(ISD::FMA, VT, Expand); in X86TargetLowering()
694 setOperationAction(ISD::FPOWI, VT, Expand); in X86TargetLowering()
695 setOperationAction(ISD::FSQRT, VT, Expand); in X86TargetLowering()
696 setOperationAction(ISD::FCOPYSIGN, VT, Expand); in X86TargetLowering()
697 setOperationAction(ISD::FFLOOR, VT, Expand); in X86TargetLowering()
698 setOperationAction(ISD::FCEIL, VT, Expand); in X86TargetLowering()
699 setOperationAction(ISD::FTRUNC, VT, Expand); in X86TargetLowering()
700 setOperationAction(ISD::FRINT, VT, Expand); in X86TargetLowering()
701 setOperationAction(ISD::FNEARBYINT, VT, Expand); in X86TargetLowering()
702 setOperationAction(ISD::SMUL_LOHI, VT, Expand); in X86TargetLowering()
703 setOperationAction(ISD::MULHS, VT, Expand); in X86TargetLowering()
704 setOperationAction(ISD::UMUL_LOHI, VT, Expand); in X86TargetLowering()
705 setOperationAction(ISD::MULHU, VT, Expand); in X86TargetLowering()
706 setOperationAction(ISD::SDIVREM, VT, Expand); in X86TargetLowering()
707 setOperationAction(ISD::UDIVREM, VT, Expand); in X86TargetLowering()
708 setOperationAction(ISD::FPOW, VT, Expand); in X86TargetLowering()
709 setOperationAction(ISD::CTPOP, VT, Expand); in X86TargetLowering()
710 setOperationAction(ISD::CTTZ, VT, Expand); in X86TargetLowering()
711 setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Expand); in X86TargetLowering()
712 setOperationAction(ISD::CTLZ, VT, Expand); in X86TargetLowering()
713 setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Expand); in X86TargetLowering()
714 setOperationAction(ISD::SHL, VT, Expand); in X86TargetLowering()
715 setOperationAction(ISD::SRA, VT, Expand); in X86TargetLowering()
716 setOperationAction(ISD::SRL, VT, Expand); in X86TargetLowering()
717 setOperationAction(ISD::ROTL, VT, Expand); in X86TargetLowering()
718 setOperationAction(ISD::ROTR, VT, Expand); in X86TargetLowering()
719 setOperationAction(ISD::BSWAP, VT, Expand); in X86TargetLowering()
720 setOperationAction(ISD::SETCC, VT, Expand); in X86TargetLowering()
721 setOperationAction(ISD::FLOG, VT, Expand); in X86TargetLowering()
722 setOperationAction(ISD::FLOG2, VT, Expand); in X86TargetLowering()
723 setOperationAction(ISD::FLOG10, VT, Expand); in X86TargetLowering()
724 setOperationAction(ISD::FEXP, VT, Expand); in X86TargetLowering()
725 setOperationAction(ISD::FEXP2, VT, Expand); in X86TargetLowering()
726 setOperationAction(ISD::FP_TO_UINT, VT, Expand); in X86TargetLowering()
727 setOperationAction(ISD::FP_TO_SINT, VT, Expand); in X86TargetLowering()
728 setOperationAction(ISD::UINT_TO_FP, VT, Expand); in X86TargetLowering()
729 setOperationAction(ISD::SINT_TO_FP, VT, Expand); in X86TargetLowering()
730 setOperationAction(ISD::SIGN_EXTEND_INREG, VT,Expand); in X86TargetLowering()
731 setOperationAction(ISD::TRUNCATE, VT, Expand); in X86TargetLowering()
732 setOperationAction(ISD::SIGN_EXTEND, VT, Expand); in X86TargetLowering()
733 setOperationAction(ISD::ZERO_EXTEND, VT, Expand); in X86TargetLowering()
734 setOperationAction(ISD::ANY_EXTEND, VT, Expand); in X86TargetLowering()
735 setOperationAction(ISD::VSELECT, VT, Expand); in X86TargetLowering()
736 setOperationAction(ISD::SELECT_CC, VT, Expand); in X86TargetLowering()
740 setLoadExtAction(ISD::SEXTLOAD, InnerVT, VT, Expand); in X86TargetLowering()
741 setLoadExtAction(ISD::ZEXTLOAD, InnerVT, VT, Expand); in X86TargetLowering()
748 setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand); in X86TargetLowering()
753 setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand); in X86TargetLowering()
767 setOperationAction(ISD::MULHS, MMXTy, Expand); in X86TargetLowering()
768 setOperationAction(ISD::AND, MMXTy, Expand); in X86TargetLowering()
769 setOperationAction(ISD::OR, MMXTy, Expand); in X86TargetLowering()
770 setOperationAction(ISD::XOR, MMXTy, Expand); in X86TargetLowering()
771 setOperationAction(ISD::SCALAR_TO_VECTOR, MMXTy, Expand); in X86TargetLowering()
772 setOperationAction(ISD::SELECT, MMXTy, Expand); in X86TargetLowering()
773 setOperationAction(ISD::BITCAST, MMXTy, Expand); in X86TargetLowering()
775 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v1i64, Expand); in X86TargetLowering()
780 setOperationAction(ISD::FADD, MVT::v4f32, Legal); in X86TargetLowering()
781 setOperationAction(ISD::FSUB, MVT::v4f32, Legal); in X86TargetLowering()
782 setOperationAction(ISD::FMUL, MVT::v4f32, Legal); in X86TargetLowering()
783 setOperationAction(ISD::FDIV, MVT::v4f32, Legal); in X86TargetLowering()
784 setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); in X86TargetLowering()
785 setOperationAction(ISD::FNEG, MVT::v4f32, Custom); in X86TargetLowering()
786 setOperationAction(ISD::FABS, MVT::v4f32, Custom); in X86TargetLowering()
787 setOperationAction(ISD::LOAD, MVT::v4f32, Legal); in X86TargetLowering()
788 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); in X86TargetLowering()
789 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom); in X86TargetLowering()
790 setOperationAction(ISD::VSELECT, MVT::v4f32, Custom); in X86TargetLowering()
791 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); in X86TargetLowering()
792 setOperationAction(ISD::SELECT, MVT::v4f32, Custom); in X86TargetLowering()
793 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom); in X86TargetLowering()
806 setOperationAction(ISD::ADD, MVT::v16i8, Legal); in X86TargetLowering()
807 setOperationAction(ISD::ADD, MVT::v8i16, Legal); in X86TargetLowering()
808 setOperationAction(ISD::ADD, MVT::v4i32, Legal); in X86TargetLowering()
809 setOperationAction(ISD::ADD, MVT::v2i64, Legal); in X86TargetLowering()
810 setOperationAction(ISD::MUL, MVT::v4i32, Custom); in X86TargetLowering()
811 setOperationAction(ISD::MUL, MVT::v2i64, Custom); in X86TargetLowering()
812 setOperationAction(ISD::UMUL_LOHI, MVT::v4i32, Custom); in X86TargetLowering()
813 setOperationAction(ISD::SMUL_LOHI, MVT::v4i32, Custom); in X86TargetLowering()
814 setOperationAction(ISD::MULHU, MVT::v8i16, Legal); in X86TargetLowering()
815 setOperationAction(ISD::MULHS, MVT::v8i16, Legal); in X86TargetLowering()
816 setOperationAction(ISD::SUB, MVT::v16i8, Legal); in X86TargetLowering()
817 setOperationAction(ISD::SUB, MVT::v8i16, Legal); in X86TargetLowering()
818 setOperationAction(ISD::SUB, MVT::v4i32, Legal); in X86TargetLowering()
819 setOperationAction(ISD::SUB, MVT::v2i64, Legal); in X86TargetLowering()
820 setOperationAction(ISD::MUL, MVT::v8i16, Legal); in X86TargetLowering()
821 setOperationAction(ISD::FADD, MVT::v2f64, Legal); in X86TargetLowering()
822 setOperationAction(ISD::FSUB, MVT::v2f64, Legal); in X86TargetLowering()
823 setOperationAction(ISD::FMUL, MVT::v2f64, Legal); in X86TargetLowering()
824 setOperationAction(ISD::FDIV, MVT::v2f64, Legal); in X86TargetLowering()
825 setOperationAction(ISD::FSQRT, MVT::v2f64, Legal); in X86TargetLowering()
826 setOperationAction(ISD::FNEG, MVT::v2f64, Custom); in X86TargetLowering()
827 setOperationAction(ISD::FABS, MVT::v2f64, Custom); in X86TargetLowering()
829 setOperationAction(ISD::SETCC, MVT::v2i64, Custom); in X86TargetLowering()
830 setOperationAction(ISD::SETCC, MVT::v16i8, Custom); in X86TargetLowering()
831 setOperationAction(ISD::SETCC, MVT::v8i16, Custom); in X86TargetLowering()
832 setOperationAction(ISD::SETCC, MVT::v4i32, Custom); in X86TargetLowering()
834 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Custom); in X86TargetLowering()
835 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Custom); in X86TargetLowering()
836 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); in X86TargetLowering()
837 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); in X86TargetLowering()
838 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); in X86TargetLowering()
844 setOperationAction(ISD::CTPOP, MVT::v4i32, Custom); in X86TargetLowering()
845 setOperationAction(ISD::CTPOP, MVT::v2i64, Custom); in X86TargetLowering()
857 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in X86TargetLowering()
858 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in X86TargetLowering()
859 setOperationAction(ISD::VSELECT, VT, Custom); in X86TargetLowering()
860 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
868 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i8, Custom); in X86TargetLowering()
869 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i16, Custom); in X86TargetLowering()
870 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v8i8, Custom); in X86TargetLowering()
871 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i8, Custom); in X86TargetLowering()
872 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i16, Custom); in X86TargetLowering()
873 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i32, Custom); in X86TargetLowering()
874 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i8, Custom); in X86TargetLowering()
875 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i16, Custom); in X86TargetLowering()
876 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v8i8, Custom); in X86TargetLowering()
879 setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Custom); in X86TargetLowering()
880 setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom); in X86TargetLowering()
881 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Custom); in X86TargetLowering()
882 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Custom); in X86TargetLowering()
883 setOperationAction(ISD::VSELECT, MVT::v2f64, Custom); in X86TargetLowering()
884 setOperationAction(ISD::VSELECT, MVT::v2i64, Custom); in X86TargetLowering()
885 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2f64, Custom); in X86TargetLowering()
886 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Custom); in X86TargetLowering()
889 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Custom); in X86TargetLowering()
890 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Custom); in X86TargetLowering()
901 setOperationAction(ISD::AND, VT, Promote); in X86TargetLowering()
902 AddPromotedToType (ISD::AND, VT, MVT::v2i64); in X86TargetLowering()
903 setOperationAction(ISD::OR, VT, Promote); in X86TargetLowering()
904 AddPromotedToType (ISD::OR, VT, MVT::v2i64); in X86TargetLowering()
905 setOperationAction(ISD::XOR, VT, Promote); in X86TargetLowering()
906 AddPromotedToType (ISD::XOR, VT, MVT::v2i64); in X86TargetLowering()
907 setOperationAction(ISD::LOAD, VT, Promote); in X86TargetLowering()
908 AddPromotedToType (ISD::LOAD, VT, MVT::v2i64); in X86TargetLowering()
909 setOperationAction(ISD::SELECT, VT, Promote); in X86TargetLowering()
910 AddPromotedToType (ISD::SELECT, VT, MVT::v2i64); in X86TargetLowering()
914 setOperationAction(ISD::LOAD, MVT::v2f64, Legal); in X86TargetLowering()
915 setOperationAction(ISD::LOAD, MVT::v2i64, Legal); in X86TargetLowering()
916 setOperationAction(ISD::SELECT, MVT::v2f64, Custom); in X86TargetLowering()
917 setOperationAction(ISD::SELECT, MVT::v2i64, Custom); in X86TargetLowering()
919 setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal); in X86TargetLowering()
920 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal); in X86TargetLowering()
922 setOperationAction(ISD::UINT_TO_FP, MVT::v4i8, Custom); in X86TargetLowering()
923 setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom); in X86TargetLowering()
927 setOperationAction(ISD::UINT_TO_FP, MVT::v2f32, Custom); in X86TargetLowering()
929 setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Custom); in X86TargetLowering()
930 setOperationAction(ISD::FP_ROUND, MVT::v2f32, Custom); in X86TargetLowering()
933 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2f32, Legal); in X86TargetLowering()
935 setOperationAction(ISD::BITCAST, MVT::v2i32, Custom); in X86TargetLowering()
936 setOperationAction(ISD::BITCAST, MVT::v4i16, Custom); in X86TargetLowering()
937 setOperationAction(ISD::BITCAST, MVT::v8i8, Custom); in X86TargetLowering()
942 setOperationAction(ISD::FFLOOR, RoundedTy, Legal); in X86TargetLowering()
943 setOperationAction(ISD::FCEIL, RoundedTy, Legal); in X86TargetLowering()
944 setOperationAction(ISD::FTRUNC, RoundedTy, Legal); in X86TargetLowering()
945 setOperationAction(ISD::FRINT, RoundedTy, Legal); in X86TargetLowering()
946 setOperationAction(ISD::FNEARBYINT, RoundedTy, Legal); in X86TargetLowering()
950 setOperationAction(ISD::MUL, MVT::v4i32, Legal); in X86TargetLowering()
954 setOperationAction(ISD::VSELECT, MVT::v16i8, Legal); in X86TargetLowering()
959 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i8, Custom); in X86TargetLowering()
960 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i16, Custom); in X86TargetLowering()
961 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i32, Custom); in X86TargetLowering()
965 setLoadExtAction(ISD::SEXTLOAD, MVT::v8i16, MVT::v8i8, Legal); in X86TargetLowering()
966 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i32, MVT::v4i8, Legal); in X86TargetLowering()
967 setLoadExtAction(ISD::SEXTLOAD, MVT::v2i64, MVT::v2i8, Legal); in X86TargetLowering()
968 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i32, MVT::v4i16, Legal); in X86TargetLowering()
969 setLoadExtAction(ISD::SEXTLOAD, MVT::v2i64, MVT::v2i16, Legal); in X86TargetLowering()
970 setLoadExtAction(ISD::SEXTLOAD, MVT::v2i64, MVT::v2i32, Legal); in X86TargetLowering()
972 setLoadExtAction(ISD::ZEXTLOAD, MVT::v8i16, MVT::v8i8, Legal); in X86TargetLowering()
973 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i32, MVT::v4i8, Legal); in X86TargetLowering()
974 setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i64, MVT::v2i8, Legal); in X86TargetLowering()
975 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i32, MVT::v4i16, Legal); in X86TargetLowering()
976 setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i64, MVT::v2i16, Legal); in X86TargetLowering()
977 setLoadExtAction(ISD::ZEXTLOAD, MVT::v2i64, MVT::v2i32, Legal); in X86TargetLowering()
983 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom); in X86TargetLowering()
984 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); in X86TargetLowering()
985 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); in X86TargetLowering()
986 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); in X86TargetLowering()
988 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i8, Custom); in X86TargetLowering()
989 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i16, Custom); in X86TargetLowering()
990 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Custom); in X86TargetLowering()
991 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom); in X86TargetLowering()
996 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v2i64, Custom); in X86TargetLowering()
997 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Custom); in X86TargetLowering()
1002 setOperationAction(ISD::SRL, MVT::v8i16, Custom); in X86TargetLowering()
1003 setOperationAction(ISD::SRL, MVT::v16i8, Custom); in X86TargetLowering()
1005 setOperationAction(ISD::SHL, MVT::v8i16, Custom); in X86TargetLowering()
1006 setOperationAction(ISD::SHL, MVT::v16i8, Custom); in X86TargetLowering()
1008 setOperationAction(ISD::SRA, MVT::v8i16, Custom); in X86TargetLowering()
1009 setOperationAction(ISD::SRA, MVT::v16i8, Custom); in X86TargetLowering()
1013 setOperationAction(ISD::SRL, MVT::v2i64, Custom); in X86TargetLowering()
1014 setOperationAction(ISD::SRL, MVT::v4i32, Custom); in X86TargetLowering()
1016 setOperationAction(ISD::SHL, MVT::v2i64, Custom); in X86TargetLowering()
1017 setOperationAction(ISD::SHL, MVT::v4i32, Custom); in X86TargetLowering()
1019 setOperationAction(ISD::SRA, MVT::v4i32, Custom); in X86TargetLowering()
1030 setOperationAction(ISD::LOAD, MVT::v8f32, Legal); in X86TargetLowering()
1031 setOperationAction(ISD::LOAD, MVT::v4f64, Legal); in X86TargetLowering()
1032 setOperationAction(ISD::LOAD, MVT::v4i64, Legal); in X86TargetLowering()
1034 setOperationAction(ISD::FADD, MVT::v8f32, Legal); in X86TargetLowering()
1035 setOperationAction(ISD::FSUB, MVT::v8f32, Legal); in X86TargetLowering()
1036 setOperationAction(ISD::FMUL, MVT::v8f32, Legal); in X86TargetLowering()
1037 setOperationAction(ISD::FDIV, MVT::v8f32, Legal); in X86TargetLowering()
1038 setOperationAction(ISD::FSQRT, MVT::v8f32, Legal); in X86TargetLowering()
1039 setOperationAction(ISD::FFLOOR, MVT::v8f32, Legal); in X86TargetLowering()
1040 setOperationAction(ISD::FCEIL, MVT::v8f32, Legal); in X86TargetLowering()
1041 setOperationAction(ISD::FTRUNC, MVT::v8f32, Legal); in X86TargetLowering()
1042 setOperationAction(ISD::FRINT, MVT::v8f32, Legal); in X86TargetLowering()
1043 setOperationAction(ISD::FNEARBYINT, MVT::v8f32, Legal); in X86TargetLowering()
1044 setOperationAction(ISD::FNEG, MVT::v8f32, Custom); in X86TargetLowering()
1045 setOperationAction(ISD::FABS, MVT::v8f32, Custom); in X86TargetLowering()
1047 setOperationAction(ISD::FADD, MVT::v4f64, Legal); in X86TargetLowering()
1048 setOperationAction(ISD::FSUB, MVT::v4f64, Legal); in X86TargetLowering()
1049 setOperationAction(ISD::FMUL, MVT::v4f64, Legal); in X86TargetLowering()
1050 setOperationAction(ISD::FDIV, MVT::v4f64, Legal); in X86TargetLowering()
1051 setOperationAction(ISD::FSQRT, MVT::v4f64, Legal); in X86TargetLowering()
1052 setOperationAction(ISD::FFLOOR, MVT::v4f64, Legal); in X86TargetLowering()
1053 setOperationAction(ISD::FCEIL, MVT::v4f64, Legal); in X86TargetLowering()
1054 setOperationAction(ISD::FTRUNC, MVT::v4f64, Legal); in X86TargetLowering()
1055 setOperationAction(ISD::FRINT, MVT::v4f64, Legal); in X86TargetLowering()
1056 setOperationAction(ISD::FNEARBYINT, MVT::v4f64, Legal); in X86TargetLowering()
1057 setOperationAction(ISD::FNEG, MVT::v4f64, Custom); in X86TargetLowering()
1058 setOperationAction(ISD::FABS, MVT::v4f64, Custom); in X86TargetLowering()
1062 setOperationAction(ISD::FP_TO_SINT, MVT::v8i16, Promote); in X86TargetLowering()
1063 setOperationAction(ISD::FP_TO_UINT, MVT::v8i16, Promote); in X86TargetLowering()
1064 setOperationAction(ISD::FP_TO_SINT, MVT::v8i32, Legal); in X86TargetLowering()
1066 setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Promote); in X86TargetLowering()
1067 setOperationAction(ISD::SINT_TO_FP, MVT::v8i32, Legal); in X86TargetLowering()
1068 setOperationAction(ISD::FP_ROUND, MVT::v4f32, Legal); in X86TargetLowering()
1070 setOperationAction(ISD::UINT_TO_FP, MVT::v8i8, Custom); in X86TargetLowering()
1071 setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Custom); in X86TargetLowering()
1074 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4f32, Legal); in X86TargetLowering()
1076 setOperationAction(ISD::SRL, MVT::v16i16, Custom); in X86TargetLowering()
1077 setOperationAction(ISD::SRL, MVT::v32i8, Custom); in X86TargetLowering()
1079 setOperationAction(ISD::SHL, MVT::v16i16, Custom); in X86TargetLowering()
1080 setOperationAction(ISD::SHL, MVT::v32i8, Custom); in X86TargetLowering()
1082 setOperationAction(ISD::SRA, MVT::v16i16, Custom); in X86TargetLowering()
1083 setOperationAction(ISD::SRA, MVT::v32i8, Custom); in X86TargetLowering()
1085 setOperationAction(ISD::SETCC, MVT::v32i8, Custom); in X86TargetLowering()
1086 setOperationAction(ISD::SETCC, MVT::v16i16, Custom); in X86TargetLowering()
1087 setOperationAction(ISD::SETCC, MVT::v8i32, Custom); in X86TargetLowering()
1088 setOperationAction(ISD::SETCC, MVT::v4i64, Custom); in X86TargetLowering()
1090 setOperationAction(ISD::SELECT, MVT::v4f64, Custom); in X86TargetLowering()
1091 setOperationAction(ISD::SELECT, MVT::v4i64, Custom); in X86TargetLowering()
1092 setOperationAction(ISD::SELECT, MVT::v8f32, Custom); in X86TargetLowering()
1094 setOperationAction(ISD::SIGN_EXTEND, MVT::v4i64, Custom); in X86TargetLowering()
1095 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i32, Custom); in X86TargetLowering()
1096 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i16, Custom); in X86TargetLowering()
1097 setOperationAction(ISD::ZERO_EXTEND, MVT::v4i64, Custom); in X86TargetLowering()
1098 setOperationAction(ISD::ZERO_EXTEND, MVT::v8i32, Custom); in X86TargetLowering()
1099 setOperationAction(ISD::ZERO_EXTEND, MVT::v16i16, Custom); in X86TargetLowering()
1100 setOperationAction(ISD::ANY_EXTEND, MVT::v4i64, Custom); in X86TargetLowering()
1101 setOperationAction(ISD::ANY_EXTEND, MVT::v8i32, Custom); in X86TargetLowering()
1102 setOperationAction(ISD::ANY_EXTEND, MVT::v16i16, Custom); in X86TargetLowering()
1103 setOperationAction(ISD::TRUNCATE, MVT::v16i8, Custom); in X86TargetLowering()
1104 setOperationAction(ISD::TRUNCATE, MVT::v8i16, Custom); in X86TargetLowering()
1105 setOperationAction(ISD::TRUNCATE, MVT::v4i32, Custom); in X86TargetLowering()
1108 setOperationAction(ISD::FMA, MVT::v8f32, Legal); in X86TargetLowering()
1109 setOperationAction(ISD::FMA, MVT::v4f64, Legal); in X86TargetLowering()
1110 setOperationAction(ISD::FMA, MVT::v4f32, Legal); in X86TargetLowering()
1111 setOperationAction(ISD::FMA, MVT::v2f64, Legal); in X86TargetLowering()
1112 setOperationAction(ISD::FMA, MVT::f32, Legal); in X86TargetLowering()
1113 setOperationAction(ISD::FMA, MVT::f64, Legal); in X86TargetLowering()
1117 setOperationAction(ISD::ADD, MVT::v4i64, Legal); in X86TargetLowering()
1118 setOperationAction(ISD::ADD, MVT::v8i32, Legal); in X86TargetLowering()
1119 setOperationAction(ISD::ADD, MVT::v16i16, Legal); in X86TargetLowering()
1120 setOperationAction(ISD::ADD, MVT::v32i8, Legal); in X86TargetLowering()
1122 setOperationAction(ISD::SUB, MVT::v4i64, Legal); in X86TargetLowering()
1123 setOperationAction(ISD::SUB, MVT::v8i32, Legal); in X86TargetLowering()
1124 setOperationAction(ISD::SUB, MVT::v16i16, Legal); in X86TargetLowering()
1125 setOperationAction(ISD::SUB, MVT::v32i8, Legal); in X86TargetLowering()
1127 setOperationAction(ISD::MUL, MVT::v4i64, Custom); in X86TargetLowering()
1128 setOperationAction(ISD::MUL, MVT::v8i32, Legal); in X86TargetLowering()
1129 setOperationAction(ISD::MUL, MVT::v16i16, Legal); in X86TargetLowering()
1132 setOperationAction(ISD::UMUL_LOHI, MVT::v8i32, Custom); in X86TargetLowering()
1133 setOperationAction(ISD::SMUL_LOHI, MVT::v8i32, Custom); in X86TargetLowering()
1134 setOperationAction(ISD::MULHU, MVT::v16i16, Legal); in X86TargetLowering()
1135 setOperationAction(ISD::MULHS, MVT::v16i16, Legal); in X86TargetLowering()
1139 setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Custom); in X86TargetLowering()
1146 setOperationAction(ISD::CTPOP, MVT::v4i64, Custom); in X86TargetLowering()
1149 setOperationAction(ISD::CTPOP, MVT::v8i32, Custom); in X86TargetLowering()
1152 setLoadExtAction(ISD::SEXTLOAD, MVT::v16i16, MVT::v16i8, Legal); in X86TargetLowering()
1153 setLoadExtAction(ISD::SEXTLOAD, MVT::v8i32, MVT::v8i8, Legal); in X86TargetLowering()
1154 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i64, MVT::v4i8, Legal); in X86TargetLowering()
1155 setLoadExtAction(ISD::SEXTLOAD, MVT::v8i32, MVT::v8i16, Legal); in X86TargetLowering()
1156 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i64, MVT::v4i16, Legal); in X86TargetLowering()
1157 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i64, MVT::v4i32, Legal); in X86TargetLowering()
1159 setLoadExtAction(ISD::ZEXTLOAD, MVT::v16i16, MVT::v16i8, Legal); in X86TargetLowering()
1160 setLoadExtAction(ISD::ZEXTLOAD, MVT::v8i32, MVT::v8i8, Legal); in X86TargetLowering()
1161 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i64, MVT::v4i8, Legal); in X86TargetLowering()
1162 setLoadExtAction(ISD::ZEXTLOAD, MVT::v8i32, MVT::v8i16, Legal); in X86TargetLowering()
1163 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i64, MVT::v4i16, Legal); in X86TargetLowering()
1164 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i64, MVT::v4i32, Legal); in X86TargetLowering()
1166 setOperationAction(ISD::ADD, MVT::v4i64, Custom); in X86TargetLowering()
1167 setOperationAction(ISD::ADD, MVT::v8i32, Custom); in X86TargetLowering()
1168 setOperationAction(ISD::ADD, MVT::v16i16, Custom); in X86TargetLowering()
1169 setOperationAction(ISD::ADD, MVT::v32i8, Custom); in X86TargetLowering()
1171 setOperationAction(ISD::SUB, MVT::v4i64, Custom); in X86TargetLowering()
1172 setOperationAction(ISD::SUB, MVT::v8i32, Custom); in X86TargetLowering()
1173 setOperationAction(ISD::SUB, MVT::v16i16, Custom); in X86TargetLowering()
1174 setOperationAction(ISD::SUB, MVT::v32i8, Custom); in X86TargetLowering()
1176 setOperationAction(ISD::MUL, MVT::v4i64, Custom); in X86TargetLowering()
1177 setOperationAction(ISD::MUL, MVT::v8i32, Custom); in X86TargetLowering()
1178 setOperationAction(ISD::MUL, MVT::v16i16, Custom); in X86TargetLowering()
1184 setOperationAction(ISD::SRL, MVT::v4i64, Custom); in X86TargetLowering()
1185 setOperationAction(ISD::SRL, MVT::v8i32, Custom); in X86TargetLowering()
1187 setOperationAction(ISD::SHL, MVT::v4i64, Custom); in X86TargetLowering()
1188 setOperationAction(ISD::SHL, MVT::v8i32, Custom); in X86TargetLowering()
1190 setOperationAction(ISD::SRA, MVT::v8i32, Custom); in X86TargetLowering()
1195 setOperationAction(ISD::MLOAD, VT, Legal); in X86TargetLowering()
1196 setOperationAction(ISD::MSTORE, VT, Legal); in X86TargetLowering()
1201 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom); in X86TargetLowering()
1207 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in X86TargetLowering()
1208 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in X86TargetLowering()
1209 setOperationAction(ISD::VSELECT, VT, Custom); in X86TargetLowering()
1210 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
1211 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
1212 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom); in X86TargetLowering()
1213 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); in X86TargetLowering()
1214 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom); in X86TargetLowering()
1218 setOperationAction(ISD::VSELECT, MVT::v32i8, Legal); in X86TargetLowering()
1229 setOperationAction(ISD::AND, VT, Promote); in X86TargetLowering()
1230 AddPromotedToType (ISD::AND, VT, MVT::v4i64); in X86TargetLowering()
1231 setOperationAction(ISD::OR, VT, Promote); in X86TargetLowering()
1232 AddPromotedToType (ISD::OR, VT, MVT::v4i64); in X86TargetLowering()
1233 setOperationAction(ISD::XOR, VT, Promote); in X86TargetLowering()
1234 AddPromotedToType (ISD::XOR, VT, MVT::v4i64); in X86TargetLowering()
1235 setOperationAction(ISD::LOAD, VT, Promote); in X86TargetLowering()
1236 AddPromotedToType (ISD::LOAD, VT, MVT::v4i64); in X86TargetLowering()
1237 setOperationAction(ISD::SELECT, VT, Promote); in X86TargetLowering()
1238 AddPromotedToType (ISD::SELECT, VT, MVT::v4i64); in X86TargetLowering()
1253 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v8f32, Legal); in X86TargetLowering()
1255 setOperationAction(ISD::BR_CC, MVT::i1, Expand); in X86TargetLowering()
1256 setOperationAction(ISD::SETCC, MVT::i1, Custom); in X86TargetLowering()
1257 setOperationAction(ISD::XOR, MVT::i1, Legal); in X86TargetLowering()
1258 setOperationAction(ISD::OR, MVT::i1, Legal); in X86TargetLowering()
1259 setOperationAction(ISD::AND, MVT::i1, Legal); in X86TargetLowering()
1260 setOperationAction(ISD::LOAD, MVT::v16f32, Legal); in X86TargetLowering()
1261 setOperationAction(ISD::LOAD, MVT::v8f64, Legal); in X86TargetLowering()
1262 setOperationAction(ISD::LOAD, MVT::v8i64, Legal); in X86TargetLowering()
1263 setOperationAction(ISD::LOAD, MVT::v16i32, Legal); in X86TargetLowering()
1264 setOperationAction(ISD::LOAD, MVT::v16i1, Legal); in X86TargetLowering()
1266 setOperationAction(ISD::FADD, MVT::v16f32, Legal); in X86TargetLowering()
1267 setOperationAction(ISD::FSUB, MVT::v16f32, Legal); in X86TargetLowering()
1268 setOperationAction(ISD::FMUL, MVT::v16f32, Legal); in X86TargetLowering()
1269 setOperationAction(ISD::FDIV, MVT::v16f32, Legal); in X86TargetLowering()
1270 setOperationAction(ISD::FSQRT, MVT::v16f32, Legal); in X86TargetLowering()
1271 setOperationAction(ISD::FNEG, MVT::v16f32, Custom); in X86TargetLowering()
1273 setOperationAction(ISD::FADD, MVT::v8f64, Legal); in X86TargetLowering()
1274 setOperationAction(ISD::FSUB, MVT::v8f64, Legal); in X86TargetLowering()
1275 setOperationAction(ISD::FMUL, MVT::v8f64, Legal); in X86TargetLowering()
1276 setOperationAction(ISD::FDIV, MVT::v8f64, Legal); in X86TargetLowering()
1277 setOperationAction(ISD::FSQRT, MVT::v8f64, Legal); in X86TargetLowering()
1278 setOperationAction(ISD::FNEG, MVT::v8f64, Custom); in X86TargetLowering()
1279 setOperationAction(ISD::FMA, MVT::v8f64, Legal); in X86TargetLowering()
1280 setOperationAction(ISD::FMA, MVT::v16f32, Legal); in X86TargetLowering()
1282 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Legal); in X86TargetLowering()
1283 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Legal); in X86TargetLowering()
1284 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Legal); in X86TargetLowering()
1285 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Legal); in X86TargetLowering()
1287 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Legal); in X86TargetLowering()
1288 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Legal); in X86TargetLowering()
1289 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Legal); in X86TargetLowering()
1290 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Legal); in X86TargetLowering()
1292 setOperationAction(ISD::FP_TO_SINT, MVT::v16i32, Legal); in X86TargetLowering()
1293 setOperationAction(ISD::FP_TO_UINT, MVT::v16i32, Legal); in X86TargetLowering()
1294 setOperationAction(ISD::FP_TO_UINT, MVT::v8i32, Legal); in X86TargetLowering()
1295 setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Legal); in X86TargetLowering()
1296 setOperationAction(ISD::SINT_TO_FP, MVT::v16i32, Legal); in X86TargetLowering()
1297 setOperationAction(ISD::SINT_TO_FP, MVT::v8i1, Custom); in X86TargetLowering()
1298 setOperationAction(ISD::SINT_TO_FP, MVT::v16i1, Custom); in X86TargetLowering()
1299 setOperationAction(ISD::SINT_TO_FP, MVT::v16i8, Promote); in X86TargetLowering()
1300 setOperationAction(ISD::SINT_TO_FP, MVT::v16i16, Promote); in X86TargetLowering()
1301 setOperationAction(ISD::UINT_TO_FP, MVT::v16i32, Legal); in X86TargetLowering()
1302 setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Legal); in X86TargetLowering()
1303 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Legal); in X86TargetLowering()
1304 setOperationAction(ISD::FP_ROUND, MVT::v8f32, Legal); in X86TargetLowering()
1305 setOperationAction(ISD::FP_EXTEND, MVT::v8f32, Legal); in X86TargetLowering()
1307 setOperationAction(ISD::TRUNCATE, MVT::i1, Custom); in X86TargetLowering()
1308 setOperationAction(ISD::TRUNCATE, MVT::v16i8, Custom); in X86TargetLowering()
1309 setOperationAction(ISD::TRUNCATE, MVT::v8i32, Custom); in X86TargetLowering()
1310 setOperationAction(ISD::TRUNCATE, MVT::v8i1, Custom); in X86TargetLowering()
1311 setOperationAction(ISD::TRUNCATE, MVT::v16i1, Custom); in X86TargetLowering()
1312 setOperationAction(ISD::TRUNCATE, MVT::v16i16, Custom); in X86TargetLowering()
1313 setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom); in X86TargetLowering()
1314 setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom); in X86TargetLowering()
1315 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom); in X86TargetLowering()
1316 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64, Custom); in X86TargetLowering()
1317 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i8, Custom); in X86TargetLowering()
1318 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i16, Custom); in X86TargetLowering()
1319 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i16, Custom); in X86TargetLowering()
1321 setOperationAction(ISD::FFLOOR, MVT::v16f32, Legal); in X86TargetLowering()
1322 setOperationAction(ISD::FFLOOR, MVT::v8f64, Legal); in X86TargetLowering()
1323 setOperationAction(ISD::FCEIL, MVT::v16f32, Legal); in X86TargetLowering()
1324 setOperationAction(ISD::FCEIL, MVT::v8f64, Legal); in X86TargetLowering()
1325 setOperationAction(ISD::FTRUNC, MVT::v16f32, Legal); in X86TargetLowering()
1326 setOperationAction(ISD::FTRUNC, MVT::v8f64, Legal); in X86TargetLowering()
1327 setOperationAction(ISD::FRINT, MVT::v16f32, Legal); in X86TargetLowering()
1328 setOperationAction(ISD::FRINT, MVT::v8f64, Legal); in X86TargetLowering()
1329 setOperationAction(ISD::FNEARBYINT, MVT::v16f32, Legal); in X86TargetLowering()
1330 setOperationAction(ISD::FNEARBYINT, MVT::v8f64, Legal); in X86TargetLowering()
1332 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8f64, Custom); in X86TargetLowering()
1333 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i64, Custom); in X86TargetLowering()
1334 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16f32, Custom); in X86TargetLowering()
1335 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16i32, Custom); in X86TargetLowering()
1336 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16i1, Legal); in X86TargetLowering()
1338 setOperationAction(ISD::SETCC, MVT::v16i1, Custom); in X86TargetLowering()
1339 setOperationAction(ISD::SETCC, MVT::v8i1, Custom); in X86TargetLowering()
1341 setOperationAction(ISD::MUL, MVT::v8i64, Custom); in X86TargetLowering()
1343 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i1, Custom); in X86TargetLowering()
1344 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i1, Custom); in X86TargetLowering()
1345 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i1, Custom); in X86TargetLowering()
1346 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i1, Custom); in X86TargetLowering()
1347 setOperationAction(ISD::BUILD_VECTOR, MVT::v8i1, Custom); in X86TargetLowering()
1348 setOperationAction(ISD::BUILD_VECTOR, MVT::v16i1, Custom); in X86TargetLowering()
1349 setOperationAction(ISD::SELECT, MVT::v8f64, Custom); in X86TargetLowering()
1350 setOperationAction(ISD::SELECT, MVT::v8i64, Custom); in X86TargetLowering()
1351 setOperationAction(ISD::SELECT, MVT::v16f32, Custom); in X86TargetLowering()
1353 setOperationAction(ISD::ADD, MVT::v8i64, Legal); in X86TargetLowering()
1354 setOperationAction(ISD::ADD, MVT::v16i32, Legal); in X86TargetLowering()
1356 setOperationAction(ISD::SUB, MVT::v8i64, Legal); in X86TargetLowering()
1357 setOperationAction(ISD::SUB, MVT::v16i32, Legal); in X86TargetLowering()
1359 setOperationAction(ISD::MUL, MVT::v16i32, Legal); in X86TargetLowering()
1361 setOperationAction(ISD::SRL, MVT::v8i64, Custom); in X86TargetLowering()
1362 setOperationAction(ISD::SRL, MVT::v16i32, Custom); in X86TargetLowering()
1364 setOperationAction(ISD::SHL, MVT::v8i64, Custom); in X86TargetLowering()
1365 setOperationAction(ISD::SHL, MVT::v16i32, Custom); in X86TargetLowering()
1367 setOperationAction(ISD::SRA, MVT::v8i64, Custom); in X86TargetLowering()
1368 setOperationAction(ISD::SRA, MVT::v16i32, Custom); in X86TargetLowering()
1370 setOperationAction(ISD::AND, MVT::v8i64, Legal); in X86TargetLowering()
1371 setOperationAction(ISD::OR, MVT::v8i64, Legal); in X86TargetLowering()
1372 setOperationAction(ISD::XOR, MVT::v8i64, Legal); in X86TargetLowering()
1373 setOperationAction(ISD::AND, MVT::v16i32, Legal); in X86TargetLowering()
1374 setOperationAction(ISD::OR, MVT::v16i32, Legal); in X86TargetLowering()
1375 setOperationAction(ISD::XOR, MVT::v16i32, Legal); in X86TargetLowering()
1378 setOperationAction(ISD::CTLZ, MVT::v8i64, Legal); in X86TargetLowering()
1379 setOperationAction(ISD::CTLZ, MVT::v16i32, Legal); in X86TargetLowering()
1388 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom); in X86TargetLowering()
1391 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal); in X86TargetLowering()
1398 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in X86TargetLowering()
1399 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
1400 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in X86TargetLowering()
1401 setOperationAction(ISD::VSELECT, VT, Legal); in X86TargetLowering()
1402 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
1403 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom); in X86TargetLowering()
1404 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); in X86TargetLowering()
1405 setOperationAction(ISD::MLOAD, VT, Legal); in X86TargetLowering()
1406 setOperationAction(ISD::MSTORE, VT, Legal); in X86TargetLowering()
1416 setOperationAction(ISD::SELECT, VT, Promote); in X86TargetLowering()
1417 AddPromotedToType (ISD::SELECT, VT, MVT::v8i64); in X86TargetLowering()
1428 setOperationAction(ISD::LOAD, MVT::v32i16, Legal); in X86TargetLowering()
1429 setOperationAction(ISD::LOAD, MVT::v64i8, Legal); in X86TargetLowering()
1430 setOperationAction(ISD::SETCC, MVT::v32i1, Custom); in X86TargetLowering()
1431 setOperationAction(ISD::SETCC, MVT::v64i1, Custom); in X86TargetLowering()
1432 setOperationAction(ISD::ADD, MVT::v32i16, Legal); in X86TargetLowering()
1433 setOperationAction(ISD::ADD, MVT::v64i8, Legal); in X86TargetLowering()
1434 setOperationAction(ISD::SUB, MVT::v32i16, Legal); in X86TargetLowering()
1435 setOperationAction(ISD::SUB, MVT::v64i8, Legal); in X86TargetLowering()
1436 setOperationAction(ISD::MUL, MVT::v32i16, Legal); in X86TargetLowering()
1437 setOperationAction(ISD::CONCAT_VECTORS, MVT::v32i1, Custom); in X86TargetLowering()
1438 setOperationAction(ISD::CONCAT_VECTORS, MVT::v64i1, Custom); in X86TargetLowering()
1439 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v32i1, Custom); in X86TargetLowering()
1440 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v64i1, Custom); in X86TargetLowering()
1452 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in X86TargetLowering()
1453 setOperationAction(ISD::VSELECT, VT, Legal); in X86TargetLowering()
1462 setOperationAction(ISD::SETCC, MVT::v4i1, Custom); in X86TargetLowering()
1463 setOperationAction(ISD::SETCC, MVT::v2i1, Custom); in X86TargetLowering()
1464 setOperationAction(ISD::CONCAT_VECTORS, MVT::v4i1, Custom); in X86TargetLowering()
1465 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i1, Custom); in X86TargetLowering()
1466 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v8i1, Custom); in X86TargetLowering()
1467 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v4i1, Custom); in X86TargetLowering()
1469 setOperationAction(ISD::AND, MVT::v8i32, Legal); in X86TargetLowering()
1470 setOperationAction(ISD::OR, MVT::v8i32, Legal); in X86TargetLowering()
1471 setOperationAction(ISD::XOR, MVT::v8i32, Legal); in X86TargetLowering()
1472 setOperationAction(ISD::AND, MVT::v4i32, Legal); in X86TargetLowering()
1473 setOperationAction(ISD::OR, MVT::v4i32, Legal); in X86TargetLowering()
1474 setOperationAction(ISD::XOR, MVT::v4i32, Legal); in X86TargetLowering()
1478 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); in X86TargetLowering()
1479 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom); in X86TargetLowering()
1480 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); in X86TargetLowering()
1482 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom); in X86TargetLowering()
1493 setOperationAction(ISD::SADDO, VT, Custom); in X86TargetLowering()
1494 setOperationAction(ISD::UADDO, VT, Custom); in X86TargetLowering()
1495 setOperationAction(ISD::SSUBO, VT, Custom); in X86TargetLowering()
1496 setOperationAction(ISD::USUBO, VT, Custom); in X86TargetLowering()
1497 setOperationAction(ISD::SMULO, VT, Custom); in X86TargetLowering()
1498 setOperationAction(ISD::UMULO, VT, Custom); in X86TargetLowering()
1516 setOperationAction(ISD::FSINCOS, MVT::f64, Custom); in X86TargetLowering()
1517 setOperationAction(ISD::FSINCOS, MVT::f32, Custom); in X86TargetLowering()
1522 setOperationAction(ISD::SDIV, MVT::i128, Custom); in X86TargetLowering()
1523 setOperationAction(ISD::UDIV, MVT::i128, Custom); in X86TargetLowering()
1524 setOperationAction(ISD::SREM, MVT::i128, Custom); in X86TargetLowering()
1525 setOperationAction(ISD::UREM, MVT::i128, Custom); in X86TargetLowering()
1526 setOperationAction(ISD::SDIVREM, MVT::i128, Custom); in X86TargetLowering()
1527 setOperationAction(ISD::UDIVREM, MVT::i128, Custom); in X86TargetLowering()
1531 setTargetDAGCombine(ISD::VECTOR_SHUFFLE); in X86TargetLowering()
1532 setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT); in X86TargetLowering()
1533 setTargetDAGCombine(ISD::BITCAST); in X86TargetLowering()
1534 setTargetDAGCombine(ISD::VSELECT); in X86TargetLowering()
1535 setTargetDAGCombine(ISD::SELECT); in X86TargetLowering()
1536 setTargetDAGCombine(ISD::SHL); in X86TargetLowering()
1537 setTargetDAGCombine(ISD::SRA); in X86TargetLowering()
1538 setTargetDAGCombine(ISD::SRL); in X86TargetLowering()
1539 setTargetDAGCombine(ISD::OR); in X86TargetLowering()
1540 setTargetDAGCombine(ISD::AND); in X86TargetLowering()
1541 setTargetDAGCombine(ISD::ADD); in X86TargetLowering()
1542 setTargetDAGCombine(ISD::FADD); in X86TargetLowering()
1543 setTargetDAGCombine(ISD::FSUB); in X86TargetLowering()
1544 setTargetDAGCombine(ISD::FMA); in X86TargetLowering()
1545 setTargetDAGCombine(ISD::SUB); in X86TargetLowering()
1546 setTargetDAGCombine(ISD::LOAD); in X86TargetLowering()
1547 setTargetDAGCombine(ISD::MLOAD); in X86TargetLowering()
1548 setTargetDAGCombine(ISD::STORE); in X86TargetLowering()
1549 setTargetDAGCombine(ISD::MSTORE); in X86TargetLowering()
1550 setTargetDAGCombine(ISD::ZERO_EXTEND); in X86TargetLowering()
1551 setTargetDAGCombine(ISD::ANY_EXTEND); in X86TargetLowering()
1552 setTargetDAGCombine(ISD::SIGN_EXTEND); in X86TargetLowering()
1553 setTargetDAGCombine(ISD::SIGN_EXTEND_INREG); in X86TargetLowering()
1554 setTargetDAGCombine(ISD::TRUNCATE); in X86TargetLowering()
1555 setTargetDAGCombine(ISD::SINT_TO_FP); in X86TargetLowering()
1556 setTargetDAGCombine(ISD::SETCC); in X86TargetLowering()
1557 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN); in X86TargetLowering()
1558 setTargetDAGCombine(ISD::BUILD_VECTOR); in X86TargetLowering()
1559 setTargetDAGCombine(ISD::MUL); in X86TargetLowering()
1560 setTargetDAGCombine(ISD::XOR); in X86TargetLowering()
1859 const SmallVectorImpl<ISD::OutputArg> &Outs, in CanLowerReturn()
1874 const SmallVectorImpl<ISD::OutputArg> &Outs, in LowerReturn()
1900 ValToCopy = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), ValToCopy); in LowerReturn()
1902 ValToCopy = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), ValToCopy); in LowerReturn()
1904 ValToCopy = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), ValToCopy); in LowerReturn()
1906 ValToCopy = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), ValToCopy); in LowerReturn()
1932 ValToCopy = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f80, ValToCopy); in LowerReturn()
1943 ValToCopy = DAG.getNode(ISD::BITCAST, dl, MVT::i64, ValToCopy); in LowerReturn()
1944 ValToCopy = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, in LowerReturn()
1949 ValToCopy = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32,ValToCopy); in LowerReturn()
2001 if (Copy->getOpcode() == ISD::CopyToReg) { in isUsedByReturnOnly()
2007 } else if (Copy->getOpcode() != ISD::FP_EXTEND) in isUsedByReturnOnly()
2034 ISD::NodeType ExtendKind) const { in getTypeForExtArgOrReturn()
2037 if (Subtarget->is64Bit() && VT == MVT::i1 && ExtendKind == ISD::ZERO_EXTEND) in getTypeForExtArgOrReturn()
2052 const SmallVectorImpl<ISD::InputArg> &Ins, in LowerCallResult()
2085 Val = DAG.getNode(ISD::FP_ROUND, dl, VA.getValVT(), Val, in LowerCallResult()
2114 callIsStructReturn(const SmallVectorImpl<ISD::OutputArg> &Outs) { in callIsStructReturn()
2118 const ISD::ArgFlagsTy &Flags = Outs[0].Flags; in callIsStructReturn()
2128 argsAreStructReturn(const SmallVectorImpl<ISD::InputArg> &Ins) { in argsAreStructReturn()
2132 const ISD::ArgFlagsTy &Flags = Ins[0].Flags; in argsAreStructReturn()
2145 ISD::ArgFlagsTy Flags, SelectionDAG &DAG, in CreateCopyOfByValArgument()
2190 const SmallVectorImpl<ISD::InputArg> &Ins, in LowerMemArgument()
2196 ISD::ArgFlagsTy Flags = Ins[i].Flags; in LowerMemArgument()
2280 const SmallVectorImpl<ISD::InputArg> &Ins, in LowerFormalArguments()
2362 ArgValue = DAG.getNode(ISD::AssertSext, dl, RegVT, ArgValue, in LowerFormalArguments()
2365 ArgValue = DAG.getNode(ISD::AssertZext, dl, RegVT, ArgValue, in LowerFormalArguments()
2368 ArgValue = DAG.getNode(ISD::BITCAST, dl, VA.getValVT(), ArgValue); in LowerFormalArguments()
2375 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue); in LowerFormalArguments()
2405 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Copy, Chain); in LowerFormalArguments()
2494 SDValue FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(), RSFIN, in LowerFormalArguments()
2521 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); in LowerFormalArguments()
2628 ISD::ArgFlagsTy Flags) const { in LowerMemOpCallTo()
2631 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, PtrOff); in LowerMemOpCallTo()
2681 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; in LowerCall()
2683 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; in LowerCall()
2793 ISD::ArgFlagsTy Flags = Outs[i].Flags; in LowerCall()
2807 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, RegVT, Arg); in LowerCall()
2810 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, RegVT, Arg); in LowerCall()
2815 Arg = DAG.getNode(ISD::BITCAST, dl, MVT::i64, Arg); in LowerCall()
2816 Arg = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Arg); in LowerCall()
2819 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, RegVT, Arg); in LowerCall()
2822 Arg = DAG.getNode(ISD::BITCAST, dl, RegVT, Arg); in LowerCall()
2862 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); in LowerCall()
2940 ISD::ArgFlagsTy Flags = Outs[i].Flags; in LowerCall()
2957 Source = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, Source); in LowerCall()
2972 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains2); in LowerCall()
2995 } else if (Callee->getOpcode() == ISD::GlobalAddress) { in LowerCall()
3007 unsigned WrapperKind = ISD::DELETED_NODE; in LowerCall()
3039 if (WrapperKind != ISD::DELETED_NODE) in LowerCall()
3069 Callee = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Callee); in LowerCall()
3206 bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags, in MatchingStackOffset()
3211 if (Arg.getOpcode() == ISD::CopyFromReg) { in MatchingStackOffset()
3244 } else if (Arg.getOpcode() == ISD::FrameIndex && Flags.isByVal()) { in MatchingStackOffset()
3267 const SmallVectorImpl<ISD::OutputArg> &Outs, in IsEligibleForTailCallOptimization()
3269 const SmallVectorImpl<ISD::InputArg> &Ins, in IsEligibleForTailCallOptimization()
3422 ISD::ArgFlagsTy Flags = Outs[i].Flags; in IsEligibleForTailCallOptimization()
3478 return Op.hasOneUse() && ISD::isNormalLoad(Op.getNode()); in MayFoldLoad()
3482 return Op.hasOneUse() && ISD::isNormalStore(*Op.getNode()->use_begin()); in MayFoldIntoStore()
3634 static unsigned TranslateX86CC(ISD::CondCode SetCCOpcode, bool isFP, in TranslateX86CC()
3638 if (SetCCOpcode == ISD::SETGT && RHSC->isAllOnesValue()) { in TranslateX86CC()
3643 if (SetCCOpcode == ISD::SETLT && RHSC->isNullValue()) { in TranslateX86CC()
3647 if (SetCCOpcode == ISD::SETLT && RHSC->getZExtValue() == 1) { in TranslateX86CC()
3656 case ISD::SETEQ: return X86::COND_E; in TranslateX86CC()
3657 case ISD::SETGT: return X86::COND_G; in TranslateX86CC()
3658 case ISD::SETGE: return X86::COND_GE; in TranslateX86CC()
3659 case ISD::SETLT: return X86::COND_L; in TranslateX86CC()
3660 case ISD::SETLE: return X86::COND_LE; in TranslateX86CC()
3661 case ISD::SETNE: return X86::COND_NE; in TranslateX86CC()
3662 case ISD::SETULT: return X86::COND_B; in TranslateX86CC()
3663 case ISD::SETUGT: return X86::COND_A; in TranslateX86CC()
3664 case ISD::SETULE: return X86::COND_BE; in TranslateX86CC()
3665 case ISD::SETUGE: return X86::COND_AE; in TranslateX86CC()
3672 if (ISD::isNON_EXTLoad(LHS.getNode()) && in TranslateX86CC()
3673 !ISD::isNON_EXTLoad(RHS.getNode())) { in TranslateX86CC()
3680 case ISD::SETOLT: in TranslateX86CC()
3681 case ISD::SETOLE: in TranslateX86CC()
3682 case ISD::SETUGT: in TranslateX86CC()
3683 case ISD::SETUGE: in TranslateX86CC()
3696 case ISD::SETUEQ: in TranslateX86CC()
3697 case ISD::SETEQ: return X86::COND_E; in TranslateX86CC()
3698 case ISD::SETOLT: // flipped in TranslateX86CC()
3699 case ISD::SETOGT: in TranslateX86CC()
3700 case ISD::SETGT: return X86::COND_A; in TranslateX86CC()
3701 case ISD::SETOLE: // flipped in TranslateX86CC()
3702 case ISD::SETOGE: in TranslateX86CC()
3703 case ISD::SETGE: return X86::COND_AE; in TranslateX86CC()
3704 case ISD::SETUGT: // flipped in TranslateX86CC()
3705 case ISD::SETULT: in TranslateX86CC()
3706 case ISD::SETLT: return X86::COND_B; in TranslateX86CC()
3707 case ISD::SETUGE: // flipped in TranslateX86CC()
3708 case ISD::SETULE: in TranslateX86CC()
3709 case ISD::SETLE: return X86::COND_BE; in TranslateX86CC()
3710 case ISD::SETONE: in TranslateX86CC()
3711 case ISD::SETNE: return X86::COND_NE; in TranslateX86CC()
3712 case ISD::SETUO: return X86::COND_P; in TranslateX86CC()
3713 case ISD::SETO: return X86::COND_NP; in TranslateX86CC()
3714 case ISD::SETOEQ: in TranslateX86CC()
3715 case ISD::SETUNE: return X86::COND_INVALID; in TranslateX86CC()
3750 ISD::LoadExtType ExtTy, in shouldReduceLoadWidth()
3775 if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT)) in isExtractSubvectorCheap()
3953 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst); in getZeroVector()
3956 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4f32, Cst, Cst, Cst, Cst); in getZeroVector()
3962 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8i32, Ops); in getZeroVector()
3968 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8f32, Ops); in getZeroVector()
3974 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v16i32, Ops); in getZeroVector()
3983 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); in getZeroVector()
3987 return DAG.getNode(ISD::BITCAST, dl, VT, Vec); in getZeroVector()
4002 if (Vec.getOpcode() == ISD::UNDEF) in ExtractSubVector()
4014 if (Vec.getOpcode() == ISD::BUILD_VECTOR) in ExtractSubVector()
4015 return DAG.getNode(ISD::BUILD_VECTOR, dl, ResultVT, in ExtractSubVector()
4020 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResultVT, Vec, VecIdx); in ExtractSubVector()
4049 if (Vec.getOpcode() == ISD::UNDEF) in InsertSubVector()
4064 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResultVT, Result, Vec, VecIdx); in InsertSubVector()
4083 Result.getOpcode() != ISD::UNDEF) { in Insert128BitVector()
4087 SDValue Vec256 = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResultVT, Undef, in Insert128BitVector()
4114 Vec256 = DAG.getNode(ISD::BITCAST, dl, CastVT, Vec256); in Insert128BitVector()
4116 return DAG.getNode(ISD::BITCAST, dl, ResultVT, Vec256); in Insert128BitVector()
4159 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v8i32, Ops); in getOnesVector()
4161 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst); in getOnesVector()
4165 Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst); in getOnesVector()
4169 return DAG.getNode(ISD::BITCAST, dl, VT, Vec); in getOnesVector()
4288 while (MaskNode->getOpcode() == ISD::BITCAST) in getTargetShuffleMask()
4291 if (MaskNode->getOpcode() == ISD::BUILD_VECTOR) { in getTargetShuffleMask()
4304 if (Op->getOpcode() == ISD::UNDEF) { in getTargetShuffleMask()
4437 if (Opcode == ISD::BITCAST) { in getShuffleScalarElt()
4446 if (V.getOpcode() == ISD::SCALAR_TO_VECTOR) in getShuffleScalarElt()
4450 if (V.getOpcode() == ISD::BUILD_VECTOR) in getShuffleScalarElt()
4482 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, in LowerBuildVectorv16i8()
4506 LastElt = DAG.getNode(ISD::ZERO_EXTEND, dl, in LowerBuildVectorv16i8()
4510 ThisElt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Op.getOperand(i)); in LowerBuildVectorv16i8()
4511 ThisElt = DAG.getNode(ISD::SHL, dl, MVT::i16, in LowerBuildVectorv16i8()
4514 ThisElt = DAG.getNode(ISD::OR, dl, MVT::i16, ThisElt, LastElt); in LowerBuildVectorv16i8()
4519 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, V, ThisElt, in LowerBuildVectorv16i8()
4524 return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, V); in LowerBuildVectorv16i8()
4550 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, in LowerBuildVectorv8i16()
4567 Zeroable[i] = (Elt.getOpcode() == ISD::UNDEF || X86::isZeroNode(Elt)); in LowerBuildVectorv4x32()
4580 if (Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT || in LowerBuildVectorv4x32()
4620 V1 = DAG.getNode(ISD::BITCAST, SDLoc(V1), VT, V1); in LowerBuildVectorv4x32()
4650 V1 = DAG.getNode(ISD::BITCAST, SDLoc(V1), MVT::v4f32, V1); in LowerBuildVectorv4x32()
4652 V2 = DAG.getNode(ISD::BITCAST, SDLoc(V2), MVT::v4f32, V2); in LowerBuildVectorv4x32()
4661 return DAG.getNode(ISD::BITCAST, SDLoc(Op), VT, Result); in LowerBuildVectorv4x32()
4671 SrcOp = DAG.getNode(ISD::BITCAST, dl, ShVT, SrcOp); in getVShift()
4675 return DAG.getNode(ISD::BITCAST, dl, VT, in getVShift()
4687 if (!ISD::isNormalLoad(LD) || LD->isVolatile()) in LowerAsSplatVectorLoad()
4732 Ptr = DAG.getNode(ISD::ADD, SDLoc(Ptr), Ptr.getValueType(), in LowerAsSplatVectorLoad()
4774 if (Elt.getNode() && Elt.getOpcode() == ISD::BITCAST) in EltsFromConsecutiveLoads()
4777 (Elt.getOpcode() != ISD::UNDEF && !ISD::isNON_EXTLoad(Elt.getNode()))) in EltsFromConsecutiveLoads()
4780 if (Elt.getNode()->getOpcode() == ISD::UNDEF) in EltsFromConsecutiveLoads()
4786 if (Elt.getOpcode() == ISD::UNDEF) in EltsFromConsecutiveLoads()
4812 !DAG.getTargetLoweringInfo().isOperationLegal(ISD::LOAD, VT)) in EltsFromConsecutiveLoads()
4823 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, in EltsFromConsecutiveLoads()
4852 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, in EltsFromConsecutiveLoads()
4859 return DAG.getNode(ISD::BITCAST, DL, VT, ResNode); in EltsFromConsecutiveLoads()
4893 case ISD::BUILD_VECTOR: { in LowerVectorBroadcast()
4904 ConstSplatVal = (Ld.getOpcode() == ISD::Constant || in LowerVectorBroadcast()
4905 Ld.getOpcode() == ISD::ConstantFP); in LowerVectorBroadcast()
4914 case ISD::VECTOR_SHUFFLE: { in LowerVectorBroadcast()
4923 if (Sc.getOpcode() != ISD::SCALAR_TO_VECTOR && in LowerVectorBroadcast()
4924 Sc.getOpcode() != ISD::BUILD_VECTOR) { in LowerVectorBroadcast()
4936 ConstSplatVal = (Ld.getOpcode() == ISD::Constant || in LowerVectorBroadcast()
4937 Ld.getOpcode() == ISD::ConstantFP); in LowerVectorBroadcast()
4999 bool IsLoad = ISD::isNormalLoad(Ld.getNode()); in LowerVectorBroadcast()
5065 if (!TLI.isOperationLegalOrCustom(ISD::INSERT_VECTOR_ELT, VT)) in buildFromShuffleMostly()
5079 if (Opc == ISD::UNDEF) in buildFromShuffleMostly()
5082 if (Opc != ISD::EXTRACT_VECTOR_ELT) { in buildFromShuffleMostly()
5125 NV = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, NV, Op.getOperand(Idx), in buildFromShuffleMostly()
5141 if (ISD::isBuildVectorAllZeros(Op.getNode())) { in LowerBUILD_VECTORvXi1()
5144 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); in LowerBUILD_VECTORvXi1()
5147 if (ISD::isBuildVectorAllOnes(Op.getNode())) { in LowerBUILD_VECTORvXi1()
5150 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); in LowerBUILD_VECTORvXi1()
5161 if (In.getOpcode() == ISD::UNDEF) in LowerBUILD_VECTORvXi1()
5177 SDValue FullMask = DAG.getNode(ISD::BITCAST, dl, MVT::v16i1, in LowerBUILD_VECTORvXi1()
5179 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, FullMask, in LowerBUILD_VECTORvXi1()
5188 DstVec = DAG.getNode(ISD::BITCAST, dl, VT, VecAsImm); in LowerBUILD_VECTORvXi1()
5192 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DstVec, in LowerBUILD_VECTORvXi1()
5201 Select = DAG.getNode(ISD::SELECT, dl, SelectVT, Op.getOperand(0), in LowerBUILD_VECTORvXi1()
5205 Select = DAG.getNode(ISD::SELECT, dl, SelectVT, Op.getOperand(0), in LowerBUILD_VECTORvXi1()
5208 return DAG.getNode(ISD::BITCAST, dl, VT, Select); in LowerBUILD_VECTORvXi1()
5235 bool IsCommutable = (Opcode == ISD::ADD || Opcode == ISD::FADD); in isHorizontalBinOp()
5247 if (Op->getOpcode() == ISD::UNDEF) { in isHorizontalBinOp()
5265 CanFold = (Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in isHorizontalBinOp()
5266 Op1.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in isHorizontalBinOp()
5277 if (V0.getOpcode() == ISD::UNDEF) in isHorizontalBinOp()
5280 if (V1.getOpcode() == ISD::UNDEF) in isHorizontalBinOp()
5353 if (!isUndefLO && V0->getOpcode() != ISD::UNDEF) in ExpandHorizontalBinOp()
5355 if (!isUndefHI && V1->getOpcode() != ISD::UNDEF) in ExpandHorizontalBinOp()
5359 if (!isUndefLO && (V0_LO->getOpcode() != ISD::UNDEF || in ExpandHorizontalBinOp()
5360 V1_LO->getOpcode() != ISD::UNDEF)) in ExpandHorizontalBinOp()
5363 if (!isUndefHI && (V0_HI->getOpcode() != ISD::UNDEF || in ExpandHorizontalBinOp()
5364 V1_HI->getOpcode() != ISD::UNDEF)) in ExpandHorizontalBinOp()
5368 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LO, HI); in ExpandHorizontalBinOp()
5388 unsigned ExpectedOpcode = ISD::FSUB; in matchAddSub()
5389 unsigned NextExpectedOpcode = ISD::FADD; in matchAddSub()
5398 if (Opcode == ISD::UNDEF) { in matchAddSub()
5413 if (Op0.getOpcode() != ISD::EXTRACT_VECTOR_ELT || in matchAddSub()
5414 Op1.getOpcode() != ISD::EXTRACT_VECTOR_ELT || in matchAddSub()
5431 if (InVec0.getOpcode() == ISD::UNDEF) in matchAddSub()
5433 if (InVec1.getOpcode() == ISD::UNDEF) in matchAddSub()
5439 if (ExpectedOpcode == ISD::FSUB) in matchAddSub()
5457 if (AddFound && SubFound && InVec0.getOpcode() != ISD::UNDEF && in matchAddSub()
5458 InVec1.getOpcode() != ISD::UNDEF) in matchAddSub()
5487 if (BV->getOperand(i)->getOpcode() == ISD::UNDEF) in PerformBUILD_VECTORCombine()
5491 if (BV->getOperand(i)->getOpcode() == ISD::UNDEF) in PerformBUILD_VECTORCombine()
5501 if (isHorizontalBinOp(BV, ISD::FADD, DAG, 0, NumElts, InVec0, InVec1)) in PerformBUILD_VECTORCombine()
5504 if (isHorizontalBinOp(BV, ISD::FSUB, DAG, 0, NumElts, InVec0, InVec1)) in PerformBUILD_VECTORCombine()
5508 if (isHorizontalBinOp(BV, ISD::ADD, DAG, 0, NumElts, InVec0, InVec1)) in PerformBUILD_VECTORCombine()
5511 if (isHorizontalBinOp(BV, ISD::SUB, DAG, 0, NumElts, InVec0, InVec1)) in PerformBUILD_VECTORCombine()
5522 if (isHorizontalBinOp(BV, ISD::FADD, DAG, 0, Half, InVec0, InVec1) && in PerformBUILD_VECTORCombine()
5523 isHorizontalBinOp(BV, ISD::FADD, DAG, Half, NumElts, InVec2, InVec3) && in PerformBUILD_VECTORCombine()
5524 ((InVec0.getOpcode() == ISD::UNDEF || in PerformBUILD_VECTORCombine()
5525 InVec2.getOpcode() == ISD::UNDEF) || InVec0 == InVec2) && in PerformBUILD_VECTORCombine()
5526 ((InVec1.getOpcode() == ISD::UNDEF || in PerformBUILD_VECTORCombine()
5527 InVec3.getOpcode() == ISD::UNDEF) || InVec1 == InVec3)) in PerformBUILD_VECTORCombine()
5530 if (isHorizontalBinOp(BV, ISD::FSUB, DAG, 0, Half, InVec0, InVec1) && in PerformBUILD_VECTORCombine()
5531 isHorizontalBinOp(BV, ISD::FSUB, DAG, Half, NumElts, InVec2, InVec3) && in PerformBUILD_VECTORCombine()
5532 ((InVec0.getOpcode() == ISD::UNDEF || in PerformBUILD_VECTORCombine()
5533 InVec2.getOpcode() == ISD::UNDEF) || InVec0 == InVec2) && in PerformBUILD_VECTORCombine()
5534 ((InVec1.getOpcode() == ISD::UNDEF || in PerformBUILD_VECTORCombine()
5535 InVec3.getOpcode() == ISD::UNDEF) || InVec1 == InVec3)) in PerformBUILD_VECTORCombine()
5543 if (isHorizontalBinOp(BV, ISD::ADD, DAG, 0, Half, InVec0, InVec1) && in PerformBUILD_VECTORCombine()
5544 isHorizontalBinOp(BV, ISD::ADD, DAG, Half, NumElts, InVec2, InVec3) && in PerformBUILD_VECTORCombine()
5545 ((InVec0.getOpcode() == ISD::UNDEF || in PerformBUILD_VECTORCombine()
5546 InVec2.getOpcode() == ISD::UNDEF) || InVec0 == InVec2) && in PerformBUILD_VECTORCombine()
5547 ((InVec1.getOpcode() == ISD::UNDEF || in PerformBUILD_VECTORCombine()
5548 InVec3.getOpcode() == ISD::UNDEF) || InVec1 == InVec3)) in PerformBUILD_VECTORCombine()
5550 else if (isHorizontalBinOp(BV, ISD::SUB, DAG, 0, Half, InVec0, InVec1) && in PerformBUILD_VECTORCombine()
5551 isHorizontalBinOp(BV, ISD::SUB, DAG, Half, NumElts, InVec2, InVec3) && in PerformBUILD_VECTORCombine()
5552 ((InVec0.getOpcode() == ISD::UNDEF || in PerformBUILD_VECTORCombine()
5553 InVec2.getOpcode() == ISD::UNDEF) || InVec0 == InVec2) && in PerformBUILD_VECTORCombine()
5554 ((InVec1.getOpcode() == ISD::UNDEF || in PerformBUILD_VECTORCombine()
5555 InVec3.getOpcode() == ISD::UNDEF) || InVec1 == InVec3)) in PerformBUILD_VECTORCombine()
5583 if (isHorizontalBinOp(BV, ISD::ADD, DAG, 0, NumElts, InVec0, InVec1)) in PerformBUILD_VECTORCombine()
5585 else if (isHorizontalBinOp(BV, ISD::SUB, DAG, 0, NumElts, InVec0, InVec1)) in PerformBUILD_VECTORCombine()
5587 else if (isHorizontalBinOp(BV, ISD::FADD, DAG, 0, NumElts, InVec0, InVec1)) in PerformBUILD_VECTORCombine()
5589 else if (isHorizontalBinOp(BV, ISD::FSUB, DAG, 0, NumElts, InVec0, InVec1)) in PerformBUILD_VECTORCombine()
5623 if (ISD::isBuildVectorAllZeros(Op.getNode())) { in LowerBUILD_VECTOR()
5635 if (Subtarget->hasSSE2() && ISD::isBuildVectorAllOnes(Op.getNode())) { in LowerBUILD_VECTOR()
5655 if (Elt.getOpcode() == ISD::UNDEF) in LowerBUILD_VECTOR()
5658 if (Elt.getOpcode() != ISD::Constant && in LowerBUILD_VECTOR()
5659 Elt.getOpcode() != ISD::ConstantFP) in LowerBUILD_VECTOR()
5692 Item = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Item); in LowerBUILD_VECTOR()
5693 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, Item); in LowerBUILD_VECTOR()
5695 ISD::BITCAST, dl, VT, in LowerBUILD_VECTOR()
5706 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); in LowerBUILD_VECTOR()
5712 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, ZeroVec, in LowerBUILD_VECTOR()
5717 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); in LowerBUILD_VECTOR()
5725 Item = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Item); in LowerBUILD_VECTOR()
5728 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v8i32, Item); in LowerBUILD_VECTOR()
5733 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Item); in LowerBUILD_VECTOR()
5739 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Item); in LowerBUILD_VECTOR()
5742 return DAG.getNode(ISD::BITCAST, dl, VT, Item); in LowerBUILD_VECTOR()
5752 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, in LowerBUILD_VECTOR()
5766 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); in LowerBUILD_VECTOR()
5804 SDValue Lower = DAG.getNode(ISD::BUILD_VECTOR, dl, HVT, in LowerBUILD_VECTOR()
5806 SDValue Upper = DAG.getNode(ISD::BUILD_VECTOR, dl, HVT, in LowerBUILD_VECTOR()
5820 SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, in LowerBUILD_VECTOR()
5851 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i)); in LowerBUILD_VECTOR()
5899 if (Op.getOperand(0).getOpcode() != ISD::UNDEF) in LowerBUILD_VECTOR()
5900 Result = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(0)); in LowerBUILD_VECTOR()
5905 if (Op.getOperand(i).getOpcode() == ISD::UNDEF) continue; in LowerBUILD_VECTOR()
5906 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Result, in LowerBUILD_VECTOR()
5916 if (Op.getOperand(i).getOpcode() != ISD::UNDEF) in LowerBUILD_VECTOR()
5917 V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i)); in LowerBUILD_VECTOR()
5934 if (V[i+EltStride].getOpcode() == ISD::UNDEF && in LowerBUILD_VECTOR()
5989 SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT, Ops); in LowerCONCAT_VECTORSvXi1()
5993 SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT, Ops); in LowerCONCAT_VECTORSvXi1()
5994 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); in LowerCONCAT_VECTORSvXi1()
5999 bool IsZeroV1 = ISD::isBuildVectorAllZeros(V1.getNode()); in LowerCONCAT_VECTORSvXi1()
6000 bool IsZeroV2 = ISD::isBuildVectorAllZeros(V2.getNode()); in LowerCONCAT_VECTORSvXi1()
6010 V2 = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Undef, V2, ZeroIdx); in LowerCONCAT_VECTORSvXi1()
6015 V1 = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Undef, V1, ZeroIdx); in LowerCONCAT_VECTORSvXi1()
6021 return DAG.getNode(ISD::OR, dl, ResVT, V1, V2); in LowerCONCAT_VECTORSvXi1()
6212 SDValue V1Mask = DAG.getNode(ISD::BUILD_VECTOR, DL, VT, MaskOps); in lowerVectorShuffleAsBitBlend()
6213 V1 = DAG.getNode(ISD::AND, DL, VT, V1, V1Mask); in lowerVectorShuffleAsBitBlend()
6216 V2 = DAG.getNode(ISD::BITCAST, DL, VT, in lowerVectorShuffleAsBitBlend()
6218 DAG.getNode(ISD::BITCAST, DL, MaskVT, V1Mask), in lowerVectorShuffleAsBitBlend()
6219 DAG.getNode(ISD::BITCAST, DL, MaskVT, V2))); in lowerVectorShuffleAsBitBlend()
6220 return DAG.getNode(ISD::OR, DL, VT, V1, V2); in lowerVectorShuffleAsBitBlend()
6270 V1 = DAG.getNode(ISD::BITCAST, DL, BlendVT, V1); in lowerVectorShuffleAsBlend()
6271 V2 = DAG.getNode(ISD::BITCAST, DL, BlendVT, V2); in lowerVectorShuffleAsBlend()
6272 return DAG.getNode(ISD::BITCAST, DL, VT, in lowerVectorShuffleAsBlend()
6287 V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1); in lowerVectorShuffleAsBlend()
6288 V2 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V2); in lowerVectorShuffleAsBlend()
6289 return DAG.getNode(ISD::BITCAST, DL, VT, in lowerVectorShuffleAsBlend()
6339 V1 = DAG.getNode(ISD::BITCAST, DL, BlendVT, V1); in lowerVectorShuffleAsBlend()
6340 V2 = DAG.getNode(ISD::BITCAST, DL, BlendVT, V2); in lowerVectorShuffleAsBlend()
6342 ISD::BITCAST, DL, VT, in lowerVectorShuffleAsBlend()
6343 DAG.getNode(ISD::VSELECT, DL, BlendVT, in lowerVectorShuffleAsBlend()
6344 DAG.getNode(ISD::BUILD_VECTOR, DL, BlendVT, VSELECTMask), in lowerVectorShuffleAsBlend()
6526 Lo = DAG.getNode(ISD::BITCAST, DL, AlignVT, Lo); in lowerVectorShuffleAsByteRotate()
6527 Hi = DAG.getNode(ISD::BITCAST, DL, AlignVT, Hi); in lowerVectorShuffleAsByteRotate()
6529 return DAG.getNode(ISD::BITCAST, DL, VT, in lowerVectorShuffleAsByteRotate()
6544 Lo = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Lo); in lowerVectorShuffleAsByteRotate()
6545 Hi = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, Hi); in lowerVectorShuffleAsByteRotate()
6551 return DAG.getNode(ISD::BITCAST, DL, VT, in lowerVectorShuffleAsByteRotate()
6552 DAG.getNode(ISD::OR, DL, MVT::v2i64, LoShift, HiShift)); in lowerVectorShuffleAsByteRotate()
6567 while (V1.getOpcode() == ISD::BITCAST) in computeZeroableShuffleElements()
6569 while (V2.getOpcode() == ISD::BITCAST) in computeZeroableShuffleElements()
6572 bool V1IsZero = ISD::isBuildVectorAllZeros(V1.getNode()); in computeZeroableShuffleElements()
6573 bool V2IsZero = ISD::isBuildVectorAllZeros(V2.getNode()); in computeZeroableShuffleElements()
6586 if (V.getOpcode() != ISD::BUILD_VECTOR || Size != (int)V.getNumOperands()) in computeZeroableShuffleElements()
6592 if (Input.getOpcode() == ISD::UNDEF || X86::isZeroNode(Input)) in computeZeroableShuffleElements()
6612 Zero = DAG.getNode(ISD::BITCAST, DL, EltVT, Zero); in lowerVectorShuffleAsBitMask()
6613 AllOnes = DAG.getNode(ISD::BITCAST, DL, EltVT, AllOnes); in lowerVectorShuffleAsBitMask()
6633 SDValue VMask = DAG.getNode(ISD::BUILD_VECTOR, DL, VT, VMaskOps); in lowerVectorShuffleAsBitMask()
6635 ? (unsigned) X86ISD::FAND : (unsigned) ISD::AND, in lowerVectorShuffleAsBitMask()
6705 V = DAG.getNode(ISD::BITCAST, DL, ShiftVT, V); in lowerVectorShuffleAsShift()
6708 return DAG.getNode(ISD::BITCAST, DL, VT, V); in lowerVectorShuffleAsShift()
6749 return DAG.getNode(ISD::BITCAST, DL, VT, in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
6758 ISD::BITCAST, DL, VT, in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
6760 DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, InputV), in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
6766 DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, InputV), in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
6770 ISD::BITCAST, DL, VT, in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
6772 DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, InputV), in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
6785 InputV = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, InputV); in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
6786 return DAG.getNode(ISD::BITCAST, DL, VT, in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
6788 DAG.getNode(ISD::BUILD_VECTOR, DL, in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
6797 InputV = DAG.getNode(ISD::BITCAST, DL, InputVT, InputV); in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
6803 return DAG.getNode(ISD::BITCAST, DL, VT, InputV); in lowerVectorShuffleAsSpecificZeroOrAnyExtend()
6901 V = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, V); in lowerVectorShuffleAsZeroOrAnyExtend()
6903 return DAG.getNode(ISD::BITCAST, DL, VT, V); in lowerVectorShuffleAsZeroOrAnyExtend()
6917 while (V.getOpcode() == ISD::BITCAST) in getScalarValueForVectorElement()
6925 if (V.getOpcode() == ISD::BUILD_VECTOR || in getScalarValueForVectorElement()
6926 (Idx == 0 && V.getOpcode() == ISD::SCALAR_TO_VECTOR)) in getScalarValueForVectorElement()
6927 return DAG.getNode(ISD::BITCAST, SDLoc(V), EltVT, V.getOperand(Idx)); in getScalarValueForVectorElement()
6937 while (V.getOpcode() == ISD::BITCAST) in isShuffleFoldableLoad()
6940 return ISD::isNON_EXTLoad(V.getNode()); in isShuffleFoldableLoad()
6972 V2S = DAG.getNode(ISD::BITCAST, DL, EltVT, V2S); in lowerVectorShuffleAsElementInsertion()
6981 V2S = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, V2S); in lowerVectorShuffleAsElementInsertion()
6983 V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, ExtVT, V2S); in lowerVectorShuffleAsElementInsertion()
7021 V2 = DAG.getNode(ISD::BITCAST, DL, VT, V2); in lowerVectorShuffleAsElementInsertion()
7033 V2 = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, V2); in lowerVectorShuffleAsElementInsertion()
7039 V2 = DAG.getNode(ISD::BITCAST, DL, VT, V2); in lowerVectorShuffleAsElementInsertion()
7075 case ISD::CONCAT_VECTORS: { in lowerVectorShuffleAsBroadcast()
7082 case ISD::INSERT_SUBVECTOR: { in lowerVectorShuffleAsBroadcast()
7105 if (V.getOpcode() == ISD::BUILD_VECTOR || in lowerVectorShuffleAsBroadcast()
7106 (V.getOpcode() == ISD::SCALAR_TO_VECTOR && BroadcastIdx == 0)) { in lowerVectorShuffleAsBroadcast()
7262 V1 = DAG.getNode(ISD::BITCAST, DL, UnpackVT, V1); in lowerVectorShuffleAsUnpack()
7263 V2 = DAG.getNode(ISD::BITCAST, DL, UnpackVT, V2); in lowerVectorShuffleAsUnpack()
7266 return DAG.getNode(ISD::BITCAST, DL, VT, in lowerVectorShuffleAsUnpack()
7380 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, V1S)); in lowerV2F64VectorShuffle()
7424 V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, V1); in lowerV2I64VectorShuffle()
7429 ISD::BITCAST, DL, MVT::v2i64, in lowerV2I64VectorShuffle()
7443 while (V.getOpcode() == ISD::BITCAST) in lowerV2I64VectorShuffle()
7450 return DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, in lowerV2I64VectorShuffle()
7505 V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, V1); in lowerV2I64VectorShuffle()
7506 V2 = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, V2); in lowerV2I64VectorShuffle()
7507 return DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, in lowerV2I64VectorShuffle()
7807 return DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, in lowerV4I32VectorShuffle()
7810 DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, V1), in lowerV4I32VectorShuffle()
7811 DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, V2), Mask)); in lowerV4I32VectorShuffle()
7989 V = DAG.getNode(ISD::BITCAST, DL, VT, in lowerV8I16GeneralSingleInputVectorShuffle()
7991 DAG.getNode(ISD::BITCAST, DL, PSHUFDVT, V), in lowerV8I16GeneralSingleInputVectorShuffle()
8233 V = DAG.getNode(ISD::BITCAST, DL, VT, in lowerV8I16GeneralSingleInputVectorShuffle()
8235 DAG.getNode(ISD::BITCAST, DL, PSHUFDVT, V), in lowerV8I16GeneralSingleInputVectorShuffle()
8297 DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, V1), in lowerVectorShuffleAsPSHUFB()
8298 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v16i8, V1Mask)); in lowerVectorShuffleAsPSHUFB()
8301 DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, V2), in lowerVectorShuffleAsPSHUFB()
8302 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v16i8, V2Mask)); in lowerVectorShuffleAsPSHUFB()
8307 V = DAG.getNode(ISD::OR, DL, MVT::v16i8, V1, V2); in lowerVectorShuffleAsPSHUFB()
8312 return DAG.getNode(ISD::BITCAST, DL, VT, V); in lowerVectorShuffleAsPSHUFB()
8614 ISD::BITCAST, DL, MVT::v16i8, in lowerV16I8VectorShuffle()
8616 DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1), in lowerV16I8VectorShuffle()
8635 ISD::BITCAST, DL, MVT::v16i8, in lowerV16I8VectorShuffle()
8637 DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1), in lowerV16I8VectorShuffle()
8731 DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, in lowerV16I8VectorShuffle()
8733 V1 = DAG.getNode(ISD::AND, DL, MVT::v16i8, V1, ByteClearMask); in lowerV16I8VectorShuffle()
8735 V2 = DAG.getNode(ISD::AND, DL, MVT::v16i8, V2, ByteClearMask); in lowerV16I8VectorShuffle()
8738 V1 = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V1); in lowerV16I8VectorShuffle()
8739 V2 = IsSingleInput ? V1 : DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V2); in lowerV16I8VectorShuffle()
8742 Result = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, Result); in lowerV16I8VectorShuffle()
8776 VLoHalf = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V); in lowerV16I8VectorShuffle()
8777 VLoHalf = DAG.getNode(ISD::AND, DL, MVT::v8i16, VLoHalf, in lowerV16I8VectorShuffle()
8793 VLoHalf = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, in lowerV16I8VectorShuffle()
8795 VHiHalf = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, in lowerV16I8VectorShuffle()
8910 while (V.getOpcode() == ISD::BITCAST) in splitAndLowerVectorShuffle()
8923 LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OrigSplitVT, V, in splitAndLowerVectorShuffle()
8925 HiV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OrigSplitVT, V, in splitAndLowerVectorShuffle()
8934 LoV = DAG.getNode(ISD::BUILD_VECTOR, DL, OrigSplitVT, LoOps); in splitAndLowerVectorShuffle()
8935 HiV = DAG.getNode(ISD::BUILD_VECTOR, DL, OrigSplitVT, HiOps); in splitAndLowerVectorShuffle()
8937 return std::make_pair(DAG.getNode(ISD::BITCAST, DL, SplitVT, LoV), in splitAndLowerVectorShuffle()
8938 DAG.getNode(ISD::BITCAST, DL, SplitVT, HiV)); in splitAndLowerVectorShuffle()
9011 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi); in splitAndLowerVectorShuffle()
9139 bool IsV1Zero = ISD::isBuildVectorAllZeros(V1.getNode()); in lowerV2X128VectorShuffle()
9140 bool IsV2Zero = ISD::isBuildVectorAllZeros(V2.getNode()); in lowerV2X128VectorShuffle()
9151 SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1, in lowerV2X128VectorShuffle()
9153 SDValue HiV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, in lowerV2X128VectorShuffle()
9155 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LoV, HiV); in lowerV2X128VectorShuffle()
9270 V1 = DAG.getNode(ISD::BITCAST, DL, LaneVT, V1); in lowerVectorShuffleByMerging128BitLanes()
9271 V2 = DAG.getNode(ISD::BITCAST, DL, LaneVT, V2); in lowerVectorShuffleByMerging128BitLanes()
9275 LaneShuffle = DAG.getNode(ISD::BITCAST, DL, VT, LaneShuffle); in lowerVectorShuffleByMerging128BitLanes()
9448 ISD::BITCAST, DL, MVT::v4i64, in lowerV4I64VectorShuffle()
9450 DAG.getNode(ISD::BITCAST, DL, MVT::v8i32, V1), in lowerV4I64VectorShuffle()
9560 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v8i32, VPermMask)); in lowerV8F32VectorShuffle()
9564 DAG.getNode(ISD::BITCAST, DL, MVT::v8f32, in lowerV8F32VectorShuffle()
9565 DAG.getNode(ISD::BUILD_VECTOR, DL, in lowerV8F32VectorShuffle()
9660 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v8i32, VPermMask), V1); in lowerV8I32VectorShuffle()
9758 ISD::BITCAST, DL, MVT::v16i16, in lowerV16I16VectorShuffle()
9761 DAG.getNode(ISD::BITCAST, DL, MVT::v32i8, V1), in lowerV16I16VectorShuffle()
9762 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v32i8, PSHUFBMask))); in lowerV16I16VectorShuffle()
9850 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v32i8, PSHUFBMask)); in lowerV32I8VectorShuffle()
9901 V1 = DAG.getNode(ISD::BITCAST, DL, FpVT, V1); in lower256BitVectorShuffle()
9902 V2 = DAG.getNode(ISD::BITCAST, DL, FpVT, V2); in lower256BitVectorShuffle()
9903 return DAG.getNode(ISD::BITCAST, DL, VT, in lower256BitVectorShuffle()
10128 bool V1IsUndef = V1.getOpcode() == ISD::UNDEF; in lowerVectorShuffle()
10129 bool V2IsUndef = V2.getOpcode() == ISD::UNDEF; in lowerVectorShuffle()
10173 V1 = DAG.getNode(ISD::BITCAST, dl, NewVT, V1); in lowerVectorShuffle()
10174 V2 = DAG.getNode(ISD::BITCAST, dl, NewVT, V2); in lowerVectorShuffle()
10175 return DAG.getNode(ISD::BITCAST, dl, VT, in lowerVectorShuffle()
10293 if (!ISD::isBuildVectorOfConstantSDNodes(Cond.getNode())) in lowerVSELECTtoVectorShuffle()
10311 if (ISD::isBuildVectorOfConstantSDNodes(Op.getOperand(0).getNode()) && in LowerVSELECT()
10312 ISD::isBuildVectorOfConstantSDNodes(Op.getOperand(1).getNode()) && in LowerVSELECT()
10313 ISD::isBuildVectorOfConstantSDNodes(Op.getOperand(2).getNode())) in LowerVSELECT()
10362 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract, in LowerEXTRACT_VECTOR_ELT_SSE4()
10364 return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert); in LowerEXTRACT_VECTOR_ELT_SSE4()
10371 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, in LowerEXTRACT_VECTOR_ELT_SSE4()
10372 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
10373 DAG.getNode(ISD::BITCAST, dl, in LowerEXTRACT_VECTOR_ELT_SSE4()
10379 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract, in LowerEXTRACT_VECTOR_ELT_SSE4()
10381 return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert); in LowerEXTRACT_VECTOR_ELT_SSE4()
10393 if ((User->getOpcode() != ISD::STORE || in LowerEXTRACT_VECTOR_ELT_SSE4()
10396 (User->getOpcode() != ISD::BITCAST || in LowerEXTRACT_VECTOR_ELT_SSE4()
10399 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
10400 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, in LowerEXTRACT_VECTOR_ELT_SSE4()
10403 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, Extract); in LowerEXTRACT_VECTOR_ELT_SSE4()
10432 SDValue Ext = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, Vec); in ExtractBitFromMaskVector()
10433 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, in ExtractBitFromMaskVector()
10435 return DAG.getNode(ISD::TRUNCATE, dl, EltVT, Elt); in ExtractBitFromMaskVector()
10477 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), in LowerEXTRACT_VECTOR_ELT()
10497 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Vec, in LowerEXTRACT_VECTOR_ELT()
10515 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, in LowerEXTRACT_VECTOR_ELT()
10516 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, in LowerEXTRACT_VECTOR_ELT()
10517 DAG.getNode(ISD::BITCAST, dl, in LowerEXTRACT_VECTOR_ELT()
10524 SDValue Assert = DAG.getNode(ISD::AssertZext, dl, EltVT, Extract, in LowerEXTRACT_VECTOR_ELT()
10526 return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert); in LowerEXTRACT_VECTOR_ELT()
10539 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec, in LowerEXTRACT_VECTOR_ELT()
10558 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec, in LowerEXTRACT_VECTOR_ELT()
10580 SDValue ExtOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, ExtVecVT, in InsertBitToMaskVector()
10581 DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVecVT, Vec), in InsertBitToMaskVector()
10582 DAG.getNode(ISD::ZERO_EXTEND, dl, ExtEltVT, Elt), Idx); in InsertBitToMaskVector()
10583 return DAG.getNode(ISD::TRUNCATE, dl, VecVT, ExtOp); in InsertBitToMaskVector()
10587 SDValue EltInVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, Elt); in InsertBitToMaskVector()
10588 if (Vec.getOpcode() == ISD::UNDEF) in InsertBitToMaskVector()
10597 return DAG.getNode(ISD::OR, dl, VecVT, Vec, EltInVec); in InsertBitToMaskVector()
10628 SDValue N1Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, N1); in LowerINSERT_VECTOR_ELT()
10641 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, V.getValueType(), V, N1, in LowerINSERT_VECTOR_ELT()
10662 N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1); in LowerINSERT_VECTOR_ELT()
10689 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1); in LowerINSERT_VECTOR_ELT()
10694 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1); in LowerINSERT_VECTOR_ELT()
10711 N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1); in LowerINSERT_VECTOR_ELT()
10731 Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT128, Op.getOperand(0)); in LowerSCALAR_TO_VECTOR()
10739 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i64, Op.getOperand(0)); in LowerSCALAR_TO_VECTOR()
10741 SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op.getOperand(0)); in LowerSCALAR_TO_VECTOR()
10743 return DAG.getNode(ISD::BITCAST, dl, OpVT, in LowerSCALAR_TO_VECTOR()
10744 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32,AnyExt)); in LowerSCALAR_TO_VECTOR()
10798 Vec.getOpcode() == ISD::INSERT_SUBVECTOR && in LowerINSERT_SUBVECTOR()
10820 if (IdxVal == 0 && Vec.getOpcode() == ISD::UNDEF) // the operation is legal in LowerINSERT_SUBVECTOR()
10832 SDValue Vec2 = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT, Undef, in LowerINSERT_SUBVECTOR()
10835 return DAG.getNode(ISD::OR, dl, OpVT, Vec, Vec2); in LowerINSERT_SUBVECTOR()
10838 SDValue Vec2 = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT, Undef, in LowerINSERT_SUBVECTOR()
10847 return DAG.getNode(ISD::OR, dl, OpVT, Vec, Vec2); in LowerINSERT_SUBVECTOR()
10884 Result = DAG.getNode(ISD::ADD, DL, getPointerTy(), in LowerConstantPool()
10917 Result = DAG.getNode(ISD::ADD, DL, getPointerTy(), in LowerJumpTable()
10956 Result = DAG.getNode(ISD::ADD, DL, getPointerTy(), in LowerExternalSymbol()
10991 Result = DAG.getNode(ISD::ADD, dl, getPointerTy(), in LowerBlockAddress()
11025 Result = DAG.getNode(ISD::ADD, dl, getPointerTy(), in LowerGlobalAddress()
11039 Result = DAG.getNode(ISD::ADD, dl, getPointerTy(), Result, in LowerGlobalAddress()
11141 return DAG.getNode(ISD::ADD, dl, PtrVT, Offset, Base); in LowerToTLSLocalDynamicModel()
11185 Offset = DAG.getNode(ISD::ADD, dl, PtrVT, in LowerToTLSExecModel()
11196 return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset); in LowerToTLSExecModel()
11247 Offset = DAG.getNode(ISD::ADD, DL, getPointerTy(), in LowerGlobalTLSAddress()
11309 IDX = DAG.getExtLoad(ISD::ZEXTLOAD, dl, getPointerTy(), Chain, in LowerGlobalTLSAddress()
11318 IDX = DAG.getNode(ISD::SHL, dl, getPointerTy(), IDX, Scale); in LowerGlobalTLSAddress()
11320 SDValue res = DAG.getNode(ISD::ADD, dl, getPointerTy(), ThreadPointer, IDX); in LowerGlobalTLSAddress()
11332 return DAG.getNode(ISD::ADD, dl, getPointerTy(), res, Offset); in LowerGlobalTLSAddress()
11345 bool isSRA = Op.getOpcode() == ISD::SRA_PARTS; in LowerShiftParts()
11352 SDValue SafeShAmt = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt, in LowerShiftParts()
11354 SDValue Tmp1 = isSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi, in LowerShiftParts()
11359 if (Op.getOpcode() == ISD::SHL_PARTS) { in LowerShiftParts()
11361 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, SafeShAmt); in LowerShiftParts()
11364 Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, SafeShAmt); in LowerShiftParts()
11370 SDValue AndNode = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt, in LowerShiftParts()
11380 if (Op.getOpcode() == ISD::SHL_PARTS) { in LowerShiftParts()
11400 return DAG.getNode(ISD::SINT_TO_FP, dl, Op.getValueType(), in LowerSINT_TO_FP()
11401 DAG.getNode(ISD::SIGN_EXTEND, dl, IntegerVT, in LowerSINT_TO_FP()
11526 SDValue XR1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, in LowerUINT_TO_FP_i64()
11532 DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, XR1), in LowerUINT_TO_FP_i64()
11538 SDValue XR2F = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Unpck1); in LowerUINT_TO_FP_i64()
11539 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, XR2F, CLod1); in LowerUINT_TO_FP_i64()
11546 SDValue S2F = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Sub); in LowerUINT_TO_FP_i64()
11549 Result = DAG.getNode(ISD::FADD, dl, MVT::v2f64, in LowerUINT_TO_FP_i64()
11550 DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Shuffle), in LowerUINT_TO_FP_i64()
11554 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Result, in LowerUINT_TO_FP_i64()
11567 SDValue Load = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, in LowerUINT_TO_FP_i32()
11573 Load = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, in LowerUINT_TO_FP_i32()
11574 DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Load), in LowerUINT_TO_FP_i32()
11578 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64, in LowerUINT_TO_FP_i32()
11579 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, in LowerUINT_TO_FP_i32()
11580 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, in LowerUINT_TO_FP_i32()
11582 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, in LowerUINT_TO_FP_i32()
11583 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, in LowerUINT_TO_FP_i32()
11585 Or = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, in LowerUINT_TO_FP_i32()
11586 DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Or), in LowerUINT_TO_FP_i32()
11590 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Or, Bias); in LowerUINT_TO_FP_i32()
11596 return DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub, in LowerUINT_TO_FP_i32()
11599 return DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub); in LowerUINT_TO_FP_i32()
11645 SDValue VecCstLow = DAG.getNode(ISD::BUILD_VECTOR, DL, VecIntVT, in lowerUINT_TO_FP_vXi32()
11651 SDValue VecCstHigh = DAG.getNode(ISD::BUILD_VECTOR, DL, VecIntVT, in lowerUINT_TO_FP_vXi32()
11658 SDValue VecCstShift = DAG.getNode(ISD::BUILD_VECTOR, DL, VecIntVT, in lowerUINT_TO_FP_vXi32()
11660 SDValue HighShift = DAG.getNode(ISD::SRL, DL, VecIntVT, V, VecCstShift); in lowerUINT_TO_FP_vXi32()
11667 DAG.getNode(ISD::BITCAST, DL, VecI16VT, VecCstLow); in lowerUINT_TO_FP_vXi32()
11668 SDValue VecBitcast = DAG.getNode(ISD::BITCAST, DL, VecI16VT, V); in lowerUINT_TO_FP_vXi32()
11676 DAG.getNode(ISD::BITCAST, DL, VecI16VT, VecCstHigh); in lowerUINT_TO_FP_vXi32()
11678 DAG.getNode(ISD::BITCAST, DL, VecI16VT, HighShift); in lowerUINT_TO_FP_vXi32()
11685 SDValue VecCstMask = DAG.getNode(ISD::BUILD_VECTOR, DL, VecIntVT, CstMask, in lowerUINT_TO_FP_vXi32()
11688 SDValue LowAnd = DAG.getNode(ISD::AND, DL, VecIntVT, V, VecCstMask); in lowerUINT_TO_FP_vXi32()
11689 Low = DAG.getNode(ISD::OR, DL, VecIntVT, LowAnd, VecCstLow); in lowerUINT_TO_FP_vXi32()
11692 High = DAG.getNode(ISD::OR, DL, VecIntVT, HighShift, VecCstHigh); in lowerUINT_TO_FP_vXi32()
11700 SDValue VecCstFAdd = DAG.getNode(ISD::BUILD_VECTOR, DL, VecFloatVT, in lowerUINT_TO_FP_vXi32()
11704 SDValue HighBitcast = DAG.getNode(ISD::BITCAST, DL, VecFloatVT, High); in lowerUINT_TO_FP_vXi32()
11706 DAG.getNode(ISD::FADD, DL, VecFloatVT, HighBitcast, VecCstFAdd); in lowerUINT_TO_FP_vXi32()
11708 SDValue LowBitcast = DAG.getNode(ISD::BITCAST, DL, VecFloatVT, Low); in lowerUINT_TO_FP_vXi32()
11709 return DAG.getNode(ISD::FADD, DL, VecFloatVT, LowBitcast, FHigh); in lowerUINT_TO_FP_vXi32()
11726 return DAG.getNode(ISD::SINT_TO_FP, dl, Op.getValueType(), in lowerUINT_TO_FP_vec()
11727 DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N0)); in lowerUINT_TO_FP_vec()
11748 return DAG.getNode(ISD::SINT_TO_FP, dl, Op.getValueType(), N0); in LowerUINT_TO_FP()
11763 SDValue OffsetSlot = DAG.getNode(ISD::ADD, dl, in LowerUINT_TO_FP()
11801 ISD::SETLT); in LowerUINT_TO_FP()
11811 SDValue Offset = DAG.getNode(ISD::SELECT, dl, Zero.getValueType(), SignSet, in LowerUINT_TO_FP()
11813 FudgePtr = DAG.getNode(ISD::ADD, dl, getPointerTy(), FudgePtr, Offset); in LowerUINT_TO_FP()
11817 SDValue Fudge = DAG.getExtLoad(ISD::EXTLOAD, dl, MVT::f80, DAG.getEntryNode(), in LowerUINT_TO_FP()
11821 SDValue Add = DAG.getNode(ISD::FADD, dl, MVT::f80, Fild, Fudge); in LowerUINT_TO_FP()
11822 return DAG.getNode(ISD::FP_ROUND, dl, DstVT, Add, DAG.getIntPtrConstant(0)); in LowerUINT_TO_FP()
11912 ? DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops) in FP_TO_INTHelper()
11948 bool NeedZero = Op.getOpcode() == ISD::ZERO_EXTEND; in LowerAVXExtend()
11955 OpLo = DAG.getNode(ISD::BITCAST, dl, HVT, OpLo); in LowerAVXExtend()
11956 OpHi = DAG.getNode(ISD::BITCAST, dl, HVT, OpHi); in LowerAVXExtend()
11958 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi); in LowerAVXExtend()
12035 In = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, In); in LowerTRUNCATE()
12036 return DAG.getNode(ISD::TRUNCATE, DL, VT, In); in LowerTRUNCATE()
12050 In = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, In); in LowerTRUNCATE()
12062 SDValue And = DAG.getNode(ISD::AND, DL, InVT, OneV, In); in LowerTRUNCATE()
12070 In = DAG.getNode(ISD::BITCAST, DL, MVT::v8i32, In); in LowerTRUNCATE()
12073 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, In, in LowerTRUNCATE()
12077 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In, in LowerTRUNCATE()
12079 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In, in LowerTRUNCATE()
12081 OpLo = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, OpLo); in LowerTRUNCATE()
12082 OpHi = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, OpHi); in LowerTRUNCATE()
12090 In = DAG.getNode(ISD::BITCAST, DL, MVT::v32i8, In); in LowerTRUNCATE()
12105 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v32i8, pshufbMask); in LowerTRUNCATE()
12107 In = DAG.getNode(ISD::BITCAST, DL, MVT::v4i64, In); in LowerTRUNCATE()
12112 In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In, in LowerTRUNCATE()
12114 return DAG.getNode(ISD::BITCAST, DL, VT, In); in LowerTRUNCATE()
12117 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In, in LowerTRUNCATE()
12120 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In, in LowerTRUNCATE()
12123 OpLo = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, OpLo); in LowerTRUNCATE()
12124 OpHi = DAG.getNode(ISD::BITCAST, DL, MVT::v16i8, OpHi); in LowerTRUNCATE()
12134 OpLo = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, OpLo); in LowerTRUNCATE()
12135 OpHi = DAG.getNode(ISD::BITCAST, DL, MVT::v4i32, OpHi); in LowerTRUNCATE()
12140 return DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, res); in LowerTRUNCATE()
12157 DAG.getNode(ISD::BITCAST, DL, NVT, In), in LowerTRUNCATE()
12159 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, in LowerTRUNCATE()
12209 DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f32, in LowerFP_EXTEND()
12216 assert((Op.getOpcode() == ISD::FABS || Op.getOpcode() == ISD::FNEG) && in LowerFABSorFNEG()
12219 bool IsFABS = (Op.getOpcode() == ISD::FABS); in LowerFABSorFNEG()
12225 if (User->getOpcode() == ISD::FNEG) in LowerFABSorFNEG()
12229 bool IsFNABS = !IsFABS && (Op0.getOpcode() == ISD::FABS); in LowerFABSorFNEG()
12266 SDValue MaskCasted = DAG.getNode(ISD::BITCAST, dl, VecVT, Mask); in LowerFABSorFNEG()
12268 DAG.getNode(ISD::BITCAST, dl, VecVT, Op0.getOperand(0)) : in LowerFABSorFNEG()
12269 DAG.getNode(ISD::BITCAST, dl, VecVT, Op0); in LowerFABSorFNEG()
12270 unsigned BitOp = IsFABS ? ISD::AND : IsFNABS ? ISD::OR : ISD::XOR; in LowerFABSorFNEG()
12271 return DAG.getNode(ISD::BITCAST, dl, VT, in LowerFABSorFNEG()
12292 Op1 = DAG.getNode(ISD::FP_EXTEND, dl, VT, Op1); in LowerFCOPYSIGN()
12297 Op1 = DAG.getNode(ISD::FP_ROUND, dl, VT, Op1, DAG.getIntPtrConstant(1)); in LowerFCOPYSIGN()
12357 return DAG.getNode(ISD::AND, dl, VT, xFGETSIGN, DAG.getConstant(1, VT)); in LowerFGETSIGN()
12363 assert(Op.getOpcode() == ISD::OR && "Only check OR'd tree."); in LowerVectorAllZeroTest()
12387 if (I->getOpcode() == ISD::OR) { in LowerVectorAllZeroTest()
12396 if (I->getOpcode() != ISD::EXTRACT_VECTOR_ELT) in LowerVectorAllZeroTest()
12437 VecIns[i] = DAG.getNode(ISD::BITCAST, DL, TestVT, VecIns[i]); in LowerVectorAllZeroTest()
12445 VecIns.push_back(DAG.getNode(ISD::OR, DL, TestVT, LHS, RHS)); in LowerVectorAllZeroTest()
12458 if (User->getOpcode() == ISD::TRUNCATE && User->hasOneUse()) { in hasNonFlagsUse()
12464 if (User->getOpcode() != ISD::BRCOND && User->getOpcode() != ISD::SETCC && in hasNonFlagsUse()
12465 !(User->getOpcode() == ISD::SELECT && UOpNo == 0)) in hasNonFlagsUse()
12476 SDValue ExtOp = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i8, Op); in EmitTest()
12497 case ISD::ADD: in EmitTest()
12498 case ISD::SUB: in EmitTest()
12499 case ISD::MUL: in EmitTest()
12500 case ISD::SHL: { in EmitTest()
12532 if (Op->getOpcode() == ISD::TRUNCATE && Op->hasOneUse()) { in EmitTest()
12538 case ISD::ADD: in EmitTest()
12539 case ISD::SUB: in EmitTest()
12540 case ISD::AND: in EmitTest()
12541 case ISD::OR: in EmitTest()
12542 case ISD::XOR: { in EmitTest()
12553 case ISD::ADD: in EmitTest()
12566 if (UI->getOpcode() != ISD::CopyToReg && in EmitTest()
12567 UI->getOpcode() != ISD::SETCC && in EmitTest()
12568 UI->getOpcode() != ISD::STORE) in EmitTest()
12592 case ISD::SHL: in EmitTest()
12593 case ISD::SRL: in EmitTest()
12604 APInt Mask = ArithOp.getOpcode() == ISD::SRL in EmitTest()
12609 SDValue New = DAG.getNode(ISD::AND, dl, VT, Op->getOperand(0), in EmitTest()
12616 case ISD::AND: in EmitTest()
12622 case ISD::SUB: in EmitTest()
12623 case ISD::OR: in EmitTest()
12624 case ISD::XOR: in EmitTest()
12629 if (UI->getOpcode() == ISD::STORE) in EmitTest()
12635 case ISD::SUB: Opcode = X86ISD::SUB; break; in EmitTest()
12636 case ISD::XOR: Opcode = X86ISD::XOR; break; in EmitTest()
12637 case ISD::AND: Opcode = X86ISD::AND; break; in EmitTest()
12638 case ISD::OR: { in EmitTest()
12676 case ISD::ADD: ConvertedOp = X86ISD::ADD; break; in EmitTest()
12677 case ISD::SUB: ConvertedOp = X86ISD::SUB; break; in EmitTest()
12678 case ISD::AND: ConvertedOp = X86ISD::AND; break; in EmitTest()
12679 case ISD::OR: ConvertedOp = X86ISD::OR; break; in EmitTest()
12680 case ISD::XOR: ConvertedOp = X86ISD::XOR; break; in EmitTest()
12686 SDValue V0 = DAG.getNode(ISD::TRUNCATE, dl, VT, WideVal.getOperand(0)); in EmitTest()
12687 SDValue V1 = DAG.getNode(ISD::TRUNCATE, dl, VT, WideVal.getOperand(1)); in EmitTest()
12729 isX86CCUnsigned(X86CC) ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND; in EmitCmp()
12758 SDValue TruncFPSW = DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Cmp); in ConvertCmpIfNecessary()
12760 SDValue Srl = DAG.getNode(ISD::SRL, dl, MVT::i16, FNStSW, in ConvertCmpIfNecessary()
12762 SDValue TruncSrl = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Srl); in ConvertCmpIfNecessary()
12843 SDValue X86TargetLowering::LowerToBT(SDValue And, ISD::CondCode CC, in LowerToBT()
12847 if (Op0.getOpcode() == ISD::TRUNCATE) in LowerToBT()
12849 if (Op1.getOpcode() == ISD::TRUNCATE) in LowerToBT()
12853 if (Op1.getOpcode() == ISD::SHL) in LowerToBT()
12855 if (Op0.getOpcode() == ISD::SHL) { in LowerToBT()
12871 } else if (Op1.getOpcode() == ISD::Constant) { in LowerToBT()
12876 if (AndRHSVal == 1 && AndLHS.getOpcode() == ISD::SRL) { in LowerToBT()
12896 LHS = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, LHS); in LowerToBT()
12901 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LHS.getValueType(), RHS); in LowerToBT()
12904 X86::CondCode Cond = CC == ISD::SETEQ ? X86::COND_AE : X86::COND_B; in LowerToBT()
12914 static int translateX86FSETCC(ISD::CondCode SetCCOpcode, SDValue &Op0, in translateX86FSETCC()
12930 case ISD::SETOEQ: in translateX86FSETCC()
12931 case ISD::SETEQ: SSECC = 0; break; in translateX86FSETCC()
12932 case ISD::SETOGT: in translateX86FSETCC()
12933 case ISD::SETGT: Swap = true; // Fallthrough in translateX86FSETCC()
12934 case ISD::SETLT: in translateX86FSETCC()
12935 case ISD::SETOLT: SSECC = 1; break; in translateX86FSETCC()
12936 case ISD::SETOGE: in translateX86FSETCC()
12937 case ISD::SETGE: Swap = true; // Fallthrough in translateX86FSETCC()
12938 case ISD::SETLE: in translateX86FSETCC()
12939 case ISD::SETOLE: SSECC = 2; break; in translateX86FSETCC()
12940 case ISD::SETUO: SSECC = 3; break; in translateX86FSETCC()
12941 case ISD::SETUNE: in translateX86FSETCC()
12942 case ISD::SETNE: SSECC = 4; break; in translateX86FSETCC()
12943 case ISD::SETULE: Swap = true; // Fallthrough in translateX86FSETCC()
12944 case ISD::SETUGE: SSECC = 5; break; in translateX86FSETCC()
12945 case ISD::SETULT: Swap = true; // Fallthrough in translateX86FSETCC()
12946 case ISD::SETUGT: SSECC = 6; break; in translateX86FSETCC()
12947 case ISD::SETO: SSECC = 7; break; in translateX86FSETCC()
12948 case ISD::SETUEQ: in translateX86FSETCC()
12949 case ISD::SETONE: SSECC = 8; break; in translateX86FSETCC()
12962 assert(VT.is256BitVector() && Op.getOpcode() == ISD::SETCC && in Lower256IntVSETCC()
12982 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, in Lower256IntVSETCC()
12999 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); in LowerIntVSETCC_AVX512()
13006 case ISD::SETNE: SSECC = 4; break; in LowerIntVSETCC_AVX512()
13007 case ISD::SETEQ: Opc = X86ISD::PCMPEQM; break; in LowerIntVSETCC_AVX512()
13008 case ISD::SETUGT: SSECC = 6; Unsigned = true; break; in LowerIntVSETCC_AVX512()
13009 case ISD::SETLT: Swap = true; //fall-through in LowerIntVSETCC_AVX512()
13010 case ISD::SETGT: Opc = X86ISD::PCMPGTM; break; in LowerIntVSETCC_AVX512()
13011 case ISD::SETULT: SSECC = 1; Unsigned = true; break; in LowerIntVSETCC_AVX512()
13012 case ISD::SETUGE: SSECC = 5; Unsigned = true; break; //NLT in LowerIntVSETCC_AVX512()
13013 case ISD::SETGE: Swap = true; SSECC = 2; break; // LE + swap in LowerIntVSETCC_AVX512()
13014 case ISD::SETULE: Unsigned = true; //fall-through in LowerIntVSETCC_AVX512()
13015 case ISD::SETLE: SSECC = 2; break; in LowerIntVSETCC_AVX512()
13054 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, ULTOp1); in ChangeVSETULTtoVSETULE()
13063 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get(); in LowerVSETCC()
13083 if (SetCCOpcode == ISD::SETUEQ) { in LowerVSETCC()
13084 CC0 = 3; CC1 = 0; CombineOpc = ISD::OR; in LowerVSETCC()
13086 assert(SetCCOpcode == ISD::SETONE); in LowerVSETCC()
13087 CC0 = 7; CC1 = 4; CombineOpc = ISD::AND; in LowerVSETCC()
13120 return DAG.getNode(ISD::TRUNCATE, dl, VT, in LowerVSETCC()
13121 DAG.getNode(ISD::SETCC, dl, OpVT, Op0, Op1, CC)); in LowerVSETCC()
13133 case ISD::SETNE: Invert = true; in LowerVSETCC()
13134 case ISD::SETEQ: Opc = X86ISD::PCMPEQ; break; in LowerVSETCC()
13135 case ISD::SETLT: Swap = true; in LowerVSETCC()
13136 case ISD::SETGT: Opc = X86ISD::PCMPGT; break; in LowerVSETCC()
13137 case ISD::SETGE: Swap = true; in LowerVSETCC()
13138 case ISD::SETLE: Opc = X86ISD::PCMPGT; in LowerVSETCC()
13140 case ISD::SETULT: Swap = true; in LowerVSETCC()
13141 case ISD::SETUGT: Opc = X86ISD::PCMPGT; in LowerVSETCC()
13143 case ISD::SETUGE: Swap = true; in LowerVSETCC()
13144 case ISD::SETULE: Opc = X86ISD::PCMPGT; in LowerVSETCC()
13157 case ISD::SETULE: Opc = X86ISD::UMIN; MinMax = true; break; in LowerVSETCC()
13158 case ISD::SETUGE: Opc = X86ISD::UMAX; MinMax = true; break; in LowerVSETCC()
13172 case ISD::SETULT: { in LowerVSETCC()
13188 case ISD::SETUGE: Subus = true; Invert = false; Swap = true; break; in LowerVSETCC()
13189 case ISD::SETULE: Subus = true; Invert = false; Swap = false; break; in LowerVSETCC()
13208 Op0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op0); in LowerVSETCC()
13209 Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op1); in LowerVSETCC()
13220 SB = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, in LowerVSETCC()
13223 Op0 = DAG.getNode(ISD::XOR, dl, MVT::v4i32, Op0, SB); in LowerVSETCC()
13224 Op1 = DAG.getNode(ISD::XOR, dl, MVT::v4i32, Op1, SB); in LowerVSETCC()
13237 SDValue Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, EQHi, GTLo); in LowerVSETCC()
13238 Result = DAG.getNode(ISD::OR, dl, MVT::v4i32, Result, GTHi); in LowerVSETCC()
13243 return DAG.getNode(ISD::BITCAST, dl, VT, Result); in LowerVSETCC()
13252 Op0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op0); in LowerVSETCC()
13253 Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op1); in LowerVSETCC()
13261 Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, Result, Shuf); in LowerVSETCC()
13266 return DAG.getNode(ISD::BITCAST, dl, VT, Result); in LowerVSETCC()
13275 Op0 = DAG.getNode(ISD::XOR, dl, VT, Op0, SB); in LowerVSETCC()
13276 Op1 = DAG.getNode(ISD::XOR, dl, VT, Op1, SB); in LowerVSETCC()
13306 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); in LowerSETCC()
13312 if (Op0.getOpcode() == ISD::AND && Op0.hasOneUse() && in LowerSETCC()
13313 Op1.getOpcode() == ISD::Constant && in LowerSETCC()
13315 (CC == ISD::SETEQ || CC == ISD::SETNE)) { in LowerSETCC()
13319 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, NewSetCC); in LowerSETCC()
13326 if (Op1.getOpcode() == ISD::Constant && in LowerSETCC()
13329 (CC == ISD::SETEQ || CC == ISD::SETNE)) { in LowerSETCC()
13335 bool Invert = (CC == ISD::SETNE) ^ in LowerSETCC()
13345 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, SetCC); in LowerSETCC()
13349 if ((Op0.getValueType() == MVT::i1) && (Op1.getOpcode() == ISD::Constant) && in LowerSETCC()
13351 (CC == ISD::SETEQ || CC == ISD::SETNE)) { in LowerSETCC()
13353 ISD::CondCode NewCC = ISD::getSetCCInverse(CC, true); in LowerSETCC()
13367 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, SetCC); in LowerSETCC()
13398 if (V.getOpcode() != ISD::TRUNCATE) in isTruncWithZeroHighBitsInput()
13419 if (Cond.getOpcode() == ISD::SETCC && in LowerSELECT()
13457 SDValue VOp1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op1); in LowerSELECT()
13458 SDValue VOp2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op2); in LowerSELECT()
13459 SDValue VCmp = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Cmp); in LowerSELECT()
13462 VCmp = DAG.getNode(ISD::BITCAST, DL, VCmpVT, VCmp); in LowerSELECT()
13464 SDValue VSel = DAG.getNode(ISD::VSELECT, DL, VecVT, VCmp, VOp1, VOp2); in LowerSELECT()
13466 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, in LowerSELECT()
13475 if (Cond.getOpcode() == ISD::SETCC) { in LowerSELECT()
13526 Res = DAG.getNode(ISD::OR, DL, Res.getValueType(), Res, Y); in LowerSELECT()
13532 if (Cond.getOpcode() == ISD::AND && in LowerSELECT()
13560 } else if (CondOpcode == ISD::USUBO || CondOpcode == ISD::SSUBO || in LowerSELECT()
13561 CondOpcode == ISD::UADDO || CondOpcode == ISD::SADDO || in LowerSELECT()
13562 ((CondOpcode == ISD::UMULO || CondOpcode == ISD::SMULO) && in LowerSELECT()
13570 case ISD::UADDO: X86Opcode = X86ISD::ADD; X86Cond = X86::COND_B; break; in LowerSELECT()
13571 case ISD::SADDO: X86Opcode = X86ISD::ADD; X86Cond = X86::COND_O; break; in LowerSELECT()
13572 case ISD::USUBO: X86Opcode = X86ISD::SUB; X86Cond = X86::COND_B; break; in LowerSELECT()
13573 case ISD::SSUBO: X86Opcode = X86ISD::SUB; X86Cond = X86::COND_O; break; in LowerSELECT()
13574 case ISD::UMULO: X86Opcode = X86ISD::UMUL; X86Cond = X86::COND_O; break; in LowerSELECT()
13575 case ISD::SMULO: X86Opcode = X86ISD::SMUL; X86Cond = X86::COND_O; break; in LowerSELECT()
13578 if (CondOpcode == ISD::UMULO) in LowerSELECT()
13586 if (CondOpcode == ISD::UMULO) in LowerSELECT()
13602 if (Cond.getOpcode() == ISD::AND && Cond.hasOneUse()) { in LowerSELECT()
13603 SDValue NewSetCC = LowerToBT(Cond, ISD::SETNE, DL, DAG); in LowerSELECT()
13639 Op1.getOpcode() == ISD::TRUNCATE && Op2.getOpcode() == ISD::TRUNCATE) { in LowerSELECT()
13643 T1.getOpcode() != ISD::CopyFromReg && T2.getOpcode()!=ISD::CopyFromReg){ in LowerSELECT()
13646 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Cmov); in LowerSELECT()
13758 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi); in LowerSIGN_EXTEND()
13784 ISD::LoadExtType Ext = Ld->getExtensionType(); in LowerExtendedLoad()
13786 assert((Ext == ISD::EXTLOAD || Ext == ISD::SEXTLOAD) in LowerExtendedLoad()
13795 if (Ext == ISD::SEXTLOAD && RegSz == 256 && !Subtarget->hasInt256()) { in LowerExtendedLoad()
13860 assert((Ext != ISD::SEXTLOAD || NumLoads == 1) && in LowerExtendedLoad()
13864 if (Ext == ISD::SEXTLOAD && RegSz == 256) in LowerExtendedLoad()
13901 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoadUnitVecVT, ScalarLoad); in LowerExtendedLoad()
13903 Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, LoadUnitVecVT, Res, in LowerExtendedLoad()
13906 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); in LowerExtendedLoad()
13909 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); in LowerExtendedLoad()
13913 SDValue SlicedVec = DAG.getNode(ISD::BITCAST, dl, WideVecVT, Res); in LowerExtendedLoad()
13916 if (Ext == ISD::SEXTLOAD) { in LowerExtendedLoad()
13927 assert(TLI.isOperationLegalOrCustom(ISD::SRA, RegVT) && in LowerExtendedLoad()
13938 Shuff = DAG.getNode(ISD::BITCAST, dl, RegVT, Shuff); in LowerExtendedLoad()
13944 DAG.getNode(ISD::SRA, dl, RegVT, Shuff, DAG.getConstant(Amt, RegVT)); in LowerExtendedLoad()
13959 Shuff = DAG.getNode(ISD::BITCAST, dl, RegVT, Shuff); in LowerExtendedLoad()
13969 if (Opc != ISD::OR && Opc != ISD::AND) in isAndOrOfSetCCs()
13980 if (Op.getOpcode() != ISD::XOR) in isXor1OfSetCC()
13999 if (Cond.getOpcode() == ISD::SETCC) { in LowerBRCOND()
14001 if (cast<CondCodeSDNode>(Cond.getOperand(2))->get() == ISD::SETEQ && in LowerBRCOND()
14005 (Cond.getOperand(0).getOpcode() == ISD::SADDO || in LowerBRCOND()
14006 Cond.getOperand(0).getOpcode() == ISD::UADDO || in LowerBRCOND()
14007 Cond.getOperand(0).getOpcode() == ISD::SSUBO || in LowerBRCOND()
14008 Cond.getOperand(0).getOpcode() == ISD::USUBO || in LowerBRCOND()
14009 Cond.getOperand(0).getOpcode() == ISD::SMULO || in LowerBRCOND()
14010 Cond.getOperand(0).getOpcode() == ISD::UMULO)) { in LowerBRCOND()
14029 if (Cond.getOpcode() == ISD::AND && in LowerBRCOND()
14063 if (CondOpcode == ISD::UADDO || CondOpcode == ISD::SADDO || in LowerBRCOND()
14064 CondOpcode == ISD::USUBO || CondOpcode == ISD::SSUBO || in LowerBRCOND()
14065 ((CondOpcode == ISD::UMULO || CondOpcode == ISD::SMULO) && in LowerBRCOND()
14076 case ISD::UADDO: X86Opcode = X86ISD::ADD; X86Cond = X86::COND_B; break; in LowerBRCOND()
14077 case ISD::SADDO: in LowerBRCOND()
14084 case ISD::USUBO: X86Opcode = X86ISD::SUB; X86Cond = X86::COND_B; break; in LowerBRCOND()
14085 case ISD::SSUBO: in LowerBRCOND()
14092 case ISD::UMULO: X86Opcode = X86ISD::UMUL; X86Cond = X86::COND_O; break; in LowerBRCOND()
14093 case ISD::SMULO: X86Opcode = X86ISD::SMUL; X86Cond = X86::COND_O; break; in LowerBRCOND()
14098 if (CondOpcode == ISD::UMULO) in LowerBRCOND()
14106 if (CondOpcode == ISD::UMULO) in LowerBRCOND()
14117 if (CondOpc == ISD::OR) { in LowerBRCOND()
14147 if (User->getOpcode() == ISD::BR) { in LowerBRCOND()
14176 } else if (Cond.getOpcode() == ISD::SETCC && in LowerBRCOND()
14177 cast<CondCodeSDNode>(Cond.getOperand(2))->get() == ISD::SETOEQ) { in LowerBRCOND()
14188 if (User->getOpcode() == ISD::BR) { in LowerBRCOND()
14207 } else if (Cond.getOpcode() == ISD::SETCC && in LowerBRCOND()
14208 cast<CondCodeSDNode>(Cond.getOperand(2))->get() == ISD::SETUNE) { in LowerBRCOND()
14219 if (User->getOpcode() == ISD::BR) { in LowerBRCOND()
14248 if (Cond.getOpcode() == ISD::AND && Cond.hasOneUse()) { in LowerBRCOND()
14249 SDValue NewSetCC = LowerToBT(Cond, ISD::SETNE, dl, DAG); in LowerBRCOND()
14306 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value in LowerDYNAMIC_STACKALLOC()
14308 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1, in LowerDYNAMIC_STACKALLOC()
14368 SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0), in LowerDYNAMIC_STACKALLOC()
14409 FIN = DAG.getNode(ISD::ADD, DL, getPointerTy(), in LowerVASTART()
14418 FIN = DAG.getNode(ISD::ADD, DL, getPointerTy(), in LowerVASTART()
14428 FIN = DAG.getNode(ISD::ADD, DL, getPointerTy(), in LowerVASTART()
14435 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); in LowerVASTART()
14542 ISD::isBuildVectorOfConstantSDNodes(SrcOp.getNode())) { in getTargetVShiftByConstNode()
14552 if (CurrentOp->getOpcode() == ISD::UNDEF) { in getTargetVShiftByConstNode()
14564 if (CurrentOp->getOpcode() == ISD::UNDEF) { in getTargetVShiftByConstNode()
14576 if (CurrentOp->getOpcode() == ISD::UNDEF) { in getTargetVShiftByConstNode()
14587 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Elts); in getTargetVShiftByConstNode()
14616 if (Subtarget.hasSSE41() && ShAmt.getOpcode() == ISD::ZERO_EXTEND && in getTargetVShiftNode()
14620 Op0 = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(Op0), MVT::v8i16, Op0); in getTargetVShiftNode()
14634 ShAmt = DAG.getNode(ISD::BUILD_VECTOR, dl, BVT, ShOps); in getTargetVShiftNode()
14642 ShAmt = DAG.getNode(ISD::BITCAST, dl, ShVT, ShAmt); in getTargetVShiftNode()
14667 SDValue VMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT, in getVectorMaskingNode()
14668 DAG.getNode(ISD::BITCAST, dl, BitcastVT, Mask), in getVectorMaskingNode()
14677 return DAG.getNode(ISD::AND, dl, VT, Op, VMask); in getVectorMaskingNode()
14679 if (PreservedSrc.getOpcode() == ISD::UNDEF) in getVectorMaskingNode()
14681 return DAG.getNode(ISD::VSELECT, dl, VT, VMask, Op, PreservedSrc); in getVectorMaskingNode()
14701 SDValue IMask = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Mask); in getScalarMaskingNode()
14703 if (PreservedSrc.getOpcode() == ISD::UNDEF) in getScalarMaskingNode()
14829 SDValue Res = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, BitcastVT, in LowerINTRINSIC_WO_CHAIN()
14832 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); in LowerINTRINSIC_WO_CHAIN()
14835 ISD::CondCode CC = (ISD::CondCode)IntrData->Opc1; in LowerINTRINSIC_WO_CHAIN()
14843 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); in LowerINTRINSIC_WO_CHAIN()
14867 SDValue VMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT, in LowerINTRINSIC_WO_CHAIN()
14868 DAG.getNode(ISD::BITCAST, dl, BitcastVT, Mask), in LowerINTRINSIC_WO_CHAIN()
14882 SDValue VMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT, in LowerINTRINSIC_WO_CHAIN()
14883 DAG.getNode(ISD::BITCAST, dl, BitcastVT, Mask), in LowerINTRINSIC_WO_CHAIN()
14976 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); in LowerINTRINSIC_WO_CHAIN()
14981 SDValue LHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i1, Op.getOperand(1)); in LowerINTRINSIC_WO_CHAIN()
14982 SDValue RHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i1, Op.getOperand(2)); in LowerINTRINSIC_WO_CHAIN()
14986 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); in LowerINTRINSIC_WO_CHAIN()
15050 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC); in LowerINTRINSIC_WO_CHAIN()
15083 MaskInReg = DAG.getNode(ISD::BITCAST, dl, MaskVT, Mask); in getGatherNode()
15087 if (Src.getOpcode() == ISD::UNDEF) in getGatherNode()
15111 MaskInReg = DAG.getNode(ISD::BITCAST, dl, MaskVT, Mask); in getScatterNode()
15134 MaskInReg = DAG.getNode(ISD::BITCAST, dl, MaskVT, Mask); in getPrefetchNode()
15172 SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI, in getReadPerformanceCounter()
15174 Results.push_back(DAG.getNode(ISD::OR, DL, MVT::i64, LO, Tmp)); in getReadPerformanceCounter()
15181 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops); in getReadPerformanceCounter()
15226 SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI, in getReadTimeStampCounter()
15228 Results.push_back(DAG.getNode(ISD::OR, DL, MVT::i64, LO, Tmp)); in getReadTimeStampCounter()
15235 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops); in getReadTimeStampCounter()
15280 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result, isValid, in LowerINTRINSIC_W_CHAIN()
15335 SDValue Ret = DAG.getNode(ISD::ZERO_EXTEND, dl, Op->getValueType(0), SetCC); in LowerINTRINSIC_W_CHAIN()
15336 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), in LowerINTRINSIC_W_CHAIN()
15374 SDValue VMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT, in LowerINTRINSIC_W_CHAIN()
15375 DAG.getNode(ISD::BITCAST, dl, BitcastVT, Mask), in LowerINTRINSIC_W_CHAIN()
15398 SDValue VMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT, in LowerINTRINSIC_W_CHAIN()
15399 DAG.getNode(ISD::BITCAST, dl, BitcastVT, Mask), in LowerINTRINSIC_W_CHAIN()
15430 DAG.getNode(ISD::ADD, dl, PtrVT, in LowerRETURNADDR()
15514 SDValue StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, Frame, in LowerEH_RETURN()
15516 StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, StoreAddr, Offset); in LowerEH_RETURN()
15574 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
15583 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
15589 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
15597 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
15604 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp, in LowerINIT_TRAMPOLINE()
15610 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); in LowerINIT_TRAMPOLINE()
15659 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerINIT_TRAMPOLINE()
15661 Disp = DAG.getNode(ISD::SUB, dl, MVT::i32, FPtr, Addr); in LowerINIT_TRAMPOLINE()
15671 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerINIT_TRAMPOLINE()
15678 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerINIT_TRAMPOLINE()
15684 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp, in LowerINIT_TRAMPOLINE()
15690 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); in LowerINIT_TRAMPOLINE()
15740 DAG.getNode(ISD::SRL, DL, MVT::i16, in LowerFLT_ROUNDS_()
15741 DAG.getNode(ISD::AND, DL, MVT::i16, in LowerFLT_ROUNDS_()
15745 DAG.getNode(ISD::SRL, DL, MVT::i16, in LowerFLT_ROUNDS_()
15746 DAG.getNode(ISD::AND, DL, MVT::i16, in LowerFLT_ROUNDS_()
15751 DAG.getNode(ISD::AND, DL, MVT::i16, in LowerFLT_ROUNDS_()
15752 DAG.getNode(ISD::ADD, DL, MVT::i16, in LowerFLT_ROUNDS_()
15753 DAG.getNode(ISD::OR, DL, MVT::i16, CWD1, CWD2), in LowerFLT_ROUNDS_()
15758 ISD::TRUNCATE : ISD::ZERO_EXTEND), DL, VT, RetVal); in LowerFLT_ROUNDS_()
15771 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op); in LowerCTLZ()
15788 Op = DAG.getNode(ISD::XOR, dl, OpVT, Op, DAG.getConstant(NumBits-1, OpVT)); in LowerCTLZ()
15791 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op); in LowerCTLZ()
15805 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op); in LowerCTLZ_ZERO_UNDEF()
15813 Op = DAG.getNode(ISD::XOR, dl, OpVT, Op, DAG.getConstant(NumBits-1, OpVT)); in LowerCTLZ_ZERO_UNDEF()
15816 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op); in LowerCTLZ_ZERO_UNDEF()
15864 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, in Lower256IntArith()
15910 Evens = DAG.getNode(ISD::BITCAST, dl, VT, Evens); in LowerMUL()
15911 Odds = DAG.getNode(ISD::BITCAST, dl, VT, Odds); in LowerMUL()
15939 A = DAG.getNode(ISD::BITCAST, dl, MulVT, A); in LowerMUL()
15940 B = DAG.getNode(ISD::BITCAST, dl, MulVT, B); in LowerMUL()
15941 Ahi = DAG.getNode(ISD::BITCAST, dl, MulVT, Ahi); in LowerMUL()
15942 Bhi = DAG.getNode(ISD::BITCAST, dl, MulVT, Bhi); in LowerMUL()
15951 SDValue Res = DAG.getNode(ISD::ADD, dl, VT, AloBlo, AloBhi); in LowerMUL()
15952 return DAG.getNode(ISD::ADD, dl, VT, Res, AhiBlo); in LowerMUL()
15965 case ISD::SDIV: isSigned = true; LC = RTLIB::SDIV_I128; break; in LowerWin64_i128OP()
15966 case ISD::UDIV: isSigned = false; LC = RTLIB::UDIV_I128; break; in LowerWin64_i128OP()
15967 case ISD::SREM: isSigned = true; LC = RTLIB::SREM_I128; break; in LowerWin64_i128OP()
15968 case ISD::UREM: isSigned = false; LC = RTLIB::UREM_I128; break; in LowerWin64_i128OP()
15969 case ISD::SDIVREM: isSigned = true; LC = RTLIB::SDIVREM_I128; break; in LowerWin64_i128OP()
15970 case ISD::UDIVREM: isSigned = false; LC = RTLIB::UDIVREM_I128; break; in LowerWin64_i128OP()
16004 return DAG.getNode(ISD::BITCAST, dl, VT, CallInfo.first); in LowerWin64_i128OP()
16037 bool IsSigned = Op->getOpcode() == ISD::SMUL_LOHI; in LowerMUL_LOHI()
16042 SDValue Mul1 = DAG.getNode(ISD::BITCAST, dl, VT, in LowerMUL_LOHI()
16046 SDValue Mul2 = DAG.getNode(ISD::BITCAST, dl, VT, in LowerMUL_LOHI()
16068 SDValue T1 = DAG.getNode(ISD::AND, dl, VT, in LowerMUL_LOHI()
16069 DAG.getNode(ISD::SRA, dl, VT, Op0, ShAmt), Op1); in LowerMUL_LOHI()
16070 SDValue T2 = DAG.getNode(ISD::AND, dl, VT, in LowerMUL_LOHI()
16071 DAG.getNode(ISD::SRA, dl, VT, Op1, ShAmt), Op0); in LowerMUL_LOHI()
16073 SDValue Fixup = DAG.getNode(ISD::ADD, dl, VT, T1, T2); in LowerMUL_LOHI()
16074 Highs = DAG.getNode(ISD::SUB, dl, VT, Highs, Fixup); in LowerMUL_LOHI()
16100 if (Op.getOpcode() == ISD::SHL) in LowerScalarImmediateShift()
16103 if (Op.getOpcode() == ISD::SRL) in LowerScalarImmediateShift()
16106 if (Op.getOpcode() == ISD::SRA && VT != MVT::v2i64 && VT != MVT::v4i64) in LowerScalarImmediateShift()
16115 if (Op.getOpcode() == ISD::SHL) { in LowerScalarImmediateShift()
16119 SHL = DAG.getNode(ISD::BITCAST, dl, VT, SHL); in LowerScalarImmediateShift()
16123 return DAG.getNode(ISD::AND, dl, VT, SHL, in LowerScalarImmediateShift()
16124 DAG.getNode(ISD::BUILD_VECTOR, dl, VT, V)); in LowerScalarImmediateShift()
16126 if (Op.getOpcode() == ISD::SRL) { in LowerScalarImmediateShift()
16130 SRL = DAG.getNode(ISD::BITCAST, dl, VT, SRL); in LowerScalarImmediateShift()
16134 return DAG.getNode(ISD::AND, dl, VT, SRL, in LowerScalarImmediateShift()
16135 DAG.getNode(ISD::BUILD_VECTOR, dl, VT, V)); in LowerScalarImmediateShift()
16137 if (Op.getOpcode() == ISD::SRA) { in LowerScalarImmediateShift()
16145 SDValue Res = DAG.getNode(ISD::SRL, dl, VT, R, Amt); in LowerScalarImmediateShift()
16148 SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, V); in LowerScalarImmediateShift()
16149 Res = DAG.getNode(ISD::XOR, dl, VT, Res, Mask); in LowerScalarImmediateShift()
16150 Res = DAG.getNode(ISD::SUB, dl, VT, Res, Mask); in LowerScalarImmediateShift()
16161 Amt.getOpcode() == ISD::BITCAST && in LowerScalarImmediateShift()
16162 Amt.getOperand(0).getOpcode() == ISD::BUILD_VECTOR) { in LowerScalarImmediateShift()
16192 case ISD::SHL: in LowerScalarImmediateShift()
16195 case ISD::SRL: in LowerScalarImmediateShift()
16198 case ISD::SRA: in LowerScalarImmediateShift()
16214 if ((VT == MVT::v2i64 && Op.getOpcode() != ISD::SRA) || in LowerScalarVariableShift()
16217 ((VT == MVT::v4i64 && Op.getOpcode() != ISD::SRA) || in LowerScalarVariableShift()
16227 if (BaseShAmt && BaseShAmt.getOpcode() == ISD::UNDEF) in LowerScalarVariableShift()
16230 if (Amt.getOpcode() == ISD::EXTRACT_SUBVECTOR) in LowerScalarVariableShift()
16237 if (InVec.getOpcode() == ISD::BUILD_VECTOR) { in LowerScalarVariableShift()
16241 } else if (InVec.getOpcode() == ISD::INSERT_VECTOR_ELT) { in LowerScalarVariableShift()
16251 BaseShAmt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InVec, in LowerScalarVariableShift()
16259 BaseShAmt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, BaseShAmt); in LowerScalarVariableShift()
16261 BaseShAmt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, BaseShAmt); in LowerScalarVariableShift()
16266 case ISD::SHL: in LowerScalarVariableShift()
16279 case ISD::SRA: in LowerScalarVariableShift()
16290 case ISD::SRL: in LowerScalarVariableShift()
16311 Amt.getOpcode() == ISD::BITCAST && in LowerScalarVariableShift()
16312 Amt.getOperand(0).getOpcode() == ISD::BUILD_VECTOR) { in LowerScalarVariableShift()
16327 case ISD::SHL: in LowerScalarVariableShift()
16329 case ISD::SRL: in LowerScalarVariableShift()
16331 case ISD::SRA: in LowerScalarVariableShift()
16360 if (Op.getOpcode() == ISD::SRL && in LowerShift()
16364 if (Op.getOpcode() == ISD::SHL && in LowerShift()
16368 if (Op.getOpcode() == ISD::SRA && (VT == MVT::v4i32 || VT == MVT::v8i32)) in LowerShift()
16374 if (VT == MVT::v2i64 && Op.getOpcode() != ISD::SRA) { in LowerShift()
16386 if (Op.getOpcode() == ISD::SHL && in LowerShift()
16389 ISD::isBuildVectorOfConstantSDNodes(Amt.getNode())) { in LowerShift()
16398 if (Op->getOpcode() == ISD::UNDEF) { in LowerShift()
16412 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Elts); in LowerShift()
16413 return DAG.getNode(ISD::MUL, dl, VT, R, BV); in LowerShift()
16417 if (VT == MVT::v4i32 && Op->getOpcode() == ISD::SHL) { in LowerShift()
16418 Op = DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(23, VT)); in LowerShift()
16420 Op = DAG.getNode(ISD::ADD, dl, VT, Op, DAG.getConstant(0x3f800000U, VT)); in LowerShift()
16421 Op = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, Op); in LowerShift()
16422 Op = DAG.getNode(ISD::FP_TO_SINT, dl, VT, Op); in LowerShift()
16423 return DAG.getNode(ISD::MUL, dl, VT, Op, R); in LowerShift()
16439 ISD::isBuildVectorOfConstantSDNodes(Amt.getNode())) { in LowerShift()
16491 SDValue BitCast1 = DAG.getNode(ISD::BITCAST, dl, CastVT, Shift1); in LowerShift()
16492 SDValue BitCast2 = DAG.getNode(ISD::BITCAST, dl, CastVT, Shift2); in LowerShift()
16495 return DAG.getNode(ISD::BITCAST, dl, VT, Result); in LowerShift()
16499 if (VT == MVT::v16i8 && Op->getOpcode() == ISD::SHL) { in LowerShift()
16503 Op = DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(5, VT)); in LowerShift()
16504 Op = DAG.getNode(ISD::BITCAST, dl, VT, Op); in LowerShift()
16508 SDValue OpVSel = DAG.getNode(ISD::AND, dl, VT, VSelM, Op); in LowerShift()
16515 SDValue M = DAG.getNode(ISD::AND, dl, VT, R, CM1); in LowerShift()
16517 M = DAG.getNode(ISD::BITCAST, dl, VT, M); in LowerShift()
16518 R = DAG.getNode(ISD::VSELECT, dl, VT, OpVSel, M, R); in LowerShift()
16521 Op = DAG.getNode(ISD::ADD, dl, VT, Op, Op); in LowerShift()
16522 OpVSel = DAG.getNode(ISD::AND, dl, VT, VSelM, Op); in LowerShift()
16526 M = DAG.getNode(ISD::AND, dl, VT, R, CM2); in LowerShift()
16528 M = DAG.getNode(ISD::BITCAST, dl, VT, M); in LowerShift()
16529 R = DAG.getNode(ISD::VSELECT, dl, VT, OpVSel, M, R); in LowerShift()
16532 Op = DAG.getNode(ISD::ADD, dl, VT, Op, Op); in LowerShift()
16533 OpVSel = DAG.getNode(ISD::AND, dl, VT, VSelM, Op); in LowerShift()
16537 R = DAG.getNode(ISD::VSELECT, dl, VT, OpVSel, in LowerShift()
16538 DAG.getNode(ISD::ADD, dl, VT, R, R), R); in LowerShift()
16548 Op.getOpcode() == ISD::SRA ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in LowerShift()
16550 Amt = DAG.getNode(ISD::ANY_EXTEND, dl, NewVT, Amt); in LowerShift()
16551 return DAG.getNode(ISD::TRUNCATE, dl, VT, in LowerShift()
16567 if (Amt.getOpcode() == ISD::BUILD_VECTOR) { in LowerShift()
16573 Amt1 = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT, Amt1Csts); in LowerShift()
16574 Amt2 = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT, Amt2Csts); in LowerShift()
16586 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, V1, V2); in LowerShift()
16605 case ISD::SADDO: in LowerXALUO()
16617 case ISD::UADDO: in LowerXALUO()
16621 case ISD::SSUBO: in LowerXALUO()
16633 case ISD::USUBO: in LowerXALUO()
16637 case ISD::SMULO: in LowerXALUO()
16641 case ISD::UMULO: { // i64, i8 = umulo lhs, rhs --> i64, i64, i32 umul lhs,rhs in LowerXALUO()
16656 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Sum, SetCC); in LowerXALUO()
16669 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Sum, SetCC); in LowerXALUO()
16897 Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, InVec, in LowerBITCAST()
16904 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, dl, NewVT, Elts); in LowerBITCAST()
16905 SDValue ToV2F64 = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, BV); in LowerBITCAST()
16906 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, ToV2F64, in LowerBITCAST()
16968 SDValue OnesV = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ones); in LowerCTPOP()
16969 SDValue Srl = DAG.getNode(ISD::SRL, dl, VT, Op, OnesV); in LowerCTPOP()
16971 Srl = DAG.getNode(ISD::BITCAST, dl, BitcastVT, Srl); in LowerCTPOP()
16974 SDValue M55 = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Mask55); in LowerCTPOP()
16976 M55 = DAG.getNode(ISD::BITCAST, dl, BitcastVT, M55); in LowerCTPOP()
16978 SDValue And = DAG.getNode(ISD::AND, dl, Srl.getValueType(), Srl, M55); in LowerCTPOP()
16980 And = DAG.getNode(ISD::BITCAST, dl, VT, And); in LowerCTPOP()
16981 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, Op, And); in LowerCTPOP()
16985 SDValue M33 = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Mask33); in LowerCTPOP()
16987 SDValue TwosV = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Twos); in LowerCTPOP()
16989 Srl = DAG.getNode(ISD::SRL, dl, VT, Sub, TwosV); in LowerCTPOP()
16991 Srl = DAG.getNode(ISD::BITCAST, dl, BitcastVT, Srl); in LowerCTPOP()
16992 M33 = DAG.getNode(ISD::BITCAST, dl, BitcastVT, M33); in LowerCTPOP()
16993 Sub = DAG.getNode(ISD::BITCAST, dl, BitcastVT, Sub); in LowerCTPOP()
16996 SDValue AndRHS = DAG.getNode(ISD::AND, dl, M33.getValueType(), Srl, M33); in LowerCTPOP()
16997 SDValue AndLHS = DAG.getNode(ISD::AND, dl, M33.getValueType(), Sub, M33); in LowerCTPOP()
16999 AndRHS = DAG.getNode(ISD::BITCAST, dl, VT, AndRHS); in LowerCTPOP()
17000 AndLHS = DAG.getNode(ISD::BITCAST, dl, VT, AndLHS); in LowerCTPOP()
17002 SDValue Add = DAG.getNode(ISD::ADD, dl, VT, AndLHS, AndRHS); in LowerCTPOP()
17006 SDValue FoursV = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Fours); in LowerCTPOP()
17007 Srl = DAG.getNode(ISD::SRL, dl, VT, Add, FoursV); in LowerCTPOP()
17008 Add = DAG.getNode(ISD::ADD, dl, VT, Add, Srl); in LowerCTPOP()
17011 SDValue M0F = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Mask0F); in LowerCTPOP()
17013 Add = DAG.getNode(ISD::BITCAST, dl, BitcastVT, Add); in LowerCTPOP()
17014 M0F = DAG.getNode(ISD::BITCAST, dl, BitcastVT, M0F); in LowerCTPOP()
17016 And = DAG.getNode(ISD::AND, dl, M0F.getValueType(), Add, M0F); in LowerCTPOP()
17018 And = DAG.getNode(ISD::BITCAST, dl, VT, And); in LowerCTPOP()
17039 SDValue CstsV = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Csts); in LowerCTPOP()
17040 Srl = DAG.getNode(ISD::SRL, dl, VT, Add, CstsV); in LowerCTPOP()
17041 Add = DAG.getNode(ISD::ADD, dl, VT, Add, Srl); in LowerCTPOP()
17048 SDValue M3F = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Cst3FV); in LowerCTPOP()
17050 Add = DAG.getNode(ISD::BITCAST, dl, BitcastVT, Add); in LowerCTPOP()
17051 M3F = DAG.getNode(ISD::BITCAST, dl, BitcastVT, M3F); in LowerCTPOP()
17053 And = DAG.getNode(ISD::AND, dl, M3F.getValueType(), Add, M3F); in LowerCTPOP()
17055 And = DAG.getNode(ISD::BITCAST, dl, VT, And); in LowerCTPOP()
17064 SDValue negOp = DAG.getNode(ISD::SUB, dl, T, in LowerLOAD_SUB()
17066 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, dl, in LowerLOAD_SUB()
17087 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, in LowerATOMIC_STORE()
17113 case ISD::ADDC: Opc = X86ISD::ADD; break; in LowerADDC_ADDE_SUBC_SUBE()
17114 case ISD::ADDE: Opc = X86ISD::ADC; ExtraOp = true; break; in LowerADDC_ADDE_SUBC_SUBE()
17115 case ISD::SUBC: Opc = X86ISD::SUB; break; in LowerADDC_ADDE_SUBC_SUBE()
17116 case ISD::SUBE: Opc = X86ISD::SBB; ExtraOp = true; break; in LowerADDC_ADDE_SUBC_SUBE()
17170 SDValue SinVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT, in LowerFSINCOS()
17172 SDValue CosVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT, in LowerFSINCOS()
17175 return DAG.getNode(ISD::MERGE_VALUES, dl, Tys, SinVal, CosVal); in LowerFSINCOS()
17183 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, Subtarget, DAG); in LowerOperation()
17184 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: in LowerOperation()
17186 case ISD::CTPOP: return LowerCTPOP(Op, Subtarget, DAG); in LowerOperation()
17187 case ISD::ATOMIC_LOAD_SUB: return LowerLOAD_SUB(Op,DAG); in LowerOperation()
17188 case ISD::ATOMIC_STORE: return LowerATOMIC_STORE(Op,DAG); in LowerOperation()
17189 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); in LowerOperation()
17190 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, Subtarget, DAG); in LowerOperation()
17191 case ISD::VECTOR_SHUFFLE: return lowerVectorShuffle(Op, Subtarget, DAG); in LowerOperation()
17192 case ISD::VSELECT: return LowerVSELECT(Op, DAG); in LowerOperation()
17193 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); in LowerOperation()
17194 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); in LowerOperation()
17195 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op,Subtarget,DAG); in LowerOperation()
17196 case ISD::INSERT_SUBVECTOR: return LowerINSERT_SUBVECTOR(Op, Subtarget,DAG); in LowerOperation()
17197 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG); in LowerOperation()
17198 case ISD::ConstantPool: return LowerConstantPool(Op, DAG); in LowerOperation()
17199 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG); in LowerOperation()
17200 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG); in LowerOperation()
17201 case ISD::ExternalSymbol: return LowerExternalSymbol(Op, DAG); in LowerOperation()
17202 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG); in LowerOperation()
17203 case ISD::SHL_PARTS: in LowerOperation()
17204 case ISD::SRA_PARTS: in LowerOperation()
17205 case ISD::SRL_PARTS: return LowerShiftParts(Op, DAG); in LowerOperation()
17206 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG); in LowerOperation()
17207 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG); in LowerOperation()
17208 case ISD::TRUNCATE: return LowerTRUNCATE(Op, DAG); in LowerOperation()
17209 case ISD::ZERO_EXTEND: return LowerZERO_EXTEND(Op, Subtarget, DAG); in LowerOperation()
17210 case ISD::SIGN_EXTEND: return LowerSIGN_EXTEND(Op, Subtarget, DAG); in LowerOperation()
17211 case ISD::ANY_EXTEND: return LowerANY_EXTEND(Op, Subtarget, DAG); in LowerOperation()
17212 case ISD::FP_TO_SINT: return LowerFP_TO_SINT(Op, DAG); in LowerOperation()
17213 case ISD::FP_TO_UINT: return LowerFP_TO_UINT(Op, DAG); in LowerOperation()
17214 case ISD::FP_EXTEND: return LowerFP_EXTEND(Op, DAG); in LowerOperation()
17215 case ISD::LOAD: return LowerExtendedLoad(Op, Subtarget, DAG); in LowerOperation()
17216 case ISD::FABS: in LowerOperation()
17217 case ISD::FNEG: return LowerFABSorFNEG(Op, DAG); in LowerOperation()
17218 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG); in LowerOperation()
17219 case ISD::FGETSIGN: return LowerFGETSIGN(Op, DAG); in LowerOperation()
17220 case ISD::SETCC: return LowerSETCC(Op, DAG); in LowerOperation()
17221 case ISD::SELECT: return LowerSELECT(Op, DAG); in LowerOperation()
17222 case ISD::BRCOND: return LowerBRCOND(Op, DAG); in LowerOperation()
17223 case ISD::JumpTable: return LowerJumpTable(Op, DAG); in LowerOperation()
17224 case ISD::VASTART: return LowerVASTART(Op, DAG); in LowerOperation()
17225 case ISD::VAARG: return LowerVAARG(Op, DAG); in LowerOperation()
17226 case ISD::VACOPY: return LowerVACOPY(Op, Subtarget, DAG); in LowerOperation()
17227 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, Subtarget, DAG); in LowerOperation()
17228 case ISD::INTRINSIC_VOID: in LowerOperation()
17229 case ISD::INTRINSIC_W_CHAIN: return LowerINTRINSIC_W_CHAIN(Op, Subtarget, DAG); in LowerOperation()
17230 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); in LowerOperation()
17231 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG); in LowerOperation()
17232 case ISD::FRAME_TO_ARGS_OFFSET: in LowerOperation()
17234 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG); in LowerOperation()
17235 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG); in LowerOperation()
17236 case ISD::EH_SJLJ_SETJMP: return lowerEH_SJLJ_SETJMP(Op, DAG); in LowerOperation()
17237 case ISD::EH_SJLJ_LONGJMP: return lowerEH_SJLJ_LONGJMP(Op, DAG); in LowerOperation()
17238 case ISD::INIT_TRAMPOLINE: return LowerINIT_TRAMPOLINE(Op, DAG); in LowerOperation()
17239 case ISD::ADJUST_TRAMPOLINE: return LowerADJUST_TRAMPOLINE(Op, DAG); in LowerOperation()
17240 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG); in LowerOperation()
17241 case ISD::CTLZ: return LowerCTLZ(Op, DAG); in LowerOperation()
17242 case ISD::CTLZ_ZERO_UNDEF: return LowerCTLZ_ZERO_UNDEF(Op, DAG); in LowerOperation()
17243 case ISD::CTTZ: return LowerCTTZ(Op, DAG); in LowerOperation()
17244 case ISD::MUL: return LowerMUL(Op, Subtarget, DAG); in LowerOperation()
17245 case ISD::UMUL_LOHI: in LowerOperation()
17246 case ISD::SMUL_LOHI: return LowerMUL_LOHI(Op, Subtarget, DAG); in LowerOperation()
17247 case ISD::SRA: in LowerOperation()
17248 case ISD::SRL: in LowerOperation()
17249 case ISD::SHL: return LowerShift(Op, Subtarget, DAG); in LowerOperation()
17250 case ISD::SADDO: in LowerOperation()
17251 case ISD::UADDO: in LowerOperation()
17252 case ISD::SSUBO: in LowerOperation()
17253 case ISD::USUBO: in LowerOperation()
17254 case ISD::SMULO: in LowerOperation()
17255 case ISD::UMULO: return LowerXALUO(Op, DAG); in LowerOperation()
17256 case ISD::READCYCLECOUNTER: return LowerREADCYCLECOUNTER(Op, Subtarget,DAG); in LowerOperation()
17257 case ISD::BITCAST: return LowerBITCAST(Op, Subtarget, DAG); in LowerOperation()
17258 case ISD::ADDC: in LowerOperation()
17259 case ISD::ADDE: in LowerOperation()
17260 case ISD::SUBC: in LowerOperation()
17261 case ISD::SUBE: return LowerADDC_ADDE_SUBC_SUBE(Op, DAG); in LowerOperation()
17262 case ISD::ADD: return LowerADD(Op, DAG); in LowerOperation()
17263 case ISD::SUB: return LowerSUB(Op, DAG); in LowerOperation()
17264 case ISD::FSINCOS: return LowerFSINCOS(Op, Subtarget, DAG); in LowerOperation()
17287 SDValue LHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, in ReplaceNodeResults()
17289 SDValue RHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, in ReplaceNodeResults()
17294 case ISD::SIGN_EXTEND_INREG: in ReplaceNodeResults()
17295 case ISD::ADDC: in ReplaceNodeResults()
17296 case ISD::ADDE: in ReplaceNodeResults()
17297 case ISD::SUBC: in ReplaceNodeResults()
17298 case ISD::SUBE: in ReplaceNodeResults()
17301 case ISD::SDIV: in ReplaceNodeResults()
17302 case ISD::UDIV: in ReplaceNodeResults()
17303 case ISD::SREM: in ReplaceNodeResults()
17304 case ISD::UREM: in ReplaceNodeResults()
17305 case ISD::SDIVREM: in ReplaceNodeResults()
17306 case ISD::UDIVREM: { in ReplaceNodeResults()
17311 case ISD::FP_TO_SINT: in ReplaceNodeResults()
17317 case ISD::FP_TO_UINT: { in ReplaceNodeResults()
17318 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT; in ReplaceNodeResults()
17338 case ISD::UINT_TO_FP: { in ReplaceNodeResults()
17343 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v2i64, in ReplaceNodeResults()
17347 SDValue VBias = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v2f64, Bias, Bias); in ReplaceNodeResults()
17348 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64, ZExtIn, in ReplaceNodeResults()
17349 DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, VBias)); in ReplaceNodeResults()
17350 Or = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Or); in ReplaceNodeResults()
17351 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, Or, VBias); in ReplaceNodeResults()
17355 case ISD::FP_ROUND: { in ReplaceNodeResults()
17362 case ISD::FP_EXTEND: { in ReplaceNodeResults()
17369 case ISD::INTRINSIC_W_CHAIN: { in ReplaceNodeResults()
17384 case ISD::READCYCLECOUNTER: { in ReplaceNodeResults()
17388 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { in ReplaceNodeResults()
17394 cpInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(2), in ReplaceNodeResults()
17396 cpInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(2), in ReplaceNodeResults()
17405 swapInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(3), in ReplaceNodeResults()
17407 swapInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(3), in ReplaceNodeResults()
17438 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, T, OpsF)); in ReplaceNodeResults()
17443 case ISD::ATOMIC_SWAP: in ReplaceNodeResults()
17444 case ISD::ATOMIC_LOAD_ADD: in ReplaceNodeResults()
17445 case ISD::ATOMIC_LOAD_SUB: in ReplaceNodeResults()
17446 case ISD::ATOMIC_LOAD_AND: in ReplaceNodeResults()
17447 case ISD::ATOMIC_LOAD_OR: in ReplaceNodeResults()
17448 case ISD::ATOMIC_LOAD_XOR: in ReplaceNodeResults()
17449 case ISD::ATOMIC_LOAD_NAND: in ReplaceNodeResults()
17450 case ISD::ATOMIC_LOAD_MIN: in ReplaceNodeResults()
17451 case ISD::ATOMIC_LOAD_MAX: in ReplaceNodeResults()
17452 case ISD::ATOMIC_LOAD_UMIN: in ReplaceNodeResults()
17453 case ISD::ATOMIC_LOAD_UMAX: in ReplaceNodeResults()
17454 case ISD::ATOMIC_LOAD: { in ReplaceNodeResults()
17459 case ISD::BITCAST: { in ReplaceNodeResults()
17471 SDValue Expanded = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, in ReplaceNodeResults()
17473 SDValue ToVecInt = DAG.getNode(ISD::BITCAST, dl, WiderVT, Expanded); in ReplaceNodeResults()
17484 Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, in ReplaceNodeResults()
17487 Results.push_back(DAG.getNode(ISD::BUILD_VECTOR, dl, DstVT, Elts)); in ReplaceNodeResults()
17801 if (Val.getOpcode() != ISD::LOAD) in isZExtFree()
19323 assert((Opc >= ISD::BUILTIN_OP_END || in computeKnownBitsForTargetNode()
19324 Opc == ISD::INTRINSIC_WO_CHAIN || in computeKnownBitsForTargetNode()
19325 Opc == ISD::INTRINSIC_W_CHAIN || in computeKnownBitsForTargetNode()
19326 Opc == ISD::INTRINSIC_VOID) && in computeKnownBitsForTargetNode()
19351 case ISD::INTRINSIC_WO_CHAIN: { in computeKnownBitsForTargetNode()
19453 if (V1.getOpcode() == ISD::CONCAT_VECTORS && in PerformShuffleCombine256()
19454 V2.getOpcode() == ISD::CONCAT_VECTORS) { in PerformShuffleCombine256()
19465 if (V2.getOperand(0).getOpcode() != ISD::BUILD_VECTOR || in PerformShuffleCombine256()
19466 V2.getOperand(1).getOpcode() != ISD::UNDEF || in PerformShuffleCombine256()
19467 V1.getOperand(1).getOpcode() != ISD::UNDEF) in PerformShuffleCombine256()
19470 if (!ISD::isBuildVectorAllZeros(V2.getOperand(0).getNode())) in PerformShuffleCombine256()
19498 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, in PerformShuffleCombine256()
19505 return DAG.getNode(ISD::BITCAST, dl, VT, ResNode); in PerformShuffleCombine256()
19555 while (Input.getOpcode() == ISD::BITCAST) in combineX86ShuffleChain()
19564 DCI.CombineTo(Root.getNode(), DAG.getNode(ISD::BITCAST, DL, RootVT, Input), in combineX86ShuffleChain()
19600 Op = DAG.getNode(ISD::BITCAST, DL, ShuffleVT, Input); in combineX86ShuffleChain()
19607 DCI.CombineTo(Root.getNode(), DAG.getNode(ISD::BITCAST, DL, RootVT, Op), in combineX86ShuffleChain()
19618 Op = DAG.getNode(ISD::BITCAST, DL, ShuffleVT, Input); in combineX86ShuffleChain()
19622 DCI.CombineTo(Root.getNode(), DAG.getNode(ISD::BITCAST, DL, RootVT, Op), in combineX86ShuffleChain()
19632 Op = DAG.getNode(ISD::BITCAST, DL, ShuffleVT, Input); in combineX86ShuffleChain()
19636 DCI.CombineTo(Root.getNode(), DAG.getNode(ISD::BITCAST, DL, RootVT, Op), in combineX86ShuffleChain()
19666 Op = DAG.getNode(ISD::BITCAST, DL, ShuffleVT, Input); in combineX86ShuffleChain()
19670 DCI.CombineTo(Root.getNode(), DAG.getNode(ISD::BITCAST, DL, RootVT, Op), in combineX86ShuffleChain()
19699 Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Input); in combineX86ShuffleChain()
19702 DAG.getNode(ISD::BUILD_VECTOR, DL, ByteVT, PSHUFBMask); in combineX86ShuffleChain()
19706 DCI.CombineTo(Root.getNode(), DAG.getNode(ISD::BITCAST, DL, RootVT, Op), in combineX86ShuffleChain()
19756 while (Op.getOpcode() == ISD::BITCAST && Op.getOperand(0).hasOneUse()) in combineX86ShufflesRecursively()
19923 case ISD::BITCAST: in combineRedundantDWordShuffle()
19981 case ISD::BITCAST: in combineRedundantDWordShuffle()
20009 V = DAG.getNode(ISD::BITCAST, DL, W.getOperand(0).getValueType(), V); in combineRedundantDWordShuffle()
20028 V = DAG.getNode(ISD::BITCAST, DL, N.getValueType(), V); in combineRedundantDWordShuffle()
20055 case ISD::BITCAST: in combineRedundantHalfShuffle()
20145 V = DAG.getNode(ISD::BITCAST, DL, DVT, V); in PerformTargetShuffleCombine()
20150 return DAG.getNode(ISD::BITCAST, DL, VT, V); in PerformTargetShuffleCombine()
20162 while (D.getOpcode() == ISD::BITCAST && D.hasOneUse()) in PerformTargetShuffleCombine()
20181 V = DAG.getNode(ISD::BITCAST, DL, VT, D.getOperand(0)); in PerformTargetShuffleCombine()
20216 if (N->getOpcode() != ISD::VECTOR_SHUFFLE) in combineShuffleToAddSub()
20227 if (V1->getOpcode() != ISD::FSUB || V2->getOpcode() != ISD::FADD) in combineShuffleToAddSub()
20279 N->getOpcode() == ISD::VECTOR_SHUFFLE) in PerformShuffleCombine()
20295 N1.getOpcode() == ISD::UNDEF && N0.hasOneUse() && in PerformShuffleCombine()
20296 N0.getOpcode() == ISD::BITCAST) { in PerformShuffleCombine()
20308 case ISD::ADD : in PerformShuffleCombine()
20309 case ISD::FADD : in PerformShuffleCombine()
20310 case ISD::SUB : in PerformShuffleCombine()
20311 case ISD::FSUB : in PerformShuffleCombine()
20312 case ISD::MUL : in PerformShuffleCombine()
20313 case ISD::FMUL : in PerformShuffleCombine()
20325 SDValue BC00 = DAG.getNode(ISD::BITCAST, dl, VT, BC0.getOperand(0)); in PerformShuffleCombine()
20326 SDValue BC01 = DAG.getNode(ISD::BITCAST, dl, VT, BC0.getOperand(1)); in PerformShuffleCombine()
20392 if (InVec.getOpcode() == ISD::BITCAST) { in XFormVExtractWithShuffleIntoLoad()
20428 if (LdNode.getOpcode() == ISD::BITCAST) { in XFormVExtractWithShuffleIntoLoad()
20437 if (!ISD::isNormalLoad(LdNode.getNode())) in XFormVExtractWithShuffleIntoLoad()
20453 if (NewAlign > Align || !TLI.isOperationLegalOrCustom(ISD::LOAD, EltVT)) in XFormVExtractWithShuffleIntoLoad()
20466 Shuffle = DAG.getNode(ISD::BITCAST, dl, OriginalVT, Shuffle); in XFormVExtractWithShuffleIntoLoad()
20467 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, N->getValueType(0), Shuffle, in XFormVExtractWithShuffleIntoLoad()
20477 N->getOperand(0)->getOpcode() != ISD::BUILD_VECTOR || in PerformBITCASTCombine()
20504 if (InputVector.getOpcode() == ISD::BITCAST && InputVector.hasOneUse() && in PerformEXTRACT_VECTOR_ELTCombine()
20517 if (MMXSrc.getOpcode() == ISD::EXTRACT_VECTOR_ELT && MMXSrc.hasOneUse() && in PerformEXTRACT_VECTOR_ELTCombine()
20519 MMXSrcOp.getOpcode() == ISD::BITCAST && in PerformEXTRACT_VECTOR_ELTCombine()
20543 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT) in PerformEXTRACT_VECTOR_ELTCombine()
20550 if (Extract->use_begin()->getOpcode() != ISD::SIGN_EXTEND && in PerformEXTRACT_VECTOR_ELTCombine()
20551 Extract->use_begin()->getOpcode() != ISD::ZERO_EXTEND) in PerformEXTRACT_VECTOR_ELTCombine()
20574 if (TLI.isOperationLegal(ISD::SRA, MVT::i64)) { in PerformEXTRACT_VECTOR_ELTCombine()
20575 SDValue Cst = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, InputVector); in PerformEXTRACT_VECTOR_ELTCombine()
20577 SDValue BottomHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Cst, in PerformEXTRACT_VECTOR_ELTCombine()
20579 SDValue TopHalf = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Cst, in PerformEXTRACT_VECTOR_ELTCombine()
20584 Vals[0] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, BottomHalf); in PerformEXTRACT_VECTOR_ELTCombine()
20585 Vals[1] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, in PerformEXTRACT_VECTOR_ELTCombine()
20586 DAG.getNode(ISD::SRA, dl, MVT::i64, BottomHalf, ShAmt)); in PerformEXTRACT_VECTOR_ELTCombine()
20587 Vals[2] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, TopHalf); in PerformEXTRACT_VECTOR_ELTCombine()
20588 Vals[3] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, in PerformEXTRACT_VECTOR_ELTCombine()
20589 DAG.getNode(ISD::SRA, dl, MVT::i64, TopHalf, ShAmt)); in PerformEXTRACT_VECTOR_ELTCombine()
20604 SDValue ScalarAddr = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), in PerformEXTRACT_VECTOR_ELTCombine()
20675 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); in matchIntegerMINMAX()
20683 case ISD::SETULT: in matchIntegerMINMAX()
20684 case ISD::SETULE: in matchIntegerMINMAX()
20686 case ISD::SETUGT: in matchIntegerMINMAX()
20687 case ISD::SETUGE: in matchIntegerMINMAX()
20689 case ISD::SETLT: in matchIntegerMINMAX()
20690 case ISD::SETLE: in matchIntegerMINMAX()
20692 case ISD::SETGT: in matchIntegerMINMAX()
20693 case ISD::SETGE: in matchIntegerMINMAX()
20701 case ISD::SETULT: in matchIntegerMINMAX()
20702 case ISD::SETULE: in matchIntegerMINMAX()
20704 case ISD::SETUGT: in matchIntegerMINMAX()
20705 case ISD::SETUGE: in matchIntegerMINMAX()
20707 case ISD::SETLT: in matchIntegerMINMAX()
20708 case ISD::SETLE: in matchIntegerMINMAX()
20710 case ISD::SETGT: in matchIntegerMINMAX()
20711 case ISD::SETGE: in matchIntegerMINMAX()
20727 if (Cond.getOpcode() == ISD::SIGN_EXTEND) { in transformVSELECTtoBlendVECTOR_SHUFFLE()
20729 if (CondSrc->getOpcode() == ISD::SIGN_EXTEND_INREG) in transformVSELECTtoBlendVECTOR_SHUFFLE()
20733 if (!ISD::isBuildVectorOfConstantSDNodes(Cond.getNode())) in transformVSELECTtoBlendVECTOR_SHUFFLE()
20738 if (ISD::isBuildVectorOfConstantSDNodes(LHS.getNode()) && in transformVSELECTtoBlendVECTOR_SHUFFLE()
20739 ISD::isBuildVectorOfConstantSDNodes(RHS.getNode())) in transformVSELECTtoBlendVECTOR_SHUFFLE()
20751 if (Cond.getOperand(i)->getOpcode() == ISD::UNDEF) in transformVSELECTtoBlendVECTOR_SHUFFLE()
20781 if (Cond.getOpcode() == ISD::SETCC && VT.isFloatingPoint() && in PerformSELECTCombine()
20785 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); in PerformSELECTCombine()
20793 case ISD::SETULT: in PerformSELECTCombine()
20805 case ISD::SETOLE: in PerformSELECTCombine()
20813 case ISD::SETULE: in PerformSELECTCombine()
20817 case ISD::SETOLT: in PerformSELECTCombine()
20818 case ISD::SETLT: in PerformSELECTCombine()
20819 case ISD::SETLE: in PerformSELECTCombine()
20823 case ISD::SETOGE: in PerformSELECTCombine()
20831 case ISD::SETUGT: in PerformSELECTCombine()
20843 case ISD::SETUGE: in PerformSELECTCombine()
20847 case ISD::SETOGT: in PerformSELECTCombine()
20848 case ISD::SETGT: in PerformSELECTCombine()
20849 case ISD::SETGE: in PerformSELECTCombine()
20858 case ISD::SETOGE: in PerformSELECTCombine()
20870 case ISD::SETUGT: in PerformSELECTCombine()
20877 case ISD::SETUGE: in PerformSELECTCombine()
20881 case ISD::SETOGT: in PerformSELECTCombine()
20882 case ISD::SETGT: in PerformSELECTCombine()
20883 case ISD::SETGE: in PerformSELECTCombine()
20887 case ISD::SETULT: in PerformSELECTCombine()
20893 case ISD::SETOLE: in PerformSELECTCombine()
20905 case ISD::SETULE: in PerformSELECTCombine()
20909 case ISD::SETOLT: in PerformSELECTCombine()
20910 case ISD::SETLT: in PerformSELECTCombine()
20911 case ISD::SETLE: in PerformSELECTCombine()
20934 Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, OpVT, Cond); in PerformSELECTCombine()
20951 (Cond.getOpcode() == ISD::SETCC || // setcc -> invertible. in PerformSELECTCombine()
20952 (Cond.getOpcode() == ISD::XOR && // xor(X, C) -> invertible. in PerformSELECTCombine()
20962 Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond, in PerformSELECTCombine()
20966 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, LHS.getValueType(), Cond); in PerformSELECTCombine()
20969 return DAG.getNode(ISD::SHL, DL, LHS.getValueType(), Cond, in PerformSELECTCombine()
20976 Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond, in PerformSELECTCombine()
20980 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, in PerformSELECTCombine()
20982 return DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond, in PerformSELECTCombine()
21011 Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond, in PerformSELECTCombine()
21015 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0), in PerformSELECTCombine()
21019 Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond, in PerformSELECTCombine()
21024 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond, in PerformSELECTCombine()
21047 if (N->getOpcode() == ISD::SELECT && Cond.getOpcode() == ISD::SETCC && in PerformSELECTCombine()
21050 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); in PerformSELECTCombine()
21053 case ISD::SETLT: in PerformSELECTCombine()
21054 case ISD::SETGT: { in PerformSELECTCombine()
21055 ISD::CondCode NewCC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGE; in PerformSELECTCombine()
21058 return DAG.getNode(ISD::SELECT, DL, VT, Cond, LHS, RHS); in PerformSELECTCombine()
21068 if (N->getOpcode() == ISD::VSELECT && Cond.getOpcode() == ISD::SETCC && in PerformSELECTCombine()
21072 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get(); in PerformSELECTCombine()
21077 if (ISD::isBuildVectorAllZeros(LHS.getNode())) { in PerformSELECTCombine()
21079 CC = ISD::getSetCCInverse(CC, true); in PerformSELECTCombine()
21080 } else if (ISD::isBuildVectorAllZeros(RHS.getNode())) { in PerformSELECTCombine()
21092 if ((CC == ISD::SETUGE || CC == ISD::SETUGT) && in PerformSELECTCombine()
21093 Other->getOpcode() == ISD::SUB && DAG.isEqualTo(OpRHS, CondRHS)) in PerformSELECTCombine()
21103 if (CC == ISD::SETUGT && Other->getOpcode() == ISD::ADD && in PerformSELECTCombine()
21115 if (CC == ISD::SETLT && Other->getOpcode() == ISD::XOR && in PerformSELECTCombine()
21116 ISD::isBuildVectorAllZeros(CondRHS.getNode()) && in PerformSELECTCombine()
21128 if (N->getOpcode() == ISD::VSELECT && Cond.getOpcode() == ISD::SETCC) { in PerformSELECTCombine()
21148 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LHS, RHS); in PerformSELECTCombine()
21155 if (N->getOpcode() == ISD::VSELECT && CondVT == VT) { in PerformSELECTCombine()
21159 bool TValIsAllOnes = ISD::isBuildVectorAllOnes(LHS.getNode()); in PerformSELECTCombine()
21160 bool FValIsAllZeros = ISD::isBuildVectorAllZeros(RHS.getNode()); in PerformSELECTCombine()
21166 Cond.getOpcode() == ISD::SETCC && in PerformSELECTCombine()
21169 bool TValIsAllZeros = ISD::isBuildVectorAllZeros(LHS.getNode()); in PerformSELECTCombine()
21170 bool FValIsAllOnes = ISD::isBuildVectorAllOnes(RHS.getNode()); in PerformSELECTCombine()
21174 ISD::CondCode NewCC = in PerformSELECTCombine()
21175 ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(), in PerformSELECTCombine()
21190 Ret = DAG.getNode(ISD::OR, DL, CondVT, Cond, in PerformSELECTCombine()
21191 DAG.getNode(ISD::BITCAST, DL, CondVT, RHS)); in PerformSELECTCombine()
21193 Ret = DAG.getNode(ISD::AND, DL, CondVT, Cond, in PerformSELECTCombine()
21194 DAG.getNode(ISD::BITCAST, DL, CondVT, LHS)); in PerformSELECTCombine()
21196 return DAG.getNode(ISD::BITCAST, DL, VT, Ret); in PerformSELECTCombine()
21211 if ((N->getOpcode() == ISD::VSELECT || in PerformSELECTCombine()
21223 if (N->getOpcode() == ISD::VSELECT && DCI.isBeforeLegalizeOps() && in PerformSELECTCombine()
21225 !ISD::isBuildVectorOfConstantSDNodes(Cond.getNode())) { in PerformSELECTCombine()
21242 if (!TLI.isOperationLegalOrCustom(ISD::VSELECT, VT)) in PerformSELECTCombine()
21278 if (I->getOpcode() != ISD::VSELECT) in PerformSELECTCombine()
21359 while (SetCC.getOpcode() == ISD::ZERO_EXTEND || in checkBoolTestSetCCCombine()
21360 SetCC.getOpcode() == ISD::TRUNCATE || in checkBoolTestSetCCCombine()
21361 SetCC.getOpcode() == ISD::AND) { in checkBoolTestSetCCCombine()
21362 if (SetCC.getOpcode() == ISD::AND) { in checkBoolTestSetCCCombine()
21407 if (Op.getOpcode() == ISD::ZERO_EXTEND || in checkBoolTestSetCCCombine()
21408 Op.getOpcode() == ISD::TRUNCATE) in checkBoolTestSetCCCombine()
21460 case ISD::AND: in checkBoolTestAndOrSetCCCombine()
21464 case ISD::OR: in checkBoolTestAndOrSetCCCombine()
21541 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, TrueC->getValueType(0), Cond); in PerformCMOVCombine()
21544 Cond = DAG.getNode(ISD::SHL, DL, Cond.getValueType(), Cond, in PerformCMOVCombine()
21558 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, in PerformCMOVCombine()
21560 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond, in PerformCMOVCombine()
21595 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0), in PerformCMOVCombine()
21599 Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond, in PerformCMOVCombine()
21604 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond, in PerformCMOVCombine()
21725 if (ISD::isBuildVectorAllZeros(Mask.getNode())) in PerformINTRINSIC_WO_CHAINCombine()
21728 if (ISD::isBuildVectorAllOnes(Mask.getNode())) in PerformINTRINSIC_WO_CHAINCombine()
21782 return DAG.getNode(ISD::SRA, SDLoc(N), VT, Op0, Splat); in PerformINTRINSIC_WO_CHAINCombine()
21823 !(N->hasOneUse() && N->use_begin()->getOpcode() == ISD::ADD)) in PerformMulCombine()
21831 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in PerformMulCombine()
21838 NewMul = DAG.getNode(ISD::SHL, DL, VT, NewMul, in PerformMulCombine()
21859 N1C && N0.getOpcode() == ISD::AND && in PerformSHLCombine()
21860 N0.getOperand(1).getOpcode() == ISD::Constant) { in PerformSHLCombine()
21863 ((N00.getOpcode() == ISD::ANY_EXTEND || in PerformSHLCombine()
21864 N00.getOpcode() == ISD::ZERO_EXTEND) && in PerformSHLCombine()
21870 return DAG.getNode(ISD::AND, SDLoc(N), VT, in PerformSHLCombine()
21886 return DAG.getNode(ISD::ADD, SDLoc(N), VT, N0, N0); in PerformSHLCombine()
21926 if (N->getOpcode() == ISD::SHL) { in PerformShiftCombine()
21931 if (N->getOpcode() != ISD::SRA) { in PerformShiftCombine()
21972 case ISD::BR_CC: in CMPEQCombine()
21973 case ISD::BRCOND: in CMPEQCombine()
21974 case ISD::SELECT: in CMPEQCombine()
21977 case ISD::CopyToReg: in CMPEQCombine()
21978 case ISD::SIGN_EXTEND: in CMPEQCombine()
21979 case ISD::ZERO_EXTEND: in CMPEQCombine()
21980 case ISD::ANY_EXTEND: in CMPEQCombine()
22003 return DAG.getNode(ISD::ZERO_EXTEND, DL, N->getValueType(0), in CMPEQCombine()
22020 SDValue Vector64 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, in CMPEQCombine()
22022 SDValue Vector32 = DAG.getNode(ISD::BITCAST, DL, MVT::v4f32, in CMPEQCombine()
22024 OnesOrZeroesF = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, in CMPEQCombine()
22029 SDValue OnesOrZeroesI = DAG.getNode(ISD::BITCAST, DL, IntVT, OnesOrZeroesF); in CMPEQCombine()
22030 SDValue ANDed = DAG.getNode(ISD::AND, DL, IntVT, OnesOrZeroesI, in CMPEQCombine()
22032 SDValue OneBitOfTruth = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, ANDed); in CMPEQCombine()
22047 if (ISD::isBuildVectorAllOnes(N)) in CanFoldXORWithAllOnes()
22051 if (N->getOpcode() == ISD::BITCAST) in CanFoldXORWithAllOnes()
22057 N->getOpcode() == ISD::INSERT_SUBVECTOR) { in CanFoldXORWithAllOnes()
22061 if (V1.getOpcode() == ISD::INSERT_SUBVECTOR && in CanFoldXORWithAllOnes()
22062 V1.getOperand(0).getOpcode() == ISD::UNDEF && in CanFoldXORWithAllOnes()
22063 ISD::isBuildVectorAllOnes(V1.getOperand(1).getNode()) && in CanFoldXORWithAllOnes()
22064 ISD::isBuildVectorAllOnes(V2.getNode())) in CanFoldXORWithAllOnes()
22082 assert((N->getOpcode() == ISD::ANY_EXTEND || in WidenMaskArithmetic()
22083 N->getOpcode() == ISD::ZERO_EXTEND || in WidenMaskArithmetic()
22084 N->getOpcode() == ISD::SIGN_EXTEND) && "Invalid Node"); in WidenMaskArithmetic()
22091 if (Narrow->getOpcode() != ISD::XOR && in WidenMaskArithmetic()
22092 Narrow->getOpcode() != ISD::AND && in WidenMaskArithmetic()
22093 Narrow->getOpcode() != ISD::OR) in WidenMaskArithmetic()
22101 if (N0.getOpcode() != ISD::TRUNCATE) in WidenMaskArithmetic()
22110 bool RHSTrunc = N1.getOpcode() == ISD::TRUNCATE; in WidenMaskArithmetic()
22125 N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT.getScalarType(), in WidenMaskArithmetic()
22128 N1 = DAG.getNode(ISD::BUILD_VECTOR, DL, WideVT, C); in WidenMaskArithmetic()
22137 case ISD::ANY_EXTEND: in WidenMaskArithmetic()
22139 case ISD::ZERO_EXTEND: { in WidenMaskArithmetic()
22143 return DAG.getNode(ISD::AND, DL, VT, in WidenMaskArithmetic()
22146 case ISD::SIGN_EXTEND: in WidenMaskArithmetic()
22147 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, in WidenMaskArithmetic()
22166 if (N0.getOpcode() != ISD::BITCAST || in VectorZextCombine()
22167 N0.getOperand(0).getOpcode() != ISD::VECTOR_SHUFFLE) in VectorZextCombine()
22172 if (N1.getOpcode() == ISD::BITCAST) in VectorZextCombine()
22174 if (N1.getOpcode() != ISD::BUILD_VECTOR) in VectorZextCombine()
22182 if (Shuffle->getOperand(1)->getOpcode() != ISD::UNDEF) in VectorZextCombine()
22241 return DAG.getNode(ISD::BITCAST, DL, N0.getValueType(), NewShuffle); in VectorZextCombine()
22266 (N0.getOpcode() == ISD::SRA || N0.getOpcode() == ISD::SRL)) { in PerformAndCombine()
22292 if (N0.getOpcode() == ISD::XOR && in PerformAndCombine()
22298 if (N1.getOpcode() == ISD::XOR && in PerformAndCombine()
22330 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == X86ISD::ANDNP) { in PerformOrCombine()
22345 if (Mask.getOpcode() == ISD::BITCAST) in PerformOrCombine()
22347 if (X.getOpcode() == ISD::BITCAST) in PerformOrCombine()
22349 if (Y.getOpcode() == ISD::BITCAST) in PerformOrCombine()
22359 if (Mask.getOpcode() == ISD::SRA) { in PerformOrCombine()
22375 if (Y.getOpcode() == ISD::SUB && Y.getOperand(1) == X && in PerformOrCombine()
22376 ISD::isBuildVectorAllZeros(Y.getOperand(0).getNode()) && in PerformOrCombine()
22381 return DAG.getNode(ISD::BITCAST, DL, VT, Mask); in PerformOrCombine()
22389 X = DAG.getNode(ISD::BITCAST, DL, BlendVT, X); in PerformOrCombine()
22390 Y = DAG.getNode(ISD::BITCAST, DL, BlendVT, Y); in PerformOrCombine()
22391 Mask = DAG.getNode(ISD::BITCAST, DL, BlendVT, Mask); in PerformOrCombine()
22392 Mask = DAG.getNode(ISD::VSELECT, DL, BlendVT, Mask, Y, X); in PerformOrCombine()
22393 return DAG.getNode(ISD::BITCAST, DL, VT, Mask); in PerformOrCombine()
22413 if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL) in PerformOrCombine()
22415 if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL) in PerformOrCombine()
22426 if (ShAmt0.getOpcode() == ISD::TRUNCATE) in PerformOrCombine()
22428 if (ShAmt1.getOpcode() == ISD::TRUNCATE) in PerformOrCombine()
22435 if (ShAmt0.getOpcode() == ISD::SUB) { in PerformOrCombine()
22442 if (ShAmt1.getOpcode() == ISD::SUB) { in PerformOrCombine()
22446 if (ShAmt1Op1.getNode()->getOpcode() == ISD::TRUNCATE) in PerformOrCombine()
22451 DAG.getNode(ISD::TRUNCATE, DL, in PerformOrCombine()
22460 DAG.getNode(ISD::TRUNCATE, DL, in PerformOrCombine()
22482 if (VT.isInteger() && N->getOpcode() == ISD::XOR && in performIntegerAbsCombine()
22483 N0.getOpcode() == ISD::ADD && in performIntegerAbsCombine()
22485 N1.getOpcode() == ISD::SRA && in performIntegerAbsCombine()
22529 ISD::LoadExtType Ext = Ld->getExtensionType(); in PerformLOADCombine()
22533 !DCI.isBeforeLegalizeOps() && !IsAligned && Ext == ISD::NON_EXTLOAD) { in PerformLOADCombine()
22547 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); in PerformLOADCombine()
22552 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, in PerformLOADCombine()
22570 if (Mld->getExtensionType() != ISD::SEXTLOAD) in PerformMLOADCombine()
22594 SDValue WideSrc0 = DAG.getNode(ISD::BITCAST, dl, WideVecVT, Mld->getSrc0()); in PerformMLOADCombine()
22595 if (Mld->getSrc0().getOpcode() != ISD::UNDEF) { in PerformMLOADCombine()
22611 NewMask = DAG.getNode(ISD::BITCAST, dl, WideVecVT, Mask); in PerformMLOADCombine()
22635 NewMask = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewMaskVT, Ops); in PerformMLOADCombine()
22641 ISD::NON_EXTLOAD); in PerformMLOADCombine()
22679 SDValue WideVec = DAG.getNode(ISD::BITCAST, dl, WideVecVT, Mst->getValue()); in PerformMSTORECombine()
22696 NewMask = DAG.getNode(ISD::BITCAST, dl, WideVecVT, Mask); in PerformMSTORECombine()
22719 NewMask = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewMaskVT, Ops); in PerformMSTORECombine()
22750 SDValue Ptr1 = DAG.getNode(ISD::ADD, dl, Ptr0.getValueType(), Ptr0, Stride); in PerformSTORECombine()
22759 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Ch0, Ch1); in PerformSTORECombine()
22788 SDValue WideVec = DAG.getNode(ISD::BITCAST, dl, WideVecVT, St->getValue()); in PerformSTORECombine()
22819 SDValue ShuffWide = DAG.getNode(ISD::BITCAST, dl, StoreVecVT, Shuff); in PerformSTORECombine()
22827 SDValue SubVec = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, in PerformSTORECombine()
22833 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); in PerformSTORECombine()
22837 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); in PerformSTORECombine()
22869 ChainVal->getOpcode() == ISD::TokenFactor) { in PerformSTORECombine()
22879 if (!Ld || !ISD::isNormalLoad(Ld)) in PerformSTORECombine()
22902 NewChain = DAG.getNode(ISD::TokenFactor, LdDL, MVT::Other, Ops); in PerformSTORECombine()
22912 SDValue HiAddr = DAG.getNode(ISD::ADD, LdDL, MVT::i32, LoAddr, in PerformSTORECombine()
22929 NewChain = DAG.getNode(ISD::TokenFactor, LdDL, MVT::Other, Ops); in PerformSTORECombine()
22933 HiAddr = DAG.getNode(ISD::ADD, StDL, MVT::i32, LoAddr, in PerformSTORECombine()
22945 return DAG.getNode(ISD::TokenFactor, StDL, MVT::Other, LoSt, HiSt); in PerformSTORECombine()
22976 if (LHS.getOpcode() != ISD::VECTOR_SHUFFLE && in isHorizontalBinOp()
22977 RHS.getOpcode() != ISD::VECTOR_SHUFFLE) in isHorizontalBinOp()
23002 if (LHS.getOpcode() == ISD::VECTOR_SHUFFLE) { in isHorizontalBinOp()
23003 if (LHS.getOperand(0).getOpcode() != ISD::UNDEF) in isHorizontalBinOp()
23005 if (LHS.getOperand(1).getOpcode() != ISD::UNDEF) in isHorizontalBinOp()
23010 if (LHS.getOpcode() != ISD::UNDEF) in isHorizontalBinOp()
23020 if (RHS.getOpcode() == ISD::VECTOR_SHUFFLE) { in isHorizontalBinOp()
23021 if (RHS.getOperand(0).getOpcode() != ISD::UNDEF) in isHorizontalBinOp()
23023 if (RHS.getOperand(1).getOpcode() != ISD::UNDEF) in isHorizontalBinOp()
23028 if (RHS.getOpcode() != ISD::UNDEF) in isHorizontalBinOp()
23194 if (Op.getOpcode() == ISD::BITCAST) in PerformVZEXT_MOVLCombine()
23200 return DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Op); in PerformVZEXT_MOVLCombine()
23221 if (VT == MVT::v4i64 && (N0.getOpcode() == ISD::ANY_EXTEND || in PerformSIGN_EXTEND_INREGCombine()
23222 N0.getOpcode() == ISD::SIGN_EXTEND)) { in PerformSIGN_EXTEND_INREGCombine()
23227 if (N00.getOpcode() == ISD::LOAD && Subtarget->hasInt256()) in PerformSIGN_EXTEND_INREGCombine()
23228 if (!ISD::isNormalLoad(N00.getNode())) in PerformSIGN_EXTEND_INREGCombine()
23232 SDValue Tmp = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::v4i32, in PerformSIGN_EXTEND_INREGCombine()
23234 return DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i64, Tmp); in PerformSIGN_EXTEND_INREGCombine()
23250 if (N0.getOpcode() == ISD::SDIVREM && N0.getResNo() == 1 && in PerformSExtCombine()
23293 bool NegA = (A.getOpcode() == ISD::FNEG); in PerformFMACombine()
23294 bool NegB = (B.getOpcode() == ISD::FNEG); in PerformFMACombine()
23295 bool NegC = (C.getOpcode() == ISD::FNEG); in PerformFMACombine()
23326 if (N0.getOpcode() == ISD::AND && in PerformZExtCombine()
23334 return DAG.getNode(ISD::AND, dl, VT, in PerformZExtCombine()
23341 if (N0.getOpcode() == ISD::TRUNCATE && in PerformZExtCombine()
23346 return DAG.getNode(ISD::AND, dl, VT, in PerformZExtCombine()
23362 if (N0.getOpcode() == ISD::UDIVREM && in PerformZExtCombine()
23379 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get(); in PerformISDSETCCCombine()
23385 if ((CC == ISD::SETNE || CC == ISD::SETEQ) && LHS.getOpcode() == ISD::SUB) in PerformISDSETCCCombine()
23388 SDValue addV = DAG.getNode(ISD::ADD, SDLoc(N), LHS.getValueType(), RHS, in PerformISDSETCCCombine()
23393 if ((CC == ISD::SETNE || CC == ISD::SETEQ) && RHS.getOpcode() == ISD::SUB) in PerformISDSETCCCombine()
23396 SDValue addV = DAG.getNode(ISD::ADD, SDLoc(N), RHS.getValueType(), LHS, in PerformISDSETCCCombine()
23403 (CC == ISD::SETNE || CC == ISD::SETEQ || ISD::isSignedIntSetCC(CC))) { in PerformISDSETCCCombine()
23405 (LHS.getOpcode() == ISD::SIGN_EXTEND) && in PerformISDSETCCCombine()
23407 bool IsVZero1 = ISD::isBuildVectorAllZeros(RHS.getNode()); in PerformISDSETCCCombine()
23412 CC = ISD::getSetCCSwappedOperands(CC); in PerformISDSETCCCombine()
23414 IsSEXT0 = (LHS.getOpcode() == ISD::SIGN_EXTEND) && in PerformISDSETCCCombine()
23416 IsVZero1 = ISD::isBuildVectorAllZeros(RHS.getNode()); in PerformISDSETCCCombine()
23422 if (CC == ISD::SETGT) in PerformISDSETCCCombine()
23424 if (CC == ISD::SETLE) in PerformISDSETCCCombine()
23426 if (CC == ISD::SETEQ || CC == ISD::SETGE) in PerformISDSETCCCombine()
23429 assert((CC == ISD::SETNE || CC == ISD::SETLT) && in PerformISDSETCCCombine()
23445 ISD::ADD, dl, Addr.getSimpleValueType(), Addr, in NarrowVectorLoadToElement()
23474 SDValue LoadScalarToVector = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Ld); in PerformINSERTPSCombine()
23515 return DAG.getNode(ISD::AND, DL, VT, in MaterializeSETB()
23520 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, in MaterializeSETB()
23605 if (!VT.isVector() || N->getOperand(0)->getOpcode() != ISD::AND || in performVectorCompareAndMaskUnaryOpCombine()
23606 N->getOperand(0)->getOperand(0)->getOpcode() != ISD::SETCC || in performVectorCompareAndMaskUnaryOpCombine()
23627 SDValue MaskConst = DAG.getNode(ISD::BITCAST, DL, IntVT, SourceConst); in performVectorCompareAndMaskUnaryOpCombine()
23628 SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT, in performVectorCompareAndMaskUnaryOpCombine()
23630 SDValue Res = DAG.getNode(ISD::BITCAST, DL, VT, NewAnd); in performVectorCompareAndMaskUnaryOpCombine()
23653 SDValue P = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Op0); in PerformSINT_TO_FPCombine()
23654 return DAG.getNode(ISD::SINT_TO_FP, dl, N->getValueType(0), P); in PerformSINT_TO_FPCombine()
23659 if (Op0.getOpcode() == ISD::LOAD) { in PerformSINT_TO_FPCombine()
23668 ISD::isNON_EXTLoad(Op0.getNode()) && Op0.hasOneUse() && in PerformSINT_TO_FPCombine()
23693 SDValue Res1 = DAG.getNode(ISD::AND, DL, VT, in PerformADCCombine()
23712 SDValue Ext = N->getOperand(N->getOpcode() == ISD::SUB ? 1 : 0); in OptimizeConditionalInDecrement()
23713 if (Ext.getOpcode() != ISD::ZERO_EXTEND || !Ext.hasOneUse()) in OptimizeConditionalInDecrement()
23734 SDValue OtherVal = N->getOperand(N->getOpcode() == ISD::SUB ? 0 : 1); in OptimizeConditionalInDecrement()
23736 return DAG.getNode(N->getOpcode() == ISD::SUB ? X86ISD::ADC : X86ISD::SBB, in OptimizeConditionalInDecrement()
23739 return DAG.getNode(N->getOpcode() == ISD::SUB ? X86ISD::SBB : X86ISD::ADC, in OptimizeConditionalInDecrement()
23771 if (Op1->hasOneUse() && Op1.getOpcode() == ISD::XOR && in PerformSubCombine()
23775 SDValue NewXor = DAG.getNode(ISD::XOR, SDLoc(Op1), VT, in PerformSubCombine()
23778 return DAG.getNode(ISD::ADD, SDLoc(N), VT, NewXor, in PerformSubCombine()
23806 while (V.getOpcode() == ISD::BITCAST) in performVZEXTCombine()
23829 DAG.getNode(ISD::BITCAST, DL, OpVT, V)); in performVZEXTCombine()
23835 if (V.getOpcode() == ISD::SCALAR_TO_VECTOR && in performVZEXTCombine()
23836 V.getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT && in performVZEXTCombine()
23848 OrigV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OrigVT, OrigV, in performVZEXTCombine()
23851 Op = DAG.getNode(ISD::BITCAST, DL, OpVT, OrigV); in performVZEXTCombine()
23864 case ISD::EXTRACT_VECTOR_ELT: in PerformDAGCombine()
23866 case ISD::VSELECT: in PerformDAGCombine()
23867 case ISD::SELECT: in PerformDAGCombine()
23870 case ISD::BITCAST: return PerformBITCASTCombine(N, DAG); in PerformDAGCombine()
23872 case ISD::ADD: return PerformAddCombine(N, DAG, Subtarget); in PerformDAGCombine()
23873 case ISD::SUB: return PerformSubCombine(N, DAG, Subtarget); in PerformDAGCombine()
23875 case ISD::MUL: return PerformMulCombine(N, DAG, DCI); in PerformDAGCombine()
23876 case ISD::SHL: in PerformDAGCombine()
23877 case ISD::SRA: in PerformDAGCombine()
23878 case ISD::SRL: return PerformShiftCombine(N, DAG, DCI, Subtarget); in PerformDAGCombine()
23879 case ISD::AND: return PerformAndCombine(N, DAG, DCI, Subtarget); in PerformDAGCombine()
23880 case ISD::OR: return PerformOrCombine(N, DAG, DCI, Subtarget); in PerformDAGCombine()
23881 case ISD::XOR: return PerformXorCombine(N, DAG, DCI, Subtarget); in PerformDAGCombine()
23882 case ISD::LOAD: return PerformLOADCombine(N, DAG, DCI, Subtarget); in PerformDAGCombine()
23883 case ISD::MLOAD: return PerformMLOADCombine(N, DAG, DCI, Subtarget); in PerformDAGCombine()
23884 case ISD::STORE: return PerformSTORECombine(N, DAG, Subtarget); in PerformDAGCombine()
23885 case ISD::MSTORE: return PerformMSTORECombine(N, DAG, Subtarget); in PerformDAGCombine()
23886 case ISD::SINT_TO_FP: return PerformSINT_TO_FPCombine(N, DAG, Subtarget); in PerformDAGCombine()
23887 case ISD::FADD: return PerformFADDCombine(N, DAG, Subtarget); in PerformDAGCombine()
23888 case ISD::FSUB: return PerformFSUBCombine(N, DAG, Subtarget); in PerformDAGCombine()
23897 case ISD::ANY_EXTEND: in PerformDAGCombine()
23898 case ISD::ZERO_EXTEND: return PerformZExtCombine(N, DAG, DCI, Subtarget); in PerformDAGCombine()
23899 case ISD::SIGN_EXTEND: return PerformSExtCombine(N, DAG, DCI, Subtarget); in PerformDAGCombine()
23900 case ISD::SIGN_EXTEND_INREG: in PerformDAGCombine()
23902 case ISD::TRUNCATE: return PerformTruncateCombine(N, DAG,DCI,Subtarget); in PerformDAGCombine()
23903 case ISD::SETCC: return PerformISDSETCCCombine(N, DAG, Subtarget); in PerformDAGCombine()
23921 case ISD::VECTOR_SHUFFLE: return PerformShuffleCombine(N, DAG, DCI,Subtarget); in PerformDAGCombine()
23922 case ISD::FMA: return PerformFMACombine(N, DAG, Subtarget); in PerformDAGCombine()
23923 case ISD::INTRINSIC_WO_CHAIN: in PerformDAGCombine()
23931 case ISD::BUILD_VECTOR: return PerformBUILD_VECTORCombine(N, DAG, Subtarget); in PerformDAGCombine()
23950 case ISD::LOAD: in isTypeDesirableForOp()
23951 case ISD::SIGN_EXTEND: in isTypeDesirableForOp()
23952 case ISD::ZERO_EXTEND: in isTypeDesirableForOp()
23953 case ISD::ANY_EXTEND: in isTypeDesirableForOp()
23954 case ISD::SHL: in isTypeDesirableForOp()
23955 case ISD::SRL: in isTypeDesirableForOp()
23956 case ISD::SUB: in isTypeDesirableForOp()
23957 case ISD::ADD: in isTypeDesirableForOp()
23958 case ISD::MUL: in isTypeDesirableForOp()
23959 case ISD::AND: in isTypeDesirableForOp()
23960 case ISD::OR: in isTypeDesirableForOp()
23961 case ISD::XOR: in isTypeDesirableForOp()
23978 case ISD::LOAD: { in IsDesirableToPromoteOp()
23982 if (LD->getExtensionType() == ISD::NON_EXTLOAD /*&& in IsDesirableToPromoteOp()
23988 if (UI->getOpcode() != ISD::CopyToReg) in IsDesirableToPromoteOp()
23995 case ISD::SIGN_EXTEND: in IsDesirableToPromoteOp()
23996 case ISD::ZERO_EXTEND: in IsDesirableToPromoteOp()
23997 case ISD::ANY_EXTEND: in IsDesirableToPromoteOp()
24000 case ISD::SHL: in IsDesirableToPromoteOp()
24001 case ISD::SRL: { in IsDesirableToPromoteOp()
24009 case ISD::ADD: in IsDesirableToPromoteOp()
24010 case ISD::MUL: in IsDesirableToPromoteOp()
24011 case ISD::AND: in IsDesirableToPromoteOp()
24012 case ISD::OR: in IsDesirableToPromoteOp()
24013 case ISD::XOR: in IsDesirableToPromoteOp()
24016 case ISD::SUB: { in IsDesirableToPromoteOp()
24436 } else if (Op.getOpcode() == ISD::ADD) { in LowerAsmOperandForConstraint()
24442 } else if (Op.getOpcode() == ISD::SUB) { in LowerAsmOperandForConstraint()