Lines Matching refs:ISD
263 TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) && in ExpandConstantFP()
276 DAG.getExtLoad(ISD::EXTLOAD, dl, OrigVT, in ExpandConstantFP()
293 assert(ST->getAddressingMode() == ISD::UNINDEXED && in ExpandUnalignedStore()
310 SDValue Result = DAG.getNode(ISD::BITCAST, dl, intVT, Val); in ExpandUnalignedStore()
351 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, in ExpandUnalignedStore()
353 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); in ExpandUnalignedStore()
363 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Store, StackPtr, in ExpandUnalignedStore()
375 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); in ExpandUnalignedStore()
391 SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount); in ExpandUnalignedStore()
399 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, in ExpandUnalignedStore()
408 DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2); in ExpandUnalignedStore()
417 assert(LD->getAddressingMode() == ISD::UNINDEXED && in ExpandUnalignedLoad()
431 SDValue Result = DAG.getNode(ISD::BITCAST, dl, LoadedVT, newLoad); in ExpandUnalignedLoad()
433 Result = DAG.getNode(VT.isFloatingPoint() ? ISD::FP_EXTEND : in ExpandUnalignedLoad()
434 ISD::ANY_EXTEND, dl, VT, Result); in ExpandUnalignedLoad()
470 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); in ExpandUnalignedLoad()
471 StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr, in ExpandUnalignedLoad()
478 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, RegVT, Chain, Ptr, in ExpandUnalignedLoad()
493 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); in ExpandUnalignedLoad()
517 ISD::LoadExtType HiExtType = LD->getExtensionType(); in ExpandUnalignedLoad()
520 if (HiExtType == ISD::NON_EXTLOAD) in ExpandUnalignedLoad()
521 HiExtType = ISD::ZEXTLOAD; in ExpandUnalignedLoad()
526 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, LD->getPointerInfo(), in ExpandUnalignedLoad()
530 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, in ExpandUnalignedLoad()
542 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, in ExpandUnalignedLoad()
544 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, VT, Chain, Ptr, in ExpandUnalignedLoad()
554 SDValue Result = DAG.getNode(ISD::SHL, dl, VT, Hi, ShiftAmount); in ExpandUnalignedLoad()
555 Result = DAG.getNode(ISD::OR, dl, VT, Result, Lo); in ExpandUnalignedLoad()
557 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), in ExpandUnalignedLoad()
595 unsigned CastOpc = IdxVT.bitsGT(PtrVT) ? ISD::TRUNCATE : ISD::ZERO_EXTEND; in PerformInsertVectorEltInMemory()
599 Tmp3 = DAG.getNode(ISD::MUL, dl, IdxVT, Tmp3,DAG.getConstant(EltSize, IdxVT)); in PerformInsertVectorEltInMemory()
600 SDValue StackPtr2 = DAG.getNode(ISD::ADD, dl, IdxVT, Tmp3, StackPtr); in PerformInsertVectorEltInMemory()
620 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, in ExpandINSERT_VECTOR_ELT()
683 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, in OptimizeFloatStore()
690 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); in OptimizeFloatStore()
717 switch (TLI.getOperationAction(ISD::STORE, VT)) { in LegalizeStoreOps()
739 MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT); in LegalizeStoreOps()
742 Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value); in LegalizeStoreOps()
794 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, in LegalizeStoreOps()
796 Hi = DAG.getNode(ISD::SRL, dl, Value.getValueType(), Value, in LegalizeStoreOps()
807 Hi = DAG.getNode(ISD::SRL, dl, Value.getValueType(), Value, in LegalizeStoreOps()
816 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, in LegalizeStoreOps()
825 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); in LegalizeStoreOps()
857 Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value); in LegalizeStoreOps()
875 ISD::LoadExtType ExtType = LD->getExtensionType(); in LegalizeLoadOps()
876 if (ExtType == ISD::NON_EXTLOAD) { in LegalizeLoadOps()
912 RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res); in LegalizeLoadOps()
958 ISD::LoadExtType NewExtType = in LegalizeLoadOps()
959 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD; in LegalizeLoadOps()
969 if (ExtType == ISD::SEXTLOAD) in LegalizeLoadOps()
971 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, in LegalizeLoadOps()
974 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType()) in LegalizeLoadOps()
976 Result = DAG.getNode(ISD::AssertZext, dl, in LegalizeLoadOps()
999 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), in LegalizeLoadOps()
1006 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, in LegalizeLoadOps()
1015 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), in LegalizeLoadOps()
1019 Hi = DAG.getNode(ISD::SHL, dl, Hi.getValueType(), Hi, in LegalizeLoadOps()
1024 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); in LegalizeLoadOps()
1035 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, in LegalizeLoadOps()
1037 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, in LegalizeLoadOps()
1045 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), in LegalizeLoadOps()
1049 Hi = DAG.getNode(ISD::SHL, dl, Hi.getValueType(), Hi, in LegalizeLoadOps()
1054 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); in LegalizeLoadOps()
1093 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, Node->getValueType(0), SrcVT)) { in LegalizeLoadOps()
1101 ISD::LoadExtType MidExtType = in LegalizeLoadOps()
1102 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType; in LegalizeLoadOps()
1107 ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType); in LegalizeLoadOps()
1121 assert(ExtType != ISD::EXTLOAD && in LegalizeLoadOps()
1125 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl, in LegalizeLoadOps()
1130 if (ExtType == ISD::SEXTLOAD) in LegalizeLoadOps()
1131 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, in LegalizeLoadOps()
1160 if (Node->getOpcode() == ISD::TargetConstant) // Allow illegal target nodes. in LegalizeOp()
1172 Node->getOperand(i).getOpcode() == ISD::TargetConstant) && in LegalizeOp()
1179 case ISD::INTRINSIC_W_CHAIN: in LegalizeOp()
1180 case ISD::INTRINSIC_WO_CHAIN: in LegalizeOp()
1181 case ISD::INTRINSIC_VOID: in LegalizeOp()
1182 case ISD::STACKSAVE: in LegalizeOp()
1185 case ISD::VAARG: in LegalizeOp()
1191 case ISD::FP_TO_FP16: in LegalizeOp()
1192 case ISD::SINT_TO_FP: in LegalizeOp()
1193 case ISD::UINT_TO_FP: in LegalizeOp()
1194 case ISD::EXTRACT_VECTOR_ELT: in LegalizeOp()
1198 case ISD::FP_ROUND_INREG: in LegalizeOp()
1199 case ISD::SIGN_EXTEND_INREG: { in LegalizeOp()
1204 case ISD::ATOMIC_STORE: { in LegalizeOp()
1209 case ISD::SELECT_CC: in LegalizeOp()
1210 case ISD::SETCC: in LegalizeOp()
1211 case ISD::BR_CC: { in LegalizeOp()
1212 unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 : in LegalizeOp()
1213 Node->getOpcode() == ISD::SETCC ? 2 : 1; in LegalizeOp()
1214 unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0; in LegalizeOp()
1216 ISD::CondCode CCCode = in LegalizeOp()
1220 if (Node->getOpcode() == ISD::SELECT_CC) in LegalizeOp()
1228 case ISD::LOAD: in LegalizeOp()
1229 case ISD::STORE: in LegalizeOp()
1234 case ISD::CALLSEQ_START: in LegalizeOp()
1235 case ISD::CALLSEQ_END: in LegalizeOp()
1241 case ISD::EXTRACT_ELEMENT: in LegalizeOp()
1242 case ISD::FLT_ROUNDS_: in LegalizeOp()
1243 case ISD::SADDO: in LegalizeOp()
1244 case ISD::SSUBO: in LegalizeOp()
1245 case ISD::UADDO: in LegalizeOp()
1246 case ISD::USUBO: in LegalizeOp()
1247 case ISD::SMULO: in LegalizeOp()
1248 case ISD::UMULO: in LegalizeOp()
1249 case ISD::FPOWI: in LegalizeOp()
1250 case ISD::MERGE_VALUES: in LegalizeOp()
1251 case ISD::EH_RETURN: in LegalizeOp()
1252 case ISD::FRAME_TO_ARGS_OFFSET: in LegalizeOp()
1253 case ISD::EH_SJLJ_SETJMP: in LegalizeOp()
1254 case ISD::EH_SJLJ_LONGJMP: in LegalizeOp()
1261 case ISD::INIT_TRAMPOLINE: in LegalizeOp()
1262 case ISD::ADJUST_TRAMPOLINE: in LegalizeOp()
1263 case ISD::FRAMEADDR: in LegalizeOp()
1264 case ISD::RETURNADDR: in LegalizeOp()
1271 case ISD::READ_REGISTER: in LegalizeOp()
1272 case ISD::WRITE_REGISTER: in LegalizeOp()
1278 case ISD::DEBUGTRAP: in LegalizeOp()
1283 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), in LegalizeOp()
1292 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) { in LegalizeOp()
1304 case ISD::SHL: in LegalizeOp()
1305 case ISD::SRL: in LegalizeOp()
1306 case ISD::SRA: in LegalizeOp()
1307 case ISD::ROTL: in LegalizeOp()
1308 case ISD::ROTR: in LegalizeOp()
1321 case ISD::SRL_PARTS: in LegalizeOp()
1322 case ISD::SRA_PARTS: in LegalizeOp()
1323 case ISD::SHL_PARTS: in LegalizeOp()
1386 case ISD::CALLSEQ_START: in LegalizeOp()
1387 case ISD::CALLSEQ_END: in LegalizeOp()
1389 case ISD::LOAD: { in LegalizeOp()
1392 case ISD::STORE: { in LegalizeOp()
1439 Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx, in ExpandExtractFromVectorThroughStack()
1443 StackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, StackPtr); in ExpandExtractFromVectorThroughStack()
1452 ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr, MachinePointerInfo(), in ExpandExtractFromVectorThroughStack()
1492 Idx = DAG.getNode(ISD::MUL, dl, Idx.getValueType(), Idx, in ExpandInsertToVectorThroughStack()
1496 SDValue SubStackPtr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, in ExpandInsertToVectorThroughStack()
1526 if (Node->getOperand(i).getOpcode() == ISD::UNDEF) continue; in ExpandVectorBuildThroughStack()
1531 Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx); in ExpandVectorBuildThroughStack()
1549 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); in ExpandVectorBuildThroughStack()
1570 SignBit = DAG.getNode(ISD::BITCAST, dl, IVT, Tmp2); in ExpandFCOPYSIGN()
1591 LoadPtr = DAG.getNode(ISD::ADD, dl, LoadPtr.getValueType(), LoadPtr, in ExpandFCOPYSIGN()
1601 SignBit = DAG.getNode(ISD::SHL, dl, LoadTy, SignBit, in ExpandFCOPYSIGN()
1609 ISD::SETLT); in ExpandFCOPYSIGN()
1611 SDValue AbsVal = DAG.getNode(ISD::FABS, dl, Tmp1.getValueType(), Tmp1); in ExpandFCOPYSIGN()
1615 DAG.getNode(ISD::FNEG, dl, AbsVal.getValueType(), AbsVal), in ExpandFCOPYSIGN()
1642 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value in ExpandDYNAMIC_STACKALLOC()
1644 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1, in ExpandDYNAMIC_STACKALLOC()
1680 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get(); in LegalizeSetCCCondCode()
1688 ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode); in LegalizeSetCCCondCode()
1694 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID; in LegalizeSetCCCondCode()
1698 case ISD::SETO: in LegalizeSetCCCondCode()
1699 assert(TLI.getCondCodeAction(ISD::SETOEQ, OpVT) in LegalizeSetCCCondCode()
1702 CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break; in LegalizeSetCCCondCode()
1703 case ISD::SETUO: in LegalizeSetCCCondCode()
1704 assert(TLI.getCondCodeAction(ISD::SETUNE, OpVT) in LegalizeSetCCCondCode()
1707 CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR; break; in LegalizeSetCCCondCode()
1708 case ISD::SETOEQ: in LegalizeSetCCCondCode()
1709 case ISD::SETOGT: in LegalizeSetCCCondCode()
1710 case ISD::SETOGE: in LegalizeSetCCCondCode()
1711 case ISD::SETOLT: in LegalizeSetCCCondCode()
1712 case ISD::SETOLE: in LegalizeSetCCCondCode()
1713 case ISD::SETONE: in LegalizeSetCCCondCode()
1714 case ISD::SETUEQ: in LegalizeSetCCCondCode()
1715 case ISD::SETUNE: in LegalizeSetCCCondCode()
1716 case ISD::SETUGT: in LegalizeSetCCCondCode()
1717 case ISD::SETUGE: in LegalizeSetCCCondCode()
1718 case ISD::SETULT: in LegalizeSetCCCondCode()
1719 case ISD::SETULE: in LegalizeSetCCCondCode()
1724 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO; in LegalizeSetCCCondCode()
1725 Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND; in LegalizeSetCCCondCode()
1726 CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10); in LegalizeSetCCCondCode()
1730 case ISD::SETLE: in LegalizeSetCCCondCode()
1731 case ISD::SETGT: in LegalizeSetCCCondCode()
1732 case ISD::SETGE: in LegalizeSetCCCondCode()
1733 case ISD::SETLT: in LegalizeSetCCCondCode()
1737 case ISD::SETNE: in LegalizeSetCCCondCode()
1738 case ISD::SETEQ: in LegalizeSetCCCondCode()
1740 InvCC = CCCode == ISD::SETEQ ? ISD::SETNE : ISD::SETEQ; in LegalizeSetCCCondCode()
1752 if (CCCode != ISD::SETO && CCCode != ISD::SETUO) { in LegalizeSetCCCondCode()
1814 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, in EmitStackConvert()
1856 if (V.getOpcode() == ISD::UNDEF) in ExpandBVWithShuffles()
1861 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V); in ExpandBVWithShuffles()
1948 if (V.getOpcode() == ISD::UNDEF) in ExpandBUILD_VECTOR()
1969 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0)); in ExpandBUILD_VECTOR()
1991 assert(Node->getOperand(i).getOpcode() == ISD::UNDEF); in ExpandBUILD_VECTOR()
2006 if (Node->getOperand(i).getOpcode() == ISD::UNDEF) in ExpandBUILD_VECTOR()
2016 if (V.getOpcode() == ISD::UNDEF) in ExpandBUILD_VECTOR()
2022 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1); in ExpandBUILD_VECTOR()
2025 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2); in ExpandBUILD_VECTOR()
2211 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; in useDivRem()
2214 OtherOpcode = isDIV ? ISD::SREM : ISD::SDIV; in useDivRem()
2216 OtherOpcode = isDIV ? ISD::UREM : ISD::UDIV; in useDivRem()
2238 bool isSigned = Opcode == ISD::SDIVREM; in ExpandDivRemLibCall()
2325 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN in useSinCos()
2326 ? ISD::FCOS : ISD::FSIN; in useSinCos()
2335 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS) in useSinCos()
2424 SDValue Lo = DAG.getNode(ISD::ADD, dl, StackSlot.getValueType(), in ExpandLegalINT_TO_FP()
2434 Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit); in ExpandLegalINT_TO_FP()
2457 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias); in ExpandLegalINT_TO_FP()
2465 Result = DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub, in ExpandLegalINT_TO_FP()
2468 Result = DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub); in ExpandLegalINT_TO_FP()
2489 SDValue Hi = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, in ExpandLegalINT_TO_FP()
2491 SDValue LoOr = DAG.getNode(ISD::OR, dl, MVT::i64, Lo, TwoP52); in ExpandLegalINT_TO_FP()
2492 SDValue HiOr = DAG.getNode(ISD::OR, dl, MVT::i64, Hi, TwoP84); in ExpandLegalINT_TO_FP()
2493 SDValue LoFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, LoOr); in ExpandLegalINT_TO_FP()
2494 SDValue HiFlt = DAG.getNode(ISD::BITCAST, dl, MVT::f64, HiOr); in ExpandLegalINT_TO_FP()
2495 SDValue HiSub = DAG.getNode(ISD::FSUB, dl, MVT::f64, HiFlt, in ExpandLegalINT_TO_FP()
2497 return DAG.getNode(ISD::FADD, dl, MVT::f64, LoFlt, HiSub); in ExpandLegalINT_TO_FP()
2506 SDValue Fast = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Op0); in ExpandLegalINT_TO_FP()
2510 SDValue Shr = DAG.getNode(ISD::SRL, dl, MVT::i64, Op0, ShiftConst); in ExpandLegalINT_TO_FP()
2512 SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, AndConst); in ExpandLegalINT_TO_FP()
2513 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, Shr); in ExpandLegalINT_TO_FP()
2515 SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Or); in ExpandLegalINT_TO_FP()
2516 SDValue Slow = DAG.getNode(ISD::FADD, dl, MVT::f32, SignCvt, SignCvt); in ExpandLegalINT_TO_FP()
2523 Op0, DAG.getConstant(0, MVT::i64), ISD::SETLT); in ExpandLegalINT_TO_FP()
2529 SDValue And = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, in ExpandLegalINT_TO_FP()
2531 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i64, And, in ExpandLegalINT_TO_FP()
2533 SDValue And2 = DAG.getNode(ISD::AND, dl, MVT::i64, Op0, in ExpandLegalINT_TO_FP()
2536 And2, DAG.getConstant(UINT64_C(0), MVT::i64), ISD::SETNE); in ExpandLegalINT_TO_FP()
2540 ISD::SETUGE); in ExpandLegalINT_TO_FP()
2544 SDValue Sh = DAG.getNode(ISD::SRL, dl, MVT::i64, Sel2, in ExpandLegalINT_TO_FP()
2546 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sh); in ExpandLegalINT_TO_FP()
2547 SDValue Fcvt = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Trunc); in ExpandLegalINT_TO_FP()
2550 SDValue Fmul = DAG.getNode(ISD::FMUL, dl, MVT::f64, TwoP32, Fcvt); in ExpandLegalINT_TO_FP()
2551 SDValue Lo = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Sel2); in ExpandLegalINT_TO_FP()
2552 SDValue Fcvt2 = DAG.getNode(ISD::UINT_TO_FP, dl, MVT::f64, Lo); in ExpandLegalINT_TO_FP()
2553 SDValue Fadd = DAG.getNode(ISD::FADD, dl, MVT::f64, Fmul, Fcvt2); in ExpandLegalINT_TO_FP()
2554 return DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Fadd, in ExpandLegalINT_TO_FP()
2558 SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0); in ExpandLegalINT_TO_FP()
2562 ISD::SETLT); in ExpandLegalINT_TO_FP()
2584 CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset); in ExpandLegalINT_TO_FP()
2592 SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, in ExpandLegalINT_TO_FP()
2601 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg); in ExpandLegalINT_TO_FP()
2624 if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) { in PromoteLegalINT_TO_FP()
2625 OpToUse = ISD::SINT_TO_FP; in PromoteLegalINT_TO_FP()
2631 if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) { in PromoteLegalINT_TO_FP()
2632 OpToUse = ISD::UINT_TO_FP; in PromoteLegalINT_TO_FP()
2642 DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, in PromoteLegalINT_TO_FP()
2667 if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) { in PromoteLegalFP_TO_INT()
2668 OpToUse = ISD::FP_TO_SINT; in PromoteLegalFP_TO_INT()
2673 if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) { in PromoteLegalFP_TO_INT()
2674 OpToUse = ISD::FP_TO_UINT; in PromoteLegalFP_TO_INT()
2687 return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation); in PromoteLegalFP_TO_INT()
2698 Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT)); in ExpandBSWAP()
2699 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT)); in ExpandBSWAP()
2700 return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2); in ExpandBSWAP()
2702 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, SHVT)); in ExpandBSWAP()
2703 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT)); in ExpandBSWAP()
2704 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT)); in ExpandBSWAP()
2705 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, SHVT)); in ExpandBSWAP()
2706 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, DAG.getConstant(0xFF0000, VT)); in ExpandBSWAP()
2707 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, VT)); in ExpandBSWAP()
2708 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); in ExpandBSWAP()
2709 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); in ExpandBSWAP()
2710 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); in ExpandBSWAP()
2712 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, SHVT)); in ExpandBSWAP()
2713 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, SHVT)); in ExpandBSWAP()
2714 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, SHVT)); in ExpandBSWAP()
2715 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, SHVT)); in ExpandBSWAP()
2716 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, SHVT)); in ExpandBSWAP()
2717 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, SHVT)); in ExpandBSWAP()
2718 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, SHVT)); in ExpandBSWAP()
2719 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, SHVT)); in ExpandBSWAP()
2720 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7, DAG.getConstant(255ULL<<48, VT)); in ExpandBSWAP()
2721 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6, DAG.getConstant(255ULL<<40, VT)); in ExpandBSWAP()
2722 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5, DAG.getConstant(255ULL<<32, VT)); in ExpandBSWAP()
2723 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4, DAG.getConstant(255ULL<<24, VT)); in ExpandBSWAP()
2724 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, DAG.getConstant(255ULL<<16, VT)); in ExpandBSWAP()
2725 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(255ULL<<8 , VT)); in ExpandBSWAP()
2726 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7); in ExpandBSWAP()
2727 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5); in ExpandBSWAP()
2728 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); in ExpandBSWAP()
2729 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); in ExpandBSWAP()
2730 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6); in ExpandBSWAP()
2731 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); in ExpandBSWAP()
2732 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4); in ExpandBSWAP()
2741 case ISD::CTPOP: { in ExpandBitCount()
2758 Op = DAG.getNode(ISD::SUB, dl, VT, Op, in ExpandBitCount()
2759 DAG.getNode(ISD::AND, dl, VT, in ExpandBitCount()
2760 DAG.getNode(ISD::SRL, dl, VT, Op, in ExpandBitCount()
2764 Op = DAG.getNode(ISD::ADD, dl, VT, in ExpandBitCount()
2765 DAG.getNode(ISD::AND, dl, VT, Op, Mask33), in ExpandBitCount()
2766 DAG.getNode(ISD::AND, dl, VT, in ExpandBitCount()
2767 DAG.getNode(ISD::SRL, dl, VT, Op, in ExpandBitCount()
2771 Op = DAG.getNode(ISD::AND, dl, VT, in ExpandBitCount()
2772 DAG.getNode(ISD::ADD, dl, VT, Op, in ExpandBitCount()
2773 DAG.getNode(ISD::SRL, dl, VT, Op, in ExpandBitCount()
2777 Op = DAG.getNode(ISD::SRL, dl, VT, in ExpandBitCount()
2778 DAG.getNode(ISD::MUL, dl, VT, Op, Mask01), in ExpandBitCount()
2783 case ISD::CTLZ_ZERO_UNDEF: in ExpandBitCount()
2785 return DAG.getNode(ISD::CTLZ, dl, Op.getValueType(), Op); in ExpandBitCount()
2786 case ISD::CTLZ: { in ExpandBitCount()
2801 Op = DAG.getNode(ISD::OR, dl, VT, Op, in ExpandBitCount()
2802 DAG.getNode(ISD::SRL, dl, VT, Op, Tmp3)); in ExpandBitCount()
2805 return DAG.getNode(ISD::CTPOP, dl, VT, Op); in ExpandBitCount()
2807 case ISD::CTTZ_ZERO_UNDEF: in ExpandBitCount()
2809 return DAG.getNode(ISD::CTTZ, dl, Op.getValueType(), Op); in ExpandBitCount()
2810 case ISD::CTTZ: { in ExpandBitCount()
2816 SDValue Tmp3 = DAG.getNode(ISD::AND, dl, VT, in ExpandBitCount()
2818 DAG.getNode(ISD::SUB, dl, VT, Op, in ExpandBitCount()
2821 if (!TLI.isOperationLegalOrCustom(ISD::CTPOP, VT) && in ExpandBitCount()
2822 TLI.isOperationLegalOrCustom(ISD::CTLZ, VT)) in ExpandBitCount()
2823 return DAG.getNode(ISD::SUB, dl, VT, in ExpandBitCount()
2825 DAG.getNode(ISD::CTLZ, dl, VT, Tmp3)); in ExpandBitCount()
2826 return DAG.getNode(ISD::CTPOP, dl, VT, Tmp3); in ExpandBitCount()
2846 case ISD::CTPOP: in ExpandNode()
2847 case ISD::CTLZ: in ExpandNode()
2848 case ISD::CTLZ_ZERO_UNDEF: in ExpandNode()
2849 case ISD::CTTZ: in ExpandNode()
2850 case ISD::CTTZ_ZERO_UNDEF: in ExpandNode()
2854 case ISD::BSWAP: in ExpandNode()
2857 case ISD::FRAMEADDR: in ExpandNode()
2858 case ISD::RETURNADDR: in ExpandNode()
2859 case ISD::FRAME_TO_ARGS_OFFSET: in ExpandNode()
2862 case ISD::FLT_ROUNDS_: in ExpandNode()
2865 case ISD::EH_RETURN: in ExpandNode()
2866 case ISD::EH_LABEL: in ExpandNode()
2867 case ISD::PREFETCH: in ExpandNode()
2868 case ISD::VAEND: in ExpandNode()
2869 case ISD::EH_SJLJ_LONGJMP: in ExpandNode()
2874 case ISD::EH_SJLJ_SETJMP: in ExpandNode()
2880 case ISD::ATOMIC_FENCE: { in ExpandNode()
2896 case ISD::ATOMIC_LOAD: { in ExpandNode()
2901 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, in ExpandNode()
2911 case ISD::ATOMIC_STORE: { in ExpandNode()
2913 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, in ExpandNode()
2926 case ISD::ATOMIC_SWAP: in ExpandNode()
2927 case ISD::ATOMIC_LOAD_ADD: in ExpandNode()
2928 case ISD::ATOMIC_LOAD_SUB: in ExpandNode()
2929 case ISD::ATOMIC_LOAD_AND: in ExpandNode()
2930 case ISD::ATOMIC_LOAD_OR: in ExpandNode()
2931 case ISD::ATOMIC_LOAD_XOR: in ExpandNode()
2932 case ISD::ATOMIC_LOAD_NAND: in ExpandNode()
2933 case ISD::ATOMIC_LOAD_MIN: in ExpandNode()
2934 case ISD::ATOMIC_LOAD_MAX: in ExpandNode()
2935 case ISD::ATOMIC_LOAD_UMIN: in ExpandNode()
2936 case ISD::ATOMIC_LOAD_UMAX: in ExpandNode()
2937 case ISD::ATOMIC_CMP_SWAP: { in ExpandNode()
2943 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { in ExpandNode()
2949 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, in ExpandNode()
2957 Res, Node->getOperand(2), ISD::SETEQ); in ExpandNode()
2964 case ISD::DYNAMIC_STACKALLOC: in ExpandNode()
2967 case ISD::MERGE_VALUES: in ExpandNode()
2971 case ISD::UNDEF: { in ExpandNode()
2981 case ISD::TRAP: { in ExpandNode()
2994 case ISD::FP_ROUND: in ExpandNode()
2995 case ISD::BITCAST: in ExpandNode()
3000 case ISD::FP_EXTEND: in ExpandNode()
3006 case ISD::SIGN_EXTEND_INREG: { in ExpandNode()
3017 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0), in ExpandNode()
3019 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst); in ExpandNode()
3023 case ISD::FP_ROUND_INREG: { in ExpandNode()
3036 case ISD::SINT_TO_FP: in ExpandNode()
3037 case ISD::UINT_TO_FP: in ExpandNode()
3038 Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP, in ExpandNode()
3042 case ISD::FP_TO_SINT: in ExpandNode()
3046 case ISD::FP_TO_UINT: { in ExpandNode()
3057 Tmp1, ISD::SETLT); in ExpandNode()
3058 True = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, Node->getOperand(0)); in ExpandNode()
3059 False = DAG.getNode(ISD::FP_TO_SINT, dl, NVT, in ExpandNode()
3060 DAG.getNode(ISD::FSUB, dl, VT, in ExpandNode()
3062 False = DAG.getNode(ISD::XOR, dl, NVT, False, in ExpandNode()
3068 case ISD::VAARG: { in ExpandNode()
3083 VAList = DAG.getNode(ISD::ADD, dl, VAList.getValueType(), VAList, in ExpandNode()
3087 VAList = DAG.getNode(ISD::AND, dl, VAList.getValueType(), VAList, in ExpandNode()
3093 Tmp3 = DAG.getNode(ISD::ADD, dl, VAList.getValueType(), VAList, in ExpandNode()
3106 case ISD::VACOPY: { in ExpandNode()
3119 case ISD::EXTRACT_VECTOR_ELT: in ExpandNode()
3122 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), in ExpandNode()
3128 case ISD::EXTRACT_SUBVECTOR: in ExpandNode()
3131 case ISD::INSERT_SUBVECTOR: in ExpandNode()
3134 case ISD::CONCAT_VECTORS: { in ExpandNode()
3138 case ISD::SCALAR_TO_VECTOR: in ExpandNode()
3141 case ISD::INSERT_VECTOR_ELT: in ExpandNode()
3146 case ISD::VECTOR_SHUFFLE: { in ExpandNode()
3174 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0); in ExpandNode()
3175 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1); in ExpandNode()
3208 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, in ExpandNode()
3212 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, in ExpandNode()
3218 Tmp1 = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); in ExpandNode()
3220 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1); in ExpandNode()
3224 case ISD::EXTRACT_ELEMENT: { in ExpandNode()
3228 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0), in ExpandNode()
3231 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1); in ExpandNode()
3234 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), in ExpandNode()
3240 case ISD::STACKSAVE: in ExpandNode()
3252 case ISD::STACKRESTORE: in ExpandNode()
3262 case ISD::FCOPYSIGN: in ExpandNode()
3265 case ISD::FNEG: in ExpandNode()
3268 Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1, in ExpandNode()
3272 case ISD::FABS: { in ExpandNode()
3278 Tmp1, Tmp2, ISD::SETUGT); in ExpandNode()
3279 Tmp3 = DAG.getNode(ISD::FNEG, dl, VT, Tmp1); in ExpandNode()
3284 case ISD::FMINNUM: in ExpandNode()
3289 case ISD::FMAXNUM: in ExpandNode()
3294 case ISD::FSQRT: in ExpandNode()
3299 case ISD::FSIN: in ExpandNode()
3300 case ISD::FCOS: { in ExpandNode()
3302 bool isSIN = Node->getOpcode() == ISD::FSIN; in ExpandNode()
3305 if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) || in ExpandNode()
3309 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0)); in ExpandNode()
3324 case ISD::FSINCOS: in ExpandNode()
3328 case ISD::FLOG: in ExpandNode()
3333 case ISD::FLOG2: in ExpandNode()
3338 case ISD::FLOG10: in ExpandNode()
3343 case ISD::FEXP: in ExpandNode()
3348 case ISD::FEXP2: in ExpandNode()
3353 case ISD::FTRUNC: in ExpandNode()
3358 case ISD::FFLOOR: in ExpandNode()
3363 case ISD::FCEIL: in ExpandNode()
3368 case ISD::FRINT: in ExpandNode()
3373 case ISD::FNEARBYINT: in ExpandNode()
3380 case ISD::FROUND: in ExpandNode()
3387 case ISD::FPOWI: in ExpandNode()
3392 case ISD::FPOW: in ExpandNode()
3397 case ISD::FDIV: in ExpandNode()
3402 case ISD::FREM: in ExpandNode()
3407 case ISD::FMA: in ExpandNode()
3412 case ISD::FMAD: in ExpandNode()
3415 case ISD::FADD: in ExpandNode()
3420 case ISD::FMUL: in ExpandNode()
3425 case ISD::FP16_TO_FP: { in ExpandNode()
3435 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0)); in ExpandNode()
3437 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res)); in ExpandNode()
3440 case ISD::FP_TO_FP16: { in ExpandNode()
3445 TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) { in ExpandNode()
3448 SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op, in ExpandNode()
3451 DAG.getNode(ISD::FP_TO_FP16, dl, MVT::i16, FloatVal)); in ExpandNode()
3462 case ISD::ConstantFP: { in ExpandNode()
3470 case ISD::FSUB: { in ExpandNode()
3472 if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) && in ExpandNode()
3473 TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) { in ExpandNode()
3474 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1)); in ExpandNode()
3475 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1); in ExpandNode()
3484 case ISD::SUB: { in ExpandNode()
3486 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) && in ExpandNode()
3487 TLI.isOperationLegalOrCustom(ISD::XOR, VT) && in ExpandNode()
3489 Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1), in ExpandNode()
3491 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, VT)); in ExpandNode()
3492 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1)); in ExpandNode()
3495 case ISD::UREM: in ExpandNode()
3496 case ISD::SREM: { in ExpandNode()
3498 bool isSigned = Node->getOpcode() == ISD::SREM; in ExpandNode()
3499 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV; in ExpandNode()
3500 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; in ExpandNode()
3513 Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3); in ExpandNode()
3514 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1); in ExpandNode()
3528 case ISD::UDIV: in ExpandNode()
3529 case ISD::SDIV: { in ExpandNode()
3530 bool isSigned = Node->getOpcode() == ISD::SDIV; in ExpandNode()
3531 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; in ExpandNode()
3552 case ISD::MULHU: in ExpandNode()
3553 case ISD::MULHS: { in ExpandNode()
3554 unsigned ExpandOpcode = Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : in ExpandNode()
3555 ISD::SMUL_LOHI; in ExpandNode()
3565 case ISD::SDIVREM: in ExpandNode()
3566 case ISD::UDIVREM: in ExpandNode()
3570 case ISD::MUL: { in ExpandNode()
3578 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT); in ExpandNode()
3579 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT); in ExpandNode()
3580 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT); in ExpandNode()
3581 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT); in ExpandNode()
3584 OpToUse = ISD::SMUL_LOHI; in ExpandNode()
3586 OpToUse = ISD::UMUL_LOHI; in ExpandNode()
3588 OpToUse = ISD::SMUL_LOHI; in ExpandNode()
3590 OpToUse = ISD::UMUL_LOHI; in ExpandNode()
3600 if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) && in ExpandNode()
3601 TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) && in ExpandNode()
3602 TLI.isOperationLegalOrCustom(ISD::SHL, VT) && in ExpandNode()
3603 TLI.isOperationLegalOrCustom(ISD::OR, VT) && in ExpandNode()
3605 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo); in ExpandNode()
3606 Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi); in ExpandNode()
3609 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); in ExpandNode()
3610 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi)); in ExpandNode()
3621 case ISD::SADDO: in ExpandNode()
3622 case ISD::SSUBO: { in ExpandNode()
3625 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? in ExpandNode()
3626 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), in ExpandNode()
3643 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE); in ExpandNode()
3644 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE); in ExpandNode()
3646 Node->getOpcode() == ISD::SADDO ? in ExpandNode()
3647 ISD::SETEQ : ISD::SETNE); in ExpandNode()
3649 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE); in ExpandNode()
3650 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE); in ExpandNode()
3652 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE); in ExpandNode()
3656 case ISD::UADDO: in ExpandNode()
3657 case ISD::USUBO: { in ExpandNode()
3660 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::UADDO ? in ExpandNode()
3661 ISD::ADD : ISD::SUB, dl, LHS.getValueType(), in ExpandNode()
3667 ISD::CondCode CC in ExpandNode()
3668 = Node->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT; in ExpandNode()
3674 case ISD::UMULO: in ExpandNode()
3675 case ISD::SMULO: { in ExpandNode()
3683 { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND }, in ExpandNode()
3684 { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }}; in ExpandNode()
3685 bool isSigned = Node->getOpcode() == ISD::SMULO; in ExpandNode()
3687 BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); in ExpandNode()
3696 Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS); in ExpandNode()
3697 BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1, in ExpandNode()
3699 TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1, in ExpandNode()
3720 SDValue HiLHS = DAG.getNode(ISD::SRA, dl, VT, RHS, in ExpandNode()
3722 SDValue HiRHS = DAG.getNode(ISD::SRA, dl, VT, LHS, in ExpandNode()
3731 BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret, in ExpandNode()
3733 TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Ret, in ExpandNode()
3746 Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1); in ExpandNode()
3748 ISD::SETNE); in ExpandNode()
3751 DAG.getConstant(0, VT), ISD::SETNE); in ExpandNode()
3757 case ISD::BUILD_PAIR: { in ExpandNode()
3759 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0)); in ExpandNode()
3760 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1)); in ExpandNode()
3761 Tmp2 = DAG.getNode(ISD::SHL, dl, PairTy, Tmp2, in ExpandNode()
3764 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2)); in ExpandNode()
3767 case ISD::SELECT: in ExpandNode()
3771 if (Tmp1.getOpcode() == ISD::SETCC) { in ExpandNode()
3778 Tmp2, Tmp3, ISD::SETNE); in ExpandNode()
3782 case ISD::BR_JT: { in ExpandNode()
3793 Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), in ExpandNode()
3795 SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(), in ExpandNode()
3799 SDValue LD = DAG.getExtLoad(ISD::SEXTLOAD, dl, PTy, Chain, Addr, in ExpandNode()
3807 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr, in ExpandNode()
3810 Tmp1 = DAG.getNode(ISD::BRIND, dl, MVT::Other, LD.getValue(1), Addr); in ExpandNode()
3814 case ISD::BRCOND: in ExpandNode()
3819 if (Tmp2.getOpcode() == ISD::SETCC) { in ExpandNode()
3820 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, in ExpandNode()
3826 Tmp3 = (Tmp2.getOpcode() == ISD::UNDEF) ? Tmp2 : in ExpandNode()
3827 DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2, in ExpandNode()
3829 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, in ExpandNode()
3830 DAG.getCondCode(ISD::SETNE), Tmp3, in ExpandNode()
3836 case ISD::SETCC: { in ExpandNode()
3847 Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), in ExpandNode()
3872 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2, in ExpandNode()
3878 case ISD::SELECT_CC: { in ExpandNode()
3885 ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get(); in ExpandNode()
3891 assert(!TLI.isOperationExpand(ISD::SELECT, VT) && in ExpandNode()
3895 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC); in ExpandNode()
3905 ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp, in ExpandNode()
3914 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC); in ExpandNode()
3938 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), in ExpandNode()
3942 CC = DAG.getCondCode(ISD::SETNE); in ExpandNode()
3943 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, in ExpandNode()
3950 case ISD::BR_CC: { in ExpandNode()
3969 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, in ExpandNode()
3973 Tmp4 = DAG.getCondCode(ISD::SETNE); in ExpandNode()
3974 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4, in ExpandNode()
3980 case ISD::BUILD_VECTOR: in ExpandNode()
3983 case ISD::SRA: in ExpandNode()
3984 case ISD::SRL: in ExpandNode()
3985 case ISD::SHL: { in ExpandNode()
3994 SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, in ExpandNode()
3998 SDValue Sh = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, in ExpandNode()
4006 DAG.getNode(ISD::BUILD_VECTOR, dl, Node->getValueType(0), Scalars); in ExpandNode()
4010 case ISD::GLOBAL_OFFSET_TABLE: in ExpandNode()
4011 case ISD::GlobalAddress: in ExpandNode()
4012 case ISD::GlobalTLSAddress: in ExpandNode()
4013 case ISD::ExternalSymbol: in ExpandNode()
4014 case ISD::ConstantPool: in ExpandNode()
4015 case ISD::JumpTable: in ExpandNode()
4016 case ISD::INTRINSIC_W_CHAIN: in ExpandNode()
4017 case ISD::INTRINSIC_WO_CHAIN: in ExpandNode()
4018 case ISD::INTRINSIC_VOID: in ExpandNode()
4031 if (Node->getOpcode() == ISD::UINT_TO_FP || in PromoteNode()
4032 Node->getOpcode() == ISD::SINT_TO_FP || in PromoteNode()
4033 Node->getOpcode() == ISD::SETCC) { in PromoteNode()
4036 if (Node->getOpcode() == ISD::BR_CC) in PromoteNode()
4042 case ISD::CTTZ: in PromoteNode()
4043 case ISD::CTTZ_ZERO_UNDEF: in PromoteNode()
4044 case ISD::CTLZ: in PromoteNode()
4045 case ISD::CTLZ_ZERO_UNDEF: in PromoteNode()
4046 case ISD::CTPOP: in PromoteNode()
4048 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4052 if (Node->getOpcode() == ISD::CTTZ) { in PromoteNode()
4056 ISD::SETEQ); in PromoteNode()
4059 } else if (Node->getOpcode() == ISD::CTLZ || in PromoteNode()
4060 Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) { in PromoteNode()
4062 Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1, in PromoteNode()
4066 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); in PromoteNode()
4068 case ISD::BSWAP: { in PromoteNode()
4070 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4071 Tmp1 = DAG.getNode(ISD::BSWAP, dl, NVT, Tmp1); in PromoteNode()
4072 Tmp1 = DAG.getNode(ISD::SRL, dl, NVT, Tmp1, in PromoteNode()
4077 case ISD::FP_TO_UINT: in PromoteNode()
4078 case ISD::FP_TO_SINT: in PromoteNode()
4080 Node->getOpcode() == ISD::FP_TO_SINT, dl); in PromoteNode()
4083 case ISD::UINT_TO_FP: in PromoteNode()
4084 case ISD::SINT_TO_FP: in PromoteNode()
4086 Node->getOpcode() == ISD::SINT_TO_FP, dl); in PromoteNode()
4089 case ISD::VAARG: { in PromoteNode()
4095 TruncOp = ISD::BITCAST; in PromoteNode()
4099 TruncOp = ISD::TRUNCATE; in PromoteNode()
4120 case ISD::AND: in PromoteNode()
4121 case ISD::OR: in PromoteNode()
4122 case ISD::XOR: { in PromoteNode()
4125 ExtOp = ISD::BITCAST; in PromoteNode()
4126 TruncOp = ISD::BITCAST; in PromoteNode()
4129 ExtOp = ISD::ANY_EXTEND; in PromoteNode()
4130 TruncOp = ISD::TRUNCATE; in PromoteNode()
4140 case ISD::SELECT: { in PromoteNode()
4144 ExtOp = ISD::BITCAST; in PromoteNode()
4145 TruncOp = ISD::BITCAST; in PromoteNode()
4147 ExtOp = ISD::ANY_EXTEND; in PromoteNode()
4148 TruncOp = ISD::TRUNCATE; in PromoteNode()
4150 ExtOp = ISD::FP_EXTEND; in PromoteNode()
4151 TruncOp = ISD::FP_ROUND; in PromoteNode()
4159 if (TruncOp != ISD::FP_ROUND) in PromoteNode()
4167 case ISD::VECTOR_SHUFFLE: { in PromoteNode()
4171 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0)); in PromoteNode()
4172 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1)); in PromoteNode()
4176 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1); in PromoteNode()
4180 case ISD::SETCC: { in PromoteNode()
4181 unsigned ExtOp = ISD::FP_EXTEND; in PromoteNode()
4183 ISD::CondCode CCCode = in PromoteNode()
4185 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in PromoteNode()
4189 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), in PromoteNode()
4193 case ISD::BR_CC: { in PromoteNode()
4194 unsigned ExtOp = ISD::FP_EXTEND; in PromoteNode()
4196 ISD::CondCode CCCode = in PromoteNode()
4198 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in PromoteNode()
4202 Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), in PromoteNode()
4207 case ISD::FADD: in PromoteNode()
4208 case ISD::FSUB: in PromoteNode()
4209 case ISD::FMUL: in PromoteNode()
4210 case ISD::FDIV: in PromoteNode()
4211 case ISD::FREM: in PromoteNode()
4212 case ISD::FMINNUM: in PromoteNode()
4213 case ISD::FMAXNUM: in PromoteNode()
4214 case ISD::FCOPYSIGN: in PromoteNode()
4215 case ISD::FPOW: { in PromoteNode()
4216 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4217 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
4219 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, in PromoteNode()
4223 case ISD::FMA: { in PromoteNode()
4224 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4225 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); in PromoteNode()
4226 Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2)); in PromoteNode()
4228 DAG.getNode(ISD::FP_ROUND, dl, OVT, in PromoteNode()
4233 case ISD::FPOWI: { in PromoteNode()
4234 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4237 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, in PromoteNode()
4241 case ISD::FFLOOR: in PromoteNode()
4242 case ISD::FCEIL: in PromoteNode()
4243 case ISD::FRINT: in PromoteNode()
4244 case ISD::FNEARBYINT: in PromoteNode()
4245 case ISD::FROUND: in PromoteNode()
4246 case ISD::FTRUNC: in PromoteNode()
4247 case ISD::FNEG: in PromoteNode()
4248 case ISD::FSQRT: in PromoteNode()
4249 case ISD::FSIN: in PromoteNode()
4250 case ISD::FCOS: in PromoteNode()
4251 case ISD::FLOG: in PromoteNode()
4252 case ISD::FLOG2: in PromoteNode()
4253 case ISD::FLOG10: in PromoteNode()
4254 case ISD::FABS: in PromoteNode()
4255 case ISD::FEXP: in PromoteNode()
4256 case ISD::FEXP2: { in PromoteNode()
4257 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); in PromoteNode()
4259 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, in PromoteNode()