Lines Matching refs:ISD
136 if (N->getOpcode() == ISD::HANDLENODE) in AddToWorklist()
213 ISD::NodeType ExtType);
321 SDValue N3, ISD::CondCode CC,
323 SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
519 if (Op.getOpcode() == ISD::FNEG) return 2; in isNegatibleForFree()
529 case ISD::ConstantFP: in isNegatibleForFree()
533 case ISD::FADD: in isNegatibleForFree()
539 !TLI.isOperationLegalOrCustom(ISD::FSUB, Op.getValueType())) in isNegatibleForFree()
549 case ISD::FSUB: in isNegatibleForFree()
556 case ISD::FMUL: in isNegatibleForFree()
557 case ISD::FDIV: in isNegatibleForFree()
568 case ISD::FP_EXTEND: in isNegatibleForFree()
569 case ISD::FP_ROUND: in isNegatibleForFree()
570 case ISD::FSIN: in isNegatibleForFree()
581 if (Op.getOpcode() == ISD::FNEG) return Op.getOperand(0); in GetNegatedExpression()
589 case ISD::ConstantFP: { in GetNegatedExpression()
594 case ISD::FADD: in GetNegatedExpression()
601 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(), in GetNegatedExpression()
606 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(), in GetNegatedExpression()
610 case ISD::FSUB: in GetNegatedExpression()
620 return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(), in GetNegatedExpression()
623 case ISD::FMUL: in GetNegatedExpression()
624 case ISD::FDIV: in GetNegatedExpression()
641 case ISD::FP_EXTEND: in GetNegatedExpression()
642 case ISD::FSIN: in GetNegatedExpression()
646 case ISD::FP_ROUND: in GetNegatedExpression()
647 return DAG.getNode(ISD::FP_ROUND, SDLoc(Op), Op.getValueType(), in GetNegatedExpression()
661 if (N.getOpcode() == ISD::SETCC) { in isSetCCEquivalent()
668 if (N.getOpcode() != ISD::SELECT_CC || in isSetCCEquivalent()
714 if (ISD::isBuildVectorOfConstantSDNodes(N.getNode())) in isConstantIntBuildVectorOrConstantInt()
724 if (ISD::isBuildVectorOfConstantFPSDNodes(N.getNode())) in isConstantFPBuildVectorOrConstantFP()
891 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, VT, SDValue(ExtLoad, 0)); in ReplaceLoadWithPromotedLoad()
910 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) in PromoteOperand()
911 ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, PVT, MemVT) ? ISD::ZEXTLOAD in PromoteOperand()
912 : ISD::EXTLOAD) in PromoteOperand()
923 case ISD::AssertSext: in PromoteOperand()
924 return DAG.getNode(ISD::AssertSext, dl, PVT, in PromoteOperand()
927 case ISD::AssertZext: in PromoteOperand()
928 return DAG.getNode(ISD::AssertZext, dl, PVT, in PromoteOperand()
931 case ISD::Constant: { in PromoteOperand()
933 Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in PromoteOperand()
938 if (!TLI.isOperationLegal(ISD::ANY_EXTEND, PVT)) in PromoteOperand()
940 return DAG.getNode(ISD::ANY_EXTEND, dl, PVT, Op); in PromoteOperand()
944 if (!TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, PVT)) in SExtPromoteOperand()
956 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NewOp.getValueType(), NewOp, in SExtPromoteOperand()
1026 return DAG.getNode(ISD::TRUNCATE, dl, VT, in PromoteIntBinOp()
1057 if (Opc == ISD::SRA) in PromoteIntShiftOp()
1059 else if (Opc == ISD::SRL) in PromoteIntShiftOp()
1073 return DAG.getNode(ISD::TRUNCATE, dl, VT, in PromoteIntShiftOp()
1132 ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) in PromoteLoad()
1133 ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, PVT, MemVT) ? ISD::ZEXTLOAD in PromoteLoad()
1134 : ISD::EXTLOAD) in PromoteLoad()
1139 SDValue Result = DAG.getNode(ISD::TRUNCATE, dl, VT, NewLD); in PromoteLoad()
1266 assert(N->getOpcode() != ISD::DELETED_NODE && in Run()
1267 RV.getNode()->getOpcode() != ISD::DELETED_NODE && in Run()
1303 case ISD::TokenFactor: return visitTokenFactor(N); in visit()
1304 case ISD::MERGE_VALUES: return visitMERGE_VALUES(N); in visit()
1305 case ISD::ADD: return visitADD(N); in visit()
1306 case ISD::SUB: return visitSUB(N); in visit()
1307 case ISD::ADDC: return visitADDC(N); in visit()
1308 case ISD::SUBC: return visitSUBC(N); in visit()
1309 case ISD::ADDE: return visitADDE(N); in visit()
1310 case ISD::SUBE: return visitSUBE(N); in visit()
1311 case ISD::MUL: return visitMUL(N); in visit()
1312 case ISD::SDIV: return visitSDIV(N); in visit()
1313 case ISD::UDIV: return visitUDIV(N); in visit()
1314 case ISD::SREM: return visitSREM(N); in visit()
1315 case ISD::UREM: return visitUREM(N); in visit()
1316 case ISD::MULHU: return visitMULHU(N); in visit()
1317 case ISD::MULHS: return visitMULHS(N); in visit()
1318 case ISD::SMUL_LOHI: return visitSMUL_LOHI(N); in visit()
1319 case ISD::UMUL_LOHI: return visitUMUL_LOHI(N); in visit()
1320 case ISD::SMULO: return visitSMULO(N); in visit()
1321 case ISD::UMULO: return visitUMULO(N); in visit()
1322 case ISD::SDIVREM: return visitSDIVREM(N); in visit()
1323 case ISD::UDIVREM: return visitUDIVREM(N); in visit()
1324 case ISD::AND: return visitAND(N); in visit()
1325 case ISD::OR: return visitOR(N); in visit()
1326 case ISD::XOR: return visitXOR(N); in visit()
1327 case ISD::SHL: return visitSHL(N); in visit()
1328 case ISD::SRA: return visitSRA(N); in visit()
1329 case ISD::SRL: return visitSRL(N); in visit()
1330 case ISD::ROTR: in visit()
1331 case ISD::ROTL: return visitRotate(N); in visit()
1332 case ISD::CTLZ: return visitCTLZ(N); in visit()
1333 case ISD::CTLZ_ZERO_UNDEF: return visitCTLZ_ZERO_UNDEF(N); in visit()
1334 case ISD::CTTZ: return visitCTTZ(N); in visit()
1335 case ISD::CTTZ_ZERO_UNDEF: return visitCTTZ_ZERO_UNDEF(N); in visit()
1336 case ISD::CTPOP: return visitCTPOP(N); in visit()
1337 case ISD::SELECT: return visitSELECT(N); in visit()
1338 case ISD::VSELECT: return visitVSELECT(N); in visit()
1339 case ISD::SELECT_CC: return visitSELECT_CC(N); in visit()
1340 case ISD::SETCC: return visitSETCC(N); in visit()
1341 case ISD::SIGN_EXTEND: return visitSIGN_EXTEND(N); in visit()
1342 case ISD::ZERO_EXTEND: return visitZERO_EXTEND(N); in visit()
1343 case ISD::ANY_EXTEND: return visitANY_EXTEND(N); in visit()
1344 case ISD::SIGN_EXTEND_INREG: return visitSIGN_EXTEND_INREG(N); in visit()
1345 case ISD::TRUNCATE: return visitTRUNCATE(N); in visit()
1346 case ISD::BITCAST: return visitBITCAST(N); in visit()
1347 case ISD::BUILD_PAIR: return visitBUILD_PAIR(N); in visit()
1348 case ISD::FADD: return visitFADD(N); in visit()
1349 case ISD::FSUB: return visitFSUB(N); in visit()
1350 case ISD::FMUL: return visitFMUL(N); in visit()
1351 case ISD::FMA: return visitFMA(N); in visit()
1352 case ISD::FDIV: return visitFDIV(N); in visit()
1353 case ISD::FREM: return visitFREM(N); in visit()
1354 case ISD::FSQRT: return visitFSQRT(N); in visit()
1355 case ISD::FCOPYSIGN: return visitFCOPYSIGN(N); in visit()
1356 case ISD::SINT_TO_FP: return visitSINT_TO_FP(N); in visit()
1357 case ISD::UINT_TO_FP: return visitUINT_TO_FP(N); in visit()
1358 case ISD::FP_TO_SINT: return visitFP_TO_SINT(N); in visit()
1359 case ISD::FP_TO_UINT: return visitFP_TO_UINT(N); in visit()
1360 case ISD::FP_ROUND: return visitFP_ROUND(N); in visit()
1361 case ISD::FP_ROUND_INREG: return visitFP_ROUND_INREG(N); in visit()
1362 case ISD::FP_EXTEND: return visitFP_EXTEND(N); in visit()
1363 case ISD::FNEG: return visitFNEG(N); in visit()
1364 case ISD::FABS: return visitFABS(N); in visit()
1365 case ISD::FFLOOR: return visitFFLOOR(N); in visit()
1366 case ISD::FMINNUM: return visitFMINNUM(N); in visit()
1367 case ISD::FMAXNUM: return visitFMAXNUM(N); in visit()
1368 case ISD::FCEIL: return visitFCEIL(N); in visit()
1369 case ISD::FTRUNC: return visitFTRUNC(N); in visit()
1370 case ISD::BRCOND: return visitBRCOND(N); in visit()
1371 case ISD::BR_CC: return visitBR_CC(N); in visit()
1372 case ISD::LOAD: return visitLOAD(N); in visit()
1373 case ISD::STORE: return visitSTORE(N); in visit()
1374 case ISD::INSERT_VECTOR_ELT: return visitINSERT_VECTOR_ELT(N); in visit()
1375 case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N); in visit()
1376 case ISD::BUILD_VECTOR: return visitBUILD_VECTOR(N); in visit()
1377 case ISD::CONCAT_VECTORS: return visitCONCAT_VECTORS(N); in visit()
1378 case ISD::EXTRACT_SUBVECTOR: return visitEXTRACT_SUBVECTOR(N); in visit()
1379 case ISD::VECTOR_SHUFFLE: return visitVECTOR_SHUFFLE(N); in visit()
1380 case ISD::SCALAR_TO_VECTOR: return visitSCALAR_TO_VECTOR(N); in visit()
1381 case ISD::INSERT_SUBVECTOR: return visitINSERT_SUBVECTOR(N); in visit()
1382 case ISD::MLOAD: return visitMLOAD(N); in visit()
1383 case ISD::MSTORE: return visitMSTORE(N); in visit()
1384 case ISD::FP_TO_FP16: return visitFP_TO_FP16(N); in visit()
1394 assert(N->getOpcode() != ISD::DELETED_NODE && in combine()
1397 if (N->getOpcode() >= ISD::BUILTIN_OP_END || in combine()
1398 TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode())) { in combine()
1412 case ISD::ADD: in combine()
1413 case ISD::SUB: in combine()
1414 case ISD::MUL: in combine()
1415 case ISD::AND: in combine()
1416 case ISD::OR: in combine()
1417 case ISD::XOR: in combine()
1420 case ISD::SHL: in combine()
1421 case ISD::SRA: in combine()
1422 case ISD::SRL: in combine()
1425 case ISD::SIGN_EXTEND: in combine()
1426 case ISD::ZERO_EXTEND: in combine()
1427 case ISD::ANY_EXTEND: in combine()
1430 case ISD::LOAD: in combine()
1507 case ISD::EntryToken: in visitTokenFactor()
1513 case ISD::TokenFactor: in visitTokenFactor()
1545 Result = DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, Ops); in visitTokenFactor()
1586 if (ISD::isBuildVectorAllZeros(N1.getNode())) in visitADD()
1588 if (ISD::isBuildVectorAllZeros(N0.getNode())) in visitADD()
1593 if (N0.getOpcode() == ISD::UNDEF) in visitADD()
1595 if (N1.getOpcode() == ISD::UNDEF) in visitADD()
1601 return DAG.FoldConstantArithmetic(ISD::ADD, VT, N0C, N1C); in visitADD()
1605 return DAG.getNode(ISD::ADD, SDLoc(N), VT, N1, N0); in visitADD()
1612 GA->getOpcode() == ISD::GlobalAddress) in visitADD()
1617 if (N1C && N0.getOpcode() == ISD::SUB) in visitADD()
1619 return DAG.getNode(ISD::SUB, SDLoc(N), VT, in visitADD()
1624 if (SDValue RADD = ReassociateOps(ISD::ADD, SDLoc(N), N0, N1)) in visitADD()
1627 if (N0.getOpcode() == ISD::SUB && isa<ConstantSDNode>(N0.getOperand(0)) && in visitADD()
1629 return DAG.getNode(ISD::SUB, SDLoc(N), VT, N1, N0.getOperand(1)); in visitADD()
1631 if (N1.getOpcode() == ISD::SUB && isa<ConstantSDNode>(N1.getOperand(0)) && in visitADD()
1633 return DAG.getNode(ISD::SUB, SDLoc(N), VT, N0, N1.getOperand(1)); in visitADD()
1635 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(1)) in visitADD()
1638 if (N0.getOpcode() == ISD::SUB && N1 == N0.getOperand(1)) in visitADD()
1641 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD && in visitADD()
1643 return DAG.getNode(ISD::SUB, SDLoc(N), VT, N1.getOperand(0), in visitADD()
1646 if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD && in visitADD()
1648 return DAG.getNode(ISD::SUB, SDLoc(N), VT, N1.getOperand(0), in visitADD()
1651 if ((N1.getOpcode() == ISD::SUB || N1.getOpcode() == ISD::ADD) && in visitADD()
1652 N1.getOperand(0).getOpcode() == ISD::SUB && in visitADD()
1658 if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB) { in visitADD()
1665 return DAG.getNode(ISD::SUB, SDLoc(N), VT, in visitADD()
1666 DAG.getNode(ISD::ADD, SDLoc(N0), VT, N00, N10), in visitADD()
1667 DAG.getNode(ISD::ADD, SDLoc(N1), VT, N01, N11)); in visitADD()
1685 if (!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) in visitADD()
1686 return DAG.getNode(ISD::OR, SDLoc(N), VT, N0, N1); in visitADD()
1692 if (N1.getOpcode() == ISD::SHL && in visitADD()
1693 N1.getOperand(0).getOpcode() == ISD::SUB) in visitADD()
1697 return DAG.getNode(ISD::SUB, SDLoc(N), VT, N0, in visitADD()
1698 DAG.getNode(ISD::SHL, SDLoc(N), VT, in visitADD()
1701 if (N0.getOpcode() == ISD::SHL && in visitADD()
1702 N0.getOperand(0).getOpcode() == ISD::SUB) in visitADD()
1706 return DAG.getNode(ISD::SUB, SDLoc(N), VT, N1, in visitADD()
1707 DAG.getNode(ISD::SHL, SDLoc(N), VT, in visitADD()
1711 if (N1.getOpcode() == ISD::AND) { in visitADD()
1721 return DAG.getNode(ISD::SUB, DL, VT, N->getOperand(0), AndOp0); in visitADD()
1726 if (N0.getOpcode() == ISD::SIGN_EXTEND && in visitADD()
1728 !TLI.isOperationLegal(ISD::SIGN_EXTEND, MVT::i1)) { in visitADD()
1730 SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)); in visitADD()
1731 return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt); in visitADD()
1735 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) { in visitADD()
1739 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0), in visitADD()
1741 return DAG.getNode(ISD::SUB, DL, VT, N0, ZExt); in visitADD()
1755 return CombineTo(N, DAG.getNode(ISD::ADD, SDLoc(N), VT, N0, N1), in visitADDC()
1756 DAG.getNode(ISD::CARRY_FALSE, in visitADDC()
1763 return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N1, N0); in visitADDC()
1767 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, in visitADDC()
1781 return CombineTo(N, DAG.getNode(ISD::OR, SDLoc(N), VT, N0, N1), in visitADDC()
1782 DAG.getNode(ISD::CARRY_FALSE, in visitADDC()
1798 return DAG.getNode(ISD::ADDE, SDLoc(N), N->getVTList(), in visitADDE()
1802 if (CarryIn.getOpcode() == ISD::CARRY_FALSE) in visitADDE()
1803 return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N0, N1); in visitADDE()
1815 if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) in tryFoldToZero()
1831 if (ISD::isBuildVectorAllZeros(N1.getNode())) in visitSUB()
1843 return DAG.FoldConstantArithmetic(ISD::SUB, VT, N0C, N1C); in visitSUB()
1846 return DAG.getNode(ISD::ADD, SDLoc(N), VT, N0, in visitSUB()
1850 return DAG.getNode(ISD::XOR, SDLoc(N), VT, N1, N0); in visitSUB()
1852 if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(0)) in visitSUB()
1855 if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1) in visitSUB()
1858 if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1) in visitSUB()
1861 ConstantSDNode *N1C1 = N1.getOpcode() != ISD::ADD ? nullptr : in visitSUB()
1863 if (N1.getOpcode() == ISD::ADD && N0C && N1C1) { in visitSUB()
1866 return DAG.getNode(ISD::SUB, SDLoc(N), VT, NewC, in visitSUB()
1870 if (N0.getOpcode() == ISD::ADD && in visitSUB()
1871 (N0.getOperand(1).getOpcode() == ISD::SUB || in visitSUB()
1872 N0.getOperand(1).getOpcode() == ISD::ADD) && in visitSUB()
1877 if (N0.getOpcode() == ISD::ADD && in visitSUB()
1878 N0.getOperand(1).getOpcode() == ISD::ADD && in visitSUB()
1880 return DAG.getNode(ISD::ADD, SDLoc(N), VT, in visitSUB()
1883 if (N0.getOpcode() == ISD::SUB && in visitSUB()
1884 N0.getOperand(1).getOpcode() == ISD::SUB && in visitSUB()
1886 return DAG.getNode(ISD::SUB, SDLoc(N), VT, in visitSUB()
1890 if (N0.getOpcode() == ISD::UNDEF) in visitSUB()
1892 if (N1.getOpcode() == ISD::UNDEF) in visitSUB()
1899 if (N1C && GA->getOpcode() == ISD::GlobalAddress) in visitSUB()
1911 if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) { in visitSUB()
1915 SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0), in visitSUB()
1917 return DAG.getNode(ISD::ADD, DL, VT, N0, ZExt); in visitSUB()
1931 return CombineTo(N, DAG.getNode(ISD::SUB, SDLoc(N), VT, N0, N1), in visitSUBC()
1932 DAG.getNode(ISD::CARRY_FALSE, SDLoc(N), in visitSUBC()
1938 DAG.getNode(ISD::CARRY_FALSE, SDLoc(N), in visitSUBC()
1945 return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, SDLoc(N), in visitSUBC()
1950 return CombineTo(N, DAG.getNode(ISD::XOR, SDLoc(N), VT, N1, N0), in visitSUBC()
1951 DAG.getNode(ISD::CARRY_FALSE, SDLoc(N), in visitSUBC()
1963 if (CarryIn.getOpcode() == ISD::CARRY_FALSE) in visitSUBE()
1964 return DAG.getNode(ISD::SUBC, SDLoc(N), N->getVTList(), N0, N1); in visitSUBE()
1975 if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF) in visitMUL()
1999 return DAG.FoldConstantArithmetic(ISD::MUL, VT, N0.getNode(), N1.getNode()); in visitMUL()
2004 return DAG.getNode(ISD::MUL, SDLoc(N), VT, N1, N0); in visitMUL()
2017 return DAG.getNode(ISD::SUB, SDLoc(N), VT, in visitMUL()
2021 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, in visitMUL()
2029 return DAG.getNode(ISD::SUB, SDLoc(N), VT, in visitMUL()
2031 DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, in visitMUL()
2038 if (N1IsConst && N0.getOpcode() == ISD::SHL && in visitMUL()
2041 SDValue C3 = DAG.getNode(ISD::SHL, SDLoc(N), VT, in visitMUL()
2044 return DAG.getNode(ISD::MUL, SDLoc(N), VT, in visitMUL()
2053 if (N0.getOpcode() == ISD::SHL && in visitMUL()
2058 } else if (N1.getOpcode() == ISD::SHL && in visitMUL()
2065 SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N), VT, in visitMUL()
2067 return DAG.getNode(ISD::SHL, SDLoc(N), VT, in visitMUL()
2073 if (N1IsConst && N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse() && in visitMUL()
2076 return DAG.getNode(ISD::ADD, SDLoc(N), VT, in visitMUL()
2077 DAG.getNode(ISD::MUL, SDLoc(N0), VT, in visitMUL()
2079 DAG.getNode(ISD::MUL, SDLoc(N1), VT, in visitMUL()
2083 if (SDValue RMUL = ReassociateOps(ISD::MUL, SDLoc(N), N0, N1)) in visitMUL()
2103 return DAG.FoldConstantArithmetic(ISD::SDIV, VT, N0C, N1C); in visitSDIV()
2109 return DAG.getNode(ISD::SUB, SDLoc(N), VT, in visitSDIV()
2115 return DAG.getNode(ISD::UDIV, SDLoc(N), N1.getValueType(), in visitSDIV()
2136 DAG.getNode(ISD::SRA, SDLoc(N), VT, N0, in visitSDIV()
2143 DAG.getNode(ISD::SRL, SDLoc(N), VT, SGN, in visitSDIV()
2146 SDValue ADD = DAG.getNode(ISD::ADD, SDLoc(N), VT, N0, SRL); in visitSDIV()
2149 SDValue SRA = DAG.getNode(ISD::SRA, SDLoc(N), VT, ADD, in visitSDIV()
2158 return DAG.getNode(ISD::SUB, SDLoc(N), VT, DAG.getConstant(0, VT), SRA); in visitSDIV()
2169 if (N0.getOpcode() == ISD::UNDEF) in visitSDIV()
2172 if (N1.getOpcode() == ISD::UNDEF) in visitSDIV()
2192 return DAG.FoldConstantArithmetic(ISD::UDIV, VT, N0C, N1C); in visitUDIV()
2195 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, in visitUDIV()
2199 if (N1.getOpcode() == ISD::SHL) { in visitUDIV()
2203 SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N), ADDVT, in visitUDIV()
2209 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, Add); in visitUDIV()
2220 if (N0.getOpcode() == ISD::UNDEF) in visitUDIV()
2223 if (N1.getOpcode() == ISD::UNDEF) in visitUDIV()
2238 return DAG.FoldConstantArithmetic(ISD::SREM, VT, N0C, N1C); in visitSREM()
2243 return DAG.getNode(ISD::UREM, SDLoc(N), VT, N0, N1); in visitSREM()
2249 SDValue Div = DAG.getNode(ISD::SDIV, SDLoc(N), VT, N0, N1); in visitSREM()
2253 SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N), VT, in visitSREM()
2255 SDValue Sub = DAG.getNode(ISD::SUB, SDLoc(N), VT, N0, Mul); in visitSREM()
2262 if (N0.getOpcode() == ISD::UNDEF) in visitSREM()
2265 if (N1.getOpcode() == ISD::UNDEF) in visitSREM()
2280 return DAG.FoldConstantArithmetic(ISD::UREM, VT, N0C, N1C); in visitUREM()
2283 return DAG.getNode(ISD::AND, SDLoc(N), VT, N0, in visitUREM()
2286 if (N1.getOpcode() == ISD::SHL) { in visitUREM()
2290 DAG.getNode(ISD::ADD, SDLoc(N), VT, N1, in visitUREM()
2294 return DAG.getNode(ISD::AND, SDLoc(N), VT, N0, Add); in visitUREM()
2302 SDValue Div = DAG.getNode(ISD::UDIV, SDLoc(N), VT, N0, N1); in visitUREM()
2306 SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N), VT, in visitUREM()
2308 SDValue Sub = DAG.getNode(ISD::SUB, SDLoc(N), VT, N0, Mul); in visitUREM()
2315 if (N0.getOpcode() == ISD::UNDEF) in visitUREM()
2318 if (N1.getOpcode() == ISD::UNDEF) in visitUREM()
2336 return DAG.getNode(ISD::SRA, SDLoc(N), N0.getValueType(), N0, in visitMULHS()
2340 if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF) in visitMULHS()
2349 if (TLI.isOperationLegal(ISD::MUL, NewVT)) { in visitMULHS()
2350 N0 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0); in visitMULHS()
2351 N1 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1); in visitMULHS()
2352 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1); in visitMULHS()
2353 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1, in visitMULHS()
2355 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1); in visitMULHS()
2376 if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF) in visitMULHU()
2385 if (TLI.isOperationLegal(ISD::MUL, NewVT)) { in visitMULHU()
2386 N0 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0); in visitMULHU()
2387 N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1); in visitMULHU()
2388 N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1); in visitMULHU()
2389 N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1, in visitMULHU()
2391 return DAG.getNode(ISD::TRUNCATE, DL, VT, N1); in visitMULHU()
2450 SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS); in visitSMUL_LOHI()
2462 if (TLI.isOperationLegal(ISD::MUL, NewVT)) { in visitSMUL_LOHI()
2463 SDValue Lo = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(0)); in visitSMUL_LOHI()
2464 SDValue Hi = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(1)); in visitSMUL_LOHI()
2465 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi); in visitSMUL_LOHI()
2467 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo, in visitSMUL_LOHI()
2469 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi); in visitSMUL_LOHI()
2471 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo); in visitSMUL_LOHI()
2480 SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU); in visitUMUL_LOHI()
2492 if (TLI.isOperationLegal(ISD::MUL, NewVT)) { in visitUMUL_LOHI()
2493 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(0)); in visitUMUL_LOHI()
2494 SDValue Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(1)); in visitUMUL_LOHI()
2495 Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi); in visitUMUL_LOHI()
2497 Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo, in visitUMUL_LOHI()
2499 Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi); in visitUMUL_LOHI()
2501 Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo); in visitUMUL_LOHI()
2513 return DAG.getNode(ISD::SADDO, SDLoc(N), N->getVTList(), in visitSMULO()
2523 return DAG.getNode(ISD::UADDO, SDLoc(N), N->getVTList(), in visitUMULO()
2530 SDValue Res = SimplifyNodeWithTwoResults(N, ISD::SDIV, ISD::SREM); in visitSDIVREM()
2537 SDValue Res = SimplifyNodeWithTwoResults(N, ISD::UDIV, ISD::UREM); in visitUDIVREM()
2563 if ((N0.getOpcode() == ISD::ZERO_EXTEND || in SimplifyBinOpWithSameOpcodeHands()
2564 N0.getOpcode() == ISD::SIGN_EXTEND || in SimplifyBinOpWithSameOpcodeHands()
2565 N0.getOpcode() == ISD::BSWAP || in SimplifyBinOpWithSameOpcodeHands()
2567 (N0.getOpcode() == ISD::ANY_EXTEND && in SimplifyBinOpWithSameOpcodeHands()
2569 (N0.getOpcode() == ISD::TRUNCATE && in SimplifyBinOpWithSameOpcodeHands()
2587 if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL || in SimplifyBinOpWithSameOpcodeHands()
2588 N0.getOpcode() == ISD::SRA || N0.getOpcode() == ISD::AND) && in SimplifyBinOpWithSameOpcodeHands()
2605 if ((N0.getOpcode() == ISD::BITCAST || in SimplifyBinOpWithSameOpcodeHands()
2606 N0.getOpcode() == ISD::SCALAR_TO_VECTOR) && in SimplifyBinOpWithSameOpcodeHands()
2635 if (N0.getOpcode() == ISD::VECTOR_SHUFFLE && Level < AfterLegalizeDAG) { in SimplifyBinOpWithSameOpcodeHands()
2652 if (N->getOpcode() == ISD::XOR && ShOp.getOpcode() != ISD::UNDEF) { in SimplifyBinOpWithSameOpcodeHands()
2673 if (N->getOpcode() == ISD::XOR && ShOp.getOpcode() != ISD::UNDEF) { in SimplifyBinOpWithSameOpcodeHands()
2705 if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF) in visitANDLike()
2710 ISD::CondCode Op0 = cast<CondCodeSDNode>(CC0)->get(); in visitANDLike()
2711 ISD::CondCode Op1 = cast<CondCodeSDNode>(CC1)->get(); in visitANDLike()
2716 if (cast<ConstantSDNode>(LR)->isNullValue() && Op1 == ISD::SETEQ) { in visitANDLike()
2717 SDValue ORNode = DAG.getNode(ISD::OR, SDLoc(N0), in visitANDLike()
2723 if (cast<ConstantSDNode>(LR)->isAllOnesValue() && Op1 == ISD::SETEQ) { in visitANDLike()
2724 SDValue ANDNode = DAG.getNode(ISD::AND, SDLoc(N0), in visitANDLike()
2730 if (cast<ConstantSDNode>(LR)->isAllOnesValue() && Op1 == ISD::SETGT) { in visitANDLike()
2731 SDValue ORNode = DAG.getNode(ISD::OR, SDLoc(N0), in visitANDLike()
2740 Op0 == ISD::SETNE && ((cast<ConstantSDNode>(LR)->isNullValue() && in visitANDLike()
2744 SDValue ADDNode = DAG.getNode(ISD::ADD, SDLoc(N0), LL.getValueType(), in visitANDLike()
2748 DAG.getConstant(2, LL.getValueType()), ISD::SETUGE); in visitANDLike()
2752 Op1 = ISD::getSetCCSwappedOperands(Op1); in visitANDLike()
2757 ISD::CondCode Result = ISD::getSetCCAndOperation(Op0, Op1, isInteger); in visitANDLike()
2758 if (Result != ISD::SETCC_INVALID && in visitANDLike()
2761 TLI.isOperationLegal(ISD::SETCC, in visitANDLike()
2768 if (N0.getOpcode() == ISD::ADD && N1.getOpcode() == ISD::SRL && in visitANDLike()
2784 DAG.getNode(ISD::ADD, SDLoc(N0), VT, in visitANDLike()
2810 if (ISD::isBuildVectorAllZeros(N0.getNode())) in visitAND()
2816 if (ISD::isBuildVectorAllZeros(N1.getNode())) in visitAND()
2824 if (ISD::isBuildVectorAllOnes(N0.getNode())) in visitAND()
2826 if (ISD::isBuildVectorAllOnes(N1.getNode())) in visitAND()
2834 return DAG.FoldConstantArithmetic(ISD::AND, VT, N0C, N1C); in visitAND()
2838 return DAG.getNode(ISD::AND, SDLoc(N), VT, N1, N0); in visitAND()
2848 if (SDValue RAND = ReassociateOps(ISD::AND, SDLoc(N), N0, N1)) in visitAND()
2851 if (N1C && N0.getOpcode() == ISD::OR) in visitAND()
2856 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) { in visitAND()
2861 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), in visitAND()
2880 if ((N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in visitAND()
2881 N0.getOperand(0).getOpcode() == ISD::LOAD) || in visitAND()
2882 N0.getOpcode() == ISD::LOAD) { in visitAND()
2883 LoadSDNode *Load = cast<LoadSDNode>( (N0.getOpcode() == ISD::LOAD) ? in visitAND()
2932 bool CanZextLoadProfitably = TLI.isLoadExtLegal(ISD::ZEXTLOAD, in visitAND()
2945 case ISD::EXTLOAD: B = CanZextLoadProfitably; break; in visitAND()
2946 case ISD::ZEXTLOAD: in visitAND()
2947 case ISD::NON_EXTLOAD: B = true; break; in visitAND()
2954 if (Load->getExtensionType() == ISD::EXTLOAD) { in visitAND()
2955 NewLoad = DAG.getLoad(Load->getAddressingMode(), ISD::ZEXTLOAD, in visitAND()
2982 if (N1C && (N0.getOpcode() == ISD::LOAD || in visitAND()
2983 (N0.getOpcode() == ISD::ANY_EXTEND && in visitAND()
2984 N0.getOperand(0).getOpcode() == ISD::LOAD))) { in visitAND()
2985 bool HasAnyExt = N0.getOpcode() == ISD::ANY_EXTEND; in visitAND()
2989 if (LN0->getExtensionType() != ISD::SEXTLOAD && in visitAND()
2998 (!LegalOperations || TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, in visitAND()
3002 DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(LN0), LoadResultTy, in visitAND()
3014 (!LegalOperations || TLI.isLoadExtLegal(ISD::ZEXTLOAD, LoadResultTy, in visitAND()
3028 NewPtr = DAG.getNode(ISD::ADD, SDLoc(LN0), PtrType, in visitAND()
3036 DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(LN0), LoadResultTy, in visitAND()
3065 if (ISD::isEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode())) { in visitAND()
3074 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) { in visitAND()
3075 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, in visitAND()
3084 if (ISD::isSEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) && in visitAND()
3094 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) { in visitAND()
3095 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, in visitAND()
3104 if (N1C && N1C->getAPIntValue() == 0xffff && N0.getOpcode() == ISD::OR) { in visitAND()
3123 if (!TLI.isOperationLegal(ISD::BSWAP, VT)) in MatchBSwapHWordLow()
3129 if (N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::SRL) in MatchBSwapHWordLow()
3131 if (N1.getOpcode() == ISD::AND && N1.getOperand(0).getOpcode() == ISD::SHL) in MatchBSwapHWordLow()
3133 if (N0.getOpcode() == ISD::AND) { in MatchBSwapHWordLow()
3143 if (N1.getOpcode() == ISD::AND) { in MatchBSwapHWordLow()
3153 if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL) in MatchBSwapHWordLow()
3155 if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL) in MatchBSwapHWordLow()
3170 if (!LookPassAnd0 && N00.getOpcode() == ISD::AND) { in MatchBSwapHWordLow()
3181 if (!LookPassAnd1 && N10.getOpcode() == ISD::AND) { in MatchBSwapHWordLow()
3212 SDValue Res = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N00); in MatchBSwapHWordLow()
3214 Res = DAG.getNode(ISD::SRL, SDLoc(N), VT, Res, in MatchBSwapHWordLow()
3230 if (Opc != ISD::AND && Opc != ISD::SHL && Opc != ISD::SRL) in isBSwapHWordElement()
3249 if (Opc == ISD::AND) { in isBSwapHWordElement()
3253 if (N0.getOpcode() != ISD::SRL) in isBSwapHWordElement()
3261 if (N0.getOpcode() != ISD::SHL) in isBSwapHWordElement()
3267 } else if (Opc == ISD::SHL) { in isBSwapHWordElement()
3305 if (!TLI.isOperationLegal(ISD::BSWAP, VT)) in MatchBSwapHWord()
3311 if (N0.getOpcode() != ISD::OR) in MatchBSwapHWord()
3317 if (N1.getOpcode() == ISD::OR && in MatchBSwapHWord()
3339 if (N00.getOpcode() != ISD::OR) in MatchBSwapHWord()
3353 SDValue BSwap = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, in MatchBSwapHWord()
3359 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT)) in MatchBSwapHWord()
3360 return DAG.getNode(ISD::ROTL, SDLoc(N), VT, BSwap, ShAmt); in MatchBSwapHWord()
3361 if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT)) in MatchBSwapHWord()
3362 return DAG.getNode(ISD::ROTR, SDLoc(N), VT, BSwap, ShAmt); in MatchBSwapHWord()
3363 return DAG.getNode(ISD::OR, SDLoc(N), VT, in MatchBSwapHWord()
3364 DAG.getNode(ISD::SHL, SDLoc(N), VT, BSwap, ShAmt), in MatchBSwapHWord()
3365 DAG.getNode(ISD::SRL, SDLoc(N), VT, BSwap, ShAmt)); in MatchBSwapHWord()
3374 (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)) { in visitORLike()
3381 ISD::CondCode Op0 = cast<CondCodeSDNode>(CC0)->get(); in visitORLike()
3382 ISD::CondCode Op1 = cast<CondCodeSDNode>(CC1)->get(); in visitORLike()
3389 (Op1 == ISD::SETNE || Op1 == ISD::SETLT)) { in visitORLike()
3390 SDValue ORNode = DAG.getNode(ISD::OR, SDLoc(LR), in visitORLike()
3398 (Op1 == ISD::SETNE || Op1 == ISD::SETGT)) { in visitORLike()
3399 SDValue ANDNode = DAG.getNode(ISD::AND, SDLoc(LR), in visitORLike()
3407 Op1 = ISD::getSetCCSwappedOperands(Op1); in visitORLike()
3412 ISD::CondCode Result = ISD::getSetCCOrOperation(Op0, Op1, isInteger); in visitORLike()
3413 if (Result != ISD::SETCC_INVALID && in visitORLike()
3416 TLI.isOperationLegal(ISD::SETCC, in visitORLike()
3424 if (N0.getOpcode() == ISD::AND && in visitORLike()
3425 N1.getOpcode() == ISD::AND && in visitORLike()
3426 N0.getOperand(1).getOpcode() == ISD::Constant && in visitORLike()
3427 N1.getOperand(1).getOpcode() == ISD::Constant && in visitORLike()
3439 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT, in visitORLike()
3441 return DAG.getNode(ISD::AND, SDLoc(LocReference), VT, X, in visitORLike()
3447 if (N0.getOpcode() == ISD::AND && in visitORLike()
3448 N1.getOpcode() == ISD::AND && in visitORLike()
3452 SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT, in visitORLike()
3454 return DAG.getNode(ISD::AND, SDLoc(LocReference), VT, N0.getOperand(0), X); in visitORLike()
3471 if (ISD::isBuildVectorAllZeros(N0.getNode())) in visitOR()
3473 if (ISD::isBuildVectorAllZeros(N1.getNode())) in visitOR()
3477 if (ISD::isBuildVectorAllOnes(N0.getNode())) in visitOR()
3483 if (ISD::isBuildVectorAllOnes(N1.getNode())) in visitOR()
3498 ISD::isBuildVectorAllZeros(N0.getOperand(1).getNode())) { in visitOR()
3551 return DAG.FoldConstantArithmetic(ISD::OR, VT, N0C, N1C); in visitOR()
3555 return DAG.getNode(ISD::OR, SDLoc(N), VT, N1, N0); in visitOR()
3578 if (SDValue ROR = ReassociateOps(ISD::OR, SDLoc(N), N0, N1)) in visitOR()
3582 if (N1C && N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() && in visitOR()
3586 if (SDValue COR = DAG.FoldConstantArithmetic(ISD::OR, VT, N1C, C1)) in visitOR()
3588 ISD::AND, SDLoc(N), VT, in visitOR()
3589 DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1), COR); in visitOR()
3613 if (Op.getOpcode() == ISD::AND) { in MatchRotateHalf()
3622 if (Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL) { in MatchRotateHalf()
3672 if (Neg.getOpcode() == ISD::AND && in matchRotateSub()
3674 Neg.getOperand(1).getOpcode() == ISD::Constant && in matchRotateSub()
3681 if (Neg.getOpcode() != ISD::SUB) in matchRotateSub()
3691 Pos.getOpcode() == ISD::AND && in matchRotateSub()
3692 Pos.getOperand(1).getOpcode() == ISD::Constant && in matchRotateSub()
3717 else if (Pos.getOpcode() == ISD::ADD && in matchRotateSub()
3719 Pos.getOperand(1).getOpcode() == ISD::Constant) in matchRotateSub()
3767 bool HasROTL = TLI.isOperationLegalOrCustom(ISD::ROTL, VT); in MatchRotate()
3768 bool HasROTR = TLI.isOperationLegalOrCustom(ISD::ROTR, VT); in MatchRotate()
3789 if (RHSShift.getOpcode() == ISD::SHL) { in MatchRotate()
3803 if (LHSShiftAmt.getOpcode() == ISD::Constant && in MatchRotate()
3804 RHSShiftAmt.getOpcode() == ISD::Constant) { in MatchRotate()
3810 SDValue Rot = DAG.getNode(HasROTL ? ISD::ROTL : ISD::ROTR, DL, VT, in MatchRotate()
3826 Rot = DAG.getNode(ISD::AND, DL, VT, Rot, DAG.getConstant(Mask, VT)); in MatchRotate()
3840 if ((LHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND || in MatchRotate()
3841 LHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND || in MatchRotate()
3842 LHSShiftAmt.getOpcode() == ISD::ANY_EXTEND || in MatchRotate()
3843 LHSShiftAmt.getOpcode() == ISD::TRUNCATE) && in MatchRotate()
3844 (RHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND || in MatchRotate()
3845 RHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND || in MatchRotate()
3846 RHSShiftAmt.getOpcode() == ISD::ANY_EXTEND || in MatchRotate()
3847 RHSShiftAmt.getOpcode() == ISD::TRUNCATE)) { in MatchRotate()
3853 LExtOp0, RExtOp0, ISD::ROTL, ISD::ROTR, DL); in MatchRotate()
3858 RExtOp0, LExtOp0, ISD::ROTR, ISD::ROTL, DL); in MatchRotate()
3876 if (ISD::isBuildVectorAllZeros(N0.getNode())) in visitXOR()
3878 if (ISD::isBuildVectorAllZeros(N1.getNode())) in visitXOR()
3883 if (N0.getOpcode() == ISD::UNDEF && N1.getOpcode() == ISD::UNDEF) in visitXOR()
3886 if (N0.getOpcode() == ISD::UNDEF) in visitXOR()
3888 if (N1.getOpcode() == ISD::UNDEF) in visitXOR()
3894 return DAG.FoldConstantArithmetic(ISD::XOR, VT, N0C, N1C); in visitXOR()
3898 return DAG.getNode(ISD::XOR, SDLoc(N), VT, N1, N0); in visitXOR()
3903 if (SDValue RXOR = ReassociateOps(ISD::XOR, SDLoc(N), N0, N1)) in visitXOR()
3910 ISD::CondCode NotCC = ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(), in visitXOR()
3918 case ISD::SETCC: in visitXOR()
3920 case ISD::SELECT_CC: in visitXOR()
3928 if (N1C && N1C->getAPIntValue() == 1 && N0.getOpcode() == ISD::ZERO_EXTEND && in visitXOR()
3932 V = DAG.getNode(ISD::XOR, SDLoc(N0), V.getValueType(), V, in visitXOR()
3935 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, V); in visitXOR()
3940 (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) { in visitXOR()
3943 unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND; in visitXOR()
3944 LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), VT, LHS, N1); // LHS = ~LHS in visitXOR()
3945 RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), VT, RHS, N1); // RHS = ~RHS in visitXOR()
3952 (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) { in visitXOR()
3955 unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND; in visitXOR()
3956 LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), VT, LHS, N1); // LHS = ~LHS in visitXOR()
3957 RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), VT, RHS, N1); // RHS = ~RHS in visitXOR()
3963 if (N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() && in visitXOR()
3968 return DAG.getNode(ISD::AND, SDLoc(N), VT, NotX, N1); in visitXOR()
3971 if (N1C && N0.getOpcode() == ISD::XOR) { in visitXOR()
3975 return DAG.getNode(ISD::XOR, SDLoc(N), VT, N0.getOperand(1), in visitXOR()
3979 return DAG.getNode(ISD::XOR, SDLoc(N), VT, N0.getOperand(0), in visitXOR()
4005 if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT)) in visitXOR()
4007 if (N0.getOpcode() == ISD::SHL) in visitXOR()
4010 return DAG.getNode(ISD::ROTL, SDLoc(N), VT, DAG.getConstant(~1, VT), in visitXOR()
4045 case ISD::OR: in visitShiftByConstant()
4046 case ISD::XOR: in visitShiftByConstant()
4049 case ISD::AND: in visitShiftByConstant()
4052 case ISD::ADD: in visitShiftByConstant()
4053 if (N->getOpcode() != ISD::SHL) in visitShiftByConstant()
4069 if ((BinOpLHSVal->getOpcode() != ISD::SHL && in visitShiftByConstant()
4070 BinOpLHSVal->getOpcode() != ISD::SRA && in visitShiftByConstant()
4071 BinOpLHSVal->getOpcode() != ISD::SRL) || in visitShiftByConstant()
4081 if (N->getOpcode() == ISD::SRA) { in visitShiftByConstant()
4106 assert(N->getOpcode() == ISD::TRUNCATE); in distributeTruncateThroughAnd()
4107 assert(N->getOperand(0).getOpcode() == ISD::AND); in distributeTruncateThroughAnd()
4119 return DAG.getNode(ISD::AND, SDLoc(N), TruncVT, in distributeTruncateThroughAnd()
4120 DAG.getNode(ISD::TRUNCATE, SDLoc(N), TruncVT, N00), in distributeTruncateThroughAnd()
4130 if (N->getOperand(1).getOpcode() == ISD::TRUNCATE && in visitRotate()
4131 N->getOperand(1).getOperand(0).getOpcode() == ISD::AND) { in visitRotate()
4156 if (N0.getOpcode() == ISD::AND) { in visitSHL()
4161 if (N01CV && N01CV->isConstant() && N00.getOpcode() == ISD::SETCC && in visitSHL()
4164 if (SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, VT, N01CV, N1CV)) in visitSHL()
4165 return DAG.getNode(ISD::AND, SDLoc(N), VT, N00, C); in visitSHL()
4176 return DAG.FoldConstantArithmetic(ISD::SHL, VT, N0C, N1C); in visitSHL()
4187 if (N0.getOpcode() == ISD::UNDEF) in visitSHL()
4194 if (N1.getOpcode() == ISD::TRUNCATE && in visitSHL()
4195 N1.getOperand(0).getOpcode() == ISD::AND) { in visitSHL()
4198 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, NewOp1); in visitSHL()
4205 if (N1C && N0.getOpcode() == ISD::SHL) { in visitSHL()
4211 return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0.getOperand(0), in visitSHL()
4221 if (N1C && (N0.getOpcode() == ISD::ZERO_EXTEND || in visitSHL()
4222 N0.getOpcode() == ISD::ANY_EXTEND || in visitSHL()
4223 N0.getOpcode() == ISD::SIGN_EXTEND) && in visitSHL()
4224 N0.getOperand(0).getOpcode() == ISD::SHL) { in visitSHL()
4234 return DAG.getNode(ISD::SHL, SDLoc(N0), VT, in visitSHL()
4245 if (N1C && N0.getOpcode() == ISD::ZERO_EXTEND && N0.hasOneUse() && in visitSHL()
4246 N0.getOperand(0).getOpcode() == ISD::SRL) { in visitSHL()
4255 SDValue NewSHL = DAG.getNode(ISD::SHL, SDLoc(N), NewOp0.getValueType(), in visitSHL()
4258 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N0), VT, NewSHL); in visitSHL()
4268 if (N1C && N0.getOpcode() == ISD::SRL && N0.hasOneUse()) { in visitSHL()
4277 Shift = DAG.getNode(ISD::SHL, SDLoc(N), VT, N0.getOperand(0), in visitSHL()
4281 Shift = DAG.getNode(ISD::SRL, SDLoc(N), VT, N0.getOperand(0), in visitSHL()
4284 return DAG.getNode(ISD::AND, SDLoc(N0), VT, Shift, in visitSHL()
4290 if (N1C && N0.getOpcode() == ISD::SRA && N1 == N0.getOperand(1)) { in visitSHL()
4295 return DAG.getNode(ISD::AND, SDLoc(N), VT, N0.getOperand(0), in visitSHL()
4303 if (N1C && N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse() && in visitSHL()
4306 SDValue Shl0 = DAG.getNode(ISD::SHL, SDLoc(N0), VT, N0.getOperand(0), N1); in visitSHL()
4307 SDValue Shl1 = DAG.getNode(ISD::SHL, SDLoc(N1), VT, N0.getOperand(1), N1); in visitSHL()
4308 return DAG.getNode(ISD::ADD, SDLoc(N), VT, Shl0, Shl1); in visitSHL()
4338 return DAG.FoldConstantArithmetic(ISD::SRA, VT, N0C, N1C); in visitSRA()
4353 if (N1C && N0.getOpcode() == ISD::SHL && N1 == N0.getOperand(1)) { in visitSRA()
4360 TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, ExtVT))) in visitSRA()
4361 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, in visitSRA()
4366 if (N1C && N0.getOpcode() == ISD::SRA) { in visitSRA()
4371 return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0.getOperand(0), in visitSRA()
4381 if (N0.getOpcode() == ISD::SHL && N1C) { in visitSRA()
4400 TLI.isOperationLegalOrCustom(ISD::SIGN_EXTEND, TruncVT) && in visitSRA()
4401 TLI.isOperationLegalOrCustom(ISD::TRUNCATE, VT) && in visitSRA()
4406 SDValue Shift = DAG.getNode(ISD::SRL, SDLoc(N0), VT, in visitSRA()
4408 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), TruncVT, in visitSRA()
4410 return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), in visitSRA()
4417 if (N1.getOpcode() == ISD::TRUNCATE && in visitSRA()
4418 N1.getOperand(0).getOpcode() == ISD::AND) { in visitSRA()
4421 return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0, NewOp1); in visitSRA()
4426 if (N0.getOpcode() == ISD::TRUNCATE && in visitSRA()
4427 (N0.getOperand(0).getOpcode() == ISD::SRL || in visitSRA()
4428 N0.getOperand(0).getOpcode() == ISD::SRA) && in visitSRA()
4441 SDValue SRA = DAG.getNode(ISD::SRA, SDLoc(N), LargeVT, in visitSRA()
4443 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, SRA); in visitSRA()
4455 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, N1); in visitSRA()
4484 return DAG.FoldConstantArithmetic(ISD::SRL, VT, N0C, N1C); in visitSRL()
4500 if (N1C && N0.getOpcode() == ISD::SRL) { in visitSRL()
4506 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0.getOperand(0), in visitSRL()
4512 if (N1C && N0.getOpcode() == ISD::TRUNCATE && in visitSRL()
4513 N0.getOperand(0).getOpcode() == ISD::SRL && in visitSRL()
4525 return DAG.getNode(ISD::TRUNCATE, SDLoc(N0), VT, in visitSRL()
4526 DAG.getNode(ISD::SRL, SDLoc(N0), InnerShiftVT, in visitSRL()
4533 if (N1C && N0.getOpcode() == ISD::SHL && N0.getOperand(1) == N1) { in visitSRL()
4537 return DAG.getNode(ISD::AND, SDLoc(N), VT, N0.getOperand(0), in visitSRL()
4543 if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) { in visitSRL()
4550 if (!LegalTypes || TLI.isTypeDesirableForOp(ISD::SRL, SmallVT)) { in visitSRL()
4552 SDValue SmallShift = DAG.getNode(ISD::SRL, SDLoc(N0), SmallVT, in visitSRL()
4557 return DAG.getNode(ISD::AND, SDLoc(N), VT, in visitSRL()
4558 DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, SmallShift), in visitSRL()
4566 if (N0.getOpcode() == ISD::SRA) in visitSRL()
4567 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0.getOperand(0), N1); in visitSRL()
4571 if (N1C && N0.getOpcode() == ISD::CTLZ && in visitSRL()
4595 Op = DAG.getNode(ISD::SRL, SDLoc(N0), VT, Op, in visitSRL()
4600 return DAG.getNode(ISD::XOR, SDLoc(N), VT, in visitSRL()
4606 if (N1.getOpcode() == ISD::TRUNCATE && in visitSRL()
4607 N1.getOperand(0).getOpcode() == ISD::AND) { in visitSRL()
4610 return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, NewOp1); in visitSRL()
4648 if (Use->getOpcode() == ISD::BRCOND) in visitSRL()
4650 else if (Use->getOpcode() == ISD::TRUNCATE && Use->hasOneUse()) { in visitSRL()
4653 if (Use->getOpcode() == ISD::BRCOND) in visitSRL()
4667 return DAG.getNode(ISD::CTLZ, SDLoc(N), VT, N0); in visitCTLZ()
4677 return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SDLoc(N), VT, N0); in visitCTLZ_ZERO_UNDEF()
4687 return DAG.getNode(ISD::CTTZ, SDLoc(N), VT, N0); in visitCTTZ()
4697 return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, SDLoc(N), VT, N0); in visitCTTZ_ZERO_UNDEF()
4707 return DAG.getNode(ISD::CTPOP, SDLoc(N), VT, N0); in visitCTPOP()
4715 ISD::CondCode CC, const TargetLowering &TLI, in combineMinNumMaxNum()
4721 case ISD::SETOLT: in combineMinNumMaxNum()
4722 case ISD::SETOLE: in combineMinNumMaxNum()
4723 case ISD::SETLT: in combineMinNumMaxNum()
4724 case ISD::SETLE: in combineMinNumMaxNum()
4725 case ISD::SETULT: in combineMinNumMaxNum()
4726 case ISD::SETULE: { in combineMinNumMaxNum()
4727 unsigned Opcode = (LHS == True) ? ISD::FMINNUM : ISD::FMAXNUM; in combineMinNumMaxNum()
4732 case ISD::SETOGT: in combineMinNumMaxNum()
4733 case ISD::SETOGE: in combineMinNumMaxNum()
4734 case ISD::SETGT: in combineMinNumMaxNum()
4735 case ISD::SETGE: in combineMinNumMaxNum()
4736 case ISD::SETUGT: in combineMinNumMaxNum()
4737 case ISD::SETUGE: { in combineMinNumMaxNum()
4738 unsigned Opcode = (LHS == True) ? ISD::FMAXNUM : ISD::FMINNUM; in combineMinNumMaxNum()
4768 return DAG.getNode(ISD::OR, SDLoc(N), VT, N0, N2); in visitSELECT()
4788 return DAG.getNode(ISD::XOR, SDLoc(N), VT0, in visitSELECT()
4790 XORNode = DAG.getNode(ISD::XOR, SDLoc(N0), VT0, in visitSELECT()
4794 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, XORNode); in visitSELECT()
4795 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, XORNode); in visitSELECT()
4801 return DAG.getNode(ISD::AND, SDLoc(N), VT, NOTNode, N2); in visitSELECT()
4807 return DAG.getNode(ISD::OR, SDLoc(N), VT, NOTNode, N1); in visitSELECT()
4811 return DAG.getNode(ISD::AND, SDLoc(N), VT, N0, N1); in visitSELECT()
4815 return DAG.getNode(ISD::OR, SDLoc(N), VT, N0, N2); in visitSELECT()
4819 return DAG.getNode(ISD::AND, SDLoc(N), VT, N0, N1); in visitSELECT()
4826 if (N0.getOpcode() == ISD::SETCC) { in visitSELECT()
4840 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get(); in visitSELECT()
4850 TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT)) || in visitSELECT()
4851 TLI.isOperationLegal(ISD::SELECT_CC, VT)) in visitSELECT()
4852 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), VT, in visitSELECT()
4862 if (N0->getOpcode() == ISD::AND && N0->hasOneUse()) { in visitSELECT()
4865 SDValue InnerSelect = DAG.getNode(ISD::SELECT, SDLoc(N), in visitSELECT()
4867 return DAG.getNode(ISD::SELECT, SDLoc(N), N1.getValueType(), Cond0, in visitSELECT()
4871 if (N0->getOpcode() == ISD::OR && N0->hasOneUse()) { in visitSELECT()
4874 SDValue InnerSelect = DAG.getNode(ISD::SELECT, SDLoc(N), in visitSELECT()
4876 return DAG.getNode(ISD::SELECT, SDLoc(N), N1.getValueType(), Cond0, N1, in visitSELECT()
4882 if (N1->getOpcode() == ISD::SELECT) { in visitSELECT()
4889 SDValue And = DAG.getNode(ISD::AND, SDLoc(N), N0.getValueType(), in visitSELECT()
4891 return DAG.getNode(ISD::SELECT, SDLoc(N), N1.getValueType(), And, in visitSELECT()
4896 return DAG.getNode(ISD::SELECT, SDLoc(N), N1.getValueType(), Combined, in visitSELECT()
4901 if (N2->getOpcode() == ISD::SELECT) { in visitSELECT()
4908 SDValue Or = DAG.getNode(ISD::OR, SDLoc(N), N0.getValueType(), in visitSELECT()
4910 return DAG.getNode(ISD::SELECT, SDLoc(N), N1.getValueType(), Or, in visitSELECT()
4915 return DAG.getNode(ISD::SELECT, SDLoc(N), N1.getValueType(), Combined, in visitSELECT()
4950 assert(LHS.getOpcode() == ISD::CONCAT_VECTORS && in ConvertSelectToConcatVector()
4951 RHS.getOpcode() == ISD::CONCAT_VECTORS && in ConvertSelectToConcatVector()
4952 Cond.getOpcode() == ISD::BUILD_VECTOR); in ConvertSelectToConcatVector()
4966 if (Cond->getOperand(i)->getOpcode() == ISD::UNDEF) in ConvertSelectToConcatVector()
4978 if (Cond->getOperand(i)->getOpcode() == ISD::UNDEF) in ConvertSelectToConcatVector()
4991 ISD::CONCAT_VECTORS, dl, VT, in ConvertSelectToConcatVector()
5010 if (Mask.getOpcode() == ISD::SETCC) { in visitMSTORE()
5050 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, in visitMSTORE()
5065 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi); in visitMSTORE()
5084 if (Mask.getOpcode() == ISD::SETCC) { in visitMLOAD()
5122 ISD::NON_EXTLOAD); in visitMLOAD()
5125 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, in visitMLOAD()
5134 ISD::NON_EXTLOAD); in visitMLOAD()
5141 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(1), in visitMLOAD()
5148 SDValue LoadRes = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi); in visitMLOAD()
5167 if (N0.getOpcode() == ISD::SETCC) { in visitVSELECT()
5169 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get(); in visitVSELECT()
5171 bool RHSIsAllZeros = ISD::isBuildVectorAllZeros(RHS.getNode()); in visitVSELECT()
5173 if (((RHSIsAllZeros && (CC == ISD::SETGT || CC == ISD::SETGE)) || in visitVSELECT()
5174 (ISD::isBuildVectorAllOnes(RHS.getNode()) && CC == ISD::SETGT)) && in visitVSELECT()
5175 N1 == LHS && N2.getOpcode() == ISD::SUB && N1 == N2.getOperand(1)) in visitVSELECT()
5176 isAbs = ISD::isBuildVectorAllZeros(N2.getOperand(0).getNode()); in visitVSELECT()
5177 else if ((RHSIsAllZeros && (CC == ISD::SETLT || CC == ISD::SETLE)) && in visitVSELECT()
5178 N2 == LHS && N1.getOpcode() == ISD::SUB && N2 == N1.getOperand(1)) in visitVSELECT()
5179 isAbs = ISD::isBuildVectorAllZeros(N1.getOperand(0).getNode()); in visitVSELECT()
5184 ISD::SRA, DL, VT, LHS, in visitVSELECT()
5186 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, LHS, Shift); in visitVSELECT()
5189 return DAG.getNode(ISD::XOR, DL, VT, Add, Shift); in visitVSELECT()
5197 if (N0.getOpcode() == ISD::SETCC) { in visitVSELECT()
5218 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi); in visitVSELECT()
5222 if (ISD::isBuildVectorAllOnes(N0.getNode())) in visitVSELECT()
5225 if (ISD::isBuildVectorAllZeros(N0.getNode())) in visitVSELECT()
5231 if (N1.getOpcode() == ISD::CONCAT_VECTORS && in visitVSELECT()
5232 N2.getOpcode() == ISD::CONCAT_VECTORS && in visitVSELECT()
5233 ISD::isBuildVectorOfConstantSDNodes(N0.getNode())) { in visitVSELECT()
5248 ISD::CondCode CC = cast<CondCodeSDNode>(N4)->get(); in visitSELECT_CC()
5265 } else if (SCC->getOpcode() == ISD::UNDEF) { in visitSELECT_CC()
5269 } else if (SCC.getOpcode() == ISD::SETCC) { in visitSELECT_CC()
5271 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N2.getValueType(), in visitSELECT_CC()
5304 assert((Opcode == ISD::SIGN_EXTEND || Opcode == ISD::ZERO_EXTEND || in tryToFoldExtendOfConstant()
5305 Opcode == ISD::ANY_EXTEND) && "Expected EXTEND dag node in input!"); in tryToFoldExtendOfConstant()
5319 ISD::isBuildVectorOfConstantSDNodes(N0.getNode()))) in tryToFoldExtendOfConstant()
5332 if (Op->getOpcode() == ISD::UNDEF) { in tryToFoldExtendOfConstant()
5339 if (Opcode == ISD::SIGN_EXTEND) in tryToFoldExtendOfConstant()
5347 return DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Elts).getNode(); in tryToFoldExtendOfConstant()
5369 if (ExtOpc != ISD::ANY_EXTEND && User->getOpcode() == ISD::SETCC) { in ExtendUsesToFormExtLoad()
5370 ISD::CondCode CC = cast<CondCodeSDNode>(User->getOperand(2))->get(); in ExtendUsesToFormExtLoad()
5371 if (ExtOpc == ISD::ZERO_EXTEND && ISD::isSignedIntSetCC(CC)) in ExtendUsesToFormExtLoad()
5392 if (User->getOpcode() == ISD::CopyToReg) in ExtendUsesToFormExtLoad()
5401 if (Use.getResNo() == 0 && Use.getUser()->getOpcode() == ISD::CopyToReg) { in ExtendUsesToFormExtLoad()
5416 ISD::NodeType ExtType) { in ExtendSetCCUses()
5431 CombineTo(SetCC, DAG.getNode(ISD::SETCC, DL, SetCC->getValueType(0), Ops)); in ExtendSetCCUses()
5441 assert((N->getOpcode() == ISD::SIGN_EXTEND || in CombineExtLoad()
5442 N->getOpcode() == ISD::ZERO_EXTEND) && in CombineExtLoad()
5462 if (N0->getOpcode() != ISD::LOAD) in CombineExtLoad()
5467 if (!ISD::isNON_EXTLoad(LN0) || !ISD::isUNINDEXEDLoad(LN0) || in CombineExtLoad()
5476 ISD::LoadExtType ExtType = in CombineExtLoad()
5477 N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SEXTLOAD : ISD::ZEXTLOAD; in CombineExtLoad()
5509 BasePtr = DAG.getNode(ISD::ADD, DL, BasePtr.getValueType(), BasePtr, in CombineExtLoad()
5516 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); in CombineExtLoad()
5517 SDValue NewValue = DAG.getNode(ISD::CONCAT_VECTORS, DL, DstVT, Loads); in CombineExtLoad()
5524 DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), NewValue); in CombineExtLoad()
5527 (ISD::NodeType)N->getOpcode()); in CombineExtLoad()
5541 if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) in visitSIGN_EXTEND()
5542 return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, in visitSIGN_EXTEND()
5545 if (N0.getOpcode() == ISD::TRUNCATE) { in visitSIGN_EXTEND()
5576 return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, Op); in visitSIGN_EXTEND()
5581 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Op); in visitSIGN_EXTEND()
5585 if (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, in visitSIGN_EXTEND()
5588 Op = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N0), VT, Op); in visitSIGN_EXTEND()
5590 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), VT, Op); in visitSIGN_EXTEND()
5591 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, Op, in visitSIGN_EXTEND()
5599 if (ISD::isNON_EXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) && in visitSIGN_EXTEND()
5602 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, N0.getValueType()))) { in visitSIGN_EXTEND()
5606 DoXform = ExtendUsesToFormExtLoad(N, N0, ISD::SIGN_EXTEND, SetCCs, TLI); in visitSIGN_EXTEND()
5611 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT, in visitSIGN_EXTEND()
5616 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), in visitSIGN_EXTEND()
5620 ISD::SIGN_EXTEND); in visitSIGN_EXTEND()
5632 if ((ISD::isSEXTLoad(N0.getNode()) || ISD::isEXTLoad(N0.getNode())) && in visitSIGN_EXTEND()
5633 ISD::isUNINDEXEDLoad(N0.getNode()) && N0.hasOneUse()) { in visitSIGN_EXTEND()
5637 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, MemVT)) { in visitSIGN_EXTEND()
5638 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT, in visitSIGN_EXTEND()
5644 DAG.getNode(ISD::TRUNCATE, SDLoc(N0), in visitSIGN_EXTEND()
5653 if ((N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR || in visitSIGN_EXTEND()
5654 N0.getOpcode() == ISD::XOR) && in visitSIGN_EXTEND()
5656 N0.getOperand(1).getOpcode() == ISD::Constant && in visitSIGN_EXTEND()
5657 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, N0.getValueType()) && in visitSIGN_EXTEND()
5660 if (LN0->getExtensionType() != ISD::ZEXTLOAD && LN0->isUnindexed()) { in visitSIGN_EXTEND()
5664 DoXform = ExtendUsesToFormExtLoad(N, N0.getOperand(0), ISD::SIGN_EXTEND, in visitSIGN_EXTEND()
5667 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(LN0), VT, in visitSIGN_EXTEND()
5675 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, in visitSIGN_EXTEND()
5681 ISD::SIGN_EXTEND); in visitSIGN_EXTEND()
5687 if (N0.getOpcode() == ISD::SETCC) { in visitSIGN_EXTEND()
5733 if (!LegalOperations || TLI.isOperationLegal(ISD::SETCC, SetCCVT)) { in visitSIGN_EXTEND()
5735 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get(); in visitSIGN_EXTEND()
5745 if ((!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, VT)) && in visitSIGN_EXTEND()
5747 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, N0); in visitSIGN_EXTEND()
5759 if (N->getOpcode() == ISD::TRUNCATE) { in isTruncateOf()
5765 if (N->getOpcode() != ISD::SETCC || N->getValueType(0) != MVT::i1 || in isTruncateOf()
5766 cast<CondCodeSDNode>(N->getOperand(2))->get() != ISD::SETNE) in isTruncateOf()
5800 if (N0.getOpcode() == ISD::ZERO_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) in visitZERO_EXTEND()
5801 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, in visitZERO_EXTEND()
5820 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, Op); in visitZERO_EXTEND()
5822 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Op); in visitZERO_EXTEND()
5830 if (N0.getOpcode() == ISD::TRUNCATE) { in visitZERO_EXTEND()
5844 if (N0.getOpcode() == ISD::TRUNCATE && in visitZERO_EXTEND()
5845 (!LegalOperations || TLI.isOperationLegal(ISD::AND, VT))) { in visitZERO_EXTEND()
5862 Op = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Op); in visitZERO_EXTEND()
5865 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Op); in visitZERO_EXTEND()
5874 if (N0.getOpcode() == ISD::AND && in visitZERO_EXTEND()
5875 N0.getOperand(0).getOpcode() == ISD::TRUNCATE && in visitZERO_EXTEND()
5876 N0.getOperand(1).getOpcode() == ISD::Constant && in visitZERO_EXTEND()
5882 X = DAG.getNode(ISD::ANY_EXTEND, SDLoc(X), VT, X); in visitZERO_EXTEND()
5884 X = DAG.getNode(ISD::TRUNCATE, SDLoc(X), VT, X); in visitZERO_EXTEND()
5888 return DAG.getNode(ISD::AND, SDLoc(N), VT, in visitZERO_EXTEND()
5895 if (ISD::isNON_EXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) && in visitZERO_EXTEND()
5898 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, N0.getValueType()))) { in visitZERO_EXTEND()
5902 DoXform = ExtendUsesToFormExtLoad(N, N0, ISD::ZERO_EXTEND, SetCCs, TLI); in visitZERO_EXTEND()
5907 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N), VT, in visitZERO_EXTEND()
5912 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), in visitZERO_EXTEND()
5917 ISD::ZERO_EXTEND); in visitZERO_EXTEND()
5929 if ((N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR || in visitZERO_EXTEND()
5930 N0.getOpcode() == ISD::XOR) && in visitZERO_EXTEND()
5932 N0.getOperand(1).getOpcode() == ISD::Constant && in visitZERO_EXTEND()
5933 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, N0.getValueType()) && in visitZERO_EXTEND()
5936 if (LN0->getExtensionType() != ISD::SEXTLOAD && LN0->isUnindexed()) { in visitZERO_EXTEND()
5940 DoXform = ExtendUsesToFormExtLoad(N, N0.getOperand(0), ISD::ZERO_EXTEND, in visitZERO_EXTEND()
5943 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(LN0), VT, in visitZERO_EXTEND()
5951 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, in visitZERO_EXTEND()
5957 ISD::ZERO_EXTEND); in visitZERO_EXTEND()
5965 if ((ISD::isZEXTLoad(N0.getNode()) || ISD::isEXTLoad(N0.getNode())) && in visitZERO_EXTEND()
5966 ISD::isUNINDEXEDLoad(N0.getNode()) && N0.hasOneUse()) { in visitZERO_EXTEND()
5970 TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT)) { in visitZERO_EXTEND()
5971 SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N), VT, in visitZERO_EXTEND()
5977 DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), in visitZERO_EXTEND()
5984 if (N0.getOpcode() == ISD::SETCC) { in visitZERO_EXTEND()
6002 return DAG.getNode(ISD::AND, SDLoc(N), VT, in visitZERO_EXTEND()
6006 DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), VT, in visitZERO_EXTEND()
6022 return DAG.getNode(ISD::AND, SDLoc(N), VT, in visitZERO_EXTEND()
6024 DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), VT, OneOps)); in visitZERO_EXTEND()
6036 if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL) && in visitZERO_EXTEND()
6038 N0.getOperand(0).getOpcode() == ISD::ZERO_EXTEND && in visitZERO_EXTEND()
6042 if (N0.getOpcode() == ISD::SHL) { in visitZERO_EXTEND()
6056 ShAmt = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, ShAmt); in visitZERO_EXTEND()
6059 DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)), in visitZERO_EXTEND()
6077 if (N0.getOpcode() == ISD::ANY_EXTEND || in visitANY_EXTEND()
6078 N0.getOpcode() == ISD::ZERO_EXTEND || in visitANY_EXTEND()
6079 N0.getOpcode() == ISD::SIGN_EXTEND) in visitANY_EXTEND()
6084 if (N0.getOpcode() == ISD::TRUNCATE) { in visitANY_EXTEND()
6098 if (N0.getOpcode() == ISD::TRUNCATE) { in visitANY_EXTEND()
6103 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, TruncOp); in visitANY_EXTEND()
6104 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, TruncOp); in visitANY_EXTEND()
6109 if (N0.getOpcode() == ISD::AND && in visitANY_EXTEND()
6110 N0.getOperand(0).getOpcode() == ISD::TRUNCATE && in visitANY_EXTEND()
6111 N0.getOperand(1).getOpcode() == ISD::Constant && in visitANY_EXTEND()
6116 X = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, X); in visitANY_EXTEND()
6118 X = DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, X); in visitANY_EXTEND()
6122 return DAG.getNode(ISD::AND, SDLoc(N), VT, in visitANY_EXTEND()
6130 if (ISD::isNON_EXTLoad(N0.getNode()) && !VT.isVector() && in visitANY_EXTEND()
6131 ISD::isUNINDEXEDLoad(N0.getNode()) && in visitANY_EXTEND()
6132 TLI.isLoadExtLegal(ISD::EXTLOAD, VT, N0.getValueType())) { in visitANY_EXTEND()
6136 DoXform = ExtendUsesToFormExtLoad(N, N0, ISD::ANY_EXTEND, SetCCs, TLI); in visitANY_EXTEND()
6139 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT, in visitANY_EXTEND()
6144 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), in visitANY_EXTEND()
6148 ISD::ANY_EXTEND); in visitANY_EXTEND()
6156 if (N0.getOpcode() == ISD::LOAD && in visitANY_EXTEND()
6157 !ISD::isNON_EXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) && in visitANY_EXTEND()
6160 ISD::LoadExtType ExtType = LN0->getExtensionType(); in visitANY_EXTEND()
6168 DAG.getNode(ISD::TRUNCATE, SDLoc(N0), in visitANY_EXTEND()
6175 if (N0.getOpcode() == ISD::SETCC) { in visitANY_EXTEND()
6223 case ISD::Constant: { in GetDemandedBits()
6232 case ISD::OR: in GetDemandedBits()
6233 case ISD::XOR: in GetDemandedBits()
6240 case ISD::SRL: in GetDemandedBits()
6253 return DAG.getNode(ISD::SRL, SDLoc(V), V.getValueType(), in GetDemandedBits()
6268 ISD::LoadExtType ExtType = ISD::NON_EXTLOAD; in ReduceLoadWidth()
6279 if (Opc == ISD::SIGN_EXTEND_INREG) { in ReduceLoadWidth()
6280 ExtType = ISD::SEXTLOAD; in ReduceLoadWidth()
6282 } else if (Opc == ISD::SRL) { in ReduceLoadWidth()
6284 ExtType = ISD::ZEXTLOAD; in ReduceLoadWidth()
6302 if (N0.getOpcode() == ISD::SRL && N0.hasOneUse()) { in ReduceLoadWidth()
6319 if (cast<LoadSDNode>(N0)->getExtensionType() == ISD::SEXTLOAD) in ReduceLoadWidth()
6333 if (ShAmt == 0 && N0.getOpcode() == ISD::SHL && N0.hasOneUse() && in ReduceLoadWidth()
6366 if (LN0->getExtensionType() != ISD::NON_EXTLOAD && in ReduceLoadWidth()
6389 SDValue NewPtr = DAG.getNode(ISD::ADD, SDLoc(LN0), in ReduceLoadWidth()
6395 if (ExtType == ISD::NON_EXTLOAD) in ReduceLoadWidth()
6423 Result = DAG.getNode(ISD::SHL, SDLoc(N0), VT, in ReduceLoadWidth()
6440 if (isa<ConstantSDNode>(N0) || N0.getOpcode() == ISD::UNDEF) in visitSIGN_EXTEND_INREG()
6441 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, N0, N1); in visitSIGN_EXTEND_INREG()
6448 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG && in visitSIGN_EXTEND_INREG()
6450 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, in visitSIGN_EXTEND_INREG()
6456 if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) { in visitSIGN_EXTEND_INREG()
6459 (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT))) in visitSIGN_EXTEND_INREG()
6460 return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00, N1); in visitSIGN_EXTEND_INREG()
6481 if (N0.getOpcode() == ISD::SRL) { in visitSIGN_EXTEND_INREG()
6488 return DAG.getNode(ISD::SRA, SDLoc(N), VT, in visitSIGN_EXTEND_INREG()
6494 if (ISD::isEXTLoad(N0.getNode()) && in visitSIGN_EXTEND_INREG()
6495 ISD::isUNINDEXEDLoad(N0.getNode()) && in visitSIGN_EXTEND_INREG()
6498 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, EVT))) { in visitSIGN_EXTEND_INREG()
6500 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT, in visitSIGN_EXTEND_INREG()
6510 if (ISD::isZEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) && in visitSIGN_EXTEND_INREG()
6514 TLI.isLoadExtLegal(ISD::SEXTLOAD, VT, EVT))) { in visitSIGN_EXTEND_INREG()
6516 SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(N), VT, in visitSIGN_EXTEND_INREG()
6526 if (EVTBits <= 16 && N0.getOpcode() == ISD::OR) { in visitSIGN_EXTEND_INREG()
6530 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT, in visitSIGN_EXTEND_INREG()
6536 if (ISD::isBuildVectorOfConstantSDNodes(N0.getNode())) { in visitSIGN_EXTEND_INREG()
6543 if (Op->getOpcode() == ISD::UNDEF) { in visitSIGN_EXTEND_INREG()
6554 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), VT, Elts); in visitSIGN_EXTEND_INREG()
6570 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0); in visitTRUNCATE()
6572 if (N0.getOpcode() == ISD::TRUNCATE) in visitTRUNCATE()
6573 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0.getOperand(0)); in visitTRUNCATE()
6575 if (N0.getOpcode() == ISD::ZERO_EXTEND || in visitTRUNCATE()
6576 N0.getOpcode() == ISD::SIGN_EXTEND || in visitTRUNCATE()
6577 N0.getOpcode() == ISD::ANY_EXTEND) { in visitTRUNCATE()
6584 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, N0.getOperand(0)); in visitTRUNCATE()
6600 if (N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in visitTRUNCATE()
6619 SDValue V = DAG.getNode(ISD::BITCAST, SDLoc(N), in visitTRUNCATE()
6622 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, in visitTRUNCATE()
6629 if (N0.getOpcode() == ISD::SELECT) { in visitTRUNCATE()
6631 if ((!LegalOperations || TLI.isOperationLegal(ISD::SELECT, SrcVT)) && in visitTRUNCATE()
6635 SDValue TruncOp0 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(1)); in visitTRUNCATE()
6636 SDValue TruncOp1 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(2)); in visitTRUNCATE()
6637 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, Cond, TruncOp0, TruncOp1); in visitTRUNCATE()
6646 N0.getOpcode() == ISD::BITCAST && N0.hasOneUse() && in visitTRUNCATE()
6647 N0.getOperand(0).getOpcode() == ISD::BUILD_VECTOR && in visitTRUNCATE()
6668 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), VT, Opnds); in visitTRUNCATE()
6682 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Shorter); in visitTRUNCATE()
6692 if (N0.hasOneUse() && ISD::isUNINDEXEDLoad(N0.getNode())) { in visitTRUNCATE()
6707 if (N0.getOpcode() == ISD::CONCAT_VECTORS && !LegalTypes) { in visitTRUNCATE()
6715 if (X.getOpcode() != ISD::UNDEF) { in visitTRUNCATE()
6739 SDValue NV = DAG.getNode(ISD::TRUNCATE, SDLoc(V), VTs[i], V); in visitTRUNCATE()
6743 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Opnds); in visitTRUNCATE()
6757 if (Elt.getOpcode() != ISD::MERGE_VALUES) in getBuildPairElt()
6765 assert(N->getOpcode() == ISD::BUILD_PAIR); in CombineConsecutiveLoads()
6769 if (!LD1 || !LD2 || !ISD::isNON_EXTLoad(LD1) || !LD1->hasOneUse() || in CombineConsecutiveLoads()
6774 if (ISD::isNON_EXTLoad(LD2) && in CombineConsecutiveLoads()
6786 (!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT))) in CombineConsecutiveLoads()
6804 N0.getOpcode() == ISD::BUILD_VECTOR && N0.getNode()->hasOneUse() && in visitBITCAST()
6822 TLI.isOperationLegal(ISD::ConstantFP, VT)) || in visitBITCAST()
6824 TLI.isOperationLegal(ISD::Constant, VT))) in visitBITCAST()
6825 return DAG.getNode(ISD::BITCAST, SDLoc(N), VT, N0); in visitBITCAST()
6829 if (N0.getOpcode() == ISD::BITCAST) in visitBITCAST()
6830 return DAG.getNode(ISD::BITCAST, SDLoc(N), VT, in visitBITCAST()
6835 if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() && in visitBITCAST()
6841 (!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT)) && in visitBITCAST()
6862 if (((N0.getOpcode() == ISD::FNEG && !TLI.isFNegFree(N0.getValueType())) || in visitBITCAST()
6863 (N0.getOpcode() == ISD::FABS && !TLI.isFAbsFree(N0.getValueType()))) && in visitBITCAST()
6866 SDValue NewConv = DAG.getNode(ISD::BITCAST, SDLoc(N0), VT, in visitBITCAST()
6871 if (N0.getOpcode() == ISD::FNEG) in visitBITCAST()
6872 return DAG.getNode(ISD::XOR, SDLoc(N), VT, in visitBITCAST()
6874 assert(N0.getOpcode() == ISD::FABS); in visitBITCAST()
6875 return DAG.getNode(ISD::AND, SDLoc(N), VT, in visitBITCAST()
6883 if (N0.getOpcode() == ISD::FCOPYSIGN && N0.getNode()->hasOneUse() && in visitBITCAST()
6889 SDValue X = DAG.getNode(ISD::BITCAST, SDLoc(N0), in visitBITCAST()
6896 X = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, X); in visitBITCAST()
6901 X = DAG.getNode(ISD::SRL, SDLoc(X), in visitBITCAST()
6905 X = DAG.getNode(ISD::TRUNCATE, SDLoc(X), VT, X); in visitBITCAST()
6910 X = DAG.getNode(ISD::AND, SDLoc(X), VT, in visitBITCAST()
6914 SDValue Cst = DAG.getNode(ISD::BITCAST, SDLoc(N0), in visitBITCAST()
6916 Cst = DAG.getNode(ISD::AND, SDLoc(Cst), VT, in visitBITCAST()
6920 return DAG.getNode(ISD::OR, SDLoc(N), VT, X, Cst); in visitBITCAST()
6925 if (N0.getOpcode() == ISD::BUILD_PAIR) { in visitBITCAST()
6959 if (BV->getOpcode() == ISD::SCALAR_TO_VECTOR) in ConstantFoldBITCASTofBUILD_VECTOR()
6960 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(BV), VT, in ConstantFoldBITCASTofBUILD_VECTOR()
6961 DAG.getNode(ISD::BITCAST, SDLoc(BV), in ConstantFoldBITCASTofBUILD_VECTOR()
6970 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(BV), SrcEltVT, Op); in ConstantFoldBITCASTofBUILD_VECTOR()
6971 Ops.push_back(DAG.getNode(ISD::BITCAST, SDLoc(BV), in ConstantFoldBITCASTofBUILD_VECTOR()
6975 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(BV), VT, Ops); in ConstantFoldBITCASTofBUILD_VECTOR()
7015 if (Op.getOpcode() == ISD::UNDEF) continue; in ConstantFoldBITCASTofBUILD_VECTOR()
7029 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(BV), VT, Ops); in ConstantFoldBITCASTofBUILD_VECTOR()
7040 if (BV->getOperand(i).getOpcode() == ISD::UNDEF) { in ConstantFoldBITCASTofBUILD_VECTOR()
7059 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(BV), VT, Ops); in ConstantFoldBITCASTofBUILD_VECTOR()
7073 if (N0.getOpcode() == ISD::FMUL && in performFaddFmulCombines()
7081 if (N1.getOpcode() == ISD::FMUL && in performFaddFmulCombines()
7090 if (N0.getOpcode() == ISD::FMA && in performFaddFmulCombines()
7091 N0.getOperand(2).getOpcode() == ISD::FMUL) { in performFaddFmulCombines()
7101 if (N1->getOpcode() == ISD::FMA && in performFaddFmulCombines()
7102 N1.getOperand(2).getOpcode() == ISD::FMUL) { in performFaddFmulCombines()
7127 if (N0.getOpcode() == ISD::FMUL && in performFsubFmulCombines()
7131 DAG.getNode(ISD::FNEG, SL, VT, N1)); in performFsubFmulCombines()
7136 if (N1.getOpcode() == ISD::FMUL && in performFsubFmulCombines()
7139 DAG.getNode(ISD::FNEG, SL, VT, in performFsubFmulCombines()
7144 if (N0.getOpcode() == ISD::FNEG && in performFsubFmulCombines()
7145 N0.getOperand(0).getOpcode() == ISD::FMUL && in performFsubFmulCombines()
7150 DAG.getNode(ISD::FNEG, SL, VT, N00), N01, in performFsubFmulCombines()
7151 DAG.getNode(ISD::FNEG, SL, VT, N1)); in performFsubFmulCombines()
7159 N0.getOperand(2).getOpcode() == ISD::FMUL) { in performFsubFmulCombines()
7165 DAG.getNode(ISD::FNEG, SDLoc(N), VT, in performFsubFmulCombines()
7172 N1.getOperand(2).getOpcode() == ISD::FMUL) { in performFsubFmulCombines()
7176 DAG.getNode(ISD::FNEG, SDLoc(N), VT, in performFsubFmulCombines()
7180 DAG.getNode(ISD::FNEG, SDLoc(N), VT, in performFsubFmulCombines()
7204 return DAG.getNode(ISD::FADD, SDLoc(N), VT, N0, N1); in visitFADD()
7208 return DAG.getNode(ISD::FADD, SDLoc(N), VT, N1, N0); in visitFADD()
7211 if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) && in visitFADD()
7213 return DAG.getNode(ISD::FSUB, SDLoc(N), VT, N0, in visitFADD()
7217 if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) && in visitFADD()
7219 return DAG.getNode(ISD::FSUB, SDLoc(N), VT, N1, in visitFADD()
7233 if (N1CFP && N0.getOpcode() == ISD::FADD && N0.getNode()->hasOneUse() && in visitFADD()
7235 return DAG.getNode(ISD::FADD, SDLoc(N), VT, N0.getOperand(0), in visitFADD()
7236 DAG.getNode(ISD::FADD, SDLoc(N), VT, in visitFADD()
7240 if (AllowNewConst && N0.getOpcode() == ISD::FNEG && N0.getOperand(0) == N1) in visitFADD()
7244 if (AllowNewConst && N1.getOpcode() == ISD::FNEG && N1.getOperand(0) == N0) in visitFADD()
7250 if (TLI.isOperationLegalOrCustom(ISD::FMUL, VT) && !N0CFP && !N1CFP) { in visitFADD()
7251 if (N0.getOpcode() == ISD::FMUL) { in visitFADD()
7257 SDValue NewCFP = DAG.getNode(ISD::FADD, SDLoc(N), VT, in visitFADD()
7260 return DAG.getNode(ISD::FMUL, SDLoc(N), VT, N1, NewCFP); in visitFADD()
7264 if (CFP01 && !CFP00 && N1.getOpcode() == ISD::FADD && in visitFADD()
7267 SDValue NewCFP = DAG.getNode(ISD::FADD, SDLoc(N), VT, in visitFADD()
7270 return DAG.getNode(ISD::FMUL, SDLoc(N), VT, in visitFADD()
7275 if (N1.getOpcode() == ISD::FMUL) { in visitFADD()
7281 SDValue NewCFP = DAG.getNode(ISD::FADD, SDLoc(N), VT, in visitFADD()
7284 return DAG.getNode(ISD::FMUL, SDLoc(N), VT, N0, NewCFP); in visitFADD()
7288 if (CFP11 && !CFP10 && N0.getOpcode() == ISD::FADD && in visitFADD()
7291 SDValue NewCFP = DAG.getNode(ISD::FADD, SDLoc(N), VT, in visitFADD()
7294 return DAG.getNode(ISD::FMUL, SDLoc(N), VT, N1.getOperand(0), NewCFP); in visitFADD()
7298 if (N0.getOpcode() == ISD::FADD && AllowNewConst) { in visitFADD()
7303 return DAG.getNode(ISD::FMUL, SDLoc(N), VT, in visitFADD()
7307 if (N1.getOpcode() == ISD::FADD && AllowNewConst) { in visitFADD()
7312 return DAG.getNode(ISD::FMUL, SDLoc(N), VT, in visitFADD()
7318 N0.getOpcode() == ISD::FADD && N1.getOpcode() == ISD::FADD && in visitFADD()
7322 return DAG.getNode(ISD::FMUL, SDLoc(N), VT, in visitFADD()
7327 if (LegalOperations && TLI.isOperationLegal(ISD::FMAD, VT)) { in visitFADD()
7330 if (SDValue Fused = performFaddFmulCombines(ISD::FMAD, true, N, TLI, DAG)) in visitFADD()
7337 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT))) { in visitFADD()
7339 if (!TLI.isOperationLegal(ISD::FMAD, VT)) { in visitFADD()
7342 = performFaddFmulCombines(ISD::FMA, in visitFADD()
7354 if (N0.getOpcode() == ISD::FP_EXTEND) { in visitFADD()
7356 if (N00.getOpcode() == ISD::FMUL) in visitFADD()
7357 return DAG.getNode(ISD::FMA, SDLoc(N), VT, in visitFADD()
7358 DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, in visitFADD()
7360 DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, in visitFADD()
7366 if (N1.getOpcode() == ISD::FP_EXTEND) { in visitFADD()
7368 if (N10.getOpcode() == ISD::FMUL) in visitFADD()
7369 return DAG.getNode(ISD::FMA, SDLoc(N), VT, in visitFADD()
7370 DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, in visitFADD()
7372 DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, in visitFADD()
7397 return DAG.getNode(ISD::FSUB, SDLoc(N), VT, N0, N1); in visitFSUB()
7401 return DAG.getNode(ISD::FADD, dl, VT, N0, in visitFSUB()
7414 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT)) in visitFSUB()
7415 return DAG.getNode(ISD::FNEG, dl, VT, N1); in visitFSUB()
7424 if (N1.getOpcode() == ISD::FADD) { in visitFSUB()
7436 if (LegalOperations && TLI.isOperationLegal(ISD::FMAD, VT)) { in visitFSUB()
7439 if (SDValue Fused = performFsubFmulCombines(ISD::FMAD, true, N, TLI, DAG)) in visitFSUB()
7446 (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT))) { in visitFSUB()
7448 if (!TLI.isOperationLegal(ISD::FMAD, VT)) { in visitFSUB()
7452 = performFsubFmulCombines(ISD::FMA, in visitFSUB()
7464 if (N0.getOpcode() == ISD::FP_EXTEND) { in visitFSUB()
7466 if (N00.getOpcode() == ISD::FMUL) in visitFSUB()
7467 return DAG.getNode(ISD::FMA, SDLoc(N), VT, in visitFSUB()
7468 DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, in visitFSUB()
7470 DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, in visitFSUB()
7472 DAG.getNode(ISD::FNEG, SDLoc(N), VT, N1)); in visitFSUB()
7478 if (N1.getOpcode() == ISD::FP_EXTEND) { in visitFSUB()
7480 if (N10.getOpcode() == ISD::FMUL) in visitFSUB()
7481 return DAG.getNode(ISD::FMA, SDLoc(N), VT, in visitFSUB()
7482 DAG.getNode(ISD::FNEG, SDLoc(N), VT, in visitFSUB()
7483 DAG.getNode(ISD::FP_EXTEND, SDLoc(N), in visitFSUB()
7485 DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, in visitFSUB()
7492 if (N0.getOpcode() == ISD::FP_EXTEND) { in visitFSUB()
7494 if (N00.getOpcode() == ISD::FNEG) { in visitFSUB()
7496 if (N000.getOpcode() == ISD::FMUL) { in visitFSUB()
7497 return DAG.getNode(ISD::FMA, dl, VT, in visitFSUB()
7498 DAG.getNode(ISD::FNEG, dl, VT, in visitFSUB()
7499 DAG.getNode(ISD::FP_EXTEND, SDLoc(N), in visitFSUB()
7501 DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, in visitFSUB()
7503 DAG.getNode(ISD::FNEG, dl, VT, N1)); in visitFSUB()
7510 if (N0.getOpcode() == ISD::FNEG) { in visitFSUB()
7512 if (N00.getOpcode() == ISD::FP_EXTEND) { in visitFSUB()
7514 if (N000.getOpcode() == ISD::FMUL) { in visitFSUB()
7515 return DAG.getNode(ISD::FMA, dl, VT, in visitFSUB()
7516 DAG.getNode(ISD::FNEG, dl, VT, in visitFSUB()
7517 DAG.getNode(ISD::FP_EXTEND, SDLoc(N), in visitFSUB()
7519 DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, in visitFSUB()
7521 DAG.getNode(ISD::FNEG, dl, VT, N1)); in visitFSUB()
7548 return DAG.getNode(ISD::FMUL, SDLoc(N), VT, N0, N1); in visitFMUL()
7553 return DAG.getNode(ISD::FMUL, SDLoc(N), VT, N1, N0); in visitFMUL()
7565 if (N0.getOpcode() == ISD::FMUL) { in visitFMUL()
7583 SDValue MulConsts = DAG.getNode(ISD::FMUL, SL, VT, N01, N1); in visitFMUL()
7584 return DAG.getNode(ISD::FMUL, SL, VT, N00, MulConsts); in visitFMUL()
7593 if (N0.getOpcode() == ISD::FADD && N0.getOperand(0) == N0.getOperand(1)) { in visitFMUL()
7596 SDValue MulConsts = DAG.getNode(ISD::FMUL, SL, VT, Two, N1); in visitFMUL()
7597 return DAG.getNode(ISD::FMUL, SDLoc(N), VT, N0.getOperand(0), MulConsts); in visitFMUL()
7603 return DAG.getNode(ISD::FADD, SDLoc(N), VT, N0, N0); in visitFMUL()
7607 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT)) in visitFMUL()
7608 return DAG.getNode(ISD::FNEG, SDLoc(N), VT, N0); in visitFMUL()
7616 return DAG.getNode(ISD::FMUL, SDLoc(N), VT, in visitFMUL()
7639 return DAG.getNode(ISD::FMA, dl, VT, N0, N1, N2); in visitFMA()
7649 return DAG.getNode(ISD::FADD, SDLoc(N), VT, N1, N2); in visitFMA()
7651 return DAG.getNode(ISD::FADD, SDLoc(N), VT, N0, N2); in visitFMA()
7655 return DAG.getNode(ISD::FMA, SDLoc(N), VT, N1, N0, N2); in visitFMA()
7659 N2.getOpcode() == ISD::FMUL && in visitFMA()
7661 N2.getOperand(1).getOpcode() == ISD::ConstantFP) { in visitFMA()
7662 return DAG.getNode(ISD::FMUL, dl, VT, N0, in visitFMA()
7663 DAG.getNode(ISD::FADD, dl, VT, N1, N2.getOperand(1))); in visitFMA()
7669 N0.getOpcode() == ISD::FMUL && N1CFP && in visitFMA()
7670 N0.getOperand(1).getOpcode() == ISD::ConstantFP) { in visitFMA()
7671 return DAG.getNode(ISD::FMA, dl, VT, in visitFMA()
7673 DAG.getNode(ISD::FMUL, dl, VT, N1, N0.getOperand(1)), in visitFMA()
7681 return DAG.getNode(ISD::FADD, dl, VT, N0, N2); in visitFMA()
7684 (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))) { in visitFMA()
7685 SDValue RHSNeg = DAG.getNode(ISD::FNEG, dl, VT, N0); in visitFMA()
7687 return DAG.getNode(ISD::FADD, dl, VT, N2, RHSNeg); in visitFMA()
7693 return DAG.getNode(ISD::FMUL, dl, VT, N0, in visitFMA()
7694 DAG.getNode(ISD::FADD, dl, VT, in visitFMA()
7699 N2.getOpcode() == ISD::FNEG && N2.getOperand(0) == N0) in visitFMA()
7700 return DAG.getNode(ISD::FMUL, dl, VT, N0, in visitFMA()
7701 DAG.getNode(ISD::FADD, dl, VT, in visitFMA()
7724 return DAG.getNode(ISD::FDIV, SDLoc(N), VT, N0, N1); in visitFDIV()
7740 TLI.isOperationLegal(llvm::ISD::ConstantFP, VT) || in visitFDIV()
7742 return DAG.getNode(ISD::FMUL, SDLoc(N), VT, N0, in visitFDIV()
7748 if (N1.getOpcode() == ISD::FSQRT) { in visitFDIV()
7750 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); in visitFDIV()
7752 } else if (N1.getOpcode() == ISD::FP_EXTEND && in visitFDIV()
7753 N1.getOperand(0).getOpcode() == ISD::FSQRT) { in visitFDIV()
7755 RV = DAG.getNode(ISD::FP_EXTEND, SDLoc(N1), VT, RV); in visitFDIV()
7757 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); in visitFDIV()
7759 } else if (N1.getOpcode() == ISD::FP_ROUND && in visitFDIV()
7760 N1.getOperand(0).getOpcode() == ISD::FSQRT) { in visitFDIV()
7762 RV = DAG.getNode(ISD::FP_ROUND, SDLoc(N1), VT, RV, N1.getOperand(1)); in visitFDIV()
7764 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); in visitFDIV()
7766 } else if (N1.getOpcode() == ISD::FMUL) { in visitFDIV()
7771 if (N1.getOperand(0).getOpcode() == ISD::FSQRT) { in visitFDIV()
7774 } else if (N1.getOperand(1).getOpcode() == ISD::FSQRT) { in visitFDIV()
7782 RV = DAG.getNode(ISD::FDIV, SDLoc(N1), VT, RV, OtherOp); in visitFDIV()
7784 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); in visitFDIV()
7792 return DAG.getNode(ISD::FMUL, DL, VT, N0, RV); in visitFDIV()
7802 return DAG.getNode(ISD::FDIV, SDLoc(N), VT, in visitFDIV()
7826 if (User->getOpcode() == ISD::FDIV && User->getOperand(1) == N1) in visitFDIV()
7832 SDValue Reciprocal = DAG.getNode(ISD::FDIV, SDLoc(N), VT, FPOne, N1); in visitFDIV()
7837 SDValue NewNode = DAG.getNode(ISD::FMUL, SDLoc(*I), VT, in visitFDIV()
7858 return DAG.getNode(ISD::FREM, SDLoc(N), VT, N0, N1); in visitFREM()
7869 RV = DAG.getNode(ISD::FMUL, SDLoc(N), VT, N->getOperand(0), RV); in visitFSQRT()
7877 N->getOperand(0), Zero, ISD::SETEQ); in visitFSQRT()
7881 RV = DAG.getNode(VT.isVector() ? ISD::VSELECT : ISD::SELECT, in visitFSQRT()
7897 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, N0, N1); in visitFCOPYSIGN()
7904 if (!LegalOperations || TLI.isOperationLegal(ISD::FABS, VT)) in visitFCOPYSIGN()
7905 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0); in visitFCOPYSIGN()
7907 if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT)) in visitFCOPYSIGN()
7908 return DAG.getNode(ISD::FNEG, SDLoc(N), VT, in visitFCOPYSIGN()
7909 DAG.getNode(ISD::FABS, SDLoc(N0), VT, N0)); in visitFCOPYSIGN()
7916 if (N0.getOpcode() == ISD::FABS || N0.getOpcode() == ISD::FNEG || in visitFCOPYSIGN()
7917 N0.getOpcode() == ISD::FCOPYSIGN) in visitFCOPYSIGN()
7918 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, in visitFCOPYSIGN()
7922 if (N1.getOpcode() == ISD::FABS) in visitFCOPYSIGN()
7923 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0); in visitFCOPYSIGN()
7926 if (N1.getOpcode() == ISD::FCOPYSIGN) in visitFCOPYSIGN()
7927 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, in visitFCOPYSIGN()
7932 if (N1.getOpcode() == ISD::FP_EXTEND || N1.getOpcode() == ISD::FP_ROUND) in visitFCOPYSIGN()
7933 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, in visitFCOPYSIGN()
7948 TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT))) in visitSINT_TO_FP()
7949 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, N0); in visitSINT_TO_FP()
7953 if (!TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, OpVT) && in visitSINT_TO_FP()
7954 TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, OpVT)) { in visitSINT_TO_FP()
7957 return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), VT, N0); in visitSINT_TO_FP()
7961 if (TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT) || !LegalOperations) { in visitSINT_TO_FP()
7963 if (N0.getOpcode() == ISD::SETCC && N0.getValueType() == MVT::i1 && in visitSINT_TO_FP()
7966 TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT))) { in visitSINT_TO_FP()
7971 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), VT, Ops); in visitSINT_TO_FP()
7976 if (N0.getOpcode() == ISD::ZERO_EXTEND && in visitSINT_TO_FP()
7977 N0.getOperand(0).getOpcode() == ISD::SETCC &&!VT.isVector() && in visitSINT_TO_FP()
7979 TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT))) { in visitSINT_TO_FP()
7984 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), VT, Ops); in visitSINT_TO_FP()
8000 TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT))) in visitUINT_TO_FP()
8001 return DAG.getNode(ISD::UINT_TO_FP, SDLoc(N), VT, N0); in visitUINT_TO_FP()
8005 if (!TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, OpVT) && in visitUINT_TO_FP()
8006 TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, OpVT)) { in visitUINT_TO_FP()
8009 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, N0); in visitUINT_TO_FP()
8013 if (TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT) || !LegalOperations) { in visitUINT_TO_FP()
8016 if (N0.getOpcode() == ISD::SETCC && !VT.isVector() && in visitUINT_TO_FP()
8018 TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT))) { in visitUINT_TO_FP()
8023 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), VT, Ops); in visitUINT_TO_FP()
8035 if (N0.getOpcode() != ISD::UINT_TO_FP && N0.getOpcode() != ISD::SINT_TO_FP) in FoldIntToFPToInt()
8040 bool IsInputSigned = N0.getOpcode() == ISD::SINT_TO_FP; in FoldIntToFPToInt()
8041 bool IsOutputSigned = N->getOpcode() == ISD::FP_TO_SINT; in FoldIntToFPToInt()
8061 unsigned ExtOp = IsInputSigned && IsOutputSigned ? ISD::SIGN_EXTEND in FoldIntToFPToInt()
8062 : ISD::ZERO_EXTEND; in FoldIntToFPToInt()
8066 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Src); in FoldIntToFPToInt()
8069 return DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Src); in FoldIntToFPToInt()
8081 return DAG.getNode(ISD::FP_TO_SINT, SDLoc(N), VT, N0); in visitFP_TO_SINT()
8093 return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), VT, N0); in visitFP_TO_UINT()
8106 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), VT, N0, N1); in visitFP_ROUND()
8109 if (N0.getOpcode() == ISD::FP_EXTEND && VT == N0.getOperand(0).getValueType()) in visitFP_ROUND()
8113 if (N0.getOpcode() == ISD::FP_ROUND) { in visitFP_ROUND()
8122 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), VT, N0.getOperand(0), in visitFP_ROUND()
8127 if (N0.getOpcode() == ISD::FCOPYSIGN && N0.getNode()->hasOneUse()) { in visitFP_ROUND()
8128 SDValue Tmp = DAG.getNode(ISD::FP_ROUND, SDLoc(N0), VT, in visitFP_ROUND()
8131 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), VT, in visitFP_ROUND()
8147 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Round); in visitFP_ROUND_INREG()
8159 N->use_begin()->getOpcode() == ISD::FP_ROUND) in visitFP_EXTEND()
8164 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, N0); in visitFP_EXTEND()
8167 if (N0.getOpcode() == ISD::FP16_TO_FP && in visitFP_EXTEND()
8168 TLI.getOperationAction(ISD::FP16_TO_FP, VT) == TargetLowering::Legal) in visitFP_EXTEND()
8169 return DAG.getNode(ISD::FP16_TO_FP, SDLoc(N), VT, N0.getOperand(0)); in visitFP_EXTEND()
8173 if (N0.getOpcode() == ISD::FP_ROUND in visitFP_EXTEND()
8178 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), VT, in visitFP_EXTEND()
8180 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, In); in visitFP_EXTEND()
8184 if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() && in visitFP_EXTEND()
8185 TLI.isLoadExtLegal(ISD::EXTLOAD, VT, N0.getValueType())) { in visitFP_EXTEND()
8187 SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, SDLoc(N), VT, in visitFP_EXTEND()
8193 DAG.getNode(ISD::FP_ROUND, SDLoc(N0), in visitFP_EXTEND()
8208 return DAG.getNode(ISD::FCEIL, SDLoc(N), VT, N0); in visitFCEIL()
8219 return DAG.getNode(ISD::FTRUNC, SDLoc(N), VT, N0); in visitFTRUNC()
8230 return DAG.getNode(ISD::FFLOOR, SDLoc(N), VT, N0); in visitFFLOOR()
8242 return DAG.getNode(ISD::FNEG, SDLoc(N), VT, N0); in visitFNEG()
8251 N0.getOpcode() == ISD::BITCAST && in visitFNEG()
8266 Int = DAG.getNode(ISD::XOR, SDLoc(N0), IntVT, Int, in visitFNEG()
8269 return DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Int); in visitFNEG()
8274 if (N0.getOpcode() == ISD::FMUL) { in visitFNEG()
8281 TLI.isOperationLegal(ISD::ConstantFP, N->getValueType(0)))) in visitFNEG()
8283 ISD::FMUL, SDLoc(N), VT, N0.getOperand(0), in visitFNEG()
8284 DAG.getNode(ISD::FNEG, SDLoc(N), VT, N0.getOperand(1))); in visitFNEG()
8306 return DAG.getNode(ISD::FMINNUM, SDLoc(N), VT, N1, N0); in visitFMINNUM()
8327 return DAG.getNode(ISD::FMAXNUM, SDLoc(N), VT, N1, N0); in visitFMAXNUM()
8339 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0); in visitFABS()
8342 if (N0.getOpcode() == ISD::FABS) in visitFABS()
8347 if (N0.getOpcode() == ISD::FNEG || N0.getOpcode() == ISD::FCOPYSIGN) in visitFABS()
8348 return DAG.getNode(ISD::FABS, SDLoc(N), VT, N0.getOperand(0)); in visitFABS()
8353 N0.getOpcode() == ISD::BITCAST && in visitFABS()
8368 Int = DAG.getNode(ISD::AND, SDLoc(N0), IntVT, Int, in visitFABS()
8371 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Int); in visitFABS()
8391 if (N1.getOpcode() == ISD::SETCC && in visitBRCOND()
8392 TLI.isOperationLegalOrCustom(ISD::BR_CC, in visitBRCOND()
8394 return DAG.getNode(ISD::BR_CC, SDLoc(N), MVT::Other, in visitBRCOND()
8399 if ((N1.hasOneUse() && N1.getOpcode() == ISD::SRL) || in visitBRCOND()
8400 ((N1.getOpcode() == ISD::TRUNCATE && N1.hasOneUse()) && in visitBRCOND()
8402 N1.getOperand(0).getOpcode() == ISD::SRL))) { in visitBRCOND()
8404 if (N1.getOpcode() == ISD::TRUNCATE) { in visitBRCOND()
8430 if (Op0.getOpcode() == ISD::AND && in visitBRCOND()
8431 Op1.getOpcode() == ISD::Constant) { in visitBRCOND()
8434 if (AndOp1.getOpcode() == ISD::Constant) { in visitBRCOND()
8443 ISD::SETNE); in visitBRCOND()
8445 SDValue NewBRCond = DAG.getNode(ISD::BRCOND, SDLoc(N), in visitBRCOND()
8469 if (N1.hasOneUse() && N1.getOpcode() == ISD::XOR) { in visitBRCOND()
8486 return DAG.getNode(ISD::BRCOND, SDLoc(N), in visitBRCOND()
8496 if (Op0.getOpcode() != ISD::SETCC && Op1.getOpcode() != ISD::SETCC) { in visitBRCOND()
8500 Op0.getOpcode() == ISD::XOR) { in visitBRCOND()
8511 Equal ? ISD::SETEQ : ISD::SETNE); in visitBRCOND()
8516 return DAG.getNode(ISD::BRCOND, SDLoc(N), in visitBRCOND()
8543 if (Simp.getNode() && Simp.getOpcode() == ISD::SETCC) in visitBR_CC()
8544 return DAG.getNode(ISD::BR_CC, SDLoc(N), MVT::Other, in visitBR_CC()
8570 if (N->getOpcode() == ISD::ADD) { in canFoldInAddressingMode()
8578 } else if (N->getOpcode() == ISD::SUB) { in canFoldInAddressingMode()
8608 if (!TLI.isIndexedLoadLegal(ISD::PRE_INC, VT) && in CombineToPreIndexedLoadStore()
8609 !TLI.isIndexedLoadLegal(ISD::PRE_DEC, VT)) in CombineToPreIndexedLoadStore()
8616 if (!TLI.isIndexedStoreLegal(ISD::PRE_INC, VT) && in CombineToPreIndexedLoadStore()
8617 !TLI.isIndexedStoreLegal(ISD::PRE_DEC, VT)) in CombineToPreIndexedLoadStore()
8627 if ((Ptr.getOpcode() != ISD::ADD && Ptr.getOpcode() != ISD::SUB) || in CombineToPreIndexedLoadStore()
8634 ISD::MemIndexedMode AM = ISD::UNINDEXED; in CombineToPreIndexedLoadStore()
8684 if (Use->getOpcode() != ISD::ADD && Use->getOpcode() != ISD::SUB) { in CombineToPreIndexedLoadStore()
8787 X0 = (OtherUses[i]->getOpcode() == ISD::SUB && OffsetIdx == 1) ? -1 : 1; in CombineToPreIndexedLoadStore()
8788 Y0 = (OtherUses[i]->getOpcode() == ISD::SUB && OffsetIdx == 0) ? -1 : 1; in CombineToPreIndexedLoadStore()
8789 X1 = (AM == ISD::PRE_DEC && !Swapped) ? -1 : 1; in CombineToPreIndexedLoadStore()
8790 Y1 = (AM == ISD::PRE_DEC && Swapped) ? -1 : 1; in CombineToPreIndexedLoadStore()
8792 unsigned Opcode = (Y0 * Y1 < 0) ? ISD::SUB : ISD::ADD; in CombineToPreIndexedLoadStore()
8832 if (!TLI.isIndexedLoadLegal(ISD::POST_INC, VT) && in CombineToPostIndexedLoadStore()
8833 !TLI.isIndexedLoadLegal(ISD::POST_DEC, VT)) in CombineToPostIndexedLoadStore()
8840 if (!TLI.isIndexedStoreLegal(ISD::POST_INC, VT) && in CombineToPostIndexedLoadStore()
8841 !TLI.isIndexedStoreLegal(ISD::POST_DEC, VT)) in CombineToPostIndexedLoadStore()
8854 (Op->getOpcode() != ISD::ADD && Op->getOpcode() != ISD::SUB)) in CombineToPostIndexedLoadStore()
8859 ISD::MemIndexedMode AM = ISD::UNINDEXED; in CombineToPostIndexedLoadStore()
8884 if (Use->getOpcode() == ISD::ADD || Use->getOpcode() == ISD::SUB){ in CombineToPostIndexedLoadStore()
8940 ISD::MemIndexedMode AM = LD->getAddressingMode(); in SplitIndexingFromLoad()
8941 assert(AM != ISD::UNINDEXED); in SplitIndexingFromLoad()
8948 assert((Inc.getOpcode() != ISD::TargetConstant || in SplitIndexingFromLoad()
8951 if (Inc.getOpcode() == ISD::TargetConstant) { in SplitIndexingFromLoad()
8958 (AM == ISD::PRE_INC || AM == ISD::POST_INC ? ISD::ADD : ISD::SUB); in SplitIndexingFromLoad()
9001 bool HasOTCInc = LD->getOperand(2).getOpcode() == ISD::TargetConstant && in visitLOAD()
9034 if (ISD::isNormalLoad(N) && !LD->isVolatile()) { in visitLOAD()
9035 if (ISD::isNON_TRUNCStore(Chain.getNode())) { in visitLOAD()
9076 if (LD->getExtensionType() == ISD::NON_EXTLOAD) { in visitLOAD()
9087 SDValue Token = DAG.getNode(ISD::TokenFactor, SDLoc(N), in visitLOAD()
9271 if (Origin->getOffset().getOpcode() != ISD::UNDEF) in isLegal()
9282 if (!TLI.isOperationLegal(ISD::LOAD, SliceType)) in isLegal()
9296 if (!TLI.isOperationLegal(ISD::ADD, PtrType)) in isLegal()
9302 !TLI.isOperationLegal(ISD::ZERO_EXTEND, TruncateType)) in isLegal()
9346 BaseAddr = DAG->getNode(ISD::ADD, SDLoc(Origin), ArithType, BaseAddr, in loadSlice()
9363 DAG->getNode(ISD::ZERO_EXTEND, SDLoc(LastInst), FinalType, LastInst); in loadSlice()
9375 if (Use->getOpcode() != ISD::BITCAST) in canMergeExpensiveCrossRegisterBankCopy()
9383 if (ArgRC == ResRC || !TLI.isOperationLegal(ISD::LOAD, ResVT)) in canMergeExpensiveCrossRegisterBankCopy()
9403 if (!TLI.isOperationLegal(ISD::LOAD, ResVT)) in canMergeExpensiveCrossRegisterBankCopy()
9562 if (LD->isVolatile() || !ISD::isNormalLoad(LD) || in SliceUpLoad()
9585 if (User->getOpcode() == ISD::SRL && User->hasOneUse() && in SliceUpLoad()
9593 if (User->getOpcode() != ISD::TRUNCATE) in SliceUpLoad()
9639 if (SliceInst.getNode()->getOpcode() != ISD::LOAD) in SliceUpLoad()
9641 assert(SliceInst->getOpcode() == ISD::LOAD && in SliceUpLoad()
9646 SDValue Chain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, in SliceUpLoad()
9660 if (V->getOpcode() != ISD::AND || in CheckForMaskedLoad()
9662 !ISD::isNormalLoad(V->getOperand(0).getNode())) in CheckForMaskedLoad()
9673 else if (Chain->getOpcode() != ISD::TokenFactor) in CheckForMaskedLoad()
9754 IVal = DAG.getNode(ISD::SRL, SDLoc(IVal), IVal.getValueType(), IVal, in ShrinkLoadReplaceStoreWithStore()
9769 Ptr = DAG.getNode(ISD::ADD, SDLoc(IVal), Ptr.getValueType(), in ShrinkLoadReplaceStoreWithStore()
9775 IVal = DAG.getNode(ISD::TRUNCATE, SDLoc(IVal), VT, IVal); in ShrinkLoadReplaceStoreWithStore()
9808 if (Opc == ISD::OR) { in ReduceLoadOpStoreWidth()
9824 if ((Opc != ISD::OR && Opc != ISD::XOR && Opc != ISD::AND) || in ReduceLoadOpStoreWidth()
9825 Value.getOperand(1).getOpcode() != ISD::Constant) in ReduceLoadOpStoreWidth()
9829 if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() && in ReduceLoadOpStoreWidth()
9841 if (Opc == ISD::AND) in ReduceLoadOpStoreWidth()
9869 if (Opc == ISD::AND) in ReduceLoadOpStoreWidth()
9882 SDValue NewPtr = DAG.getNode(ISD::ADD, SDLoc(LD), in ReduceLoadOpStoreWidth()
9918 if (ISD::isNormalStore(ST) && ISD::isNormalLoad(Value.getNode()) && in TransformFPLoadStorePair()
9932 if (!TLI.isOperationLegal(ISD::LOAD, IntVT) || in TransformFPLoadStorePair()
9933 !TLI.isOperationLegal(ISD::STORE, IntVT) || in TransformFPLoadStorePair()
9934 !TLI.isDesirableToTransformToIntegerOp(ISD::LOAD, VT) || in TransformFPLoadStorePair()
9935 !TLI.isDesirableToTransformToIntegerOp(ISD::STORE, VT)) in TransformFPLoadStorePair()
10004 if (Ptr->getOpcode() != ISD::ADD) in match()
10020 if (Ptr->getOperand(1)->getOpcode() == ISD::MUL) in match()
10028 if (IndexOffset->getOpcode() == ISD::SIGN_EXTEND) { in match()
10034 if (IndexOffset->getOpcode() != ISD::ADD) in match()
10045 if (Index->getOpcode() == ISD::SIGN_EXTEND) { in match()
10104 StoredVal = DAG.getNode(ISD::BUILD_VECTOR, DL, Ty, Ops); in MergeStoresOfConstantsOrVecElts()
10186 bool IsExtractVecEltSrc = (StoredVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT); in MergeConsecutiveStores()
10193 if (Chain->hasOneUse() && Chain->use_begin()->getOpcode() == ISD::STORE) in MergeConsecutiveStores()
10205 if (BasePtr.Base.getOpcode() == ISD::UNDEF) in MergeConsecutiveStores()
10395 if (StoredVal.getOpcode() != ISD::EXTRACT_VECTOR_ELT) in MergeConsecutiveStores()
10436 if (Ld->getExtensionType() != ISD::NON_EXTLOAD) in MergeConsecutiveStores()
10503 TLI.isLoadExtLegal(ISD::ZEXTLOAD, LegalizedStoredValueTy, StoreTy) && in MergeConsecutiveStores()
10504 TLI.isLoadExtLegal(ISD::SEXTLOAD, LegalizedStoredValueTy, StoreTy) && in MergeConsecutiveStores()
10505 TLI.isLoadExtLegal(ISD::EXTLOAD, LegalizedStoredValueTy, StoreTy)) in MergeConsecutiveStores()
10598 if (Value.getOpcode() == ISD::BITCAST && !ST->isTruncatingStore() && in visitSTORE()
10606 TLI.isOperationLegalOrCustom(ISD::STORE, SVT))) in visitSTORE()
10614 if (Value.getOpcode() == ISD::UNDEF && ST->isUnindexed()) in visitSTORE()
10623 if (Value.getOpcode() != ISD::TargetConstantFP) { in visitSTORE()
10634 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i32)) { in visitSTORE()
10644 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i64)) { in visitSTORE()
10652 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i32)) { in visitSTORE()
10670 Ptr = DAG.getNode(ISD::ADD, SDLoc(N), Ptr.getValueType(), Ptr, in visitSTORE()
10677 return DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, in visitSTORE()
10732 SDValue Token = DAG.getNode(ISD::TokenFactor, SDLoc(N), in visitSTORE()
10798 if ((Value.getOpcode() == ISD::FP_ROUND || Value.getOpcode() == ISD::TRUNCATE) in visitSTORE()
10818 } while (ST->getOpcode() != ISD::DELETED_NODE); in visitSTORE()
10834 if (InVal.getOpcode() == ISD::UNDEF) in visitINSERT_VECTOR_ELT()
10840 if (LegalOperations && !TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) in visitINSERT_VECTOR_ELT()
10855 if (InVec.getOpcode() == ISD::INSERT_VECTOR_ELT && InVec.hasOneUse() in visitINSERT_VECTOR_ELT()
10861 SDValue NewOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N), VT, in visitINSERT_VECTOR_ELT()
10864 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(InVec.getNode()), in visitINSERT_VECTOR_ELT()
10875 if (InVec.getOpcode() == ISD::BUILD_VECTOR && InVec.hasOneUse()) { in visitINSERT_VECTOR_ELT()
10878 } else if (InVec.getOpcode() == ISD::UNDEF) { in visitINSERT_VECTOR_ELT()
10892 DAG.getNode(ISD::ANY_EXTEND, dl, OpVT, InVal) : in visitINSERT_VECTOR_ELT()
10893 DAG.getNode(ISD::TRUNCATE, dl, OpVT, InVal); in visitINSERT_VECTOR_ELT()
10898 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); in visitINSERT_VECTOR_ELT()
10909 if (NewAlign > Align || !TLI.isOperationLegalOrCustom(ISD::LOAD, VecEltVT)) in ReplaceExtractVectorEltOfLoadWithNarrowedLoad()
10927 ISD::MUL, SDLoc(EVE), EltNo.getValueType(), EltNo, in ReplaceExtractVectorEltOfLoadWithNarrowedLoad()
10931 ISD::SUB, SDLoc(EVE), EltNo.getValueType(), in ReplaceExtractVectorEltOfLoadWithNarrowedLoad()
10935 NewPtr = DAG.getNode(ISD::ADD, SDLoc(EVE), PtrType, NewPtr, Offset); in ReplaceExtractVectorEltOfLoadWithNarrowedLoad()
10948 ISD::LoadExtType ExtType = TLI.isLoadExtLegal(ISD::ZEXTLOAD, ResultVT, in ReplaceExtractVectorEltOfLoadWithNarrowedLoad()
10950 ? ISD::ZEXTLOAD in ReplaceExtractVectorEltOfLoadWithNarrowedLoad()
10951 : ISD::EXTLOAD; in ReplaceExtractVectorEltOfLoadWithNarrowedLoad()
10964 Load = DAG.getNode(ISD::TRUNCATE, SDLoc(EVE), ResultVT, Load); in ReplaceExtractVectorEltOfLoadWithNarrowedLoad()
10966 Load = DAG.getNode(ISD::BITCAST, SDLoc(EVE), ResultVT, Load); in ReplaceExtractVectorEltOfLoadWithNarrowedLoad()
10988 if (InVec.getOpcode() == ISD::SCALAR_TO_VECTOR) { in visitEXTRACT_VECTOR_ELT()
11009 if (InVec.getOpcode() == ISD::VECTOR_SHUFFLE in visitEXTRACT_VECTOR_ELT()
11030 if (SVInVec.getOpcode() == ISD::BUILD_VECTOR) { in visitEXTRACT_VECTOR_ELT()
11045 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), NVT, in visitEXTRACT_VECTOR_ELT()
11059 if (InVec.getOpcode() == ISD::BITCAST) { in visitEXTRACT_VECTOR_ELT()
11075 ISD::isNormalLoad(InVec.getNode()) && in visitEXTRACT_VECTOR_ELT()
11096 if (ISD::isNormalLoad(InVec.getNode())) { in visitEXTRACT_VECTOR_ELT()
11098 } else if (InVec.getOpcode() == ISD::SCALAR_TO_VECTOR && in visitEXTRACT_VECTOR_ELT()
11100 ISD::isNormalLoad(InVec.getOperand(0).getNode())) { in visitEXTRACT_VECTOR_ELT()
11125 if (InVec.getOpcode() == ISD::BITCAST) { in visitEXTRACT_VECTOR_ELT()
11132 if (ISD::isNormalLoad(InVec.getNode())) { in visitEXTRACT_VECTOR_ELT()
11180 if (In.getOpcode() == ISD::UNDEF) continue; in reduceBuildVecExtToExtBuildVec()
11182 bool AnyExt = In.getOpcode() == ISD::ANY_EXTEND; in reduceBuildVecExtToExtBuildVec()
11183 bool ZeroExt = In.getOpcode() == ISD::ZERO_EXTEND; in reduceBuildVecExtToExtBuildVec()
11233 assert((Cast.getOpcode() == ISD::ANY_EXTEND || in reduceBuildVecExtToExtBuildVec()
11234 Cast.getOpcode() == ISD::ZERO_EXTEND || in reduceBuildVecExtToExtBuildVec()
11235 Cast.getOpcode() == ISD::UNDEF) && "Invalid cast opcode"); in reduceBuildVecExtToExtBuildVec()
11237 if (Cast.getOpcode() == ISD::UNDEF) in reduceBuildVecExtToExtBuildVec()
11256 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, dl, VecVT, Ops); in reduceBuildVecExtToExtBuildVec()
11261 return DAG.getNode(ISD::BITCAST, dl, VT, BV); in reduceBuildVecExtToExtBuildVec()
11271 unsigned Opcode = ISD::DELETED_NODE; in reduceBuildVecConvertToConvertBuildVec()
11278 if (Opc == ISD::UNDEF) in reduceBuildVecConvertToConvertBuildVec()
11282 if (Opcode == ISD::DELETED_NODE && in reduceBuildVecConvertToConvertBuildVec()
11283 (Opc == ISD::UINT_TO_FP || Opc == ISD::SINT_TO_FP)) { in reduceBuildVecConvertToConvertBuildVec()
11306 assert((Opcode == ISD::UINT_TO_FP || Opcode == ISD::SINT_TO_FP) in reduceBuildVecConvertToConvertBuildVec()
11324 if (In.getOpcode() == ISD::UNDEF) in reduceBuildVecConvertToConvertBuildVec()
11329 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, Opnds); in reduceBuildVecConvertToConvertBuildVec()
11341 if (ISD::allOperandsUndef(N)) in visitBUILD_VECTOR()
11359 if (LegalOperations && !TLI.isOperationLegal(ISD::VECTOR_SHUFFLE, VT)) in visitBUILD_VECTOR()
11367 if (Op.getOpcode() == ISD::UNDEF) continue; in visitBUILD_VECTOR()
11370 if (!VecIn2.getNode() && ((Op.getOpcode() == ISD::Constant && in visitBUILD_VECTOR()
11372 (Op.getOpcode() == ISD::ConstantFP && in visitBUILD_VECTOR()
11380 if (Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT || in visitBUILD_VECTOR()
11408 if (Opcode == ISD::UNDEF) { in visitBUILD_VECTOR()
11414 if (Opcode != ISD::EXTRACT_VECTOR_ELT) { in visitBUILD_VECTOR()
11416 (Opcode == ISD::Constant || Opcode == ISD::ConstantFP) && in visitBUILD_VECTOR()
11457 VecIn1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, VecIn1, in visitBUILD_VECTOR()
11461 VecIn1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, VecIn1, VecIn2); in visitBUILD_VECTOR()
11478 VecIn2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, VecIn1, in visitBUILD_VECTOR()
11480 VecIn1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, VecIn1, in visitBUILD_VECTOR()
11522 SDValue ScalarUndef = DAG.getNode(ISD::UNDEF, DL, SVT); in combineConcatVectorOfScalars()
11528 if (ISD::BITCAST == Op.getOpcode() && in combineConcatVectorOfScalars()
11531 else if (ISD::UNDEF == Op.getOpcode()) in combineConcatVectorOfScalars()
11552 ScalarUndef = DAG.getNode(ISD::UNDEF, DL, SVT); in combineConcatVectorOfScalars()
11557 if (Op.getOpcode() == ISD::UNDEF) in combineConcatVectorOfScalars()
11560 Op = DAG.getNode(ISD::BITCAST, DL, SVT, Op); in combineConcatVectorOfScalars()
11567 return DAG.getNode(ISD::BITCAST, DL, VT, in combineConcatVectorOfScalars()
11568 DAG.getNode(ISD::BUILD_VECTOR, DL, VecVT, Ops)); in combineConcatVectorOfScalars()
11583 if (ISD::allOperandsUndef(N)) in visitCONCAT_VECTORS()
11588 return Op.getOpcode() == ISD::UNDEF; in visitCONCAT_VECTORS()
11594 if (In->getOpcode() == ISD::BITCAST && in visitCONCAT_VECTORS()
11601 if (Scalar->getOpcode() == ISD::TRUNCATE && in visitCONCAT_VECTORS()
11617 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NVT, Scalar); in visitCONCAT_VECTORS()
11618 return DAG.getNode(ISD::BITCAST, dl, VT, Res); in visitCONCAT_VECTORS()
11627 return ISD::UNDEF == Op.getOpcode() || ISD::BUILD_VECTOR == Op.getOpcode(); in visitCONCAT_VECTORS()
11641 if (ISD::BUILD_VECTOR == Op.getOpcode()) { in visitCONCAT_VECTORS()
11653 if (ISD::UNDEF == Op.getOpcode()) in visitCONCAT_VECTORS()
11656 if (ISD::BUILD_VECTOR == Op.getOpcode()) { in visitCONCAT_VECTORS()
11663 DAG.getNode(ISD::TRUNCATE, SDLoc(N), MinVT, Op.getOperand(i))); in visitCONCAT_VECTORS()
11670 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), VT, Opnds); in visitCONCAT_VECTORS()
11687 if (Op.getOpcode() == ISD::UNDEF) in visitCONCAT_VECTORS()
11691 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR) in visitCONCAT_VECTORS()
11729 if (V->getOpcode() == ISD::CONCAT_VECTORS) { in visitEXTRACT_SUBVECTOR()
11746 if (V->getOpcode() == ISD::BITCAST) in visitEXTRACT_SUBVECTOR()
11749 if (V->getOpcode() == ISD::INSERT_SUBVECTOR) { in visitEXTRACT_SUBVECTOR()
11772 return DAG.getNode(ISD::BITCAST, dl, NVT, V->getOperand(1)); in visitEXTRACT_SUBVECTOR()
11773 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, in visitEXTRACT_SUBVECTOR()
11774 DAG.getNode(ISD::BITCAST, dl, in visitEXTRACT_SUBVECTOR()
11792 case ISD::CONCAT_VECTORS: { in simplifyShuffleOperandRecursively()
11814 V = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, NewOps); in simplifyShuffleOperandRecursively()
11818 case ISD::INSERT_SUBVECTOR: { in simplifyShuffleOperandRecursively()
11843 V = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in simplifyShuffleOperandRecursively()
11887 if (NumElemsPerConcat * 2 == NumElts && N1.getOpcode() == ISD::UNDEF && in partitionShuffleOfConcats()
11893 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, N0, N1); in partitionShuffleOfConcats()
11930 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops); in partitionShuffleOfConcats()
11943 if (N0.getOpcode() == ISD::UNDEF && N1.getOpcode() == ISD::UNDEF) in visitVECTOR_SHUFFLE()
11961 if (N0.getOpcode() == ISD::UNDEF) { in visitVECTOR_SHUFFLE()
11978 if (N1.getOpcode() == ISD::UNDEF) { in visitVECTOR_SHUFFLE()
12001 if (V->getOpcode() == ISD::BITCAST) { in visitVECTOR_SHUFFLE()
12008 if (V->getOpcode() == ISD::BUILD_VECTOR) { in visitVECTOR_SHUFFLE()
12014 if (V->getOperand(i).getOpcode() != ISD::UNDEF) { in visitVECTOR_SHUFFLE()
12035 SDValue NewBV = DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), in visitVECTOR_SHUFFLE()
12041 NewBV = DAG.getNode(ISD::BITCAST, SDLoc(N), VT, NewBV); in visitVECTOR_SHUFFLE()
12052 if (N0.getOpcode() == ISD::CONCAT_VECTORS && in visitVECTOR_SHUFFLE()
12054 (N1.getOpcode() == ISD::UNDEF || in visitVECTOR_SHUFFLE()
12055 (N1.getOpcode() == ISD::CONCAT_VECTORS && in visitVECTOR_SHUFFLE()
12072 if (S.getOpcode() == ISD::BUILD_VECTOR && S.hasOneUse()) { in visitVECTOR_SHUFFLE()
12074 } else if (S.getOpcode() == ISD::SCALAR_TO_VECTOR && S.hasOneUse()) { in visitVECTOR_SHUFFLE()
12096 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), VT, Ops); in visitVECTOR_SHUFFLE()
12103 if (N0.getOpcode() == ISD::BITCAST && N0.hasOneUse() && in visitVECTOR_SHUFFLE()
12104 N1.getOpcode() == ISD::UNDEF && Level < AfterLegalizeVectorOps && in visitVECTOR_SHUFFLE()
12109 while (BC0.getOpcode() == ISD::BITCAST) { in visitVECTOR_SHUFFLE()
12126 if (BC0.getOpcode() == ISD::VECTOR_SHUFFLE && BC0.hasOneUse()) { in visitVECTOR_SHUFFLE()
12165 SV0 = DAG.getNode(ISD::BITCAST, SDLoc(N), ScaleVT, SV0); in visitVECTOR_SHUFFLE()
12166 SV1 = DAG.getNode(ISD::BITCAST, SDLoc(N), ScaleVT, SV1); in visitVECTOR_SHUFFLE()
12168 ISD::BITCAST, SDLoc(N), VT, in visitVECTOR_SHUFFLE()
12179 if (N1.getOpcode() == ISD::VECTOR_SHUFFLE && in visitVECTOR_SHUFFLE()
12180 N0.getOpcode() != ISD::VECTOR_SHUFFLE && Level < AfterLegalizeDAG && in visitVECTOR_SHUFFLE()
12190 bool IsSV1Undef = SV1.getOpcode() == ISD::UNDEF; in visitVECTOR_SHUFFLE()
12203 if (N0.getOpcode() == ISD::VECTOR_SHUFFLE && N->isOnlyUserOf(N0.getNode()) && in visitVECTOR_SHUFFLE()
12243 if (CurrentVec.getOpcode() == ISD::UNDEF) { in visitVECTOR_SHUFFLE()
12310 if (InVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT) { in visitSCALAR_TO_VECTOR()
12338 if (N0.getOpcode() == ISD::CONCAT_VECTORS && in visitINSERT_SUBVECTOR()
12339 N0->getNumOperands() == 2 && N2.getOpcode() == ISD::Constant) { in visitINSERT_SUBVECTOR()
12346 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, in visitINSERT_SUBVECTOR()
12352 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, in visitINSERT_SUBVECTOR()
12363 if (N0->getOpcode() == ISD::FP16_TO_FP) in visitFP_TO_FP16()
12384 if (N->getOpcode() != ISD::AND) in XformToShuffleWithZero()
12387 if (RHS.getOpcode() == ISD::BITCAST) in XformToShuffleWithZero()
12390 if (RHS.getOpcode() == ISD::BUILD_VECTOR) { in XformToShuffleWithZero()
12416 SDValue Zero = DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), RVT, ZeroOps); in XformToShuffleWithZero()
12417 LHS = DAG.getNode(ISD::BITCAST, dl, RVT, LHS); in XformToShuffleWithZero()
12419 return DAG.getNode(ISD::BITCAST, dl, VT, Shuf); in XformToShuffleWithZero()
12438 if (LHS.getOpcode() == ISD::BUILD_VECTOR && in SimplifyVBinOp()
12439 RHS.getOpcode() == ISD::BUILD_VECTOR) { in SimplifyVBinOp()
12451 if (N->getOpcode() == ISD::SDIV || N->getOpcode() == ISD::UDIV || in SimplifyVBinOp()
12452 N->getOpcode() == ISD::FDIV) { in SimplifyVBinOp()
12453 if ((RHSOp.getOpcode() == ISD::Constant && in SimplifyVBinOp()
12455 (RHSOp.getOpcode() == ISD::ConstantFP && in SimplifyVBinOp()
12468 RHSOp = DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, RHSOp); in SimplifyVBinOp()
12470 LHSOp = DAG.getNode(ISD::TRUNCATE, SDLoc(N), RVT, LHSOp); in SimplifyVBinOp()
12476 if (FoldOp.getOpcode() != ISD::UNDEF && in SimplifyVBinOp()
12477 FoldOp.getOpcode() != ISD::Constant && in SimplifyVBinOp()
12478 FoldOp.getOpcode() != ISD::ConstantFP) in SimplifyVBinOp()
12485 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), LHS.getValueType(), Ops); in SimplifyVBinOp()
12493 LHS.getOperand(1).getOpcode() == ISD::UNDEF && in SimplifyVBinOp()
12494 RHS.getOperand(1).getOpcode() == ISD::UNDEF) { in SimplifyVBinOp()
12514 assert(N0.getOpcode() ==ISD::SETCC && "First argument must be a SetCC node!"); in SimplifySelect()
12525 if (SCC.getOpcode() == ISD::SELECT_CC) { in SimplifySelect()
12526 SDValue SETCC = DAG.getNode(ISD::SETCC, SDLoc(N0), in SimplifySelect()
12561 if (LHS.getOpcode() == ISD::LOAD) { in SimplifySelectOps()
12574 LLD->getExtensionType() != ISD::EXTLOAD && in SimplifySelectOps()
12575 RLD->getExtensionType() != ISD::EXTLOAD) || in SimplifySelectOps()
12591 if (TheSelect->getOpcode() == ISD::SELECT) { in SimplifySelectOps()
12614 Addr = DAG.getNode(ISD::SELECT_CC, SDLoc(TheSelect), in SimplifySelectOps()
12628 if (LLD->getExtensionType() == ISD::NON_EXTLOAD) { in SimplifySelectOps()
12636 Load = DAG.getExtLoad(LLD->getExtensionType() == ISD::EXTLOAD ? in SimplifySelectOps()
12663 ISD::CondCode CC, bool NotExtCompare) { in SimplifySelectCC()
12690 if ((CC == ISD::SETGE || CC == ISD::SETGT) && in SimplifySelectCC()
12691 N0 == N2 && N3.getOpcode() == ISD::FNEG && in SimplifySelectCC()
12693 return DAG.getNode(ISD::FABS, DL, VT, N0); in SimplifySelectCC()
12696 if ((CC == ISD::SETLT || CC == ISD::SETLE) && in SimplifySelectCC()
12697 N0 == N3 && N2.getOpcode() == ISD::FNEG && in SimplifySelectCC()
12699 return DAG.getNode(ISD::FABS, DL, VT, N3); in SimplifySelectCC()
12714 (TLI.getOperationAction(ISD::ConstantFP, N2.getValueType()) != in SimplifySelectCC()
12747 CPIdx = DAG.getNode(ISD::ADD, DL, CPIdx.getValueType(), CPIdx, in SimplifySelectCC()
12759 if (N1C && N3C && N3C->isNullValue() && CC == ISD::SETLT && in SimplifySelectCC()
12772 SDValue Shift = DAG.getNode(ISD::SRL, SDLoc(N0), in SimplifySelectCC()
12777 Shift = DAG.getNode(ISD::TRUNCATE, DL, AType, Shift); in SimplifySelectCC()
12781 return DAG.getNode(ISD::AND, DL, AType, Shift, N2); in SimplifySelectCC()
12784 SDValue Shift = DAG.getNode(ISD::SRA, SDLoc(N0), in SimplifySelectCC()
12791 Shift = DAG.getNode(ISD::TRUNCATE, DL, AType, Shift); in SimplifySelectCC()
12795 return DAG.getNode(ISD::AND, DL, AType, Shift, N2); in SimplifySelectCC()
12805 if (CC == ISD::SETEQ && N0->getOpcode() == ISD::AND && in SimplifySelectCC()
12817 SDValue Shl = DAG.getNode(ISD::SHL, SDLoc(N0), VT, AndLHS, ShlAmt); in SimplifySelectCC()
12824 SDValue Shr = DAG.getNode(ISD::SRA, SDLoc(N0), VT, Shl, ShrAmt); in SimplifySelectCC()
12826 return DAG.getNode(ISD::AND, DL, VT, Shr, N3); in SimplifySelectCC()
12843 TLI.isOperationLegal(ISD::SETCC, in SimplifySelectCC()
12854 Temp = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N2), in SimplifySelectCC()
12858 Temp = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N2), in SimplifySelectCC()
12870 ISD::SHL, DL, N2.getValueType(), Temp, in SimplifySelectCC()
12882 TLI.isOperationLegal(ISD::SETCC, getSetCCResultType(XType))) { in SimplifySelectCC()
12885 Res = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Res); in SimplifySelectCC()
12890 if (N1C && N1C->isNullValue() && CC == ISD::SETEQ && in SimplifySelectCC()
12892 TLI.isOperationLegal(ISD::CTLZ, XType))) { in SimplifySelectCC()
12893 SDValue Ctlz = DAG.getNode(ISD::CTLZ, SDLoc(N0), XType, N0); in SimplifySelectCC()
12894 return DAG.getNode(ISD::SRL, DL, XType, Ctlz, in SimplifySelectCC()
12899 if (N1C && N1C->isNullValue() && CC == ISD::SETGT) { in SimplifySelectCC()
12900 SDValue NegN0 = DAG.getNode(ISD::SUB, SDLoc(N0), in SimplifySelectCC()
12903 return DAG.getNode(ISD::SRL, DL, XType, in SimplifySelectCC()
12904 DAG.getNode(ISD::AND, DL, XType, NegN0, NotN0), in SimplifySelectCC()
12909 if (N1C && N1C->isAllOnesValue() && CC == ISD::SETGT) { in SimplifySelectCC()
12910 SDValue Sign = DAG.getNode(ISD::SRL, SDLoc(N0), XType, N0, in SimplifySelectCC()
12913 return DAG.getNode(ISD::XOR, DL, XType, Sign, DAG.getConstant(1, XType)); in SimplifySelectCC()
12925 if (((N1C->isNullValue() && (CC == ISD::SETGT || CC == ISD::SETGE)) || in SimplifySelectCC()
12926 (N1C->isAllOnesValue() && CC == ISD::SETGT)) && in SimplifySelectCC()
12927 N0 == N2 && N3.getOpcode() == ISD::SUB && N0 == N3.getOperand(1)) in SimplifySelectCC()
12929 else if (((N1C->isNullValue() && (CC == ISD::SETLT || CC == ISD::SETLE)) || in SimplifySelectCC()
12930 (N1C->isOne() && CC == ISD::SETLT)) && in SimplifySelectCC()
12931 N0 == N3 && N2.getOpcode() == ISD::SUB && N0 == N2.getOperand(1)) in SimplifySelectCC()
12936 SDValue Shift = DAG.getNode(ISD::SRA, SDLoc(N0), XType, in SimplifySelectCC()
12940 SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N0), in SimplifySelectCC()
12944 return DAG.getNode(ISD::XOR, DL, XType, Add, Shift); in SimplifySelectCC()
12953 SDValue N1, ISD::CondCode Cond, in SimplifySetCC()
13047 SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Op, Est); in BuildReciprocalEstimate()
13050 NewEst = DAG.getNode(ISD::FSUB, DL, VT, FPOne, NewEst); in BuildReciprocalEstimate()
13053 NewEst = DAG.getNode(ISD::FMUL, DL, VT, Est, NewEst); in BuildReciprocalEstimate()
13056 Est = DAG.getNode(ISD::FADD, DL, VT, Est, NewEst); in BuildReciprocalEstimate()
13080 SDValue HalfArg = DAG.getNode(ISD::FMUL, DL, VT, ThreeHalves, Arg); in BuildRsqrtNROneConst()
13083 HalfArg = DAG.getNode(ISD::FSUB, DL, VT, HalfArg, Arg); in BuildRsqrtNROneConst()
13088 SDValue NewEst = DAG.getNode(ISD::FMUL, DL, VT, Est, Est); in BuildRsqrtNROneConst()
13091 NewEst = DAG.getNode(ISD::FMUL, DL, VT, HalfArg, NewEst); in BuildRsqrtNROneConst()
13094 NewEst = DAG.getNode(ISD::FSUB, DL, VT, ThreeHalves, NewEst); in BuildRsqrtNROneConst()
13097 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, NewEst); in BuildRsqrtNROneConst()
13117 SDValue HalfEst = DAG.getNode(ISD::FMUL, DL, VT, Est, MinusHalf); in BuildRsqrtNRTwoConst()
13120 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, Est); in BuildRsqrtNRTwoConst()
13123 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, Arg); in BuildRsqrtNRTwoConst()
13126 Est = DAG.getNode(ISD::FADD, DL, VT, Est, MinusThree); in BuildRsqrtNRTwoConst()
13129 Est = DAG.getNode(ISD::FMUL, DL, VT, Est, HalfEst); in BuildRsqrtNRTwoConst()
13164 if (Base.getOpcode() == ISD::ADD) { in FindBaseOffset()
13324 case ISD::EntryToken: in GatherAllAliases()
13328 case ISD::LOAD: in GatherAllAliases()
13329 case ISD::STORE: { in GatherAllAliases()
13346 case ISD::TokenFactor: in GatherAllAliases()
13442 return DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, Aliases); in FindBetterChain()