Lines Matching refs:MVT
34 static bool allocateStack(unsigned ValNo, MVT ValVT, MVT LocVT, in allocateStack()
53 return EVT::getVectorVT(Ctx, MVT::i32, StoreSize / 32); in getEquivalentMemType()
62 return EVT::getVectorVT(Ctx, MVT::i32, StoreSize / 32); in getEquivalentLoadRegType()
68 setOperationAction(ISD::Constant, MVT::i32, Legal); in AMDGPUTargetLowering()
69 setOperationAction(ISD::Constant, MVT::i64, Legal); in AMDGPUTargetLowering()
70 setOperationAction(ISD::ConstantFP, MVT::f32, Legal); in AMDGPUTargetLowering()
71 setOperationAction(ISD::ConstantFP, MVT::f64, Legal); in AMDGPUTargetLowering()
73 setOperationAction(ISD::BR_JT, MVT::Other, Expand); in AMDGPUTargetLowering()
74 setOperationAction(ISD::BRIND, MVT::Other, Expand); in AMDGPUTargetLowering()
77 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom); in AMDGPUTargetLowering()
80 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); in AMDGPUTargetLowering()
84 setOperationAction(ISD::FCEIL, MVT::f32, Legal); in AMDGPUTargetLowering()
85 setOperationAction(ISD::FEXP2, MVT::f32, Legal); in AMDGPUTargetLowering()
86 setOperationAction(ISD::FPOW, MVT::f32, Legal); in AMDGPUTargetLowering()
87 setOperationAction(ISD::FLOG2, MVT::f32, Legal); in AMDGPUTargetLowering()
88 setOperationAction(ISD::FABS, MVT::f32, Legal); in AMDGPUTargetLowering()
89 setOperationAction(ISD::FFLOOR, MVT::f32, Legal); in AMDGPUTargetLowering()
90 setOperationAction(ISD::FRINT, MVT::f32, Legal); in AMDGPUTargetLowering()
91 setOperationAction(ISD::FTRUNC, MVT::f32, Legal); in AMDGPUTargetLowering()
92 setOperationAction(ISD::FMINNUM, MVT::f32, Legal); in AMDGPUTargetLowering()
93 setOperationAction(ISD::FMAXNUM, MVT::f32, Legal); in AMDGPUTargetLowering()
95 setOperationAction(ISD::FROUND, MVT::f32, Custom); in AMDGPUTargetLowering()
96 setOperationAction(ISD::FROUND, MVT::f64, Custom); in AMDGPUTargetLowering()
98 setOperationAction(ISD::FREM, MVT::f32, Custom); in AMDGPUTargetLowering()
99 setOperationAction(ISD::FREM, MVT::f64, Custom); in AMDGPUTargetLowering()
103 setOperationAction(ISD::FMAD, MVT::f32, Legal); in AMDGPUTargetLowering()
106 setOperationAction(ISD::FSUB, MVT::f64, Expand); in AMDGPUTargetLowering()
110 setOperationAction(ISD::STORE, MVT::f32, Promote); in AMDGPUTargetLowering()
111 AddPromotedToType(ISD::STORE, MVT::f32, MVT::i32); in AMDGPUTargetLowering()
113 setOperationAction(ISD::STORE, MVT::v2f32, Promote); in AMDGPUTargetLowering()
114 AddPromotedToType(ISD::STORE, MVT::v2f32, MVT::v2i32); in AMDGPUTargetLowering()
116 setOperationAction(ISD::STORE, MVT::v4f32, Promote); in AMDGPUTargetLowering()
117 AddPromotedToType(ISD::STORE, MVT::v4f32, MVT::v4i32); in AMDGPUTargetLowering()
119 setOperationAction(ISD::STORE, MVT::v8f32, Promote); in AMDGPUTargetLowering()
120 AddPromotedToType(ISD::STORE, MVT::v8f32, MVT::v8i32); in AMDGPUTargetLowering()
122 setOperationAction(ISD::STORE, MVT::v16f32, Promote); in AMDGPUTargetLowering()
123 AddPromotedToType(ISD::STORE, MVT::v16f32, MVT::v16i32); in AMDGPUTargetLowering()
125 setOperationAction(ISD::STORE, MVT::f64, Promote); in AMDGPUTargetLowering()
126 AddPromotedToType(ISD::STORE, MVT::f64, MVT::i64); in AMDGPUTargetLowering()
128 setOperationAction(ISD::STORE, MVT::v2f64, Promote); in AMDGPUTargetLowering()
129 AddPromotedToType(ISD::STORE, MVT::v2f64, MVT::v2i64); in AMDGPUTargetLowering()
133 setOperationAction(ISD::STORE, MVT::v4i32, Custom); in AMDGPUTargetLowering()
135 setTruncStoreAction(MVT::v2i32, MVT::v2i16, Custom); in AMDGPUTargetLowering()
136 setTruncStoreAction(MVT::v2i32, MVT::v2i8, Custom); in AMDGPUTargetLowering()
137 setTruncStoreAction(MVT::v4i32, MVT::v4i8, Custom); in AMDGPUTargetLowering()
141 setTruncStoreAction(MVT::v4i32, MVT::v4i16, Expand); in AMDGPUTargetLowering()
143 setTruncStoreAction(MVT::i64, MVT::i16, Expand); in AMDGPUTargetLowering()
144 setTruncStoreAction(MVT::i64, MVT::i8, Expand); in AMDGPUTargetLowering()
145 setTruncStoreAction(MVT::i64, MVT::i1, Expand); in AMDGPUTargetLowering()
146 setTruncStoreAction(MVT::v2i64, MVT::v2i1, Expand); in AMDGPUTargetLowering()
147 setTruncStoreAction(MVT::v4i64, MVT::v4i1, Expand); in AMDGPUTargetLowering()
150 setOperationAction(ISD::LOAD, MVT::f32, Promote); in AMDGPUTargetLowering()
151 AddPromotedToType(ISD::LOAD, MVT::f32, MVT::i32); in AMDGPUTargetLowering()
153 setOperationAction(ISD::LOAD, MVT::v2f32, Promote); in AMDGPUTargetLowering()
154 AddPromotedToType(ISD::LOAD, MVT::v2f32, MVT::v2i32); in AMDGPUTargetLowering()
156 setOperationAction(ISD::LOAD, MVT::v4f32, Promote); in AMDGPUTargetLowering()
157 AddPromotedToType(ISD::LOAD, MVT::v4f32, MVT::v4i32); in AMDGPUTargetLowering()
159 setOperationAction(ISD::LOAD, MVT::v8f32, Promote); in AMDGPUTargetLowering()
160 AddPromotedToType(ISD::LOAD, MVT::v8f32, MVT::v8i32); in AMDGPUTargetLowering()
162 setOperationAction(ISD::LOAD, MVT::v16f32, Promote); in AMDGPUTargetLowering()
163 AddPromotedToType(ISD::LOAD, MVT::v16f32, MVT::v16i32); in AMDGPUTargetLowering()
165 setOperationAction(ISD::LOAD, MVT::f64, Promote); in AMDGPUTargetLowering()
166 AddPromotedToType(ISD::LOAD, MVT::f64, MVT::i64); in AMDGPUTargetLowering()
168 setOperationAction(ISD::LOAD, MVT::v2f64, Promote); in AMDGPUTargetLowering()
169 AddPromotedToType(ISD::LOAD, MVT::v2f64, MVT::v2i64); in AMDGPUTargetLowering()
171 setOperationAction(ISD::CONCAT_VECTORS, MVT::v4i32, Custom); in AMDGPUTargetLowering()
172 setOperationAction(ISD::CONCAT_VECTORS, MVT::v4f32, Custom); in AMDGPUTargetLowering()
173 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i32, Custom); in AMDGPUTargetLowering()
174 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8f32, Custom); in AMDGPUTargetLowering()
175 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v2f32, Custom); in AMDGPUTargetLowering()
176 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v2i32, Custom); in AMDGPUTargetLowering()
177 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v4f32, Custom); in AMDGPUTargetLowering()
178 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v4i32, Custom); in AMDGPUTargetLowering()
179 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v8f32, Custom); in AMDGPUTargetLowering()
180 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v8i32, Custom); in AMDGPUTargetLowering()
184 for (MVT VT : MVT::integer_valuetypes()) { in AMDGPUTargetLowering()
185 setLoadExtAction(ISD::EXTLOAD, MVT::i64, VT, Expand); in AMDGPUTargetLowering()
186 setLoadExtAction(ISD::SEXTLOAD, MVT::i64, VT, Expand); in AMDGPUTargetLowering()
187 setLoadExtAction(ISD::ZEXTLOAD, MVT::i64, VT, Expand); in AMDGPUTargetLowering()
190 for (MVT VT : MVT::integer_vector_valuetypes()) { in AMDGPUTargetLowering()
191 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i8, Expand); in AMDGPUTargetLowering()
192 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i8, Expand); in AMDGPUTargetLowering()
193 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v2i8, Expand); in AMDGPUTargetLowering()
194 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i8, Expand); in AMDGPUTargetLowering()
195 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i8, Expand); in AMDGPUTargetLowering()
196 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v4i8, Expand); in AMDGPUTargetLowering()
197 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i16, Expand); in AMDGPUTargetLowering()
198 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i16, Expand); in AMDGPUTargetLowering()
199 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v2i16, Expand); in AMDGPUTargetLowering()
200 setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i16, Expand); in AMDGPUTargetLowering()
201 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i16, Expand); in AMDGPUTargetLowering()
202 setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v4i16, Expand); in AMDGPUTargetLowering()
205 setOperationAction(ISD::BR_CC, MVT::i1, Expand); in AMDGPUTargetLowering()
208 setOperationAction(ISD::FCEIL, MVT::f64, Custom); in AMDGPUTargetLowering()
209 setOperationAction(ISD::FTRUNC, MVT::f64, Custom); in AMDGPUTargetLowering()
210 setOperationAction(ISD::FRINT, MVT::f64, Custom); in AMDGPUTargetLowering()
211 setOperationAction(ISD::FFLOOR, MVT::f64, Custom); in AMDGPUTargetLowering()
216 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); in AMDGPUTargetLowering()
217 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); in AMDGPUTargetLowering()
220 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand); in AMDGPUTargetLowering()
222 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand); in AMDGPUTargetLowering()
223 setLoadExtAction(ISD::EXTLOAD, MVT::v2f32, MVT::v2f16, Expand); in AMDGPUTargetLowering()
224 setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4f16, Expand); in AMDGPUTargetLowering()
225 setLoadExtAction(ISD::EXTLOAD, MVT::v8f32, MVT::v8f16, Expand); in AMDGPUTargetLowering()
227 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand); in AMDGPUTargetLowering()
228 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f16, Expand); in AMDGPUTargetLowering()
229 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f16, Expand); in AMDGPUTargetLowering()
230 setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f16, Expand); in AMDGPUTargetLowering()
232 setTruncStoreAction(MVT::f32, MVT::f16, Expand); in AMDGPUTargetLowering()
233 setTruncStoreAction(MVT::v2f32, MVT::v2f16, Expand); in AMDGPUTargetLowering()
234 setTruncStoreAction(MVT::v4f32, MVT::v4f16, Expand); in AMDGPUTargetLowering()
235 setTruncStoreAction(MVT::v8f32, MVT::v8f16, Expand); in AMDGPUTargetLowering()
237 setTruncStoreAction(MVT::f64, MVT::f16, Expand); in AMDGPUTargetLowering()
238 setTruncStoreAction(MVT::f64, MVT::f32, Expand); in AMDGPUTargetLowering()
240 const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 }; in AMDGPUTargetLowering()
241 for (MVT VT : ScalarIntVTs) { in AMDGPUTargetLowering()
259 setOperationAction(ISD::CTPOP, MVT::i32, Expand); in AMDGPUTargetLowering()
262 setOperationAction(ISD::CTPOP, MVT::i64, Expand); in AMDGPUTargetLowering()
265 setOperationAction(ISD::ROTL, MVT::i32, Expand); in AMDGPUTargetLowering()
266 setOperationAction(ISD::ROTL, MVT::i64, Expand); in AMDGPUTargetLowering()
267 setOperationAction(ISD::ROTR, MVT::i64, Expand); in AMDGPUTargetLowering()
269 setOperationAction(ISD::MUL, MVT::i64, Expand); in AMDGPUTargetLowering()
270 setOperationAction(ISD::MULHU, MVT::i64, Expand); in AMDGPUTargetLowering()
271 setOperationAction(ISD::MULHS, MVT::i64, Expand); in AMDGPUTargetLowering()
272 setOperationAction(ISD::UDIV, MVT::i32, Expand); in AMDGPUTargetLowering()
273 setOperationAction(ISD::UREM, MVT::i32, Expand); in AMDGPUTargetLowering()
274 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom); in AMDGPUTargetLowering()
275 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom); in AMDGPUTargetLowering()
276 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); in AMDGPUTargetLowering()
277 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom); in AMDGPUTargetLowering()
278 setOperationAction(ISD::SELECT_CC, MVT::i64, Expand); in AMDGPUTargetLowering()
280 setOperationAction(ISD::SMIN, MVT::i32, Legal); in AMDGPUTargetLowering()
281 setOperationAction(ISD::UMIN, MVT::i32, Legal); in AMDGPUTargetLowering()
282 setOperationAction(ISD::SMAX, MVT::i32, Legal); in AMDGPUTargetLowering()
283 setOperationAction(ISD::UMAX, MVT::i32, Legal); in AMDGPUTargetLowering()
286 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Expand); in AMDGPUTargetLowering()
289 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Expand); in AMDGPUTargetLowering()
291 static const MVT::SimpleValueType VectorIntTypes[] = { in AMDGPUTargetLowering()
292 MVT::v2i32, MVT::v4i32 in AMDGPUTargetLowering()
295 for (MVT VT : VectorIntTypes) { in AMDGPUTargetLowering()
336 static const MVT::SimpleValueType FloatVectorTypes[] = { in AMDGPUTargetLowering()
337 MVT::v2f32, MVT::v4f32 in AMDGPUTargetLowering()
340 for (MVT VT : FloatVectorTypes) { in AMDGPUTargetLowering()
369 setOperationAction(ISD::FNEARBYINT, MVT::f32, Custom); in AMDGPUTargetLowering()
370 setOperationAction(ISD::FNEARBYINT, MVT::f64, Custom); in AMDGPUTargetLowering()
417 MVT AMDGPUTargetLowering::getVectorIdxTy(const DataLayout &) const { in getVectorIdxTy()
418 return MVT::i32; in getVectorIdxTy()
429 return (ScalarVT == MVT::f32 || ScalarVT == MVT::f64); in isFPImmLegal()
435 return (ScalarVT != MVT::f32 && ScalarVT != MVT::f64); in ShouldShrinkFPConstant()
492 return VT == MVT::f32 || VT == MVT::f64; in isFAbsFree()
497 return VT == MVT::f32 || VT == MVT::f64; in isFNegFree()
541 return Src == MVT::i32 && Dest == MVT::i64; in isZExtFree()
575 return DAG.getNode(AMDGPUISD::RET_FLAG, DL, MVT::Other, Chain); in LowerReturn()
725 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); in LowerConstantInitializer()
749 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); in LowerConstantInitializer()
813 MVT PrivPtrVT = getPointerTy(DL, AMDGPUAS::PRIVATE_ADDRESS); in LowerGlobalAddress()
814 MVT ConstPtrVT = getPointerTy(DL, AMDGPUAS::CONSTANT_ADDRESS); in LowerGlobalAddress()
1074 DAG.getConstantFP(1.0f, DL, MVT::f32), in LowerIntrinsicLRP()
1197 DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Chains) in ScalarizeVectorLoad()
1251 DAG.getNode(ISD::TokenFactor, SL, MVT::Other, in SplitVectorLoad()
1280 SDValue Mask = DAG.getConstant((1 << MemEltBits) - 1, DL, MVT::i32); in MergeVectorStore()
1287 DAG.getConstant(i, DL, MVT::i32)); in MergeVectorStore()
1288 Elt = DAG.getZExtOrTrunc(Elt, DL, MVT::i32); in MergeVectorStore()
1289 Elt = DAG.getNode(ISD::AND, DL, MVT::i32, Elt, Mask); // getZeroExtendInReg in MergeVectorStore()
1291 SDValue Shift = DAG.getConstant(MemEltBits * i, DL, MVT::i32); in MergeVectorStore()
1292 Elt = DAG.getNode(ISD::SHL, DL, MVT::i32, Elt, Shift); in MergeVectorStore()
1297 PackedValue = DAG.getNode(ISD::OR, DL, MVT::i32, PackedValue, Elt); in MergeVectorStore()
1333 DAG.getConstant(i, SL, MVT::i32)); in ScalarizeVectorStore()
1345 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, Chains); in ScalarizeVectorStore()
1399 return DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoStore, HiStore); in SplitVectorStore()
1411 assert(VT == MVT::i1 && "Only i1 non-extloads expected"); in LowerLOAD()
1419 SDValue NewLD = DAG.getExtLoad(ISD::EXTLOAD, DL, MVT::i32, Chain, in LowerLOAD()
1420 BasePtr, MVT::i8, MMO); in LowerLOAD()
1432 ExtType == ISD::NON_EXTLOAD || Load->getMemoryVT().bitsGE(MVT::i32)) in LowerLOAD()
1439 SDValue Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, Load->getBasePtr(), in LowerLOAD()
1440 DAG.getConstant(2, DL, MVT::i32)); in LowerLOAD()
1444 DAG.getTargetConstant(0, DL, MVT::i32), in LowerLOAD()
1448 SDValue ByteIdx = DAG.getNode(ISD::AND, DL, MVT::i32, in LowerLOAD()
1450 DAG.getConstant(0x3, DL, MVT::i32)); in LowerLOAD()
1453 SDValue ShiftAmt = DAG.getNode(ISD::SHL, DL, MVT::i32, ByteIdx, in LowerLOAD()
1454 DAG.getConstant(3, DL, MVT::i32)); in LowerLOAD()
1457 Ret = DAG.getNode(ISD::SRL, DL, MVT::i32, Ret, ShiftAmt); in LowerLOAD()
1467 DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, Ret, MemEltVTNode), in LowerLOAD()
1500 MemVT.bitsLT(MVT::i32)) { in LowerSTORE()
1502 if (Store->getMemoryVT() == MVT::i8) { in LowerSTORE()
1504 } else if (Store->getMemoryVT() == MVT::i16) { in LowerSTORE()
1508 SDValue Ptr = DAG.getNode(ISD::SRL, DL, MVT::i32, BasePtr, in LowerSTORE()
1509 DAG.getConstant(2, DL, MVT::i32)); in LowerSTORE()
1510 SDValue Dst = DAG.getNode(AMDGPUISD::REGISTER_LOAD, DL, MVT::i32, in LowerSTORE()
1512 DAG.getTargetConstant(0, DL, MVT::i32)); in LowerSTORE()
1514 SDValue ByteIdx = DAG.getNode(ISD::AND, DL, MVT::i32, BasePtr, in LowerSTORE()
1515 DAG.getConstant(0x3, DL, MVT::i32)); in LowerSTORE()
1517 SDValue ShiftAmt = DAG.getNode(ISD::SHL, DL, MVT::i32, ByteIdx, in LowerSTORE()
1518 DAG.getConstant(3, DL, MVT::i32)); in LowerSTORE()
1520 SDValue SExtValue = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i32, in LowerSTORE()
1525 SDValue ShiftedValue = DAG.getNode(ISD::SHL, DL, MVT::i32, in LowerSTORE()
1528 SDValue DstMask = DAG.getNode(ISD::SHL, DL, MVT::i32, in LowerSTORE()
1529 DAG.getConstant(Mask, DL, MVT::i32), in LowerSTORE()
1531 DstMask = DAG.getNode(ISD::XOR, DL, MVT::i32, DstMask, in LowerSTORE()
1532 DAG.getConstant(0xffffffff, DL, MVT::i32)); in LowerSTORE()
1533 Dst = DAG.getNode(ISD::AND, DL, MVT::i32, Dst, DstMask); in LowerSTORE()
1535 SDValue Value = DAG.getNode(ISD::OR, DL, MVT::i32, Dst, ShiftedValue); in LowerSTORE()
1536 return DAG.getNode(AMDGPUISD::REGISTER_STORE, DL, MVT::Other, in LowerSTORE()
1538 DAG.getTargetConstant(0, DL, MVT::i32)); in LowerSTORE()
1551 MVT IntVT = MVT::i32; in LowerDIVREM24()
1552 MVT FltVT = MVT::f32; in LowerDIVREM24()
1559 IntVT = MVT::getVectorVT(MVT::i32, NElts); in LowerDIVREM24()
1560 FltVT = MVT::getVectorVT(MVT::f32, NElts); in LowerDIVREM24()
1648 assert(Op.getValueType() == MVT::i64); in LowerUDIVREM64()
1666 if (VT == MVT::i64 && in LowerUDIVREM64()
1725 if (VT == MVT::i64) { in LowerUDIVREM()
1734 if (VT == MVT::i32) { in LowerUDIVREM()
1851 if (VT == MVT::i32 && in LowerSDIVREM()
1856 if (VT == MVT::i64 && in LowerSDIVREM()
1924 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src); in LowerFCEIL()
1926 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64); in LowerFCEIL()
1927 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f64); in LowerFCEIL()
1930 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64); in LowerFCEIL()
1936 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, One, Zero); in LowerFCEIL()
1938 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add); in LowerFCEIL()
1945 SDValue ExpPart = DAG.getNode(AMDGPUISD::BFE_U32, SL, MVT::i32, in extractF64Exponent()
1947 DAG.getConstant(FractBits - 32, SL, MVT::i32), in extractF64Exponent()
1948 DAG.getConstant(ExpBits, SL, MVT::i32)); in extractF64Exponent()
1949 SDValue Exp = DAG.getNode(ISD::SUB, SL, MVT::i32, ExpPart, in extractF64Exponent()
1950 DAG.getConstant(1023, SL, MVT::i32)); in extractF64Exponent()
1959 assert(Op.getValueType() == MVT::f64); in LowerFTRUNC()
1961 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in LowerFTRUNC()
1962 const SDValue One = DAG.getConstant(1, SL, MVT::i32); in LowerFTRUNC()
1964 SDValue VecSrc = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src); in LowerFTRUNC()
1968 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, VecSrc, One); in LowerFTRUNC()
1975 const SDValue SignBitMask = DAG.getConstant(UINT32_C(1) << 31, SL, MVT::i32); in LowerFTRUNC()
1976 SDValue SignBit = DAG.getNode(ISD::AND, SL, MVT::i32, Hi, SignBitMask); in LowerFTRUNC()
1979 SDValue SignBit64 = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, in LowerFTRUNC()
1981 SignBit64 = DAG.getNode(ISD::BITCAST, SL, MVT::i64, SignBit64); in LowerFTRUNC()
1983 SDValue BcInt = DAG.getNode(ISD::BITCAST, SL, MVT::i64, Src); in LowerFTRUNC()
1985 = DAG.getConstant((UINT64_C(1) << FractBits) - 1, SL, MVT::i64); in LowerFTRUNC()
1987 SDValue Shr = DAG.getNode(ISD::SRA, SL, MVT::i64, FractMask, Exp); in LowerFTRUNC()
1988 SDValue Not = DAG.getNOT(SL, Shr, MVT::i64); in LowerFTRUNC()
1989 SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, BcInt, Not); in LowerFTRUNC()
1992 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i32); in LowerFTRUNC()
1994 const SDValue FiftyOne = DAG.getConstant(FractBits - 1, SL, MVT::i32); in LowerFTRUNC()
1999 SDValue Tmp1 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpLt0, SignBit64, Tmp0); in LowerFTRUNC()
2000 SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, ExpGt51, BcInt, Tmp1); in LowerFTRUNC()
2002 return DAG.getNode(ISD::BITCAST, SL, MVT::f64, Tmp2); in LowerFTRUNC()
2009 assert(Op.getValueType() == MVT::f64); in LowerFRINT()
2012 SDValue C1 = DAG.getConstantFP(C1Val, SL, MVT::f64); in LowerFRINT()
2013 SDValue CopySign = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, C1, Src); in LowerFRINT()
2017 SDValue Tmp1 = DAG.getNode(ISD::FADD, SL, MVT::f64, Src, CopySign); in LowerFRINT()
2018 SDValue Tmp2 = DAG.getNode(ISD::FSUB, SL, MVT::f64, Tmp1, CopySign); in LowerFRINT()
2020 SDValue Fabs = DAG.getNode(ISD::FABS, SL, MVT::f64, Src); in LowerFRINT()
2023 SDValue C2 = DAG.getConstantFP(C2Val, SL, MVT::f64); in LowerFRINT()
2026 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64); in LowerFRINT()
2029 return DAG.getSelect(SL, MVT::f64, Cond, Src, Tmp2); in LowerFRINT()
2044 SDValue T = DAG.getNode(ISD::FTRUNC, SL, MVT::f32, X); in LowerFROUND32()
2048 SDValue Diff = DAG.getNode(ISD::FSUB, SL, MVT::f32, X, T); in LowerFROUND32()
2050 SDValue AbsDiff = DAG.getNode(ISD::FABS, SL, MVT::f32, Diff); in LowerFROUND32()
2052 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f32); in LowerFROUND32()
2053 const SDValue One = DAG.getConstantFP(1.0, SL, MVT::f32); in LowerFROUND32()
2054 const SDValue Half = DAG.getConstantFP(0.5, SL, MVT::f32); in LowerFROUND32()
2056 SDValue SignOne = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f32, One, X); in LowerFROUND32()
2059 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f32); in LowerFROUND32()
2063 SDValue Sel = DAG.getNode(ISD::SELECT, SL, MVT::f32, Cmp, SignOne, Zero); in LowerFROUND32()
2065 return DAG.getNode(ISD::FADD, SL, MVT::f32, T, Sel); in LowerFROUND32()
2072 SDValue L = DAG.getNode(ISD::BITCAST, SL, MVT::i64, X); in LowerFROUND64()
2074 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in LowerFROUND64()
2075 const SDValue One = DAG.getConstant(1, SL, MVT::i32); in LowerFROUND64()
2076 const SDValue NegOne = DAG.getConstant(-1, SL, MVT::i32); in LowerFROUND64()
2077 const SDValue FiftyOne = DAG.getConstant(51, SL, MVT::i32); in LowerFROUND64()
2079 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i32); in LowerFROUND64()
2081 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, X); in LowerFROUND64()
2083 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BC, One); in LowerFROUND64()
2088 MVT::i64); in LowerFROUND64()
2090 SDValue M = DAG.getNode(ISD::SRA, SL, MVT::i64, Mask, Exp); in LowerFROUND64()
2091 SDValue D = DAG.getNode(ISD::SRA, SL, MVT::i64, in LowerFROUND64()
2093 MVT::i64), in LowerFROUND64()
2096 SDValue Tmp0 = DAG.getNode(ISD::AND, SL, MVT::i64, L, M); in LowerFROUND64()
2098 DAG.getConstant(0, SL, MVT::i64), Tmp0, in LowerFROUND64()
2101 SDValue Tmp2 = DAG.getNode(ISD::SELECT, SL, MVT::i64, Tmp1, in LowerFROUND64()
2102 D, DAG.getConstant(0, SL, MVT::i64)); in LowerFROUND64()
2103 SDValue K = DAG.getNode(ISD::ADD, SL, MVT::i64, L, Tmp2); in LowerFROUND64()
2105 K = DAG.getNode(ISD::AND, SL, MVT::i64, K, DAG.getNOT(SL, M, MVT::i64)); in LowerFROUND64()
2106 K = DAG.getNode(ISD::BITCAST, SL, MVT::f64, K); in LowerFROUND64()
2112 SDValue Mag = DAG.getNode(ISD::SELECT, SL, MVT::f64, in LowerFROUND64()
2114 DAG.getConstantFP(1.0, SL, MVT::f64), in LowerFROUND64()
2115 DAG.getConstantFP(0.0, SL, MVT::f64)); in LowerFROUND64()
2117 SDValue S = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, Mag, X); in LowerFROUND64()
2119 K = DAG.getNode(ISD::SELECT, SL, MVT::f64, ExpLt0, S, K); in LowerFROUND64()
2120 K = DAG.getNode(ISD::SELECT, SL, MVT::f64, ExpGt51, X, K); in LowerFROUND64()
2128 if (VT == MVT::f32) in LowerFROUND()
2131 if (VT == MVT::f64) in LowerFROUND()
2145 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src); in LowerFFLOOR()
2147 const SDValue Zero = DAG.getConstantFP(0.0, SL, MVT::f64); in LowerFFLOOR()
2148 const SDValue NegOne = DAG.getConstantFP(-1.0, SL, MVT::f64); in LowerFFLOOR()
2151 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::f64); in LowerFFLOOR()
2157 SDValue Add = DAG.getNode(ISD::SELECT, SL, MVT::f64, And, NegOne, Zero); in LowerFFLOOR()
2159 return DAG.getNode(ISD::FADD, SL, MVT::f64, Trunc, Add); in LowerFFLOOR()
2167 SDValue BC = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Src); in LowerINT_TO_FP64()
2169 SDValue Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BC, in LowerINT_TO_FP64()
2170 DAG.getConstant(0, SL, MVT::i32)); in LowerINT_TO_FP64()
2171 SDValue Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::i32, BC, in LowerINT_TO_FP64()
2172 DAG.getConstant(1, SL, MVT::i32)); in LowerINT_TO_FP64()
2175 SL, MVT::f64, Hi); in LowerINT_TO_FP64()
2177 SDValue CvtLo = DAG.getNode(ISD::UINT_TO_FP, SL, MVT::f64, Lo); in LowerINT_TO_FP64()
2179 SDValue LdExp = DAG.getNode(AMDGPUISD::LDEXP, SL, MVT::f64, CvtHi, in LowerINT_TO_FP64()
2180 DAG.getConstant(32, SL, MVT::i32)); in LowerINT_TO_FP64()
2182 return DAG.getNode(ISD::FADD, SL, MVT::f64, LdExp, CvtLo); in LowerINT_TO_FP64()
2188 if (S0.getValueType() != MVT::i64) in LowerUINT_TO_FP()
2192 if (DestVT == MVT::f64) in LowerUINT_TO_FP()
2195 assert(DestVT == MVT::f32); in LowerUINT_TO_FP()
2200 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, S0, in LowerUINT_TO_FP()
2201 DAG.getConstant(0, DL, MVT::i32)); in LowerUINT_TO_FP()
2202 SDValue FloatLo = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, Lo); in LowerUINT_TO_FP()
2203 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, S0, in LowerUINT_TO_FP()
2204 DAG.getConstant(1, DL, MVT::i32)); in LowerUINT_TO_FP()
2205 SDValue FloatHi = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::f32, Hi); in LowerUINT_TO_FP()
2207 FloatHi = DAG.getNode(ISD::FMUL, DL, MVT::f32, FloatHi, in LowerUINT_TO_FP()
2208 DAG.getConstantFP(4294967296.0f, DL, MVT::f32)); // 2^32 in LowerUINT_TO_FP()
2209 return DAG.getNode(ISD::FADD, DL, MVT::f32, FloatLo, FloatHi); in LowerUINT_TO_FP()
2215 if (Src.getValueType() == MVT::i64 && Op.getValueType() == MVT::f64) in LowerSINT_TO_FP()
2227 SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, MVT::f64, Src); in LowerFP64_TO_INT()
2230 MVT::f64); in LowerFP64_TO_INT()
2232 MVT::f64); in LowerFP64_TO_INT()
2234 SDValue Mul = DAG.getNode(ISD::FMUL, SL, MVT::f64, Trunc, K0); in LowerFP64_TO_INT()
2236 SDValue FloorMul = DAG.getNode(ISD::FFLOOR, SL, MVT::f64, Mul); in LowerFP64_TO_INT()
2239 SDValue Fma = DAG.getNode(ISD::FMA, SL, MVT::f64, FloorMul, K1, Trunc); in LowerFP64_TO_INT()
2242 MVT::i32, FloorMul); in LowerFP64_TO_INT()
2243 SDValue Lo = DAG.getNode(ISD::FP_TO_UINT, SL, MVT::i32, Fma); in LowerFP64_TO_INT()
2245 SDValue Result = DAG.getNode(ISD::BUILD_VECTOR, SL, MVT::v2i32, Lo, Hi); in LowerFP64_TO_INT()
2247 return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Result); in LowerFP64_TO_INT()
2254 if (Op.getValueType() == MVT::i64 && Src.getValueType() == MVT::f64) in LowerFP_TO_SINT()
2264 if (Op.getValueType() == MVT::i64 && Src.getValueType() == MVT::f64) in LowerFP_TO_UINT()
2273 MVT VT = Op.getSimpleValueType(); in LowerSIGN_EXTEND_INREG()
2274 MVT ScalarVT = VT.getScalarType(); in LowerSIGN_EXTEND_INREG()
2335 return DAG.getConstant(Result, DL, MVT::i32); in constantFoldBFE()
2338 return DAG.getConstant(Src0 >> Offset, DL, MVT::i32); in constantFoldBFE()
2394 if (N->getValueType(0) != MVT::i64) in performShlCombine()
2413 SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LHS); in performShlCombine()
2415 const SDValue Zero = DAG.getConstant(0, SL, MVT::i32); in performShlCombine()
2416 return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64, Zero, Lo); in performShlCombine()
2434 N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32); in performMulCombine()
2435 N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32); in performMulCombine()
2436 Mul = DAG.getNode(AMDGPUISD::MUL_U24, DL, MVT::i32, N0, N1); in performMulCombine()
2438 N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32); in performMulCombine()
2439 N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32); in performMulCombine()
2440 Mul = DAG.getNode(AMDGPUISD::MUL_I24, DL, MVT::i32, N0, N1); in performMulCombine()
2485 if (VT == MVT::f32) in PerformDAGCombine()
2501 return DAG.getConstant(0, DL, MVT::i32); in PerformDAGCombine()
2528 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, BitsFrom, in PerformDAGCombine()
2552 SDValue ShiftVal = DAG.getConstant(OffsetVal, DL, MVT::i32); in PerformDAGCombine()
2553 return DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, MVT::i32, in PerformDAGCombine()
2742 if (VT == MVT::f32) { in getRsqrtEstimate()
2759 if (VT == MVT::f32) { in getRecipEstimate()