Lines Matching refs:MVT
63 static const MVT MVT_CC = MVT::i32;
77 addRegisterClass(MVT::i32, &AArch64::GPR32allRegClass); in AArch64TargetLowering()
78 addRegisterClass(MVT::i64, &AArch64::GPR64allRegClass); in AArch64TargetLowering()
81 addRegisterClass(MVT::f16, &AArch64::FPR16RegClass); in AArch64TargetLowering()
82 addRegisterClass(MVT::f32, &AArch64::FPR32RegClass); in AArch64TargetLowering()
83 addRegisterClass(MVT::f64, &AArch64::FPR64RegClass); in AArch64TargetLowering()
84 addRegisterClass(MVT::f128, &AArch64::FPR128RegClass); in AArch64TargetLowering()
88 addRegisterClass(MVT::v16i8, &AArch64::FPR8RegClass); in AArch64TargetLowering()
89 addRegisterClass(MVT::v8i16, &AArch64::FPR16RegClass); in AArch64TargetLowering()
91 addDRTypeForNEON(MVT::v2f32); in AArch64TargetLowering()
92 addDRTypeForNEON(MVT::v8i8); in AArch64TargetLowering()
93 addDRTypeForNEON(MVT::v4i16); in AArch64TargetLowering()
94 addDRTypeForNEON(MVT::v2i32); in AArch64TargetLowering()
95 addDRTypeForNEON(MVT::v1i64); in AArch64TargetLowering()
96 addDRTypeForNEON(MVT::v1f64); in AArch64TargetLowering()
97 addDRTypeForNEON(MVT::v4f16); in AArch64TargetLowering()
99 addQRTypeForNEON(MVT::v4f32); in AArch64TargetLowering()
100 addQRTypeForNEON(MVT::v2f64); in AArch64TargetLowering()
101 addQRTypeForNEON(MVT::v16i8); in AArch64TargetLowering()
102 addQRTypeForNEON(MVT::v8i16); in AArch64TargetLowering()
103 addQRTypeForNEON(MVT::v4i32); in AArch64TargetLowering()
104 addQRTypeForNEON(MVT::v2i64); in AArch64TargetLowering()
105 addQRTypeForNEON(MVT::v8f16); in AArch64TargetLowering()
112 setOperationAction(ISD::GlobalAddress, MVT::i64, Custom); in AArch64TargetLowering()
113 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); in AArch64TargetLowering()
114 setOperationAction(ISD::SETCC, MVT::i32, Custom); in AArch64TargetLowering()
115 setOperationAction(ISD::SETCC, MVT::i64, Custom); in AArch64TargetLowering()
116 setOperationAction(ISD::SETCC, MVT::f32, Custom); in AArch64TargetLowering()
117 setOperationAction(ISD::SETCC, MVT::f64, Custom); in AArch64TargetLowering()
118 setOperationAction(ISD::BRCOND, MVT::Other, Expand); in AArch64TargetLowering()
119 setOperationAction(ISD::BR_CC, MVT::i32, Custom); in AArch64TargetLowering()
120 setOperationAction(ISD::BR_CC, MVT::i64, Custom); in AArch64TargetLowering()
121 setOperationAction(ISD::BR_CC, MVT::f32, Custom); in AArch64TargetLowering()
122 setOperationAction(ISD::BR_CC, MVT::f64, Custom); in AArch64TargetLowering()
123 setOperationAction(ISD::SELECT, MVT::i32, Custom); in AArch64TargetLowering()
124 setOperationAction(ISD::SELECT, MVT::i64, Custom); in AArch64TargetLowering()
125 setOperationAction(ISD::SELECT, MVT::f32, Custom); in AArch64TargetLowering()
126 setOperationAction(ISD::SELECT, MVT::f64, Custom); in AArch64TargetLowering()
127 setOperationAction(ISD::SELECT_CC, MVT::i32, Custom); in AArch64TargetLowering()
128 setOperationAction(ISD::SELECT_CC, MVT::i64, Custom); in AArch64TargetLowering()
129 setOperationAction(ISD::SELECT_CC, MVT::f32, Custom); in AArch64TargetLowering()
130 setOperationAction(ISD::SELECT_CC, MVT::f64, Custom); in AArch64TargetLowering()
131 setOperationAction(ISD::BR_JT, MVT::Other, Expand); in AArch64TargetLowering()
132 setOperationAction(ISD::JumpTable, MVT::i64, Custom); in AArch64TargetLowering()
134 setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom); in AArch64TargetLowering()
135 setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom); in AArch64TargetLowering()
136 setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom); in AArch64TargetLowering()
138 setOperationAction(ISD::FREM, MVT::f32, Expand); in AArch64TargetLowering()
139 setOperationAction(ISD::FREM, MVT::f64, Expand); in AArch64TargetLowering()
140 setOperationAction(ISD::FREM, MVT::f80, Expand); in AArch64TargetLowering()
144 setOperationAction(ISD::XOR, MVT::i32, Custom); in AArch64TargetLowering()
145 setOperationAction(ISD::XOR, MVT::i64, Custom); in AArch64TargetLowering()
149 setOperationAction(ISD::FABS, MVT::f128, Expand); in AArch64TargetLowering()
150 setOperationAction(ISD::FADD, MVT::f128, Custom); in AArch64TargetLowering()
151 setOperationAction(ISD::FCOPYSIGN, MVT::f128, Expand); in AArch64TargetLowering()
152 setOperationAction(ISD::FCOS, MVT::f128, Expand); in AArch64TargetLowering()
153 setOperationAction(ISD::FDIV, MVT::f128, Custom); in AArch64TargetLowering()
154 setOperationAction(ISD::FMA, MVT::f128, Expand); in AArch64TargetLowering()
155 setOperationAction(ISD::FMUL, MVT::f128, Custom); in AArch64TargetLowering()
156 setOperationAction(ISD::FNEG, MVT::f128, Expand); in AArch64TargetLowering()
157 setOperationAction(ISD::FPOW, MVT::f128, Expand); in AArch64TargetLowering()
158 setOperationAction(ISD::FREM, MVT::f128, Expand); in AArch64TargetLowering()
159 setOperationAction(ISD::FRINT, MVT::f128, Expand); in AArch64TargetLowering()
160 setOperationAction(ISD::FSIN, MVT::f128, Expand); in AArch64TargetLowering()
161 setOperationAction(ISD::FSINCOS, MVT::f128, Expand); in AArch64TargetLowering()
162 setOperationAction(ISD::FSQRT, MVT::f128, Expand); in AArch64TargetLowering()
163 setOperationAction(ISD::FSUB, MVT::f128, Custom); in AArch64TargetLowering()
164 setOperationAction(ISD::FTRUNC, MVT::f128, Expand); in AArch64TargetLowering()
165 setOperationAction(ISD::SETCC, MVT::f128, Custom); in AArch64TargetLowering()
166 setOperationAction(ISD::BR_CC, MVT::f128, Custom); in AArch64TargetLowering()
167 setOperationAction(ISD::SELECT, MVT::f128, Custom); in AArch64TargetLowering()
168 setOperationAction(ISD::SELECT_CC, MVT::f128, Custom); in AArch64TargetLowering()
169 setOperationAction(ISD::FP_EXTEND, MVT::f128, Custom); in AArch64TargetLowering()
173 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); in AArch64TargetLowering()
174 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); in AArch64TargetLowering()
175 setOperationAction(ISD::FP_TO_SINT, MVT::i128, Custom); in AArch64TargetLowering()
176 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); in AArch64TargetLowering()
177 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom); in AArch64TargetLowering()
178 setOperationAction(ISD::FP_TO_UINT, MVT::i128, Custom); in AArch64TargetLowering()
179 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom); in AArch64TargetLowering()
180 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom); in AArch64TargetLowering()
181 setOperationAction(ISD::SINT_TO_FP, MVT::i128, Custom); in AArch64TargetLowering()
182 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom); in AArch64TargetLowering()
183 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom); in AArch64TargetLowering()
184 setOperationAction(ISD::UINT_TO_FP, MVT::i128, Custom); in AArch64TargetLowering()
185 setOperationAction(ISD::FP_ROUND, MVT::f32, Custom); in AArch64TargetLowering()
186 setOperationAction(ISD::FP_ROUND, MVT::f64, Custom); in AArch64TargetLowering()
189 setOperationAction(ISD::VASTART, MVT::Other, Custom); in AArch64TargetLowering()
190 setOperationAction(ISD::VAARG, MVT::Other, Custom); in AArch64TargetLowering()
191 setOperationAction(ISD::VACOPY, MVT::Other, Custom); in AArch64TargetLowering()
192 setOperationAction(ISD::VAEND, MVT::Other, Expand); in AArch64TargetLowering()
195 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); in AArch64TargetLowering()
196 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); in AArch64TargetLowering()
197 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Expand); in AArch64TargetLowering()
200 setOperationAction(ISD::ConstantPool, MVT::i64, Custom); in AArch64TargetLowering()
203 setOperationAction(ISD::BlockAddress, MVT::i64, Custom); in AArch64TargetLowering()
206 setOperationAction(ISD::ADDC, MVT::i32, Custom); in AArch64TargetLowering()
207 setOperationAction(ISD::ADDE, MVT::i32, Custom); in AArch64TargetLowering()
208 setOperationAction(ISD::SUBC, MVT::i32, Custom); in AArch64TargetLowering()
209 setOperationAction(ISD::SUBE, MVT::i32, Custom); in AArch64TargetLowering()
210 setOperationAction(ISD::ADDC, MVT::i64, Custom); in AArch64TargetLowering()
211 setOperationAction(ISD::ADDE, MVT::i64, Custom); in AArch64TargetLowering()
212 setOperationAction(ISD::SUBC, MVT::i64, Custom); in AArch64TargetLowering()
213 setOperationAction(ISD::SUBE, MVT::i64, Custom); in AArch64TargetLowering()
216 setOperationAction(ISD::ROTL, MVT::i32, Expand); in AArch64TargetLowering()
217 setOperationAction(ISD::ROTL, MVT::i64, Expand); in AArch64TargetLowering()
218 for (MVT VT : MVT::vector_valuetypes()) { in AArch64TargetLowering()
224 setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand); in AArch64TargetLowering()
225 setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand); in AArch64TargetLowering()
230 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Expand); in AArch64TargetLowering()
231 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Expand); in AArch64TargetLowering()
232 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Expand); in AArch64TargetLowering()
233 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Expand); in AArch64TargetLowering()
235 setOperationAction(ISD::CTPOP, MVT::i32, Custom); in AArch64TargetLowering()
236 setOperationAction(ISD::CTPOP, MVT::i64, Custom); in AArch64TargetLowering()
238 setOperationAction(ISD::SDIVREM, MVT::i32, Expand); in AArch64TargetLowering()
239 setOperationAction(ISD::SDIVREM, MVT::i64, Expand); in AArch64TargetLowering()
240 for (MVT VT : MVT::vector_valuetypes()) { in AArch64TargetLowering()
244 setOperationAction(ISD::SREM, MVT::i32, Expand); in AArch64TargetLowering()
245 setOperationAction(ISD::SREM, MVT::i64, Expand); in AArch64TargetLowering()
246 setOperationAction(ISD::UDIVREM, MVT::i32, Expand); in AArch64TargetLowering()
247 setOperationAction(ISD::UDIVREM, MVT::i64, Expand); in AArch64TargetLowering()
248 setOperationAction(ISD::UREM, MVT::i32, Expand); in AArch64TargetLowering()
249 setOperationAction(ISD::UREM, MVT::i64, Expand); in AArch64TargetLowering()
252 setOperationAction(ISD::SADDO, MVT::i32, Custom); in AArch64TargetLowering()
253 setOperationAction(ISD::SADDO, MVT::i64, Custom); in AArch64TargetLowering()
254 setOperationAction(ISD::UADDO, MVT::i32, Custom); in AArch64TargetLowering()
255 setOperationAction(ISD::UADDO, MVT::i64, Custom); in AArch64TargetLowering()
256 setOperationAction(ISD::SSUBO, MVT::i32, Custom); in AArch64TargetLowering()
257 setOperationAction(ISD::SSUBO, MVT::i64, Custom); in AArch64TargetLowering()
258 setOperationAction(ISD::USUBO, MVT::i32, Custom); in AArch64TargetLowering()
259 setOperationAction(ISD::USUBO, MVT::i64, Custom); in AArch64TargetLowering()
260 setOperationAction(ISD::SMULO, MVT::i32, Custom); in AArch64TargetLowering()
261 setOperationAction(ISD::SMULO, MVT::i64, Custom); in AArch64TargetLowering()
262 setOperationAction(ISD::UMULO, MVT::i32, Custom); in AArch64TargetLowering()
263 setOperationAction(ISD::UMULO, MVT::i64, Custom); in AArch64TargetLowering()
265 setOperationAction(ISD::FSIN, MVT::f32, Expand); in AArch64TargetLowering()
266 setOperationAction(ISD::FSIN, MVT::f64, Expand); in AArch64TargetLowering()
267 setOperationAction(ISD::FCOS, MVT::f32, Expand); in AArch64TargetLowering()
268 setOperationAction(ISD::FCOS, MVT::f64, Expand); in AArch64TargetLowering()
269 setOperationAction(ISD::FPOW, MVT::f32, Expand); in AArch64TargetLowering()
270 setOperationAction(ISD::FPOW, MVT::f64, Expand); in AArch64TargetLowering()
271 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom); in AArch64TargetLowering()
272 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); in AArch64TargetLowering()
275 setOperationAction(ISD::SETCC, MVT::f16, Promote); in AArch64TargetLowering()
276 setOperationAction(ISD::BR_CC, MVT::f16, Promote); in AArch64TargetLowering()
277 setOperationAction(ISD::SELECT_CC, MVT::f16, Promote); in AArch64TargetLowering()
278 setOperationAction(ISD::SELECT, MVT::f16, Promote); in AArch64TargetLowering()
279 setOperationAction(ISD::FADD, MVT::f16, Promote); in AArch64TargetLowering()
280 setOperationAction(ISD::FSUB, MVT::f16, Promote); in AArch64TargetLowering()
281 setOperationAction(ISD::FMUL, MVT::f16, Promote); in AArch64TargetLowering()
282 setOperationAction(ISD::FDIV, MVT::f16, Promote); in AArch64TargetLowering()
283 setOperationAction(ISD::FREM, MVT::f16, Promote); in AArch64TargetLowering()
284 setOperationAction(ISD::FMA, MVT::f16, Promote); in AArch64TargetLowering()
285 setOperationAction(ISD::FNEG, MVT::f16, Promote); in AArch64TargetLowering()
286 setOperationAction(ISD::FABS, MVT::f16, Promote); in AArch64TargetLowering()
287 setOperationAction(ISD::FCEIL, MVT::f16, Promote); in AArch64TargetLowering()
288 setOperationAction(ISD::FCOPYSIGN, MVT::f16, Promote); in AArch64TargetLowering()
289 setOperationAction(ISD::FCOS, MVT::f16, Promote); in AArch64TargetLowering()
290 setOperationAction(ISD::FFLOOR, MVT::f16, Promote); in AArch64TargetLowering()
291 setOperationAction(ISD::FNEARBYINT, MVT::f16, Promote); in AArch64TargetLowering()
292 setOperationAction(ISD::FPOW, MVT::f16, Promote); in AArch64TargetLowering()
293 setOperationAction(ISD::FPOWI, MVT::f16, Promote); in AArch64TargetLowering()
294 setOperationAction(ISD::FRINT, MVT::f16, Promote); in AArch64TargetLowering()
295 setOperationAction(ISD::FSIN, MVT::f16, Promote); in AArch64TargetLowering()
296 setOperationAction(ISD::FSINCOS, MVT::f16, Promote); in AArch64TargetLowering()
297 setOperationAction(ISD::FSQRT, MVT::f16, Promote); in AArch64TargetLowering()
298 setOperationAction(ISD::FEXP, MVT::f16, Promote); in AArch64TargetLowering()
299 setOperationAction(ISD::FEXP2, MVT::f16, Promote); in AArch64TargetLowering()
300 setOperationAction(ISD::FLOG, MVT::f16, Promote); in AArch64TargetLowering()
301 setOperationAction(ISD::FLOG2, MVT::f16, Promote); in AArch64TargetLowering()
302 setOperationAction(ISD::FLOG10, MVT::f16, Promote); in AArch64TargetLowering()
303 setOperationAction(ISD::FROUND, MVT::f16, Promote); in AArch64TargetLowering()
304 setOperationAction(ISD::FTRUNC, MVT::f16, Promote); in AArch64TargetLowering()
305 setOperationAction(ISD::FMINNUM, MVT::f16, Promote); in AArch64TargetLowering()
306 setOperationAction(ISD::FMAXNUM, MVT::f16, Promote); in AArch64TargetLowering()
307 setOperationAction(ISD::FMINNAN, MVT::f16, Promote); in AArch64TargetLowering()
308 setOperationAction(ISD::FMAXNAN, MVT::f16, Promote); in AArch64TargetLowering()
312 setOperationAction(ISD::FADD, MVT::v4f16, Promote); in AArch64TargetLowering()
313 setOperationAction(ISD::FSUB, MVT::v4f16, Promote); in AArch64TargetLowering()
314 setOperationAction(ISD::FMUL, MVT::v4f16, Promote); in AArch64TargetLowering()
315 setOperationAction(ISD::FDIV, MVT::v4f16, Promote); in AArch64TargetLowering()
316 setOperationAction(ISD::FP_EXTEND, MVT::v4f16, Promote); in AArch64TargetLowering()
317 setOperationAction(ISD::FP_ROUND, MVT::v4f16, Promote); in AArch64TargetLowering()
318 AddPromotedToType(ISD::FADD, MVT::v4f16, MVT::v4f32); in AArch64TargetLowering()
319 AddPromotedToType(ISD::FSUB, MVT::v4f16, MVT::v4f32); in AArch64TargetLowering()
320 AddPromotedToType(ISD::FMUL, MVT::v4f16, MVT::v4f32); in AArch64TargetLowering()
321 AddPromotedToType(ISD::FDIV, MVT::v4f16, MVT::v4f32); in AArch64TargetLowering()
322 AddPromotedToType(ISD::FP_EXTEND, MVT::v4f16, MVT::v4f32); in AArch64TargetLowering()
323 AddPromotedToType(ISD::FP_ROUND, MVT::v4f16, MVT::v4f32); in AArch64TargetLowering()
328 setOperationAction(ISD::FABS, MVT::v4f16, Expand); in AArch64TargetLowering()
329 setOperationAction(ISD::FCEIL, MVT::v4f16, Expand); in AArch64TargetLowering()
330 setOperationAction(ISD::FCOPYSIGN, MVT::v4f16, Expand); in AArch64TargetLowering()
331 setOperationAction(ISD::FCOS, MVT::v4f16, Expand); in AArch64TargetLowering()
332 setOperationAction(ISD::FFLOOR, MVT::v4f16, Expand); in AArch64TargetLowering()
333 setOperationAction(ISD::FMA, MVT::v4f16, Expand); in AArch64TargetLowering()
334 setOperationAction(ISD::FNEARBYINT, MVT::v4f16, Expand); in AArch64TargetLowering()
335 setOperationAction(ISD::FNEG, MVT::v4f16, Expand); in AArch64TargetLowering()
336 setOperationAction(ISD::FPOW, MVT::v4f16, Expand); in AArch64TargetLowering()
337 setOperationAction(ISD::FPOWI, MVT::v4f16, Expand); in AArch64TargetLowering()
338 setOperationAction(ISD::FREM, MVT::v4f16, Expand); in AArch64TargetLowering()
339 setOperationAction(ISD::FROUND, MVT::v4f16, Expand); in AArch64TargetLowering()
340 setOperationAction(ISD::FRINT, MVT::v4f16, Expand); in AArch64TargetLowering()
341 setOperationAction(ISD::FSIN, MVT::v4f16, Expand); in AArch64TargetLowering()
342 setOperationAction(ISD::FSINCOS, MVT::v4f16, Expand); in AArch64TargetLowering()
343 setOperationAction(ISD::FSQRT, MVT::v4f16, Expand); in AArch64TargetLowering()
344 setOperationAction(ISD::FTRUNC, MVT::v4f16, Expand); in AArch64TargetLowering()
345 setOperationAction(ISD::SETCC, MVT::v4f16, Expand); in AArch64TargetLowering()
346 setOperationAction(ISD::BR_CC, MVT::v4f16, Expand); in AArch64TargetLowering()
347 setOperationAction(ISD::SELECT, MVT::v4f16, Expand); in AArch64TargetLowering()
348 setOperationAction(ISD::SELECT_CC, MVT::v4f16, Expand); in AArch64TargetLowering()
349 setOperationAction(ISD::FEXP, MVT::v4f16, Expand); in AArch64TargetLowering()
350 setOperationAction(ISD::FEXP2, MVT::v4f16, Expand); in AArch64TargetLowering()
351 setOperationAction(ISD::FLOG, MVT::v4f16, Expand); in AArch64TargetLowering()
352 setOperationAction(ISD::FLOG2, MVT::v4f16, Expand); in AArch64TargetLowering()
353 setOperationAction(ISD::FLOG10, MVT::v4f16, Expand); in AArch64TargetLowering()
357 setOperationAction(ISD::FABS, MVT::v8f16, Expand); in AArch64TargetLowering()
358 setOperationAction(ISD::FADD, MVT::v8f16, Expand); in AArch64TargetLowering()
359 setOperationAction(ISD::FCEIL, MVT::v8f16, Expand); in AArch64TargetLowering()
360 setOperationAction(ISD::FCOPYSIGN, MVT::v8f16, Expand); in AArch64TargetLowering()
361 setOperationAction(ISD::FCOS, MVT::v8f16, Expand); in AArch64TargetLowering()
362 setOperationAction(ISD::FDIV, MVT::v8f16, Expand); in AArch64TargetLowering()
363 setOperationAction(ISD::FFLOOR, MVT::v8f16, Expand); in AArch64TargetLowering()
364 setOperationAction(ISD::FMA, MVT::v8f16, Expand); in AArch64TargetLowering()
365 setOperationAction(ISD::FMUL, MVT::v8f16, Expand); in AArch64TargetLowering()
366 setOperationAction(ISD::FNEARBYINT, MVT::v8f16, Expand); in AArch64TargetLowering()
367 setOperationAction(ISD::FNEG, MVT::v8f16, Expand); in AArch64TargetLowering()
368 setOperationAction(ISD::FPOW, MVT::v8f16, Expand); in AArch64TargetLowering()
369 setOperationAction(ISD::FPOWI, MVT::v8f16, Expand); in AArch64TargetLowering()
370 setOperationAction(ISD::FREM, MVT::v8f16, Expand); in AArch64TargetLowering()
371 setOperationAction(ISD::FROUND, MVT::v8f16, Expand); in AArch64TargetLowering()
372 setOperationAction(ISD::FRINT, MVT::v8f16, Expand); in AArch64TargetLowering()
373 setOperationAction(ISD::FSIN, MVT::v8f16, Expand); in AArch64TargetLowering()
374 setOperationAction(ISD::FSINCOS, MVT::v8f16, Expand); in AArch64TargetLowering()
375 setOperationAction(ISD::FSQRT, MVT::v8f16, Expand); in AArch64TargetLowering()
376 setOperationAction(ISD::FSUB, MVT::v8f16, Expand); in AArch64TargetLowering()
377 setOperationAction(ISD::FTRUNC, MVT::v8f16, Expand); in AArch64TargetLowering()
378 setOperationAction(ISD::SETCC, MVT::v8f16, Expand); in AArch64TargetLowering()
379 setOperationAction(ISD::BR_CC, MVT::v8f16, Expand); in AArch64TargetLowering()
380 setOperationAction(ISD::SELECT, MVT::v8f16, Expand); in AArch64TargetLowering()
381 setOperationAction(ISD::SELECT_CC, MVT::v8f16, Expand); in AArch64TargetLowering()
382 setOperationAction(ISD::FP_EXTEND, MVT::v8f16, Expand); in AArch64TargetLowering()
383 setOperationAction(ISD::FEXP, MVT::v8f16, Expand); in AArch64TargetLowering()
384 setOperationAction(ISD::FEXP2, MVT::v8f16, Expand); in AArch64TargetLowering()
385 setOperationAction(ISD::FLOG, MVT::v8f16, Expand); in AArch64TargetLowering()
386 setOperationAction(ISD::FLOG2, MVT::v8f16, Expand); in AArch64TargetLowering()
387 setOperationAction(ISD::FLOG10, MVT::v8f16, Expand); in AArch64TargetLowering()
390 for (MVT Ty : {MVT::f32, MVT::f64}) { in AArch64TargetLowering()
403 setOperationAction(ISD::PREFETCH, MVT::Other, Custom); in AArch64TargetLowering()
408 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal); in AArch64TargetLowering()
414 setOperationAction(ISD::FSINCOS, MVT::f64, Custom); in AArch64TargetLowering()
415 setOperationAction(ISD::FSINCOS, MVT::f32, Custom); in AArch64TargetLowering()
417 setOperationAction(ISD::FSINCOS, MVT::f64, Expand); in AArch64TargetLowering()
418 setOperationAction(ISD::FSINCOS, MVT::f32, Expand); in AArch64TargetLowering()
424 setOperationAction(ISD::ConstantFP, MVT::f32, Legal); in AArch64TargetLowering()
425 setOperationAction(ISD::ConstantFP, MVT::f64, Legal); in AArch64TargetLowering()
430 for (MVT VT : MVT::fp_valuetypes()) { in AArch64TargetLowering()
431 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand); in AArch64TargetLowering()
432 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand); in AArch64TargetLowering()
433 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f64, Expand); in AArch64TargetLowering()
434 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f80, Expand); in AArch64TargetLowering()
436 for (MVT VT : MVT::integer_valuetypes()) in AArch64TargetLowering()
437 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Expand); in AArch64TargetLowering()
439 setTruncStoreAction(MVT::f32, MVT::f16, Expand); in AArch64TargetLowering()
440 setTruncStoreAction(MVT::f64, MVT::f32, Expand); in AArch64TargetLowering()
441 setTruncStoreAction(MVT::f64, MVT::f16, Expand); in AArch64TargetLowering()
442 setTruncStoreAction(MVT::f128, MVT::f80, Expand); in AArch64TargetLowering()
443 setTruncStoreAction(MVT::f128, MVT::f64, Expand); in AArch64TargetLowering()
444 setTruncStoreAction(MVT::f128, MVT::f32, Expand); in AArch64TargetLowering()
445 setTruncStoreAction(MVT::f128, MVT::f16, Expand); in AArch64TargetLowering()
447 setOperationAction(ISD::BITCAST, MVT::i16, Custom); in AArch64TargetLowering()
448 setOperationAction(ISD::BITCAST, MVT::f16, Custom); in AArch64TargetLowering()
453 setIndexedLoadAction(im, MVT::i8, Legal); in AArch64TargetLowering()
454 setIndexedLoadAction(im, MVT::i16, Legal); in AArch64TargetLowering()
455 setIndexedLoadAction(im, MVT::i32, Legal); in AArch64TargetLowering()
456 setIndexedLoadAction(im, MVT::i64, Legal); in AArch64TargetLowering()
457 setIndexedLoadAction(im, MVT::f64, Legal); in AArch64TargetLowering()
458 setIndexedLoadAction(im, MVT::f32, Legal); in AArch64TargetLowering()
459 setIndexedLoadAction(im, MVT::f16, Legal); in AArch64TargetLowering()
460 setIndexedStoreAction(im, MVT::i8, Legal); in AArch64TargetLowering()
461 setIndexedStoreAction(im, MVT::i16, Legal); in AArch64TargetLowering()
462 setIndexedStoreAction(im, MVT::i32, Legal); in AArch64TargetLowering()
463 setIndexedStoreAction(im, MVT::i64, Legal); in AArch64TargetLowering()
464 setIndexedStoreAction(im, MVT::f64, Legal); in AArch64TargetLowering()
465 setIndexedStoreAction(im, MVT::f32, Legal); in AArch64TargetLowering()
466 setIndexedStoreAction(im, MVT::f16, Legal); in AArch64TargetLowering()
470 setOperationAction(ISD::TRAP, MVT::Other, Legal); in AArch64TargetLowering()
525 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); in AArch64TargetLowering()
530 setOperationAction(ISD::FABS, MVT::v1f64, Expand); in AArch64TargetLowering()
531 setOperationAction(ISD::FADD, MVT::v1f64, Expand); in AArch64TargetLowering()
532 setOperationAction(ISD::FCEIL, MVT::v1f64, Expand); in AArch64TargetLowering()
533 setOperationAction(ISD::FCOPYSIGN, MVT::v1f64, Expand); in AArch64TargetLowering()
534 setOperationAction(ISD::FCOS, MVT::v1f64, Expand); in AArch64TargetLowering()
535 setOperationAction(ISD::FDIV, MVT::v1f64, Expand); in AArch64TargetLowering()
536 setOperationAction(ISD::FFLOOR, MVT::v1f64, Expand); in AArch64TargetLowering()
537 setOperationAction(ISD::FMA, MVT::v1f64, Expand); in AArch64TargetLowering()
538 setOperationAction(ISD::FMUL, MVT::v1f64, Expand); in AArch64TargetLowering()
539 setOperationAction(ISD::FNEARBYINT, MVT::v1f64, Expand); in AArch64TargetLowering()
540 setOperationAction(ISD::FNEG, MVT::v1f64, Expand); in AArch64TargetLowering()
541 setOperationAction(ISD::FPOW, MVT::v1f64, Expand); in AArch64TargetLowering()
542 setOperationAction(ISD::FREM, MVT::v1f64, Expand); in AArch64TargetLowering()
543 setOperationAction(ISD::FROUND, MVT::v1f64, Expand); in AArch64TargetLowering()
544 setOperationAction(ISD::FRINT, MVT::v1f64, Expand); in AArch64TargetLowering()
545 setOperationAction(ISD::FSIN, MVT::v1f64, Expand); in AArch64TargetLowering()
546 setOperationAction(ISD::FSINCOS, MVT::v1f64, Expand); in AArch64TargetLowering()
547 setOperationAction(ISD::FSQRT, MVT::v1f64, Expand); in AArch64TargetLowering()
548 setOperationAction(ISD::FSUB, MVT::v1f64, Expand); in AArch64TargetLowering()
549 setOperationAction(ISD::FTRUNC, MVT::v1f64, Expand); in AArch64TargetLowering()
550 setOperationAction(ISD::SETCC, MVT::v1f64, Expand); in AArch64TargetLowering()
551 setOperationAction(ISD::BR_CC, MVT::v1f64, Expand); in AArch64TargetLowering()
552 setOperationAction(ISD::SELECT, MVT::v1f64, Expand); in AArch64TargetLowering()
553 setOperationAction(ISD::SELECT_CC, MVT::v1f64, Expand); in AArch64TargetLowering()
554 setOperationAction(ISD::FP_EXTEND, MVT::v1f64, Expand); in AArch64TargetLowering()
556 setOperationAction(ISD::FP_TO_SINT, MVT::v1i64, Expand); in AArch64TargetLowering()
557 setOperationAction(ISD::FP_TO_UINT, MVT::v1i64, Expand); in AArch64TargetLowering()
558 setOperationAction(ISD::SINT_TO_FP, MVT::v1i64, Expand); in AArch64TargetLowering()
559 setOperationAction(ISD::UINT_TO_FP, MVT::v1i64, Expand); in AArch64TargetLowering()
560 setOperationAction(ISD::FP_ROUND, MVT::v1f64, Expand); in AArch64TargetLowering()
562 setOperationAction(ISD::MUL, MVT::v1i64, Expand); in AArch64TargetLowering()
566 setOperationAction(ISD::UINT_TO_FP, MVT::v4i8, Promote); in AArch64TargetLowering()
567 setOperationAction(ISD::SINT_TO_FP, MVT::v4i8, Promote); in AArch64TargetLowering()
568 setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Promote); in AArch64TargetLowering()
569 setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Promote); in AArch64TargetLowering()
572 setOperationAction(ISD::SINT_TO_FP, MVT::v8i8, Promote); in AArch64TargetLowering()
573 setOperationAction(ISD::UINT_TO_FP, MVT::v8i8, Promote); in AArch64TargetLowering()
574 setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Promote); in AArch64TargetLowering()
575 setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Promote); in AArch64TargetLowering()
577 setOperationAction(ISD::SINT_TO_FP, MVT::v2i32, Custom); in AArch64TargetLowering()
578 setOperationAction(ISD::UINT_TO_FP, MVT::v2i32, Custom); in AArch64TargetLowering()
579 setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Custom); in AArch64TargetLowering()
580 setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Custom); in AArch64TargetLowering()
583 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Custom); in AArch64TargetLowering()
584 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom); in AArch64TargetLowering()
587 setOperationAction(ISD::MUL, MVT::v2i64, Expand); in AArch64TargetLowering()
589 setOperationAction(ISD::MUL, MVT::v8i16, Custom); in AArch64TargetLowering()
590 setOperationAction(ISD::MUL, MVT::v4i32, Custom); in AArch64TargetLowering()
591 setOperationAction(ISD::MUL, MVT::v2i64, Custom); in AArch64TargetLowering()
593 setOperationAction(ISD::ANY_EXTEND, MVT::v4i32, Legal); in AArch64TargetLowering()
594 setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand); in AArch64TargetLowering()
597 for (MVT VT : MVT::vector_valuetypes()) { in AArch64TargetLowering()
607 for (MVT InnerVT : MVT::vector_valuetypes()) { in AArch64TargetLowering()
616 for (MVT Ty : {MVT::v2f32, MVT::v4f32, MVT::v2f64}) { in AArch64TargetLowering()
632 if (VT == MVT::v2f32 || VT == MVT::v4f16) { in addTypeForNEON()
634 AddPromotedToType(ISD::LOAD, VT.getSimpleVT(), MVT::v2i32); in addTypeForNEON()
637 AddPromotedToType(ISD::STORE, VT.getSimpleVT(), MVT::v2i32); in addTypeForNEON()
638 } else if (VT == MVT::v2f64 || VT == MVT::v4f32 || VT == MVT::v8f16) { in addTypeForNEON()
640 AddPromotedToType(ISD::LOAD, VT.getSimpleVT(), MVT::v2i64); in addTypeForNEON()
643 AddPromotedToType(ISD::STORE, VT.getSimpleVT(), MVT::v2i64); in addTypeForNEON()
647 if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64) { in addTypeForNEON()
678 for (MVT InnerVT : MVT::all_valuetypes()) in addTypeForNEON()
682 if (VT != MVT::v8i8 && VT != MVT::v16i8) in addTypeForNEON()
696 VT.getSimpleVT() != MVT::v2i64 && VT.getSimpleVT() != MVT::v1i64) in addTypeForNEON()
701 if (VT.isFloatingPoint() && VT.getVectorElementType() != MVT::f16) in addTypeForNEON()
715 void AArch64TargetLowering::addDRTypeForNEON(MVT VT) { in addDRTypeForNEON()
717 addTypeForNEON(VT, MVT::v2i32); in addDRTypeForNEON()
720 void AArch64TargetLowering::addQRTypeForNEON(MVT VT) { in addQRTypeForNEON()
722 addTypeForNEON(VT, MVT::v4i32); in addQRTypeForNEON()
728 return MVT::i32; in getSetCCResultType()
777 MVT VT = Op.getOperand(1).getValueType().getSimpleVT(); in computeKnownBitsForTargetNode()
779 if (VT == MVT::v8i8 || VT == MVT::v16i8) { in computeKnownBitsForTargetNode()
783 } else if (VT == MVT::v4i16 || VT == MVT::v8i16) { in computeKnownBitsForTargetNode()
795 MVT AArch64TargetLowering::getScalarShiftAmountTy(const DataLayout &DL, in getScalarShiftAmountTy()
797 return MVT::i64; in getScalarShiftAmountTy()
823 VT == MVT::v2i64; in allowsMisalignedMemoryAccesses()
1277 SDValue NZCVOp = DAG.getConstant(NZCV, DL, MVT::i32); in emitConditionalComparison()
1447 if ((VT == MVT::i32 && C != 0x80000000 && in getAArch64Cmp()
1449 (VT == MVT::i64 && C != 0x80000000ULL && in getAArch64Cmp()
1452 C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1; in getAArch64Cmp()
1458 if ((VT == MVT::i32 && C != 0 && in getAArch64Cmp()
1460 (VT == MVT::i64 && C != 0ULL && isLegalArithImmed(C - 1ULL))) { in getAArch64Cmp()
1462 C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1; in getAArch64Cmp()
1468 if ((VT == MVT::i32 && C != INT32_MAX && in getAArch64Cmp()
1470 (VT == MVT::i64 && C != INT64_MAX && in getAArch64Cmp()
1473 C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1; in getAArch64Cmp()
1479 if ((VT == MVT::i32 && C != UINT32_MAX && in getAArch64Cmp()
1481 (VT == MVT::i64 && C != UINT64_MAX && in getAArch64Cmp()
1484 C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1; in getAArch64Cmp()
1514 cast<LoadSDNode>(LHS)->getMemoryVT() == MVT::i16 && in getAArch64Cmp()
1520 DAG.getValueType(MVT::i16)); in getAArch64Cmp()
1546 assert((Op.getValueType() == MVT::i32 || Op.getValueType() == MVT::i64) && in getAArch64XALUOOp()
1577 if (Op.getValueType() == MVT::i32) { in getAArch64XALUOOp()
1583 LHS = DAG.getNode(ExtendOpc, DL, MVT::i64, LHS); in getAArch64XALUOOp()
1584 RHS = DAG.getNode(ExtendOpc, DL, MVT::i64, RHS); in getAArch64XALUOOp()
1585 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS); in getAArch64XALUOOp()
1586 SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Mul, in getAArch64XALUOOp()
1587 DAG.getConstant(0, DL, MVT::i64)); in getAArch64XALUOOp()
1592 Value = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Add); in getAArch64XALUOOp()
1599 SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Add, in getAArch64XALUOOp()
1600 DAG.getConstant(32, DL, MVT::i64)); in getAArch64XALUOOp()
1601 UpperBits = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, UpperBits); in getAArch64XALUOOp()
1602 SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i32, Value, in getAArch64XALUOOp()
1603 DAG.getConstant(31, DL, MVT::i64)); in getAArch64XALUOOp()
1606 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32); in getAArch64XALUOOp()
1615 SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul, in getAArch64XALUOOp()
1616 DAG.getConstant(32, DL, MVT::i64)); in getAArch64XALUOOp()
1617 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32); in getAArch64XALUOOp()
1620 DAG.getConstant(0, DL, MVT::i64), in getAArch64XALUOOp()
1625 assert(Op.getValueType() == MVT::i64 && "Expected an i64 value type"); in getAArch64XALUOOp()
1627 Value = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS); in getAArch64XALUOOp()
1629 SDValue UpperBits = DAG.getNode(ISD::MULHS, DL, MVT::i64, LHS, RHS); in getAArch64XALUOOp()
1630 SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i64, Value, in getAArch64XALUOOp()
1631 DAG.getConstant(63, DL, MVT::i64)); in getAArch64XALUOOp()
1634 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32); in getAArch64XALUOOp()
1638 SDValue UpperBits = DAG.getNode(ISD::MULHU, DL, MVT::i64, LHS, RHS); in getAArch64XALUOOp()
1639 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32); in getAArch64XALUOOp()
1642 DAG.getConstant(0, DL, MVT::i64), in getAArch64XALUOOp()
1650 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::i32); in getAArch64XALUOOp()
1662 return makeLibCall(DAG, Call, MVT::f128, Ops, false, SDLoc(Op)).first; in LowerF128Call()
1690 if (LHS.getValueType() != MVT::i32 && LHS.getValueType() != MVT::i64) in LowerXOR()
1731 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in LowerADDC_ADDE_SUBC_SUBE()
1772 SDValue TVal = DAG.getConstant(1, dl, MVT::i32); in LowerXALUO()
1773 SDValue FVal = DAG.getConstant(0, dl, MVT::i32); in LowerXALUO()
1778 SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), dl, MVT::i32); in LowerXALUO()
1779 Overflow = DAG.getNode(AArch64ISD::CSEL, dl, MVT::i32, FVal, TVal, in LowerXALUO()
1782 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32); in LowerXALUO()
1813 return DAG.getNode(AArch64ISD::PREFETCH, DL, MVT::Other, Op.getOperand(0), in LowerPREFETCH()
1814 DAG.getConstant(PrfOp, DL, MVT::i32), Op.getOperand(1)); in LowerPREFETCH()
1819 assert(Op.getValueType() == MVT::f128 && "Unexpected lowering"); in LowerFP_EXTEND()
1829 if (Op.getOperand(0).getValueType() != MVT::f128) { in LowerFP_ROUND()
1854 if (InVT.getVectorElementType() == MVT::f16) { in LowerVectorFP_TO_INT()
1855 MVT NewVT = MVT::getVectorVT(MVT::f32, NumElts); in LowerVectorFP_TO_INT()
1872 MVT ExtVT = in LowerVectorFP_TO_INT()
1873 MVT::getVectorVT(MVT::getFloatingPointVT(VT.getScalarSizeInBits()), in LowerVectorFP_TO_INT()
1889 if (Op.getOperand(0).getValueType() == MVT::f16) { in LowerFP_TO_INT()
1893 DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, Op.getOperand(0))); in LowerFP_TO_INT()
1896 if (Op.getOperand(0).getValueType() != MVT::f128) { in LowerFP_TO_INT()
1921 MVT CastVT = in LowerVectorINT_TO_FP()
1922 MVT::getVectorVT(MVT::getFloatingPointVT(InVT.getScalarSizeInBits()), in LowerVectorINT_TO_FP()
1945 if (Op.getValueType() == MVT::f16) { in LowerINT_TO_FP()
1948 ISD::FP_ROUND, dl, MVT::f16, in LowerINT_TO_FP()
1949 DAG.getNode(Op.getOpcode(), dl, MVT::f32, Op.getOperand(0)), in LowerINT_TO_FP()
1954 if (Op.getOperand(0).getValueType() == MVT::i128) in LowerINT_TO_FP()
1959 if (Op.getValueType() != MVT::f128) in LowerINT_TO_FP()
1990 (ArgVT == MVT::f64) ? "__sincos_stret" : "__sincosf_stret"; in LowerFSINCOS()
2004 if (Op.getValueType() != MVT::f16) in LowerBITCAST()
2007 assert(Op.getOperand(0).getValueType() == MVT::i16); in LowerBITCAST()
2010 Op = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op.getOperand(0)); in LowerBITCAST()
2011 Op = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Op); in LowerBITCAST()
2013 DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, MVT::f16, Op, in LowerBITCAST()
2014 DAG.getTargetConstant(AArch64::hsub, DL, MVT::i32)), in LowerBITCAST()
2024 MVT::SimpleValueType OrigSimpleTy = OrigVT.getSimpleVT().SimpleTy; in getExtensionTo64Bits()
2027 case MVT::v2i8: in getExtensionTo64Bits()
2028 case MVT::v2i16: in getExtensionTo64Bits()
2029 return MVT::v2i32; in getExtensionTo64Bits()
2030 case MVT::v4i8: in getExtensionTo64Bits()
2031 return MVT::v4i16; in getExtensionTo64Bits()
2090 MVT TruncVT = MVT::getIntegerVT(EltSize); in skipExtensionForVectorMULL()
2097 Ops.push_back(DAG.getConstant(CInt.zextOrTrunc(32), dl, MVT::i32)); in skipExtensionForVectorMULL()
2100 MVT::getVectorVT(TruncVT, NumElts), Ops); in skipExtensionForVectorMULL()
2177 if (VT == MVT::v2i64) in LowerMUL()
2387 MVT ValVT = Ins[i].VT; in LowerFormalArguments()
2395 MVT ActualMVT = ActualVT.isSimple() ? ActualVT.getSimpleVT() : MVT::Other; in LowerFormalArguments()
2397 if (ActualMVT == MVT::i1 || ActualMVT == MVT::i8) in LowerFormalArguments()
2398 ValVT = MVT::i8; in LowerFormalArguments()
2399 else if (ActualMVT == MVT::i16) in LowerFormalArguments()
2400 ValVT = MVT::i16; in LowerFormalArguments()
2437 if (RegVT == MVT::i32) in LowerFormalArguments()
2439 else if (RegVT == MVT::i64) in LowerFormalArguments()
2441 else if (RegVT == MVT::f16) in LowerFormalArguments()
2443 else if (RegVT == MVT::f32) in LowerFormalArguments()
2445 else if (RegVT == MVT::f64 || RegVT.is64BitVector()) in LowerFormalArguments()
2447 else if (RegVT == MVT::f128 || RegVT.is128BitVector()) in LowerFormalArguments()
2496 MVT MemVT = VA.getValVT(); in LowerFormalArguments()
2590 SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i64); in saveVarArgRegisters()
2619 SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f128); in saveVarArgRegisters()
2635 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); in saveVarArgRegisters()
2662 assert(!VA.needsCustom() && VA.getLocVT() == MVT::i64 && in LowerCallResult()
2839 return DAG.getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains); in addTokenForArgument()
2904 MVT ArgVT = Outs[i].VT; in LowerCall()
2921 MVT ValVT = Outs[i].VT; in LowerCall()
2926 MVT ActualMVT = ActualVT.isSimple() ? ActualVT.getSimpleVT() : ValVT; in LowerCall()
2929 if (ActualMVT == MVT::i1 || ActualMVT == MVT::i8) in LowerCall()
2930 ValVT = MVT::i8; in LowerCall()
2931 else if (ActualMVT == MVT::i16) in LowerCall()
2932 ValVT = MVT::i16; in LowerCall()
3011 if (Outs[realArgIdx].ArgVT == MVT::i1) { in LowerCall()
3013 Arg = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Arg); in LowerCall()
3014 Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i8, Arg); in LowerCall()
3027 if (realArgIdx == 0 && Flags.isReturned() && Outs[0].VT == MVT::i64) { in LowerCall()
3028 assert(VA.getLocVT() == MVT::i64 && in LowerCall()
3030 assert(!Ins.empty() && Ins[0].VT == MVT::i64 && in LowerCall()
3079 DAG.getConstant(Outs[i].Flags.getByValSize(), DL, MVT::i64); in LowerCall()
3091 if (VA.getValVT() == MVT::i1 || VA.getValVT() == MVT::i8 || in LowerCall()
3092 VA.getValVT() == MVT::i16) in LowerCall()
3103 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains); in LowerCall()
3161 Ops.push_back(DAG.getTargetConstant(FPDiff, DL, MVT::i32)); in LowerCall()
3189 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerCall()
3257 if (Outs[i].ArgVT == MVT::i1) { in LowerReturn()
3261 Arg = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Arg); in LowerReturn()
3280 RetOps.push_back(DAG.getRegister(*I, MVT::i64)); in LowerReturn()
3282 RetOps.push_back(DAG.getRegister(*I, MVT::getFloatingPointVT(64))); in LowerReturn()
3294 return DAG.getNode(AArch64ISD::RET_FLAG, DL, MVT::Other, RetOps); in LowerReturn()
3396 MVT PtrVT = getPointerTy(DAG.getDataLayout()); in LowerDarwinGlobalTLSAddress()
3407 DAG.getLoad(MVT::i64, DL, Chain, DescAddr, in LowerDarwinGlobalTLSAddress()
3426 DAG.getNode(AArch64ISD::CALL, DL, DAG.getVTList(MVT::Other, MVT::Glue), in LowerDarwinGlobalTLSAddress()
3427 Chain, FuncTLVGet, DAG.getRegister(AArch64::X0, MVT::i64), in LowerDarwinGlobalTLSAddress()
3455 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); in LowerELFTLSDescCallSeq()
3507 DAG.getTargetConstant(0, DL, MVT::i32)), in LowerELFGlobalTLSAddress()
3512 DAG.getTargetConstant(0, DL, MVT::i32)), in LowerELFGlobalTLSAddress()
3542 GV, DL, MVT::i64, 0, AArch64II::MO_TLS | AArch64II::MO_HI12); in LowerELFGlobalTLSAddress()
3544 GV, DL, MVT::i64, 0, in LowerELFGlobalTLSAddress()
3548 DAG.getTargetConstant(0, DL, MVT::i32)), in LowerELFGlobalTLSAddress()
3551 DAG.getTargetConstant(0, DL, MVT::i32)), in LowerELFGlobalTLSAddress()
3588 if (LHS.getValueType() == MVT::f128) { in LowerBR_CC()
3589 softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl); in LowerBR_CC()
3618 SDValue CCVal = DAG.getConstant(OFCC, dl, MVT::i32); in LowerBR_CC()
3620 return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, in LowerBR_CC()
3626 (LHS.getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64)); in LowerBR_CC()
3642 return DAG.getNode(AArch64ISD::TBZ, dl, MVT::Other, Chain, Test, in LowerBR_CC()
3643 DAG.getConstant(Log2_64(Mask), dl, MVT::i64), in LowerBR_CC()
3647 return DAG.getNode(AArch64ISD::CBZ, dl, MVT::Other, Chain, LHS, Dest); in LowerBR_CC()
3658 return DAG.getNode(AArch64ISD::TBNZ, dl, MVT::Other, Chain, Test, in LowerBR_CC()
3659 DAG.getConstant(Log2_64(Mask), dl, MVT::i64), in LowerBR_CC()
3663 return DAG.getNode(AArch64ISD::CBNZ, dl, MVT::Other, Chain, LHS, Dest); in LowerBR_CC()
3669 return DAG.getNode(AArch64ISD::TBNZ, dl, MVT::Other, Chain, LHS, in LowerBR_CC()
3670 DAG.getConstant(Mask, dl, MVT::i64), Dest); in LowerBR_CC()
3679 return DAG.getNode(AArch64ISD::TBZ, dl, MVT::Other, Chain, LHS, in LowerBR_CC()
3680 DAG.getConstant(Mask, dl, MVT::i64), Dest); in LowerBR_CC()
3685 return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, in LowerBR_CC()
3689 assert(LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64); in LowerBR_CC()
3696 SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32); in LowerBR_CC()
3698 DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CC1Val, Cmp); in LowerBR_CC()
3700 SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32); in LowerBR_CC()
3701 return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, BR1, Dest, CC2Val, in LowerBR_CC()
3726 if (VT == MVT::f32 || VT == MVT::v2f32 || VT == MVT::v4f32) { in LowerFCOPYSIGN()
3727 EltVT = MVT::i32; in LowerFCOPYSIGN()
3728 VecVT = (VT == MVT::v2f32 ? MVT::v2i32 : MVT::v4i32); in LowerFCOPYSIGN()
3740 } else if (VT == MVT::f64 || VT == MVT::v2f64) { in LowerFCOPYSIGN()
3741 EltVT = MVT::i64; in LowerFCOPYSIGN()
3742 VecVT = MVT::v2i64; in LowerFCOPYSIGN()
3766 if (VT == MVT::f64 || VT == MVT::v2f64) { in LowerFCOPYSIGN()
3767 BuildVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, BuildVec); in LowerFCOPYSIGN()
3768 BuildVec = DAG.getNode(ISD::FNEG, DL, MVT::v2f64, BuildVec); in LowerFCOPYSIGN()
3769 BuildVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, BuildVec); in LowerFCOPYSIGN()
3775 if (VT == MVT::f32) in LowerFCOPYSIGN()
3777 else if (VT == MVT::f64) in LowerFCOPYSIGN()
3803 if (VT == MVT::i32) in LowerCTPOP()
3804 Val = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Val); in LowerCTPOP()
3805 Val = DAG.getNode(ISD::BITCAST, DL, MVT::v8i8, Val); in LowerCTPOP()
3807 SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v8i8, Val); in LowerCTPOP()
3809 ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32, in LowerCTPOP()
3810 DAG.getConstant(Intrinsic::aarch64_neon_uaddlv, DL, MVT::i32), CtPop); in LowerCTPOP()
3812 if (VT == MVT::i64) in LowerCTPOP()
3813 UaddLV = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, UaddLV); in LowerCTPOP()
3834 if (LHS.getValueType() == MVT::f128) { in LowerSETCC()
3835 softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl); in LowerSETCC()
3857 assert(LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64); in LowerSETCC()
3867 SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32); in LowerSETCC()
3880 SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32); in LowerSETCC()
3884 SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32); in LowerSETCC()
3895 if (LHS.getValueType() == MVT::f128) { in LowerSELECT_CC()
3896 softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl); in LowerSELECT_CC()
3907 if (LHS.getValueType() == MVT::f16) { in LowerSELECT_CC()
3908 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, LHS); in LowerSELECT_CC()
3909 RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, RHS); in LowerSELECT_CC()
3915 (LHS.getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64)); in LowerSELECT_CC()
3960 } else if (TVal.getValueType() == MVT::i32) { in LowerSELECT_CC()
4007 assert(LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64); in LowerSELECT_CC()
4016 SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32); in LowerSELECT_CC()
4022 SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32); in LowerSELECT_CC()
4061 SDValue CCVal = DAG.getConstant(OFCC, DL, MVT::i32); in LowerSELECT()
4241 DAG.getConstant(-GPRSize, DL, MVT::i32), in LowerAAPCS_VASTART()
4249 DAG.getConstant(-FPRSize, DL, MVT::i32), in LowerAAPCS_VASTART()
4253 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); in LowerAAPCS_VASTART()
4273 DAG.getConstant(VaListSize, DL, MVT::i32), in LowerVACOPY()
4312 if (VT.isFloatingPoint() && !VT.isVector() && VT != MVT::f64) { in LowerVAARG()
4327 SDValue WideFP = DAG.getLoad(MVT::f64, DL, APStore, VAList, in LowerVAARG()
4407 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, in LowerShiftRightParts()
4408 DAG.getConstant(VTBits, dl, MVT::i64), ShAmt); in LowerShiftRightParts()
4413 SDValue Cmp = emitComparison(ShAmt, DAG.getConstant(0, dl, MVT::i64), in LowerShiftRightParts()
4415 SDValue CCVal = DAG.getConstant(AArch64CC::EQ, dl, MVT::i32); in LowerShiftRightParts()
4417 DAG.getNode(AArch64ISD::CSEL, dl, VT, DAG.getConstant(0, dl, MVT::i64), in LowerShiftRightParts()
4420 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, ShAmt, in LowerShiftRightParts()
4421 DAG.getConstant(VTBits, dl, MVT::i64)); in LowerShiftRightParts()
4427 Cmp = emitComparison(ExtraShAmt, DAG.getConstant(0, dl, MVT::i64), ISD::SETGE, in LowerShiftRightParts()
4429 CCVal = DAG.getConstant(AArch64CC::GE, dl, MVT::i32); in LowerShiftRightParts()
4440 DAG.getConstant(VTBits - 1, dl, MVT::i64)) in LowerShiftRightParts()
4463 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, in LowerShiftLeftParts()
4464 DAG.getConstant(VTBits, dl, MVT::i64), ShAmt); in LowerShiftLeftParts()
4469 SDValue Cmp = emitComparison(ShAmt, DAG.getConstant(0, dl, MVT::i64), in LowerShiftLeftParts()
4471 SDValue CCVal = DAG.getConstant(AArch64CC::EQ, dl, MVT::i32); in LowerShiftLeftParts()
4473 DAG.getNode(AArch64ISD::CSEL, dl, VT, DAG.getConstant(0, dl, MVT::i64), in LowerShiftLeftParts()
4476 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, ShAmt, in LowerShiftLeftParts()
4477 DAG.getConstant(VTBits, dl, MVT::i64)); in LowerShiftLeftParts()
4484 Cmp = emitComparison(ExtraShAmt, DAG.getConstant(0, dl, MVT::i64), ISD::SETGE, in LowerShiftLeftParts()
4486 CCVal = DAG.getConstant(AArch64CC::GE, dl, MVT::i32); in LowerShiftLeftParts()
4510 if (Imm.isPosZero() && (VT == MVT::f64 || VT == MVT::f32)) in isFPImmLegal()
4513 if (VT == MVT::f64) in isFPImmLegal()
4515 else if (VT == MVT::f32) in isFPImmLegal()
4606 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const { in getRegForInlineAsmConstraint()
4614 if (VT == MVT::f32) in getRegForInlineAsmConstraint()
4680 if (Op.getValueType() == MVT::i64) in LowerAsmOperandForConstraint()
4681 Result = DAG.getRegister(AArch64::XZR, MVT::i64); in LowerAsmOperandForConstraint()
4683 Result = DAG.getRegister(AArch64::WZR, MVT::i32); in LowerAsmOperandForConstraint()
4782 Result = DAG.getTargetConstant(CVal, SDLoc(Op), MVT::i64); in LowerAsmOperandForConstraint()
4803 MVT EltTy = VT.getVectorElementType().getSimpleVT(); in WidenVector()
4804 MVT WideTy = MVT::getVectorVT(EltTy, 2 * NarrowSize); in WidenVector()
4808 V64Reg, DAG.getConstant(0, DL, MVT::i32)); in WidenVector()
4823 MVT EltTy = VT.getVectorElementType().getSimpleVT(); in NarrowVector()
4824 MVT NarrowTy = MVT::getVectorVT(EltTy, WideSize / 2); in NarrowVector()
4940 DAG.getConstant(NumSrcElts, dl, MVT::i64)); in ReconstructShuffle()
4946 DAG.getConstant(0, dl, MVT::i64)); in ReconstructShuffle()
4951 DAG.getConstant(0, dl, MVT::i64)); in ReconstructShuffle()
4954 DAG.getConstant(NumSrcElts, dl, MVT::i64)); in ReconstructShuffle()
4959 DAG.getConstant(Imm, dl, MVT::i32)); in ReconstructShuffle()
5294 DAG.getConstant(0, DL, MVT::i64)); in tryFormConcatFromShuffle()
5298 DAG.getConstant(0, DL, MVT::i64)); in tryFormConcatFromShuffle()
5347 if (VT.getVectorElementType() == MVT::i32 || in GeneratePerfectShuffle()
5348 VT.getVectorElementType() == MVT::f32) in GeneratePerfectShuffle()
5351 if (VT.getVectorElementType() == MVT::i16 || in GeneratePerfectShuffle()
5352 VT.getVectorElementType() == MVT::f16) in GeneratePerfectShuffle()
5355 assert(VT.getVectorElementType() == MVT::i8); in GeneratePerfectShuffle()
5363 if (EltTy == MVT::i8) in GeneratePerfectShuffle()
5365 else if (EltTy == MVT::i16 || EltTy == MVT::f16) in GeneratePerfectShuffle()
5367 else if (EltTy == MVT::i32 || EltTy == MVT::f32) in GeneratePerfectShuffle()
5369 else if (EltTy == MVT::i64 || EltTy == MVT::f64) in GeneratePerfectShuffle()
5376 SDValue Lane = DAG.getConstant(OpNum - OP_VDUP0, dl, MVT::i64); in GeneratePerfectShuffle()
5384 DAG.getConstant(Imm, dl, MVT::i32)); in GeneratePerfectShuffle()
5421 TBLMask.push_back(DAG.getConstant(Offset, DL, MVT::i32)); in GenerateTBL()
5425 MVT IndexVT = MVT::v8i8; in GenerateTBL()
5428 IndexVT = MVT::v16i8; in GenerateTBL()
5438 V1Cst = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V1Cst, V1Cst); in GenerateTBL()
5441 DAG.getConstant(Intrinsic::aarch64_neon_tbl1, DL, MVT::i32), V1Cst, in GenerateTBL()
5446 V1Cst = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V1Cst, V2Cst); in GenerateTBL()
5449 DAG.getConstant(Intrinsic::aarch64_neon_tbl1, DL, MVT::i32), V1Cst, in GenerateTBL()
5461 DAG.getConstant(Intrinsic::aarch64_neon_tbl2, DL, MVT::i32), in GenerateTBL()
5471 if (EltType == MVT::i8) in getDUPLANEOp()
5473 if (EltType == MVT::i16 || EltType == MVT::f16) in getDUPLANEOp()
5475 if (EltType == MVT::i32 || EltType == MVT::f32) in getDUPLANEOp()
5477 if (EltType == MVT::i64 || EltType == MVT::f64) in getDUPLANEOp()
5531 return DAG.getNode(Opcode, dl, VT, V1, DAG.getConstant(Lane, dl, MVT::i64)); in LowerVECTOR_SHUFFLE()
5548 DAG.getConstant(Imm, dl, MVT::i32)); in LowerVECTOR_SHUFFLE()
5553 DAG.getConstant(Imm, dl, MVT::i32)); in LowerVECTOR_SHUFFLE()
5592 SDValue DstLaneV = DAG.getConstant(Anomaly, dl, MVT::i64); in LowerVECTOR_SHUFFLE()
5600 SDValue SrcLaneV = DAG.getConstant(SrcLane, dl, MVT::i64); in LowerVECTOR_SHUFFLE()
5605 ScalarVT = MVT::i32; in LowerVECTOR_SHUFFLE()
5688 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerVectorAND()
5690 DAG.getConstant(CnstVal, dl, MVT::i32), in LowerVectorAND()
5691 DAG.getConstant(0, dl, MVT::i32)); in LowerVectorAND()
5697 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerVectorAND()
5699 DAG.getConstant(CnstVal, dl, MVT::i32), in LowerVectorAND()
5700 DAG.getConstant(8, dl, MVT::i32)); in LowerVectorAND()
5706 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerVectorAND()
5708 DAG.getConstant(CnstVal, dl, MVT::i32), in LowerVectorAND()
5709 DAG.getConstant(16, dl, MVT::i32)); in LowerVectorAND()
5715 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerVectorAND()
5717 DAG.getConstant(CnstVal, dl, MVT::i32), in LowerVectorAND()
5718 DAG.getConstant(24, dl, MVT::i32)); in LowerVectorAND()
5724 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; in LowerVectorAND()
5726 DAG.getConstant(CnstVal, dl, MVT::i32), in LowerVectorAND()
5727 DAG.getConstant(0, dl, MVT::i32)); in LowerVectorAND()
5733 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; in LowerVectorAND()
5735 DAG.getConstant(CnstVal, dl, MVT::i32), in LowerVectorAND()
5736 DAG.getConstant(8, dl, MVT::i32)); in LowerVectorAND()
5840 DAG.getConstant(Intrin, DL, MVT::i32), X, Y, in tryLowerToSLI()
5889 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerVectorOR()
5891 DAG.getConstant(CnstVal, dl, MVT::i32), in LowerVectorOR()
5892 DAG.getConstant(0, dl, MVT::i32)); in LowerVectorOR()
5898 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerVectorOR()
5900 DAG.getConstant(CnstVal, dl, MVT::i32), in LowerVectorOR()
5901 DAG.getConstant(8, dl, MVT::i32)); in LowerVectorOR()
5907 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerVectorOR()
5909 DAG.getConstant(CnstVal, dl, MVT::i32), in LowerVectorOR()
5910 DAG.getConstant(16, dl, MVT::i32)); in LowerVectorOR()
5916 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerVectorOR()
5918 DAG.getConstant(CnstVal, dl, MVT::i32), in LowerVectorOR()
5919 DAG.getConstant(24, dl, MVT::i32)); in LowerVectorOR()
5925 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; in LowerVectorOR()
5927 DAG.getConstant(CnstVal, dl, MVT::i32), in LowerVectorOR()
5928 DAG.getConstant(0, dl, MVT::i32)); in LowerVectorOR()
5934 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; in LowerVectorOR()
5936 DAG.getConstant(CnstVal, dl, MVT::i32), in LowerVectorOR()
5937 DAG.getConstant(8, dl, MVT::i32)); in LowerVectorOR()
5971 Lane = DAG.getConstant(LowBits.getZExtValue(), dl, MVT::i32); in NormalizeBuildVector()
6008 SDValue Mov = DAG.getNode(AArch64ISD::MOVIedit, dl, MVT::v2i64, in LowerBUILD_VECTOR()
6009 DAG.getConstant(CnstVal, dl, MVT::i32)); in LowerBUILD_VECTOR()
6014 SDValue Mov = DAG.getNode(AArch64ISD::MOVIedit, dl, MVT::f64, in LowerBUILD_VECTOR()
6015 DAG.getConstant(CnstVal, dl, MVT::i32)); in LowerBUILD_VECTOR()
6021 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
6023 DAG.getConstant(CnstVal, dl, MVT::i32), in LowerBUILD_VECTOR()
6024 DAG.getConstant(0, dl, MVT::i32)); in LowerBUILD_VECTOR()
6030 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
6032 DAG.getConstant(CnstVal, dl, MVT::i32), in LowerBUILD_VECTOR()
6033 DAG.getConstant(8, dl, MVT::i32)); in LowerBUILD_VECTOR()
6039 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
6041 DAG.getConstant(CnstVal, dl, MVT::i32), in LowerBUILD_VECTOR()
6042 DAG.getConstant(16, dl, MVT::i32)); in LowerBUILD_VECTOR()
6048 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
6050 DAG.getConstant(CnstVal, dl, MVT::i32), in LowerBUILD_VECTOR()
6051 DAG.getConstant(24, dl, MVT::i32)); in LowerBUILD_VECTOR()
6057 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; in LowerBUILD_VECTOR()
6059 DAG.getConstant(CnstVal, dl, MVT::i32), in LowerBUILD_VECTOR()
6060 DAG.getConstant(0, dl, MVT::i32)); in LowerBUILD_VECTOR()
6066 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; in LowerBUILD_VECTOR()
6068 DAG.getConstant(CnstVal, dl, MVT::i32), in LowerBUILD_VECTOR()
6069 DAG.getConstant(8, dl, MVT::i32)); in LowerBUILD_VECTOR()
6075 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
6077 DAG.getConstant(CnstVal, dl, MVT::i32), in LowerBUILD_VECTOR()
6078 DAG.getConstant(264, dl, MVT::i32)); in LowerBUILD_VECTOR()
6084 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
6086 DAG.getConstant(CnstVal, dl, MVT::i32), in LowerBUILD_VECTOR()
6087 DAG.getConstant(272, dl, MVT::i32)); in LowerBUILD_VECTOR()
6093 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v16i8 : MVT::v8i8; in LowerBUILD_VECTOR()
6095 DAG.getConstant(CnstVal, dl, MVT::i32)); in LowerBUILD_VECTOR()
6102 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4f32 : MVT::v2f32; in LowerBUILD_VECTOR()
6104 DAG.getConstant(CnstVal, dl, MVT::i32)); in LowerBUILD_VECTOR()
6111 SDValue Mov = DAG.getNode(AArch64ISD::FMOV, dl, MVT::v2f64, in LowerBUILD_VECTOR()
6112 DAG.getConstant(CnstVal, dl, MVT::i32)); in LowerBUILD_VECTOR()
6120 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
6122 DAG.getConstant(CnstVal, dl, MVT::i32), in LowerBUILD_VECTOR()
6123 DAG.getConstant(0, dl, MVT::i32)); in LowerBUILD_VECTOR()
6129 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
6131 DAG.getConstant(CnstVal, dl, MVT::i32), in LowerBUILD_VECTOR()
6132 DAG.getConstant(8, dl, MVT::i32)); in LowerBUILD_VECTOR()
6138 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
6140 DAG.getConstant(CnstVal, dl, MVT::i32), in LowerBUILD_VECTOR()
6141 DAG.getConstant(16, dl, MVT::i32)); in LowerBUILD_VECTOR()
6147 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
6149 DAG.getConstant(CnstVal, dl, MVT::i32), in LowerBUILD_VECTOR()
6150 DAG.getConstant(24, dl, MVT::i32)); in LowerBUILD_VECTOR()
6156 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; in LowerBUILD_VECTOR()
6158 DAG.getConstant(CnstVal, dl, MVT::i32), in LowerBUILD_VECTOR()
6159 DAG.getConstant(0, dl, MVT::i32)); in LowerBUILD_VECTOR()
6165 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16; in LowerBUILD_VECTOR()
6167 DAG.getConstant(CnstVal, dl, MVT::i32), in LowerBUILD_VECTOR()
6168 DAG.getConstant(8, dl, MVT::i32)); in LowerBUILD_VECTOR()
6174 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
6176 DAG.getConstant(CnstVal, dl, MVT::i32), in LowerBUILD_VECTOR()
6177 DAG.getConstant(264, dl, MVT::i32)); in LowerBUILD_VECTOR()
6183 MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32; in LowerBUILD_VECTOR()
6185 DAG.getConstant(CnstVal, dl, MVT::i32), in LowerBUILD_VECTOR()
6186 DAG.getConstant(272, dl, MVT::i32)); in LowerBUILD_VECTOR()
6270 assert ((EltTy == MVT::f16 || EltTy == MVT::f32 || EltTy == MVT::f64) && in LowerBUILD_VECTOR()
6272 MVT NewType = MVT::getIntegerVT(EltTy.getSizeInBits()); in LowerBUILD_VECTOR()
6292 SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i64); in LowerBUILD_VECTOR()
6336 DAG.getTargetConstant(SubIdx, dl, MVT::i32)); in LowerBUILD_VECTOR()
6344 SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i64); in LowerBUILD_VECTOR()
6366 if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 || in LowerINSERT_VECTOR_ELT()
6367 VT == MVT::v2i64 || VT == MVT::v4f32 || VT == MVT::v2f64 || in LowerINSERT_VECTOR_ELT()
6368 VT == MVT::v8f16) in LowerINSERT_VECTOR_ELT()
6371 if (VT != MVT::v8i8 && VT != MVT::v4i16 && VT != MVT::v2i32 && in LowerINSERT_VECTOR_ELT()
6372 VT != MVT::v1i64 && VT != MVT::v2f32 && VT != MVT::v4f16) in LowerINSERT_VECTOR_ELT()
6400 if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 || in LowerEXTRACT_VECTOR_ELT()
6401 VT == MVT::v2i64 || VT == MVT::v4f32 || VT == MVT::v2f64 || in LowerEXTRACT_VECTOR_ELT()
6402 VT == MVT::v8f16) in LowerEXTRACT_VECTOR_ELT()
6405 if (VT != MVT::v8i8 && VT != MVT::v4i16 && VT != MVT::v2i32 && in LowerEXTRACT_VECTOR_ELT()
6406 VT != MVT::v1i64 && VT != MVT::v2f32 && VT != MVT::v4f16) in LowerEXTRACT_VECTOR_ELT()
6416 if (ExtrTy == MVT::i16 || ExtrTy == MVT::i8) in LowerEXTRACT_VECTOR_ELT()
6417 ExtrTy = MVT::i32; in LowerEXTRACT_VECTOR_ELT()
6549 DAG.getConstant(Cnt, DL, MVT::i32)); in LowerVectorSRA_SRL_SHL()
6552 MVT::i32), in LowerVectorSRA_SRL_SHL()
6561 DAG.getConstant(Cnt, DL, MVT::i32)); in LowerVectorSRA_SRL_SHL()
6573 DAG.getConstant(Opc, DL, MVT::i32), Op.getOperand(0), in LowerVectorSRA_SRL_SHL()
6692 if (LHS.getValueType().getVectorElementType() == MVT::f16) in LowerVSETCC()
6695 assert(LHS.getValueType().getVectorElementType() == MVT::f32 || in LowerVSETCC()
6696 LHS.getValueType().getVectorElementType() == MVT::f64); in LowerVSETCC()
6750 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts); in getTgtMemIntrinsic()
6777 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts); in getTgtMemIntrinsic()
6790 Info.memVT = MVT::getVT(PtrTy->getElementType()); in getTgtMemIntrinsic()
6803 Info.memVT = MVT::getVT(PtrTy->getElementType()); in getTgtMemIntrinsic()
6815 Info.memVT = MVT::i128; in getTgtMemIntrinsic()
6827 Info.memVT = MVT::i128; in getTgtMemIntrinsic()
7168 (allowsMisalignedMemoryAccesses(MVT::f128, 0, 1, &Fast) && Fast))) in getOptimalMemOpType()
7169 return MVT::f128; in getOptimalMemOpType()
7173 (allowsMisalignedMemoryAccesses(MVT::i64, 0, 1, &Fast) && Fast))) in getOptimalMemOpType()
7174 return MVT::i64; in getOptimalMemOpType()
7178 (allowsMisalignedMemoryAccesses(MVT::i32, 0, 1, &Fast) && Fast))) in getOptimalMemOpType()
7179 return MVT::i32; in getOptimalMemOpType()
7181 return MVT::Other; in getOptimalMemOpType()
7277 case MVT::f32: in isFMAFasterThanFMulAndFAdd()
7278 case MVT::f64: in isFMAFasterThanFMulAndFAdd()
7303 if (N->getOpcode() == ISD::AND && (VT == MVT::i32 || VT == MVT::i64) && in isDesirableToCommuteWithShift()
7356 DAG.getNode(AArch64ISD::SUBS, DL, DAG.getVTList(VT, MVT::i32), in performIntegerAbsCombine()
7359 DAG.getConstant(AArch64CC::PL, DL, MVT::i32), in performIntegerAbsCombine()
7381 if ((VT != MVT::i32 && VT != MVT::i64) || in BuildSDIVPow2()
7405 DAG.getNode(ISD::SRA, DL, VT, CSel, DAG.getConstant(Lg2, DL, MVT::i64)); in BuildSDIVPow2()
7438 DAG.getConstant(VM1.logBase2(), DL, MVT::i64)); in performMulCombine()
7447 DAG.getConstant(VP1.logBase2(), DL, MVT::i64)); in performMulCombine()
7457 DAG.getConstant(VNP1.logBase2(), DL, MVT::i64)); in performMulCombine()
7466 DAG.getConstant(VNM1.logBase2(), DL, MVT::i64)); in performMulCombine()
7530 if (VT != MVT::f32 && VT != MVT::f64) in performIntToFpCombine()
7577 MVT FloatTy = Op.getSimpleValueType().getVectorElementType(); in performFpToIntCombine()
7582 MVT IntTy = N->getSimpleValueType(0).getVectorElementType(); in performFpToIntCombine()
7598 MVT ResTy; in performFpToIntCombine()
7604 ResTy = FloatBits == 32 ? MVT::v2i32 : MVT::v2i64; in performFpToIntCombine()
7607 ResTy = MVT::v4i32; in performFpToIntCombine()
7617 DAG.getConstant(IntrinsicOpcode, DL, MVT::i32), in performFpToIntCombine()
7618 Op->getOperand(0), DAG.getConstant(C, DL, MVT::i32)); in performFpToIntCombine()
7643 MVT IntTy = Op.getOperand(0).getSimpleValueType().getVectorElementType(); in performFDivCombine()
7648 MVT FloatTy = N->getSimpleValueType(0).getVectorElementType(); in performFDivCombine()
7663 MVT ResTy; in performFDivCombine()
7669 ResTy = FloatBits == 32 ? MVT::v2i32 : MVT::v2i64; in performFDivCombine()
7672 ResTy = MVT::v4i32; in performFDivCombine()
7686 DAG.getConstant(IntrinsicOpcode, DL, MVT::i32), ConvInput, in performFDivCombine()
7687 DAG.getConstant(C, DL, MVT::i32)); in performFDivCombine()
7722 if (VT != MVT::i32 && VT != MVT::i64) in tryCombineToEXTR()
7751 DAG.getConstant(ShiftRHS, DL, MVT::i64)); in tryCombineToEXTR()
7879 SDValue HalfIdx = DAG.getConstant(NumElements, dl, MVT::i64); in performBitcastCombine()
7882 SDValue SubReg = DAG.getTargetConstant(AArch64::dsub, dl, MVT::i32); in performBitcastCombine()
7915 (N00VT == MVT::v2i64 || N00VT == MVT::v4i32) && in performConcatVectorsCombine()
7917 MVT MidVT = (N00VT == MVT::v2i64 ? MVT::v4i32 : MVT::v8i16); in performConcatVectorsCombine()
7940 DAG.getConstant(0, dl, MVT::i64)); in performConcatVectorsCombine()
7955 MVT RHSTy = RHS.getValueType().getSimpleVT(); in performConcatVectorsCombine()
7962 MVT ConcatTy = MVT::getVectorVT(RHSTy.getVectorElementType(), in performConcatVectorsCombine()
8003 if (Vec.getValueType() == MVT::v4i32) in tryCombineFixedPointConvert()
8004 VecResTy = MVT::v4f32; in tryCombineFixedPointConvert()
8005 else if (Vec.getValueType() == MVT::v2i64) in tryCombineFixedPointConvert()
8006 VecResTy = MVT::v2f64; in tryCombineFixedPointConvert()
8053 MVT NarrowTy = N.getSimpleValueType(); in tryExtendDUPToExtractHigh()
8057 MVT ElementTy = NarrowTy.getVectorElementType(); in tryExtendDUPToExtractHigh()
8059 MVT NewVT = MVT::getVectorVT(ElementTy, NumElems * 2); in tryExtendDUPToExtractHigh()
8064 DAG.getConstant(NumElems, dl, MVT::i64)); in tryExtendDUPToExtractHigh()
8181 if (CmpVT != MVT::i32 && CmpVT != MVT::i64) in performSetccAddFolding()
8190 MVT::i32); in performSetccAddFolding()
8220 MVT VT = N->getSimpleValueType(0); in performAddSubLongCombine()
8293 MVT ElemTy = N->getSimpleValueType(0).getScalarType(); in tryCombineShiftImm()
8342 DAG.getConstant(-ShiftAmount, dl, MVT::i32)); in tryCombineShiftImm()
8346 DAG.getConstant(ShiftAmount, dl, MVT::i32)); in tryCombineShiftImm()
8364 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SDLoc(N), MVT::i32, in tryCombineCRC32()
8375 DAG.getConstant(0, dl, MVT::i64)); in combineAcrossLanesIntrinsic()
8505 SrcVT = MVT::getVectorVT(MVT::getIntegerVT(SrcEltSize * 2), ElementCount); in performExtendCombine()
8521 DAG.getConstant(0, DL, MVT::i64)); in performExtendCombine()
8523 DAG.getConstant(InNVT.getVectorNumElements(), DL, MVT::i64)); in performExtendCombine()
8582 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr, in replaceSplatVectorStore()
8583 DAG.getConstant(Offset, DL, MVT::i64)); in replaceSplatVectorStore()
8620 if (!VT.isVector() || VT.getVectorNumElements() < 2 || VT == MVT::v2i64) in split16BStores()
8644 DAG.getConstant(0, DL, MVT::i64)); in split16BStores()
8646 DAG.getConstant(NumElts, DL, MVT::i64)); in split16BStores()
8651 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr, in split16BStores()
8652 DAG.getConstant(8, DL, MVT::i64)); in split16BStores()
8717 Inc = DAG.getRegister(AArch64::XZR, MVT::i64); in performPostLD1Combine()
8736 EVT Tys[3] = { VT, MVT::i64, MVT::Other }; in performPostLD1Combine()
8891 DAG.getConstant(Opcode, DL, MVT::i32), PreOp) in tryMatchAcrossLaneShuffleForReduction()
8895 DAG.getConstant(0, DL, MVT::i64)); in tryMatchAcrossLaneShuffleForReduction()
8940 SetCCVT.getVectorElementType() != MVT::i1) in performAcrossLaneMinMaxReductionCombine()
8959 if (EltTy != MVT::f32) in performAcrossLaneMinMaxReductionCombine()
8962 if (EltTy != MVT::i32 && EltTy != MVT::i16 && EltTy != MVT::i8) in performAcrossLaneMinMaxReductionCombine()
9040 if (EltTy != MVT::i32 && EltTy != MVT::i16 && EltTy != MVT::i8) in performAcrossLaneAddReductionCombine()
9141 Inc = DAG.getRegister(AArch64::XZR, MVT::i64); in performNEONPostLDSTCombine()
9158 Tys[n++] = MVT::i64; // Type of write back register in performNEONPostLDSTCombine()
9159 Tys[n] = MVT::Other; // Type of the chain in performNEONPostLDSTCombine()
9191 if ((LoadNode->getMemoryVT() == MVT::i8 && width == 8) in checkValueWidth()
9192 || (LoadNode->getMemoryVT() == MVT::i16 && width == 16)) { in checkValueWidth()
9200 if ((TypeNode->getVT() == MVT::i8 && width == 8) in checkValueWidth()
9201 || (TypeNode->getVT() == MVT::i16 && width == 16)) { in checkValueWidth()
9209 if ((TypeNode->getVT() == MVT::i8 && width == 8) in checkValueWidth()
9210 || (TypeNode->getVT() == MVT::i16 && width == 16)) { in checkValueWidth()
9471 if (LHS.getValueType() != MVT::i32 && LHS.getValueType() != MVT::i64) in performBRCONDCombine()
9487 BR = DAG.getNode(AArch64ISD::CBZ, SDLoc(N), MVT::Other, Chain, LHS, Dest); in performBRCONDCombine()
9489 BR = DAG.getNode(AArch64ISD::CBNZ, SDLoc(N), MVT::Other, Chain, LHS, Dest); in performBRCONDCombine()
9507 CCVT.getVectorElementType() != MVT::i1) in performVSelectCombine()
9543 assert((N0.getValueType() == MVT::i1 || N0.getValueType() == MVT::i32) && in performSelectCombine()
9553 if (SrcVT == MVT::i1) in performSelectCombine()
9707 MVT::Glue) in isUsedByReturnOnly()
9812 if (N->getValueType(0) != MVT::i16 || Op.getValueType() != MVT::f16) in ReplaceBITCASTResults()
9816 DAG.getMachineNode(TargetOpcode::INSERT_SUBREG, DL, MVT::f32, in ReplaceBITCASTResults()
9817 DAG.getUNDEF(MVT::i32), Op, in ReplaceBITCASTResults()
9818 DAG.getTargetConstant(AArch64::hsub, DL, MVT::i32)), in ReplaceBITCASTResults()
9820 Op = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Op); in ReplaceBITCASTResults()
9821 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Op)); in ReplaceBITCASTResults()
9866 assert(N->getValueType(0) == MVT::i128 && "unexpected illegal conversion"); in ReplaceNodeResults()
9884 MVT SVT = VT.getSimpleVT(); in getPreferredVectorAction()
9887 if (SVT == MVT::v1i8 || SVT == MVT::v1i16 || SVT == MVT::v1i32 in getPreferredVectorAction()
9888 || SVT == MVT::v1f32) in getPreferredVectorAction()