Lines Matching refs:ISD
125 setOperationAction(ISD::GlobalAddress, MVT::i64, Custom); in AArch64TargetLowering()
126 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); in AArch64TargetLowering()
127 setOperationAction(ISD::SETCC, MVT::i32, Custom); in AArch64TargetLowering()
128 setOperationAction(ISD::SETCC, MVT::i64, Custom); in AArch64TargetLowering()
129 setOperationAction(ISD::SETCC, MVT::f32, Custom); in AArch64TargetLowering()
130 setOperationAction(ISD::SETCC, MVT::f64, Custom); in AArch64TargetLowering()
131 setOperationAction(ISD::BRCOND, MVT::Other, Expand); in AArch64TargetLowering()
132 setOperationAction(ISD::BR_CC, MVT::i32, Custom); in AArch64TargetLowering()
133 setOperationAction(ISD::BR_CC, MVT::i64, Custom); in AArch64TargetLowering()
134 setOperationAction(ISD::BR_CC, MVT::f32, Custom); in AArch64TargetLowering()
135 setOperationAction(ISD::BR_CC, MVT::f64, Custom); in AArch64TargetLowering()
136 setOperationAction(ISD::SELECT, MVT::i32, Custom); in AArch64TargetLowering()
137 setOperationAction(ISD::SELECT, MVT::i64, Custom); in AArch64TargetLowering()
138 setOperationAction(ISD::SELECT, MVT::f32, Custom); in AArch64TargetLowering()
139 setOperationAction(ISD::SELECT, MVT::f64, Custom); in AArch64TargetLowering()
140 setOperationAction(ISD::SELECT_CC, MVT::i32, Custom); in AArch64TargetLowering()
141 setOperationAction(ISD::SELECT_CC, MVT::i64, Custom); in AArch64TargetLowering()
142 setOperationAction(ISD::SELECT_CC, MVT::f32, Custom); in AArch64TargetLowering()
143 setOperationAction(ISD::SELECT_CC, MVT::f64, Custom); in AArch64TargetLowering()
144 setOperationAction(ISD::BR_JT, MVT::Other, Expand); in AArch64TargetLowering()
145 setOperationAction(ISD::JumpTable, MVT::i64, Custom); in AArch64TargetLowering()
147 setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom); in AArch64TargetLowering()
148 setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom); in AArch64TargetLowering()
149 setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom); in AArch64TargetLowering()
151 setOperationAction(ISD::FREM, MVT::f32, Expand); in AArch64TargetLowering()
152 setOperationAction(ISD::FREM, MVT::f64, Expand); in AArch64TargetLowering()
153 setOperationAction(ISD::FREM, MVT::f80, Expand); in AArch64TargetLowering()
157 setOperationAction(ISD::XOR, MVT::i32, Custom); in AArch64TargetLowering()
158 setOperationAction(ISD::XOR, MVT::i64, Custom); in AArch64TargetLowering()
162 setOperationAction(ISD::FABS, MVT::f128, Expand); in AArch64TargetLowering()
163 setOperationAction(ISD::FADD, MVT::f128, Custom); in AArch64TargetLowering()
164 setOperationAction(ISD::FCOPYSIGN, MVT::f128, Expand); in AArch64TargetLowering()
165 setOperationAction(ISD::FCOS, MVT::f128, Expand); in AArch64TargetLowering()
166 setOperationAction(ISD::FDIV, MVT::f128, Custom); in AArch64TargetLowering()
167 setOperationAction(ISD::FMA, MVT::f128, Expand); in AArch64TargetLowering()
168 setOperationAction(ISD::FMUL, MVT::f128, Custom); in AArch64TargetLowering()
169 setOperationAction(ISD::FNEG, MVT::f128, Expand); in AArch64TargetLowering()
170 setOperationAction(ISD::FPOW, MVT::f128, Expand); in AArch64TargetLowering()
171 setOperationAction(ISD::FREM, MVT::f128, Expand); in AArch64TargetLowering()
172 setOperationAction(ISD::FRINT, MVT::f128, Expand); in AArch64TargetLowering()
173 setOperationAction(ISD::FSIN, MVT::f128, Expand); in AArch64TargetLowering()
174 setOperationAction(ISD::FSINCOS, MVT::f128, Expand); in AArch64TargetLowering()
175 setOperationAction(ISD::FSQRT, MVT::f128, Expand); in AArch64TargetLowering()
176 setOperationAction(ISD::FSUB, MVT::f128, Custom); in AArch64TargetLowering()
177 setOperationAction(ISD::FTRUNC, MVT::f128, Expand); in AArch64TargetLowering()
178 setOperationAction(ISD::SETCC, MVT::f128, Custom); in AArch64TargetLowering()
179 setOperationAction(ISD::BR_CC, MVT::f128, Custom); in AArch64TargetLowering()
180 setOperationAction(ISD::SELECT, MVT::f128, Custom); in AArch64TargetLowering()
181 setOperationAction(ISD::SELECT_CC, MVT::f128, Custom); in AArch64TargetLowering()
182 setOperationAction(ISD::FP_EXTEND, MVT::f128, Custom); in AArch64TargetLowering()
186 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); in AArch64TargetLowering()
187 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); in AArch64TargetLowering()
188 setOperationAction(ISD::FP_TO_SINT, MVT::i128, Custom); in AArch64TargetLowering()
189 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); in AArch64TargetLowering()
190 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom); in AArch64TargetLowering()
191 setOperationAction(ISD::FP_TO_UINT, MVT::i128, Custom); in AArch64TargetLowering()
192 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom); in AArch64TargetLowering()
193 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom); in AArch64TargetLowering()
194 setOperationAction(ISD::SINT_TO_FP, MVT::i128, Custom); in AArch64TargetLowering()
195 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom); in AArch64TargetLowering()
196 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom); in AArch64TargetLowering()
197 setOperationAction(ISD::UINT_TO_FP, MVT::i128, Custom); in AArch64TargetLowering()
198 setOperationAction(ISD::FP_ROUND, MVT::f32, Custom); in AArch64TargetLowering()
199 setOperationAction(ISD::FP_ROUND, MVT::f64, Custom); in AArch64TargetLowering()
202 setOperationAction(ISD::VASTART, MVT::Other, Custom); in AArch64TargetLowering()
203 setOperationAction(ISD::VAARG, MVT::Other, Custom); in AArch64TargetLowering()
204 setOperationAction(ISD::VACOPY, MVT::Other, Custom); in AArch64TargetLowering()
205 setOperationAction(ISD::VAEND, MVT::Other, Expand); in AArch64TargetLowering()
208 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); in AArch64TargetLowering()
209 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); in AArch64TargetLowering()
210 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Expand); in AArch64TargetLowering()
218 setOperationAction(ISD::ConstantPool, MVT::i64, Custom); in AArch64TargetLowering()
221 setOperationAction(ISD::BlockAddress, MVT::i64, Custom); in AArch64TargetLowering()
224 setOperationAction(ISD::ADDC, MVT::i32, Custom); in AArch64TargetLowering()
225 setOperationAction(ISD::ADDE, MVT::i32, Custom); in AArch64TargetLowering()
226 setOperationAction(ISD::SUBC, MVT::i32, Custom); in AArch64TargetLowering()
227 setOperationAction(ISD::SUBE, MVT::i32, Custom); in AArch64TargetLowering()
228 setOperationAction(ISD::ADDC, MVT::i64, Custom); in AArch64TargetLowering()
229 setOperationAction(ISD::ADDE, MVT::i64, Custom); in AArch64TargetLowering()
230 setOperationAction(ISD::SUBC, MVT::i64, Custom); in AArch64TargetLowering()
231 setOperationAction(ISD::SUBE, MVT::i64, Custom); in AArch64TargetLowering()
234 setOperationAction(ISD::ROTL, MVT::i32, Expand); in AArch64TargetLowering()
235 setOperationAction(ISD::ROTL, MVT::i64, Expand); in AArch64TargetLowering()
238 setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand); in AArch64TargetLowering()
239 setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand); in AArch64TargetLowering()
244 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Expand); in AArch64TargetLowering()
245 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Expand); in AArch64TargetLowering()
246 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Expand); in AArch64TargetLowering()
247 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Expand); in AArch64TargetLowering()
249 setOperationAction(ISD::CTPOP, MVT::i32, Custom); in AArch64TargetLowering()
250 setOperationAction(ISD::CTPOP, MVT::i64, Custom); in AArch64TargetLowering()
252 setOperationAction(ISD::SDIVREM, MVT::i32, Expand); in AArch64TargetLowering()
253 setOperationAction(ISD::SDIVREM, MVT::i64, Expand); in AArch64TargetLowering()
254 setOperationAction(ISD::SREM, MVT::i32, Expand); in AArch64TargetLowering()
255 setOperationAction(ISD::SREM, MVT::i64, Expand); in AArch64TargetLowering()
256 setOperationAction(ISD::UDIVREM, MVT::i32, Expand); in AArch64TargetLowering()
257 setOperationAction(ISD::UDIVREM, MVT::i64, Expand); in AArch64TargetLowering()
258 setOperationAction(ISD::UREM, MVT::i32, Expand); in AArch64TargetLowering()
259 setOperationAction(ISD::UREM, MVT::i64, Expand); in AArch64TargetLowering()
262 setOperationAction(ISD::SADDO, MVT::i32, Custom); in AArch64TargetLowering()
263 setOperationAction(ISD::SADDO, MVT::i64, Custom); in AArch64TargetLowering()
264 setOperationAction(ISD::UADDO, MVT::i32, Custom); in AArch64TargetLowering()
265 setOperationAction(ISD::UADDO, MVT::i64, Custom); in AArch64TargetLowering()
266 setOperationAction(ISD::SSUBO, MVT::i32, Custom); in AArch64TargetLowering()
267 setOperationAction(ISD::SSUBO, MVT::i64, Custom); in AArch64TargetLowering()
268 setOperationAction(ISD::USUBO, MVT::i32, Custom); in AArch64TargetLowering()
269 setOperationAction(ISD::USUBO, MVT::i64, Custom); in AArch64TargetLowering()
270 setOperationAction(ISD::SMULO, MVT::i32, Custom); in AArch64TargetLowering()
271 setOperationAction(ISD::SMULO, MVT::i64, Custom); in AArch64TargetLowering()
272 setOperationAction(ISD::UMULO, MVT::i32, Custom); in AArch64TargetLowering()
273 setOperationAction(ISD::UMULO, MVT::i64, Custom); in AArch64TargetLowering()
275 setOperationAction(ISD::FSIN, MVT::f32, Expand); in AArch64TargetLowering()
276 setOperationAction(ISD::FSIN, MVT::f64, Expand); in AArch64TargetLowering()
277 setOperationAction(ISD::FCOS, MVT::f32, Expand); in AArch64TargetLowering()
278 setOperationAction(ISD::FCOS, MVT::f64, Expand); in AArch64TargetLowering()
279 setOperationAction(ISD::FPOW, MVT::f32, Expand); in AArch64TargetLowering()
280 setOperationAction(ISD::FPOW, MVT::f64, Expand); in AArch64TargetLowering()
281 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom); in AArch64TargetLowering()
282 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); in AArch64TargetLowering()
285 setOperationAction(ISD::SETCC, MVT::f16, Promote); in AArch64TargetLowering()
286 setOperationAction(ISD::BR_CC, MVT::f16, Promote); in AArch64TargetLowering()
287 setOperationAction(ISD::SELECT_CC, MVT::f16, Promote); in AArch64TargetLowering()
288 setOperationAction(ISD::SELECT, MVT::f16, Promote); in AArch64TargetLowering()
289 setOperationAction(ISD::FADD, MVT::f16, Promote); in AArch64TargetLowering()
290 setOperationAction(ISD::FSUB, MVT::f16, Promote); in AArch64TargetLowering()
291 setOperationAction(ISD::FMUL, MVT::f16, Promote); in AArch64TargetLowering()
292 setOperationAction(ISD::FDIV, MVT::f16, Promote); in AArch64TargetLowering()
293 setOperationAction(ISD::FREM, MVT::f16, Promote); in AArch64TargetLowering()
294 setOperationAction(ISD::FMA, MVT::f16, Promote); in AArch64TargetLowering()
295 setOperationAction(ISD::FNEG, MVT::f16, Promote); in AArch64TargetLowering()
296 setOperationAction(ISD::FABS, MVT::f16, Promote); in AArch64TargetLowering()
297 setOperationAction(ISD::FCEIL, MVT::f16, Promote); in AArch64TargetLowering()
298 setOperationAction(ISD::FCOPYSIGN, MVT::f16, Promote); in AArch64TargetLowering()
299 setOperationAction(ISD::FCOS, MVT::f16, Promote); in AArch64TargetLowering()
300 setOperationAction(ISD::FFLOOR, MVT::f16, Promote); in AArch64TargetLowering()
301 setOperationAction(ISD::FNEARBYINT, MVT::f16, Promote); in AArch64TargetLowering()
302 setOperationAction(ISD::FPOW, MVT::f16, Promote); in AArch64TargetLowering()
303 setOperationAction(ISD::FPOWI, MVT::f16, Promote); in AArch64TargetLowering()
304 setOperationAction(ISD::FRINT, MVT::f16, Promote); in AArch64TargetLowering()
305 setOperationAction(ISD::FSIN, MVT::f16, Promote); in AArch64TargetLowering()
306 setOperationAction(ISD::FSINCOS, MVT::f16, Promote); in AArch64TargetLowering()
307 setOperationAction(ISD::FSQRT, MVT::f16, Promote); in AArch64TargetLowering()
308 setOperationAction(ISD::FEXP, MVT::f16, Promote); in AArch64TargetLowering()
309 setOperationAction(ISD::FEXP2, MVT::f16, Promote); in AArch64TargetLowering()
310 setOperationAction(ISD::FLOG, MVT::f16, Promote); in AArch64TargetLowering()
311 setOperationAction(ISD::FLOG2, MVT::f16, Promote); in AArch64TargetLowering()
312 setOperationAction(ISD::FLOG10, MVT::f16, Promote); in AArch64TargetLowering()
313 setOperationAction(ISD::FROUND, MVT::f16, Promote); in AArch64TargetLowering()
314 setOperationAction(ISD::FTRUNC, MVT::f16, Promote); in AArch64TargetLowering()
315 setOperationAction(ISD::FMINNUM, MVT::f16, Promote); in AArch64TargetLowering()
316 setOperationAction(ISD::FMAXNUM, MVT::f16, Promote); in AArch64TargetLowering()
320 setOperationAction(ISD::FADD, MVT::v4f16, Promote); in AArch64TargetLowering()
321 setOperationAction(ISD::FSUB, MVT::v4f16, Promote); in AArch64TargetLowering()
322 setOperationAction(ISD::FMUL, MVT::v4f16, Promote); in AArch64TargetLowering()
323 setOperationAction(ISD::FDIV, MVT::v4f16, Promote); in AArch64TargetLowering()
324 setOperationAction(ISD::FP_EXTEND, MVT::v4f16, Promote); in AArch64TargetLowering()
325 setOperationAction(ISD::FP_ROUND, MVT::v4f16, Promote); in AArch64TargetLowering()
326 AddPromotedToType(ISD::FADD, MVT::v4f16, MVT::v4f32); in AArch64TargetLowering()
327 AddPromotedToType(ISD::FSUB, MVT::v4f16, MVT::v4f32); in AArch64TargetLowering()
328 AddPromotedToType(ISD::FMUL, MVT::v4f16, MVT::v4f32); in AArch64TargetLowering()
329 AddPromotedToType(ISD::FDIV, MVT::v4f16, MVT::v4f32); in AArch64TargetLowering()
330 AddPromotedToType(ISD::FP_EXTEND, MVT::v4f16, MVT::v4f32); in AArch64TargetLowering()
331 AddPromotedToType(ISD::FP_ROUND, MVT::v4f16, MVT::v4f32); in AArch64TargetLowering()
336 setOperationAction(ISD::FABS, MVT::v4f16, Expand); in AArch64TargetLowering()
337 setOperationAction(ISD::FCEIL, MVT::v4f16, Expand); in AArch64TargetLowering()
338 setOperationAction(ISD::FCOPYSIGN, MVT::v4f16, Expand); in AArch64TargetLowering()
339 setOperationAction(ISD::FCOS, MVT::v4f16, Expand); in AArch64TargetLowering()
340 setOperationAction(ISD::FFLOOR, MVT::v4f16, Expand); in AArch64TargetLowering()
341 setOperationAction(ISD::FMA, MVT::v4f16, Expand); in AArch64TargetLowering()
342 setOperationAction(ISD::FNEARBYINT, MVT::v4f16, Expand); in AArch64TargetLowering()
343 setOperationAction(ISD::FNEG, MVT::v4f16, Expand); in AArch64TargetLowering()
344 setOperationAction(ISD::FPOW, MVT::v4f16, Expand); in AArch64TargetLowering()
345 setOperationAction(ISD::FPOWI, MVT::v4f16, Expand); in AArch64TargetLowering()
346 setOperationAction(ISD::FREM, MVT::v4f16, Expand); in AArch64TargetLowering()
347 setOperationAction(ISD::FROUND, MVT::v4f16, Expand); in AArch64TargetLowering()
348 setOperationAction(ISD::FRINT, MVT::v4f16, Expand); in AArch64TargetLowering()
349 setOperationAction(ISD::FSIN, MVT::v4f16, Expand); in AArch64TargetLowering()
350 setOperationAction(ISD::FSINCOS, MVT::v4f16, Expand); in AArch64TargetLowering()
351 setOperationAction(ISD::FSQRT, MVT::v4f16, Expand); in AArch64TargetLowering()
352 setOperationAction(ISD::FTRUNC, MVT::v4f16, Expand); in AArch64TargetLowering()
353 setOperationAction(ISD::SETCC, MVT::v4f16, Expand); in AArch64TargetLowering()
354 setOperationAction(ISD::BR_CC, MVT::v4f16, Expand); in AArch64TargetLowering()
355 setOperationAction(ISD::SELECT, MVT::v4f16, Expand); in AArch64TargetLowering()
356 setOperationAction(ISD::SELECT_CC, MVT::v4f16, Expand); in AArch64TargetLowering()
357 setOperationAction(ISD::FEXP, MVT::v4f16, Expand); in AArch64TargetLowering()
358 setOperationAction(ISD::FEXP2, MVT::v4f16, Expand); in AArch64TargetLowering()
359 setOperationAction(ISD::FLOG, MVT::v4f16, Expand); in AArch64TargetLowering()
360 setOperationAction(ISD::FLOG2, MVT::v4f16, Expand); in AArch64TargetLowering()
361 setOperationAction(ISD::FLOG10, MVT::v4f16, Expand); in AArch64TargetLowering()
365 setOperationAction(ISD::FABS, MVT::v8f16, Expand); in AArch64TargetLowering()
366 setOperationAction(ISD::FADD, MVT::v8f16, Expand); in AArch64TargetLowering()
367 setOperationAction(ISD::FCEIL, MVT::v8f16, Expand); in AArch64TargetLowering()
368 setOperationAction(ISD::FCOPYSIGN, MVT::v8f16, Expand); in AArch64TargetLowering()
369 setOperationAction(ISD::FCOS, MVT::v8f16, Expand); in AArch64TargetLowering()
370 setOperationAction(ISD::FDIV, MVT::v8f16, Expand); in AArch64TargetLowering()
371 setOperationAction(ISD::FFLOOR, MVT::v8f16, Expand); in AArch64TargetLowering()
372 setOperationAction(ISD::FMA, MVT::v8f16, Expand); in AArch64TargetLowering()
373 setOperationAction(ISD::FMUL, MVT::v8f16, Expand); in AArch64TargetLowering()
374 setOperationAction(ISD::FNEARBYINT, MVT::v8f16, Expand); in AArch64TargetLowering()
375 setOperationAction(ISD::FNEG, MVT::v8f16, Expand); in AArch64TargetLowering()
376 setOperationAction(ISD::FPOW, MVT::v8f16, Expand); in AArch64TargetLowering()
377 setOperationAction(ISD::FPOWI, MVT::v8f16, Expand); in AArch64TargetLowering()
378 setOperationAction(ISD::FREM, MVT::v8f16, Expand); in AArch64TargetLowering()
379 setOperationAction(ISD::FROUND, MVT::v8f16, Expand); in AArch64TargetLowering()
380 setOperationAction(ISD::FRINT, MVT::v8f16, Expand); in AArch64TargetLowering()
381 setOperationAction(ISD::FSIN, MVT::v8f16, Expand); in AArch64TargetLowering()
382 setOperationAction(ISD::FSINCOS, MVT::v8f16, Expand); in AArch64TargetLowering()
383 setOperationAction(ISD::FSQRT, MVT::v8f16, Expand); in AArch64TargetLowering()
384 setOperationAction(ISD::FSUB, MVT::v8f16, Expand); in AArch64TargetLowering()
385 setOperationAction(ISD::FTRUNC, MVT::v8f16, Expand); in AArch64TargetLowering()
386 setOperationAction(ISD::SETCC, MVT::v8f16, Expand); in AArch64TargetLowering()
387 setOperationAction(ISD::BR_CC, MVT::v8f16, Expand); in AArch64TargetLowering()
388 setOperationAction(ISD::SELECT, MVT::v8f16, Expand); in AArch64TargetLowering()
389 setOperationAction(ISD::SELECT_CC, MVT::v8f16, Expand); in AArch64TargetLowering()
390 setOperationAction(ISD::FP_EXTEND, MVT::v8f16, Expand); in AArch64TargetLowering()
391 setOperationAction(ISD::FEXP, MVT::v8f16, Expand); in AArch64TargetLowering()
392 setOperationAction(ISD::FEXP2, MVT::v8f16, Expand); in AArch64TargetLowering()
393 setOperationAction(ISD::FLOG, MVT::v8f16, Expand); in AArch64TargetLowering()
394 setOperationAction(ISD::FLOG2, MVT::v8f16, Expand); in AArch64TargetLowering()
395 setOperationAction(ISD::FLOG10, MVT::v8f16, Expand); in AArch64TargetLowering()
399 setOperationAction(ISD::FFLOOR, Ty, Legal); in AArch64TargetLowering()
400 setOperationAction(ISD::FNEARBYINT, Ty, Legal); in AArch64TargetLowering()
401 setOperationAction(ISD::FCEIL, Ty, Legal); in AArch64TargetLowering()
402 setOperationAction(ISD::FRINT, Ty, Legal); in AArch64TargetLowering()
403 setOperationAction(ISD::FTRUNC, Ty, Legal); in AArch64TargetLowering()
404 setOperationAction(ISD::FROUND, Ty, Legal); in AArch64TargetLowering()
407 setOperationAction(ISD::PREFETCH, MVT::Other, Custom); in AArch64TargetLowering()
413 setOperationAction(ISD::FSINCOS, MVT::f64, Custom); in AArch64TargetLowering()
414 setOperationAction(ISD::FSINCOS, MVT::f32, Custom); in AArch64TargetLowering()
416 setOperationAction(ISD::FSINCOS, MVT::f64, Expand); in AArch64TargetLowering()
417 setOperationAction(ISD::FSINCOS, MVT::f32, Expand); in AArch64TargetLowering()
423 setOperationAction(ISD::ConstantFP, MVT::f32, Legal); in AArch64TargetLowering()
424 setOperationAction(ISD::ConstantFP, MVT::f64, Legal); in AArch64TargetLowering()
430 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand); in AArch64TargetLowering()
431 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand); in AArch64TargetLowering()
432 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f64, Expand); in AArch64TargetLowering()
433 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f80, Expand); in AArch64TargetLowering()
436 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Expand); in AArch64TargetLowering()
446 setOperationAction(ISD::BITCAST, MVT::i16, Custom); in AArch64TargetLowering()
447 setOperationAction(ISD::BITCAST, MVT::f16, Custom); in AArch64TargetLowering()
450 for (unsigned im = (unsigned)ISD::PRE_INC; in AArch64TargetLowering()
451 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) { in AArch64TargetLowering()
467 setOperationAction(ISD::TRAP, MVT::Other, Legal); in AArch64TargetLowering()
470 setTargetDAGCombine(ISD::OR); in AArch64TargetLowering()
474 setTargetDAGCombine(ISD::ADD); in AArch64TargetLowering()
475 setTargetDAGCombine(ISD::SUB); in AArch64TargetLowering()
477 setTargetDAGCombine(ISD::XOR); in AArch64TargetLowering()
478 setTargetDAGCombine(ISD::SINT_TO_FP); in AArch64TargetLowering()
479 setTargetDAGCombine(ISD::UINT_TO_FP); in AArch64TargetLowering()
481 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN); in AArch64TargetLowering()
483 setTargetDAGCombine(ISD::ANY_EXTEND); in AArch64TargetLowering()
484 setTargetDAGCombine(ISD::ZERO_EXTEND); in AArch64TargetLowering()
485 setTargetDAGCombine(ISD::SIGN_EXTEND); in AArch64TargetLowering()
486 setTargetDAGCombine(ISD::BITCAST); in AArch64TargetLowering()
487 setTargetDAGCombine(ISD::CONCAT_VECTORS); in AArch64TargetLowering()
488 setTargetDAGCombine(ISD::STORE); in AArch64TargetLowering()
490 setTargetDAGCombine(ISD::MUL); in AArch64TargetLowering()
492 setTargetDAGCombine(ISD::SELECT); in AArch64TargetLowering()
493 setTargetDAGCombine(ISD::VSELECT); in AArch64TargetLowering()
495 setTargetDAGCombine(ISD::INTRINSIC_VOID); in AArch64TargetLowering()
496 setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN); in AArch64TargetLowering()
497 setTargetDAGCombine(ISD::INSERT_VECTOR_ELT); in AArch64TargetLowering()
520 setOperationAction(ISD::FABS, MVT::v1f64, Expand); in AArch64TargetLowering()
521 setOperationAction(ISD::FADD, MVT::v1f64, Expand); in AArch64TargetLowering()
522 setOperationAction(ISD::FCEIL, MVT::v1f64, Expand); in AArch64TargetLowering()
523 setOperationAction(ISD::FCOPYSIGN, MVT::v1f64, Expand); in AArch64TargetLowering()
524 setOperationAction(ISD::FCOS, MVT::v1f64, Expand); in AArch64TargetLowering()
525 setOperationAction(ISD::FDIV, MVT::v1f64, Expand); in AArch64TargetLowering()
526 setOperationAction(ISD::FFLOOR, MVT::v1f64, Expand); in AArch64TargetLowering()
527 setOperationAction(ISD::FMA, MVT::v1f64, Expand); in AArch64TargetLowering()
528 setOperationAction(ISD::FMUL, MVT::v1f64, Expand); in AArch64TargetLowering()
529 setOperationAction(ISD::FNEARBYINT, MVT::v1f64, Expand); in AArch64TargetLowering()
530 setOperationAction(ISD::FNEG, MVT::v1f64, Expand); in AArch64TargetLowering()
531 setOperationAction(ISD::FPOW, MVT::v1f64, Expand); in AArch64TargetLowering()
532 setOperationAction(ISD::FREM, MVT::v1f64, Expand); in AArch64TargetLowering()
533 setOperationAction(ISD::FROUND, MVT::v1f64, Expand); in AArch64TargetLowering()
534 setOperationAction(ISD::FRINT, MVT::v1f64, Expand); in AArch64TargetLowering()
535 setOperationAction(ISD::FSIN, MVT::v1f64, Expand); in AArch64TargetLowering()
536 setOperationAction(ISD::FSINCOS, MVT::v1f64, Expand); in AArch64TargetLowering()
537 setOperationAction(ISD::FSQRT, MVT::v1f64, Expand); in AArch64TargetLowering()
538 setOperationAction(ISD::FSUB, MVT::v1f64, Expand); in AArch64TargetLowering()
539 setOperationAction(ISD::FTRUNC, MVT::v1f64, Expand); in AArch64TargetLowering()
540 setOperationAction(ISD::SETCC, MVT::v1f64, Expand); in AArch64TargetLowering()
541 setOperationAction(ISD::BR_CC, MVT::v1f64, Expand); in AArch64TargetLowering()
542 setOperationAction(ISD::SELECT, MVT::v1f64, Expand); in AArch64TargetLowering()
543 setOperationAction(ISD::SELECT_CC, MVT::v1f64, Expand); in AArch64TargetLowering()
544 setOperationAction(ISD::FP_EXTEND, MVT::v1f64, Expand); in AArch64TargetLowering()
546 setOperationAction(ISD::FP_TO_SINT, MVT::v1i64, Expand); in AArch64TargetLowering()
547 setOperationAction(ISD::FP_TO_UINT, MVT::v1i64, Expand); in AArch64TargetLowering()
548 setOperationAction(ISD::SINT_TO_FP, MVT::v1i64, Expand); in AArch64TargetLowering()
549 setOperationAction(ISD::UINT_TO_FP, MVT::v1i64, Expand); in AArch64TargetLowering()
550 setOperationAction(ISD::FP_ROUND, MVT::v1f64, Expand); in AArch64TargetLowering()
552 setOperationAction(ISD::MUL, MVT::v1i64, Expand); in AArch64TargetLowering()
556 setOperationAction(ISD::UINT_TO_FP, MVT::v4i8, Promote); in AArch64TargetLowering()
557 setOperationAction(ISD::SINT_TO_FP, MVT::v4i8, Promote); in AArch64TargetLowering()
558 setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Promote); in AArch64TargetLowering()
559 setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Promote); in AArch64TargetLowering()
562 setOperationAction(ISD::SINT_TO_FP, MVT::v8i8, Promote); in AArch64TargetLowering()
563 setOperationAction(ISD::UINT_TO_FP, MVT::v8i8, Promote); in AArch64TargetLowering()
564 setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Promote); in AArch64TargetLowering()
565 setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Promote); in AArch64TargetLowering()
567 setOperationAction(ISD::SINT_TO_FP, MVT::v2i32, Custom); in AArch64TargetLowering()
568 setOperationAction(ISD::UINT_TO_FP, MVT::v2i32, Custom); in AArch64TargetLowering()
569 setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Custom); in AArch64TargetLowering()
570 setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Custom); in AArch64TargetLowering()
573 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Custom); in AArch64TargetLowering()
574 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom); in AArch64TargetLowering()
577 setOperationAction(ISD::MUL, MVT::v2i64, Expand); in AArch64TargetLowering()
579 setOperationAction(ISD::MUL, MVT::v8i16, Custom); in AArch64TargetLowering()
580 setOperationAction(ISD::MUL, MVT::v4i32, Custom); in AArch64TargetLowering()
581 setOperationAction(ISD::MUL, MVT::v2i64, Custom); in AArch64TargetLowering()
583 setOperationAction(ISD::ANY_EXTEND, MVT::v4i32, Legal); in AArch64TargetLowering()
588 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand); in AArch64TargetLowering()
590 setOperationAction(ISD::MULHS, VT, Expand); in AArch64TargetLowering()
591 setOperationAction(ISD::SMUL_LOHI, VT, Expand); in AArch64TargetLowering()
592 setOperationAction(ISD::MULHU, VT, Expand); in AArch64TargetLowering()
593 setOperationAction(ISD::UMUL_LOHI, VT, Expand); in AArch64TargetLowering()
595 setOperationAction(ISD::BSWAP, VT, Expand); in AArch64TargetLowering()
599 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand); in AArch64TargetLowering()
600 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand); in AArch64TargetLowering()
601 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand); in AArch64TargetLowering()
607 setOperationAction(ISD::FFLOOR, Ty, Legal); in AArch64TargetLowering()
608 setOperationAction(ISD::FNEARBYINT, Ty, Legal); in AArch64TargetLowering()
609 setOperationAction(ISD::FCEIL, Ty, Legal); in AArch64TargetLowering()
610 setOperationAction(ISD::FRINT, Ty, Legal); in AArch64TargetLowering()
611 setOperationAction(ISD::FTRUNC, Ty, Legal); in AArch64TargetLowering()
612 setOperationAction(ISD::FROUND, Ty, Legal); in AArch64TargetLowering()
623 setOperationAction(ISD::LOAD, VT.getSimpleVT(), Promote); in addTypeForNEON()
624 AddPromotedToType(ISD::LOAD, VT.getSimpleVT(), MVT::v2i32); in addTypeForNEON()
626 setOperationAction(ISD::STORE, VT.getSimpleVT(), Promote); in addTypeForNEON()
627 AddPromotedToType(ISD::STORE, VT.getSimpleVT(), MVT::v2i32); in addTypeForNEON()
629 setOperationAction(ISD::LOAD, VT.getSimpleVT(), Promote); in addTypeForNEON()
630 AddPromotedToType(ISD::LOAD, VT.getSimpleVT(), MVT::v2i64); in addTypeForNEON()
632 setOperationAction(ISD::STORE, VT.getSimpleVT(), Promote); in addTypeForNEON()
633 AddPromotedToType(ISD::STORE, VT.getSimpleVT(), MVT::v2i64); in addTypeForNEON()
638 setOperationAction(ISD::FSIN, VT.getSimpleVT(), Expand); in addTypeForNEON()
639 setOperationAction(ISD::FCOS, VT.getSimpleVT(), Expand); in addTypeForNEON()
640 setOperationAction(ISD::FPOWI, VT.getSimpleVT(), Expand); in addTypeForNEON()
641 setOperationAction(ISD::FPOW, VT.getSimpleVT(), Expand); in addTypeForNEON()
642 setOperationAction(ISD::FLOG, VT.getSimpleVT(), Expand); in addTypeForNEON()
643 setOperationAction(ISD::FLOG2, VT.getSimpleVT(), Expand); in addTypeForNEON()
644 setOperationAction(ISD::FLOG10, VT.getSimpleVT(), Expand); in addTypeForNEON()
645 setOperationAction(ISD::FEXP, VT.getSimpleVT(), Expand); in addTypeForNEON()
646 setOperationAction(ISD::FEXP2, VT.getSimpleVT(), Expand); in addTypeForNEON()
649 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT.getSimpleVT(), Custom); in addTypeForNEON()
650 setOperationAction(ISD::INSERT_VECTOR_ELT, VT.getSimpleVT(), Custom); in addTypeForNEON()
651 setOperationAction(ISD::BUILD_VECTOR, VT.getSimpleVT(), Custom); in addTypeForNEON()
652 setOperationAction(ISD::VECTOR_SHUFFLE, VT.getSimpleVT(), Custom); in addTypeForNEON()
653 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT.getSimpleVT(), Custom); in addTypeForNEON()
654 setOperationAction(ISD::SRA, VT.getSimpleVT(), Custom); in addTypeForNEON()
655 setOperationAction(ISD::SRL, VT.getSimpleVT(), Custom); in addTypeForNEON()
656 setOperationAction(ISD::SHL, VT.getSimpleVT(), Custom); in addTypeForNEON()
657 setOperationAction(ISD::AND, VT.getSimpleVT(), Custom); in addTypeForNEON()
658 setOperationAction(ISD::OR, VT.getSimpleVT(), Custom); in addTypeForNEON()
659 setOperationAction(ISD::SETCC, VT.getSimpleVT(), Custom); in addTypeForNEON()
660 setOperationAction(ISD::CONCAT_VECTORS, VT.getSimpleVT(), Legal); in addTypeForNEON()
662 setOperationAction(ISD::SELECT, VT.getSimpleVT(), Expand); in addTypeForNEON()
663 setOperationAction(ISD::SELECT_CC, VT.getSimpleVT(), Expand); in addTypeForNEON()
664 setOperationAction(ISD::VSELECT, VT.getSimpleVT(), Expand); in addTypeForNEON()
666 setLoadExtAction(ISD::EXTLOAD, InnerVT, VT.getSimpleVT(), Expand); in addTypeForNEON()
670 setOperationAction(ISD::CTPOP, VT.getSimpleVT(), Expand); in addTypeForNEON()
672 setOperationAction(ISD::UDIV, VT.getSimpleVT(), Expand); in addTypeForNEON()
673 setOperationAction(ISD::SDIV, VT.getSimpleVT(), Expand); in addTypeForNEON()
674 setOperationAction(ISD::UREM, VT.getSimpleVT(), Expand); in addTypeForNEON()
675 setOperationAction(ISD::SREM, VT.getSimpleVT(), Expand); in addTypeForNEON()
676 setOperationAction(ISD::FREM, VT.getSimpleVT(), Expand); in addTypeForNEON()
678 setOperationAction(ISD::FP_TO_SINT, VT.getSimpleVT(), Custom); in addTypeForNEON()
679 setOperationAction(ISD::FP_TO_UINT, VT.getSimpleVT(), Custom); in addTypeForNEON()
682 for (unsigned im = (unsigned)ISD::PRE_INC; in addTypeForNEON()
683 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) { in addTypeForNEON()
723 case ISD::INTRINSIC_W_CHAIN: { in computeKnownBitsForTargetNode()
739 case ISD::INTRINSIC_WO_CHAIN: in computeKnownBitsForTargetNode()
740 case ISD::INTRINSIC_VOID: { in computeKnownBitsForTargetNode()
994 static AArch64CC::CondCode changeIntCCToAArch64CC(ISD::CondCode CC) { in changeIntCCToAArch64CC()
998 case ISD::SETNE: in changeIntCCToAArch64CC()
1000 case ISD::SETEQ: in changeIntCCToAArch64CC()
1002 case ISD::SETGT: in changeIntCCToAArch64CC()
1004 case ISD::SETGE: in changeIntCCToAArch64CC()
1006 case ISD::SETLT: in changeIntCCToAArch64CC()
1008 case ISD::SETLE: in changeIntCCToAArch64CC()
1010 case ISD::SETUGT: in changeIntCCToAArch64CC()
1012 case ISD::SETUGE: in changeIntCCToAArch64CC()
1014 case ISD::SETULT: in changeIntCCToAArch64CC()
1016 case ISD::SETULE: in changeIntCCToAArch64CC()
1022 static void changeFPCCToAArch64CC(ISD::CondCode CC, in changeFPCCToAArch64CC()
1029 case ISD::SETEQ: in changeFPCCToAArch64CC()
1030 case ISD::SETOEQ: in changeFPCCToAArch64CC()
1033 case ISD::SETGT: in changeFPCCToAArch64CC()
1034 case ISD::SETOGT: in changeFPCCToAArch64CC()
1037 case ISD::SETGE: in changeFPCCToAArch64CC()
1038 case ISD::SETOGE: in changeFPCCToAArch64CC()
1041 case ISD::SETOLT: in changeFPCCToAArch64CC()
1044 case ISD::SETOLE: in changeFPCCToAArch64CC()
1047 case ISD::SETONE: in changeFPCCToAArch64CC()
1051 case ISD::SETO: in changeFPCCToAArch64CC()
1054 case ISD::SETUO: in changeFPCCToAArch64CC()
1057 case ISD::SETUEQ: in changeFPCCToAArch64CC()
1061 case ISD::SETUGT: in changeFPCCToAArch64CC()
1064 case ISD::SETUGE: in changeFPCCToAArch64CC()
1067 case ISD::SETLT: in changeFPCCToAArch64CC()
1068 case ISD::SETULT: in changeFPCCToAArch64CC()
1071 case ISD::SETLE: in changeFPCCToAArch64CC()
1072 case ISD::SETULE: in changeFPCCToAArch64CC()
1075 case ISD::SETNE: in changeFPCCToAArch64CC()
1076 case ISD::SETUNE: in changeFPCCToAArch64CC()
1086 static void changeVectorFPCCToAArch64CC(ISD::CondCode CC, in changeVectorFPCCToAArch64CC()
1096 case ISD::SETUO: in changeVectorFPCCToAArch64CC()
1098 case ISD::SETO: in changeVectorFPCCToAArch64CC()
1102 case ISD::SETUEQ: in changeVectorFPCCToAArch64CC()
1103 case ISD::SETULT: in changeVectorFPCCToAArch64CC()
1104 case ISD::SETULE: in changeVectorFPCCToAArch64CC()
1105 case ISD::SETUGT: in changeVectorFPCCToAArch64CC()
1106 case ISD::SETUGE: in changeVectorFPCCToAArch64CC()
1120 static SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC, in emitComparison()
1133 if (RHS.getOpcode() == ISD::SUB && isa<ConstantSDNode>(RHS.getOperand(0)) && in emitComparison()
1135 (CC == ISD::SETEQ || CC == ISD::SETNE)) { in emitComparison()
1148 } else if (LHS.getOpcode() == ISD::AND && isa<ConstantSDNode>(RHS) && in emitComparison()
1163 static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC, in getAArch64Cmp()
1175 case ISD::SETLT: in getAArch64Cmp()
1176 case ISD::SETGE: in getAArch64Cmp()
1181 CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT; in getAArch64Cmp()
1186 case ISD::SETULT: in getAArch64Cmp()
1187 case ISD::SETUGE: in getAArch64Cmp()
1191 CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT; in getAArch64Cmp()
1196 case ISD::SETLE: in getAArch64Cmp()
1197 case ISD::SETGT: in getAArch64Cmp()
1202 CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE; in getAArch64Cmp()
1207 case ISD::SETULE: in getAArch64Cmp()
1208 case ISD::SETUGT: in getAArch64Cmp()
1213 CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE; in getAArch64Cmp()
1237 if ((CC == ISD::SETEQ || CC == ISD::SETNE) && isa<ConstantSDNode>(RHS)) { in getAArch64Cmp()
1240 if (cast<LoadSDNode>(LHS)->getExtensionType() == ISD::ZEXTLOAD && in getAArch64Cmp()
1246 DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, LHS.getValueType(), LHS, in getAArch64Cmp()
1276 case ISD::SADDO: in getAArch64XALUOOp()
1280 case ISD::UADDO: in getAArch64XALUOOp()
1284 case ISD::SSUBO: in getAArch64XALUOOp()
1288 case ISD::USUBO: in getAArch64XALUOOp()
1293 case ISD::SMULO: in getAArch64XALUOOp()
1294 case ISD::UMULO: { in getAArch64XALUOOp()
1296 bool IsSigned = Op.getOpcode() == ISD::SMULO; in getAArch64XALUOOp()
1298 unsigned ExtendOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in getAArch64XALUOOp()
1305 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS); in getAArch64XALUOOp()
1306 SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Mul, in getAArch64XALUOOp()
1312 Value = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Add); in getAArch64XALUOOp()
1319 SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Add, in getAArch64XALUOOp()
1321 UpperBits = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, UpperBits); in getAArch64XALUOOp()
1322 SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i32, Value, in getAArch64XALUOOp()
1335 SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul, in getAArch64XALUOOp()
1346 Value = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS); in getAArch64XALUOOp()
1348 SDValue UpperBits = DAG.getNode(ISD::MULHS, DL, MVT::i64, LHS, RHS); in getAArch64XALUOOp()
1349 SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i64, Value, in getAArch64XALUOOp()
1357 SDValue UpperBits = DAG.getNode(ISD::MULHU, DL, MVT::i64, LHS, RHS); in getAArch64XALUOOp()
1389 if (Sel.getOpcode() != ISD::SELECT_CC) in LowerXOR()
1391 if (Sel.getOpcode() != ISD::SELECT_CC) in LowerXOR()
1401 ISD::CondCode CC = cast<CondCodeSDNode>(Sel.getOperand(4))->get(); in LowerXOR()
1424 CC = ISD::getSetCCInverse(CC, true); in LowerXOR()
1433 TVal = DAG.getNode(ISD::XOR, dl, Other.getValueType(), Other, in LowerXOR()
1457 case ISD::ADDC: in LowerADDC_ADDE_SUBC_SUBE()
1460 case ISD::SUBC: in LowerADDC_ADDE_SUBC_SUBE()
1463 case ISD::ADDE: in LowerADDC_ADDE_SUBC_SUBE()
1467 case ISD::SUBE: in LowerADDC_ADDE_SUBC_SUBE()
1501 return DAG.getNode(ISD::MERGE_VALUES, SDLoc(Op), VTs, Value, Overflow); in LowerXALUO()
1575 return DAG.getNode(ISD::TRUNCATE, dl, VT, Cv); in LowerVectorFP_TO_INT()
1583 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, dl, ExtVT, Op.getOperand(0)); in LowerVectorFP_TO_INT()
1601 DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, Op.getOperand(0))); in LowerFP_TO_INT()
1610 if (Op.getOpcode() == ISD::FP_TO_SINT) in LowerFP_TO_INT()
1634 return DAG.getNode(ISD::FP_ROUND, dl, VT, In, DAG.getIntPtrConstant(0)); in LowerVectorINT_TO_FP()
1639 Op.getOpcode() == ISD::SINT_TO_FP ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; in LowerVectorINT_TO_FP()
1657 ISD::FP_ROUND, dl, MVT::f16, in LowerINT_TO_FP()
1672 if (Op.getOpcode() == ISD::SINT_TO_FP) in LowerINT_TO_FP()
1718 Op = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op.getOperand(0)); in LowerBITCAST()
1719 Op = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Op); in LowerBITCAST()
1764 if (N->getOpcode() != ISD::BUILD_VECTOR) in isExtendedBUILD_VECTOR()
1788 if (N->getOpcode() == ISD::SIGN_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND) in skipExtensionForVectorMULL()
1794 assert(N->getOpcode() == ISD::BUILD_VECTOR && "expected BUILD_VECTOR"); in skipExtensionForVectorMULL()
1807 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), in skipExtensionForVectorMULL()
1812 if (N->getOpcode() == ISD::SIGN_EXTEND) in isSignExtended()
1820 if (N->getOpcode() == ISD::ZERO_EXTEND) in isZeroExtended()
1829 if (Opcode == ISD::ADD || Opcode == ISD::SUB) { in isAddSubSExt()
1840 if (Opcode == ISD::ADD || Opcode == ISD::SUB) { in isAddSubZExt()
1913 DAG.getNode(ISD::BITCAST, DL, Op1VT, N00), Op1), in LowerMUL()
1915 DAG.getNode(ISD::BITCAST, DL, Op1VT, N01), Op1)); in LowerMUL()
1924 case ISD::BITCAST: in LowerOperation()
1926 case ISD::GlobalAddress: in LowerOperation()
1928 case ISD::GlobalTLSAddress: in LowerOperation()
1930 case ISD::SETCC: in LowerOperation()
1932 case ISD::BR_CC: in LowerOperation()
1934 case ISD::SELECT: in LowerOperation()
1936 case ISD::SELECT_CC: in LowerOperation()
1938 case ISD::JumpTable: in LowerOperation()
1940 case ISD::ConstantPool: in LowerOperation()
1942 case ISD::BlockAddress: in LowerOperation()
1944 case ISD::VASTART: in LowerOperation()
1946 case ISD::VACOPY: in LowerOperation()
1948 case ISD::VAARG: in LowerOperation()
1950 case ISD::ADDC: in LowerOperation()
1951 case ISD::ADDE: in LowerOperation()
1952 case ISD::SUBC: in LowerOperation()
1953 case ISD::SUBE: in LowerOperation()
1955 case ISD::SADDO: in LowerOperation()
1956 case ISD::UADDO: in LowerOperation()
1957 case ISD::SSUBO: in LowerOperation()
1958 case ISD::USUBO: in LowerOperation()
1959 case ISD::SMULO: in LowerOperation()
1960 case ISD::UMULO: in LowerOperation()
1962 case ISD::FADD: in LowerOperation()
1964 case ISD::FSUB: in LowerOperation()
1966 case ISD::FMUL: in LowerOperation()
1968 case ISD::FDIV: in LowerOperation()
1970 case ISD::FP_ROUND: in LowerOperation()
1972 case ISD::FP_EXTEND: in LowerOperation()
1974 case ISD::FRAMEADDR: in LowerOperation()
1976 case ISD::RETURNADDR: in LowerOperation()
1978 case ISD::INSERT_VECTOR_ELT: in LowerOperation()
1980 case ISD::EXTRACT_VECTOR_ELT: in LowerOperation()
1982 case ISD::BUILD_VECTOR: in LowerOperation()
1984 case ISD::VECTOR_SHUFFLE: in LowerOperation()
1986 case ISD::EXTRACT_SUBVECTOR: in LowerOperation()
1988 case ISD::SRA: in LowerOperation()
1989 case ISD::SRL: in LowerOperation()
1990 case ISD::SHL: in LowerOperation()
1992 case ISD::SHL_PARTS: in LowerOperation()
1994 case ISD::SRL_PARTS: in LowerOperation()
1995 case ISD::SRA_PARTS: in LowerOperation()
1997 case ISD::CTPOP: in LowerOperation()
1999 case ISD::FCOPYSIGN: in LowerOperation()
2001 case ISD::AND: in LowerOperation()
2003 case ISD::OR: in LowerOperation()
2005 case ISD::XOR: in LowerOperation()
2007 case ISD::PREFETCH: in LowerOperation()
2009 case ISD::SINT_TO_FP: in LowerOperation()
2010 case ISD::UINT_TO_FP: in LowerOperation()
2012 case ISD::FP_TO_SINT: in LowerOperation()
2013 case ISD::FP_TO_UINT: in LowerOperation()
2015 case ISD::FSINCOS: in LowerOperation()
2017 case ISD::MUL: in LowerOperation()
2053 const SmallVectorImpl<ISD::InputArg> &Ins, SDLoc DL, SelectionDAG &DAG, in LowerFormalArguments()
2150 ArgValue = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), ArgValue); in LowerFormalArguments()
2180 ISD::LoadExtType ExtType = ISD::NON_EXTLOAD; in LowerFormalArguments()
2190 ExtType = ISD::SEXTLOAD; in LowerFormalArguments()
2193 ExtType = ISD::ZEXTLOAD; in LowerFormalArguments()
2196 ExtType = ISD::EXTLOAD; in LowerFormalArguments()
2278 FIN = DAG.getNode(ISD::ADD, DL, getPointerTy(), FIN, in saveVarArgRegisters()
2307 FIN = DAG.getNode(ISD::ADD, DL, getPointerTy(), FIN, in saveVarArgRegisters()
2316 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); in saveVarArgRegisters()
2324 const SmallVectorImpl<ISD::InputArg> &Ins, SDLoc DL, SelectionDAG &DAG, in LowerCallResult()
2360 Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val); in LowerCallResult()
2373 const SmallVectorImpl<ISD::OutputArg> &Outs, in isEligibleForTailCallOptimization()
2375 const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const { in isEligibleForTailCallOptimization()
2520 return DAG.getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains); in addTokenForArgument()
2539 SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs; in LowerCall()
2541 SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins; in LowerCall()
2586 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags; in LowerCall()
2607 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags; in LowerCall()
2673 ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags; in LowerCall()
2682 Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg); in LowerCall()
2685 Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg); in LowerCall()
2690 Arg = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Arg); in LowerCall()
2691 Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i8, Arg); in LowerCall()
2693 Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg); in LowerCall()
2696 Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg); in LowerCall()
2699 Arg = DAG.getNode(ISD::FP_EXTEND, DL, VA.getLocVT(), Arg); in LowerCall()
2732 PtrOff = DAG.getNode(ISD::ADD, DL, getPointerTy(), StackPtr, PtrOff); in LowerCall()
2748 DstAddr = DAG.getNode(ISD::ADD, DL, getPointerTy(), StackPtr, PtrOff); in LowerCall()
2768 Arg = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Arg); in LowerCall()
2778 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains); in LowerCall()
2895 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const { in CanLowerReturn()
2907 const SmallVectorImpl<ISD::OutputArg> &Outs, in LowerReturn()
2935 Arg = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Arg); in LowerReturn()
2936 Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg); in LowerReturn()
2940 Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg); in LowerReturn()
2996 return DAG.getNode(ISD::ADD, DL, PtrVT, GlobalAddr, in LowerGlobalAddress()
3218 return DAG.getNode(ISD::ADD, DL, PtrVT, ThreadBase, TPOff); in LowerELFGlobalTLSAddress()
3232 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get(); in LowerBR_CC()
3248 CC = ISD::SETNE; in LowerBR_CC()
3257 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO || in LowerBR_CC()
3258 Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO)) { in LowerBR_CC()
3259 assert((CC == ISD::SETEQ || CC == ISD::SETNE) && in LowerBR_CC()
3270 if (CC == ISD::SETNE) in LowerBR_CC()
3286 if (CC == ISD::SETEQ) { in LowerBR_CC()
3291 if (LHS.getOpcode() == ISD::AND && in LowerBR_CC()
3301 } else if (CC == ISD::SETNE) { in LowerBR_CC()
3306 if (LHS.getOpcode() == ISD::AND && in LowerBR_CC()
3316 } else if (CC == ISD::SETLT && LHS.getOpcode() != ISD::AND) { in LowerBR_CC()
3325 if (RHSC && RHSC->getSExtValue() == -1 && CC == ISD::SETGT && in LowerBR_CC()
3326 LHS.getOpcode() != ISD::AND) { in LowerBR_CC()
3370 In2 = DAG.getNode(ISD::FP_EXTEND, DL, VT, In2); in LowerFCOPYSIGN()
3372 In2 = DAG.getNode(ISD::FP_ROUND, DL, VT, In2, DAG.getIntPtrConstant(0)); in LowerFCOPYSIGN()
3394 VecVal1 = DAG.getNode(ISD::BITCAST, DL, VecVT, In1); in LowerFCOPYSIGN()
3395 VecVal2 = DAG.getNode(ISD::BITCAST, DL, VecVT, In2); in LowerFCOPYSIGN()
3412 VecVal1 = DAG.getNode(ISD::BITCAST, DL, VecVT, In1); in LowerFCOPYSIGN()
3413 VecVal2 = DAG.getNode(ISD::BITCAST, DL, VecVT, In2); in LowerFCOPYSIGN()
3424 BuildVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, BuildVec); in LowerFCOPYSIGN()
3425 BuildVec = DAG.getNode(ISD::FNEG, DL, MVT::v2f64, BuildVec); in LowerFCOPYSIGN()
3426 BuildVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, BuildVec); in LowerFCOPYSIGN()
3437 return DAG.getNode(ISD::BITCAST, DL, VT, Sel); in LowerFCOPYSIGN()
3461 Val = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Val); in LowerCTPOP()
3462 Val = DAG.getNode(ISD::BITCAST, DL, MVT::v8i8, Val); in LowerCTPOP()
3464 SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v8i8, Val); in LowerCTPOP()
3466 ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32, in LowerCTPOP()
3470 UaddLV = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, UaddLV); in LowerCTPOP()
3481 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); in LowerSETCC()
3505 getAArch64Cmp(LHS, RHS, ISD::getSetCCInverse(CC, true), CCVal, DAG, dl); in LowerSETCC()
3523 changeFPCCToAArch64CC(ISD::getSetCCInverse(CC, false), CC1, CC2); in LowerSETCC()
3568 return Result->getOpcode() == ISD::FP_EXTEND && Result->getOperand(0) == Cmp; in selectCCOpsAreFMaxCompatible()
3571 SDValue AArch64TargetLowering::LowerSELECT_CC(ISD::CondCode CC, SDValue LHS, in LowerSELECT_CC()
3584 CC = ISD::SETNE; in LowerSELECT_CC()
3603 CC = ISD::getSetCCInverse(CC, true); in LowerSELECT_CC()
3607 CC = ISD::getSetCCInverse(CC, true); in LowerSELECT_CC()
3608 } else if (TVal.getOpcode() == ISD::XOR) { in LowerSELECT_CC()
3616 CC = ISD::getSetCCInverse(CC, true); in LowerSELECT_CC()
3618 } else if (TVal.getOpcode() == ISD::SUB) { in LowerSELECT_CC()
3626 CC = ISD::getSetCCInverse(CC, true); in LowerSELECT_CC()
3669 CC = ISD::getSetCCInverse(CC, true); in LowerSELECT_CC()
3699 CC = ISD::getSetCCSwappedOperands(CC); in LowerSELECT_CC()
3708 case ISD::SETGT: in LowerSELECT_CC()
3709 case ISD::SETGE: in LowerSELECT_CC()
3710 case ISD::SETUGT: in LowerSELECT_CC()
3711 case ISD::SETUGE: in LowerSELECT_CC()
3712 case ISD::SETOGT: in LowerSELECT_CC()
3713 case ISD::SETOGE: in LowerSELECT_CC()
3716 case ISD::SETLT: in LowerSELECT_CC()
3717 case ISD::SETLE: in LowerSELECT_CC()
3718 case ISD::SETULT: in LowerSELECT_CC()
3719 case ISD::SETULE: in LowerSELECT_CC()
3720 case ISD::SETOLT: in LowerSELECT_CC()
3721 case ISD::SETOLE: in LowerSELECT_CC()
3752 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get(); in LowerSELECT_CC()
3772 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO || in LowerSELECT()
3773 Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO)) { in LowerSELECT()
3788 ISD::CondCode CC; in LowerSELECT()
3790 if (CCVal.getOpcode() == ISD::SETCC) { in LowerSELECT()
3797 CC = ISD::SETNE; in LowerSELECT()
3931 GRTopAddr = DAG.getNode(ISD::ADD, DL, getPointerTy(), VAList, in LowerAAPCS_VASTART()
3935 GRTop = DAG.getNode(ISD::ADD, DL, getPointerTy(), GRTop, in LowerAAPCS_VASTART()
3946 VRTopAddr = DAG.getNode(ISD::ADD, DL, getPointerTy(), VAList, in LowerAAPCS_VASTART()
3950 VRTop = DAG.getNode(ISD::ADD, DL, getPointerTy(), VRTop, in LowerAAPCS_VASTART()
3958 SDValue GROffsAddr = DAG.getNode(ISD::ADD, DL, getPointerTy(), VAList, in LowerAAPCS_VASTART()
3965 SDValue VROffsAddr = DAG.getNode(ISD::ADD, DL, getPointerTy(), VAList, in LowerAAPCS_VASTART()
3971 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps); in LowerAAPCS_VASTART()
4011 VAList = DAG.getNode(ISD::ADD, DL, getPointerTy(), VAList, in LowerVAARG()
4013 VAList = DAG.getNode(ISD::AND, DL, getPointerTy(), VAList, in LowerVAARG()
4033 SDValue VANext = DAG.getNode(ISD::ADD, DL, getPointerTy(), VAList, in LowerVAARG()
4045 SDValue NarrowFP = DAG.getNode(ISD::FP_ROUND, DL, VT, WideFP.getValue(0), in LowerVAARG()
4097 DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset), in LowerRETURNADDR()
4118 unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL; in LowerShiftRightParts()
4120 assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS); in LowerShiftRightParts()
4122 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, in LowerShiftRightParts()
4124 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt); in LowerShiftRightParts()
4125 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, ShAmt, in LowerShiftRightParts()
4127 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt); in LowerShiftRightParts()
4130 ISD::SETGE, dl, DAG); in LowerShiftRightParts()
4133 SDValue FalseValLo = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2); in LowerShiftRightParts()
4141 SDValue TrueValHi = Opc == ISD::SRA in LowerShiftRightParts()
4165 assert(Op.getOpcode() == ISD::SHL_PARTS); in LowerShiftLeftParts()
4166 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, in LowerShiftLeftParts()
4168 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt); in LowerShiftLeftParts()
4169 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, ShAmt, in LowerShiftLeftParts()
4171 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt); in LowerShiftLeftParts()
4172 SDValue Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt); in LowerShiftLeftParts()
4174 SDValue FalseVal = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2); in LowerShiftLeftParts()
4177 ISD::SETGE, dl, DAG); in LowerShiftLeftParts()
4185 SDValue FalseValLo = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt); in LowerShiftLeftParts()
4502 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideTy, DAG.getUNDEF(WideTy), in WidenVector()
4529 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!"); in ReconstructShuffle()
4560 if (V.getOpcode() == ISD::UNDEF) in ReconstructShuffle()
4562 else if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT) { in ReconstructShuffle()
4619 DAG.getNode(ISD::CONCAT_VECTORS, dl, DestVT, Src.ShuffleVec, in ReconstructShuffle()
4634 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, in ReconstructShuffle()
4640 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, in ReconstructShuffle()
4645 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, in ReconstructShuffle()
4648 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec, in ReconstructShuffle()
4666 Src.ShuffleVec = DAG.getNode(ISD::BITCAST, dl, ShuffleVT, Src.ShuffleVec); in ReconstructShuffle()
4682 if (Entry.getOpcode() == ISD::UNDEF) in ReconstructShuffle()
4716 return DAG.getNode(ISD::BITCAST, dl, VT, Shuffle); in ReconstructShuffle()
4987 V0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, CastVT, V0, in tryFormConcatFromShuffle()
4991 V1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, CastVT, V1, in tryFormConcatFromShuffle()
4994 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, V0, V1); in tryFormConcatFromShuffle()
5126 SDValue V1Cst = DAG.getNode(ISD::BITCAST, DL, IndexVT, V1); in GenerateTBL()
5127 SDValue V2Cst = DAG.getNode(ISD::BITCAST, DL, IndexVT, V2); in GenerateTBL()
5130 if (V2.getNode()->getOpcode() == ISD::UNDEF) { in GenerateTBL()
5132 V1Cst = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V1Cst, V1Cst); in GenerateTBL()
5134 ISD::INTRINSIC_WO_CHAIN, DL, IndexVT, in GenerateTBL()
5136 DAG.getNode(ISD::BUILD_VECTOR, DL, IndexVT, in GenerateTBL()
5140 V1Cst = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V1Cst, V2Cst); in GenerateTBL()
5142 ISD::INTRINSIC_WO_CHAIN, DL, IndexVT, in GenerateTBL()
5144 DAG.getNode(ISD::BUILD_VECTOR, DL, IndexVT, in GenerateTBL()
5154 ISD::INTRINSIC_WO_CHAIN, DL, IndexVT, in GenerateTBL()
5156 DAG.getNode(ISD::BUILD_VECTOR, DL, IndexVT, in GenerateTBL()
5160 return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Shuffle); in GenerateTBL()
5199 if (Lane == 0 && V1.getOpcode() == ISD::SCALAR_TO_VECTOR) in LowerVECTOR_SHUFFLE()
5204 if (V1.getOpcode() == ISD::BUILD_VECTOR && in LowerVECTOR_SHUFFLE()
5214 if (V1.getOpcode() == ISD::EXTRACT_SUBVECTOR) { in LowerVECTOR_SHUFFLE()
5217 } else if (V1.getOpcode() == ISD::CONCAT_VECTORS) { in LowerVECTOR_SHUFFLE()
5242 } else if (V2->getOpcode() == ISD::UNDEF && in LowerVECTOR_SHUFFLE()
5301 ISD::INSERT_VECTOR_ELT, dl, VT, DstVec, in LowerVECTOR_SHUFFLE()
5302 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, SrcVec, SrcLaneV), in LowerVECTOR_SHUFFLE()
5471 case ISD::INTRINSIC_WO_CHAIN: { in getIntrinsicID()
5494 if (And.getOpcode() != ISD::AND) in tryLowerToSLI()
5532 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, in tryLowerToSLI()
5650 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!"); in NormalizeBuildVector()
5661 if (Lane.getOpcode() == ISD::Constant) { in NormalizeBuildVector()
5668 return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, Ops); in NormalizeBuildVector()
5913 if (V.getOpcode() == ISD::UNDEF) in LowerBUILD_VECTOR()
5938 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value); in LowerBUILD_VECTOR()
5944 if (Value.getOpcode() != ISD::EXTRACT_VECTOR_ELT || in LowerBUILD_VECTOR()
5967 Ops.push_back(DAG.getNode(ISD::BITCAST, dl, NewType, Op.getOperand(i))); in LowerBUILD_VECTOR()
5969 SDValue Val = DAG.getNode(ISD::BUILD_VECTOR, dl, VecVT, Ops); in LowerBUILD_VECTOR()
5972 return DAG.getNode(ISD::BITCAST, dl, VT, Val); in LowerBUILD_VECTOR()
5989 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Val, V, LaneIdx); in LowerBUILD_VECTOR()
6023 if (Op0.getOpcode() != ISD::UNDEF && (ElemSize == 32 || ElemSize == 64)) { in LowerBUILD_VECTOR()
6033 if (V.getOpcode() == ISD::UNDEF) in LowerBUILD_VECTOR()
6036 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Vec, V, LaneIdx); in LowerBUILD_VECTOR()
6047 assert(Op.getOpcode() == ISD::INSERT_VECTOR_ELT && "Unknown opcode!"); in LowerINSERT_VECTOR_ELT()
6072 SDValue Node = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, WideTy, WideVec, in LowerINSERT_VECTOR_ELT()
6081 assert(Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT && "Unknown opcode!"); in LowerEXTRACT_VECTOR_ELT()
6111 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtrTy, WideVec, in LowerEXTRACT_VECTOR_ELT()
6199 while (Op.getOpcode() == ISD::BITCAST) in getVShiftImm()
6256 case ISD::SHL: in LowerVectorSRA_SRL_SHL()
6260 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, in LowerVectorSRA_SRL_SHL()
6263 case ISD::SRA: in LowerVectorSRA_SRL_SHL()
6264 case ISD::SRL: in LowerVectorSRA_SRL_SHL()
6269 (Op.getOpcode() == ISD::SRA) ? AArch64ISD::VASHR : AArch64ISD::VLSHR; in LowerVectorSRA_SRL_SHL()
6277 unsigned Opc = (Op.getOpcode() == ISD::SRA) ? Intrinsic::aarch64_neon_sshl in LowerVectorSRA_SRL_SHL()
6282 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, in LowerVectorSRA_SRL_SHL()
6387 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get(); in LowerVSETCC()
6422 Cmp = DAG.getNode(ISD::OR, dl, CmpVT, Cmp, Cmp2); in LowerVSETCC()
6452 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
6473 Info.opc = ISD::INTRINSIC_VOID; in getTgtMemIntrinsic()
6494 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
6507 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
6519 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
6531 Info.opc = ISD::INTRINSIC_W_CHAIN; in getTgtMemIntrinsic()
6585 isOperationLegalOrCustom(ISD::FMA, VT) && in isProfitableToHoist()
6615 if (Val.getOpcode() != ISD::LOAD) in isZExtFree()
6850 if (N->getOpcode() == ISD::AND && (VT == MVT::i32 || VT == MVT::i64) && in isDesirableToCommuteWithShift()
6854 N->getOperand(0).getOpcode() == ISD::SRL && in isDesirableToCommuteWithShift()
6894 if (VT.isInteger() && N->getOpcode() == ISD::XOR && in performIntegerAbsCombine()
6895 N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1 && in performIntegerAbsCombine()
6896 N1.getOpcode() == ISD::SRA && N1.getOperand(0) == N0.getOperand(0)) in performIntegerAbsCombine()
6899 SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, VT), in performIntegerAbsCombine()
6940 SDValue Cmp = getAArch64Cmp(N0, Zero, ISD::SETLT, CCVal, DAG, DL); in BuildSDIVPow2()
6941 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne); in BuildSDIVPow2()
6952 DAG.getNode(ISD::SRA, DL, VT, CSel, DAG.getConstant(Lg2, MVT::i64)); in BuildSDIVPow2()
6961 return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, VT), SRA); in BuildSDIVPow2()
6983 DAG.getNode(ISD::SHL, SDLoc(N), VT, N->getOperand(0), in performMulCombine()
6985 return DAG.getNode(ISD::ADD, SDLoc(N), VT, ShiftedVal, in performMulCombine()
6992 DAG.getNode(ISD::SHL, SDLoc(N), VT, N->getOperand(0), in performMulCombine()
6994 return DAG.getNode(ISD::SUB, SDLoc(N), VT, ShiftedVal, in performMulCombine()
7002 DAG.getNode(ISD::SHL, SDLoc(N), VT, N->getOperand(0), in performMulCombine()
7004 return DAG.getNode(ISD::SUB, SDLoc(N), VT, N->getOperand(0), in performMulCombine()
7011 DAG.getNode(ISD::SHL, SDLoc(N), VT, N->getOperand(0), in performMulCombine()
7014 DAG.getNode(ISD::ADD, SDLoc(N), VT, ShiftedVal, N->getOperand(0)); in performMulCombine()
7015 return DAG.getNode(ISD::SUB, SDLoc(N), VT, DAG.getConstant(0, VT), Add); in performMulCombine()
7036 if (!VT.isVector() || N->getOperand(0)->getOpcode() != ISD::AND || in performVectorCompareAndMaskUnaryOpCombine()
7037 N->getOperand(0)->getOperand(0)->getOpcode() != ISD::SETCC || in performVectorCompareAndMaskUnaryOpCombine()
7058 SDValue MaskConst = DAG.getNode(ISD::BITCAST, DL, IntVT, SourceConst); in performVectorCompareAndMaskUnaryOpCombine()
7059 SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT, in performVectorCompareAndMaskUnaryOpCombine()
7061 SDValue Res = DAG.getNode(ISD::BITCAST, DL, VT, NewAnd); in performVectorCompareAndMaskUnaryOpCombine()
7088 if (Subtarget->hasNEON() && ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() && in performIntToFpCombine()
7102 (N->getOpcode() == ISD::SINT_TO_FP) ? AArch64ISD::SITOF : AArch64ISD::UITOF; in performIntToFpCombine()
7113 if (N.getOpcode() == ISD::SHL) in findEXTRHalf()
7115 else if (N.getOpcode() == ISD::SRL) in findEXTRHalf()
7139 assert(N->getOpcode() == ISD::OR && "Unexpected root"); in tryCombineToEXTR()
7183 if (N0.getOpcode() != ISD::AND) in tryCombineToBSL()
7187 if (N1.getOpcode() != ISD::AND) in tryCombineToBSL()
7267 if (Op0->getOpcode() != ISD::EXTRACT_SUBVECTOR && in performBitcastCombine()
7272 if (Op0->getOpcode() == ISD::EXTRACT_SUBVECTOR) { in performBitcastCombine()
7282 if (Op0->getOperand(0)->getOpcode() != ISD::BITCAST) in performBitcastCombine()
7299 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Source, HalfIdx); in performBitcastCombine()
7327 N0->getOpcode() == ISD::TRUNCATE && in performConcatVectorsCombine()
7328 N1->getOpcode() == ISD::TRUNCATE) { in performConcatVectorsCombine()
7340 return DAG.getNode(ISD::TRUNCATE, dl, VT, in performConcatVectorsCombine()
7343 DAG.getNode(ISD::BITCAST, dl, MidVT, N00), in performConcatVectorsCombine()
7344 DAG.getNode(ISD::BITCAST, dl, MidVT, N10), Mask)); in performConcatVectorsCombine()
7371 if (N1->getOpcode() != ISD::BITCAST) in performConcatVectorsCombine()
7383 return DAG.getNode(ISD::BITCAST, dl, VT, in performConcatVectorsCombine()
7384 DAG.getNode(ISD::CONCAT_VECTORS, dl, ConcatTy, in performConcatVectorsCombine()
7385 DAG.getNode(ISD::BITCAST, dl, RHSTy, N0), in performConcatVectorsCombine()
7407 if (Op1.getOpcode() == ISD::EXTRACT_VECTOR_ELT) { in tryCombineFixedPointConvert()
7430 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VecResTy, IID, Vec, Shift); in tryCombineFixedPointConvert()
7431 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResTy, Convert, Lane); in tryCombineFixedPointConvert()
7482 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N.getNode()), NarrowTy, in tryExtendDUPToExtractHigh()
7487 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR) in isEssentiallyExtractSubvector()
7490 return N.getOpcode() == ISD::BITCAST && in isEssentiallyExtractSubvector()
7491 N.getOperand(0).getOpcode() == ISD::EXTRACT_SUBVECTOR; in isEssentiallyExtractSubvector()
7498 ISD::CondCode CC;
7529 if (Op.getOpcode() == ISD::SETCC) { in isSetCC()
7573 return ((Op.getOpcode() == ISD::ZERO_EXTEND) && in isSetCCOrZExtSetCC()
7584 assert(Op && Op->getOpcode() == ISD::ADD && "Unexpected operation!"); in performSetccAddFolding()
7613 ISD::getSetCCInverse(InfoAndKind.Info.Generic.CC, true), in performSetccAddFolding()
7617 LHS = DAG.getNode(ISD::ADD, dl, VT, RHS, DAG.getConstant(1, VT)); in performSetccAddFolding()
7640 if (N->getOpcode() == ISD::ADD) in performAddSubLongCombine()
7648 if ((LHS.getOpcode() != ISD::ZERO_EXTEND && in performAddSubLongCombine()
7649 LHS.getOpcode() != ISD::SIGN_EXTEND) || in performAddSubLongCombine()
7706 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SDLoc(N), N->getValueType(0), in tryCombineLongOpWithDup()
7772 if (AndN.getOpcode() != ISD::AND) in tryCombineCRC32()
7779 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SDLoc(N), MVT::i32, in tryCombineCRC32()
7785 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), N->getValueType(0), in combineAcrossLanesIntrinsic()
7850 if (!DCI.isBeforeLegalizeOps() && N->getOpcode() == ISD::ZERO_EXTEND && in performExtendCombine()
7851 N->getOperand(0).getOpcode() == ISD::INTRINSIC_WO_CHAIN) { in performExtendCombine()
7860 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), N->getValueType(0), in performExtendCombine()
7929 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, Src, in performExtendCombine()
7931 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, Src, in performExtendCombine()
7938 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi); in performExtendCombine()
7956 if (StVal.getOpcode() != ISD::INSERT_VECTOR_ELT) in replaceSplatVectorStore()
7969 if (NextInsertElt.getOpcode() != ISD::INSERT_VECTOR_ELT) in replaceSplatVectorStore()
7991 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr, in replaceSplatVectorStore()
8052 SDValue SubVector0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, StVal, in performSTORECombine()
8054 SDValue SubVector1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, StVal, in performSTORECombine()
8060 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr, in performSTORECombine()
8081 if (LD->getOpcode() != ISD::LOAD) in performPostLD1Combine()
8106 if (User->getOpcode() != ISD::ADD in performPostLD1Combine()
8173 if (User->getOpcode() != ISD::ADD || in performNEONPostLDSTCombine()
8293 bool checkValueWidth(SDValue V, unsigned width, ISD::LoadExtType &ExtType) { in checkValueWidth()
8294 ExtType = ISD::NON_EXTLOAD; in checkValueWidth()
8298 case ISD::LOAD: { in checkValueWidth()
8307 case ISD::AssertSext: { in checkValueWidth()
8311 ExtType = ISD::SEXTLOAD; in checkValueWidth()
8316 case ISD::AssertZext: { in checkValueWidth()
8320 ExtType = ISD::ZEXTLOAD; in checkValueWidth()
8325 case ISD::Constant: in checkValueWidth()
8326 case ISD::TargetConstant: { in checkValueWidth()
8400 ISD::LoadExtType ExtType, signed AddConstant, in isEquivalentMaskless()
8412 if (ExtType == ISD::SEXTLOAD) in isEquivalentMaskless()
8493 if (AndNode->getOpcode() != ISD::AND) in performCONDCombine()
8509 if (AddValue.getOpcode() != ISD::ADD) in performCONDCombine()
8525 ISD::LoadExtType ExtType; in performCONDCombine()
8589 if (LHS.getOpcode() == ISD::SHL || LHS.getOpcode() == ISD::SRA || in performBRCONDCombine()
8590 LHS.getOpcode() == ISD::SRL) in performBRCONDCombine()
8615 if (N0.getOpcode() != ISD::SETCC || CCVT.getVectorNumElements() != 1 || in performVSelectCombine()
8632 return DAG.getNode(ISD::VSELECT, SDLoc(N), ResVT, SetCC, in performVSelectCombine()
8644 if (N0.getOpcode() != ISD::SETCC || N0.getValueType() != MVT::i1) in performSelectCombine()
8668 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, SrcVT, N0.getOperand(0)); in performSelectCombine()
8670 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, SrcVT, N0.getOperand(1)); in performSelectCombine()
8671 SDValue SetCC = DAG.getNode(ISD::SETCC, DL, CCVT, LHS, RHS, N0.getOperand(2)); in performSelectCombine()
8676 Mask = DAG.getNode(ISD::BITCAST, DL, in performSelectCombine()
8688 case ISD::ADD: in PerformDAGCombine()
8689 case ISD::SUB: in PerformDAGCombine()
8691 case ISD::XOR: in PerformDAGCombine()
8693 case ISD::MUL: in PerformDAGCombine()
8695 case ISD::SINT_TO_FP: in PerformDAGCombine()
8696 case ISD::UINT_TO_FP: in PerformDAGCombine()
8698 case ISD::OR: in PerformDAGCombine()
8700 case ISD::INTRINSIC_WO_CHAIN: in PerformDAGCombine()
8702 case ISD::ANY_EXTEND: in PerformDAGCombine()
8703 case ISD::ZERO_EXTEND: in PerformDAGCombine()
8704 case ISD::SIGN_EXTEND: in PerformDAGCombine()
8706 case ISD::BITCAST: in PerformDAGCombine()
8708 case ISD::CONCAT_VECTORS: in PerformDAGCombine()
8710 case ISD::SELECT: in PerformDAGCombine()
8712 case ISD::VSELECT: in PerformDAGCombine()
8714 case ISD::STORE: in PerformDAGCombine()
8722 case ISD::INSERT_VECTOR_ELT: in PerformDAGCombine()
8724 case ISD::INTRINSIC_VOID: in PerformDAGCombine()
8725 case ISD::INTRINSIC_W_CHAIN: in PerformDAGCombine()
8769 if (Copy->getOpcode() == ISD::CopyToReg) { in isUsedByReturnOnly()
8776 } else if (Copy->getOpcode() != ISD::FP_EXTEND) in isUsedByReturnOnly()
8806 ISD::MemIndexedMode &AM, in getIndexedAddressParts()
8809 if (Op->getOpcode() != ISD::ADD && Op->getOpcode() != ISD::SUB) in getIndexedAddressParts()
8819 IsInc = (Op->getOpcode() == ISD::ADD); in getIndexedAddressParts()
8828 ISD::MemIndexedMode &AM, in getPreIndexedAddressParts()
8844 AM = IsInc ? ISD::PRE_INC : ISD::PRE_DEC; in getPreIndexedAddressParts()
8850 ISD::MemIndexedMode &AM, SelectionDAG &DAG) const { in getPostIndexedAddressParts()
8869 AM = IsInc ? ISD::POST_INC : ISD::POST_DEC; in getPostIndexedAddressParts()
8886 Op = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Op); in ReplaceBITCASTResults()
8887 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Op)); in ReplaceBITCASTResults()
8895 case ISD::BITCAST: in ReplaceNodeResults()
8898 case ISD::FP_TO_UINT: in ReplaceNodeResults()
8899 case ISD::FP_TO_SINT: in ReplaceNodeResults()