1 //===-- ARMISelLowering.cpp - ARM DAG Lowering Implementation -------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file defines the interfaces that ARM uses to lower LLVM code into a
11 // selection DAG.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #include "ARMISelLowering.h"
16 #include "ARMCallingConv.h"
17 #include "ARMConstantPoolValue.h"
18 #include "ARMMachineFunctionInfo.h"
19 #include "ARMPerfectShuffle.h"
20 #include "ARMSubtarget.h"
21 #include "ARMTargetMachine.h"
22 #include "ARMTargetObjectFile.h"
23 #include "MCTargetDesc/ARMAddressingModes.h"
24 #include "llvm/ADT/Statistic.h"
25 #include "llvm/ADT/StringExtras.h"
26 #include "llvm/ADT/StringSwitch.h"
27 #include "llvm/CodeGen/CallingConvLower.h"
28 #include "llvm/CodeGen/IntrinsicLowering.h"
29 #include "llvm/CodeGen/MachineBasicBlock.h"
30 #include "llvm/CodeGen/MachineFrameInfo.h"
31 #include "llvm/CodeGen/MachineFunction.h"
32 #include "llvm/CodeGen/MachineInstrBuilder.h"
33 #include "llvm/CodeGen/MachineJumpTableInfo.h"
34 #include "llvm/CodeGen/MachineModuleInfo.h"
35 #include "llvm/CodeGen/MachineRegisterInfo.h"
36 #include "llvm/CodeGen/SelectionDAG.h"
37 #include "llvm/IR/CallingConv.h"
38 #include "llvm/IR/Constants.h"
39 #include "llvm/IR/Function.h"
40 #include "llvm/IR/GlobalValue.h"
41 #include "llvm/IR/IRBuilder.h"
42 #include "llvm/IR/Instruction.h"
43 #include "llvm/IR/Instructions.h"
44 #include "llvm/IR/IntrinsicInst.h"
45 #include "llvm/IR/Intrinsics.h"
46 #include "llvm/IR/Type.h"
47 #include "llvm/MC/MCSectionMachO.h"
48 #include "llvm/Support/CommandLine.h"
49 #include "llvm/Support/Debug.h"
50 #include "llvm/Support/ErrorHandling.h"
51 #include "llvm/Support/MathExtras.h"
52 #include "llvm/Support/raw_ostream.h"
53 #include "llvm/Target/TargetOptions.h"
54 #include <utility>
55 using namespace llvm;
56
57 #define DEBUG_TYPE "arm-isel"
58
59 STATISTIC(NumTailCalls, "Number of tail calls");
60 STATISTIC(NumMovwMovt, "Number of GAs materialized with movw + movt");
61 STATISTIC(NumLoopByVals, "Number of loops generated for byval arguments");
62
63 static cl::opt<bool>
64 ARMInterworking("arm-interworking", cl::Hidden,
65 cl::desc("Enable / disable ARM interworking (for debugging only)"),
66 cl::init(true));
67
68 namespace {
69 class ARMCCState : public CCState {
70 public:
ARMCCState(CallingConv::ID CC,bool isVarArg,MachineFunction & MF,SmallVectorImpl<CCValAssign> & locs,LLVMContext & C,ParmContext PC)71 ARMCCState(CallingConv::ID CC, bool isVarArg, MachineFunction &MF,
72 SmallVectorImpl<CCValAssign> &locs, LLVMContext &C,
73 ParmContext PC)
74 : CCState(CC, isVarArg, MF, locs, C) {
75 assert(((PC == Call) || (PC == Prologue)) &&
76 "ARMCCState users must specify whether their context is call"
77 "or prologue generation.");
78 CallOrPrologue = PC;
79 }
80 };
81 }
82
83 // The APCS parameter registers.
84 static const MCPhysReg GPRArgRegs[] = {
85 ARM::R0, ARM::R1, ARM::R2, ARM::R3
86 };
87
addTypeForNEON(MVT VT,MVT PromotedLdStVT,MVT PromotedBitwiseVT)88 void ARMTargetLowering::addTypeForNEON(MVT VT, MVT PromotedLdStVT,
89 MVT PromotedBitwiseVT) {
90 if (VT != PromotedLdStVT) {
91 setOperationAction(ISD::LOAD, VT, Promote);
92 AddPromotedToType (ISD::LOAD, VT, PromotedLdStVT);
93
94 setOperationAction(ISD::STORE, VT, Promote);
95 AddPromotedToType (ISD::STORE, VT, PromotedLdStVT);
96 }
97
98 MVT ElemTy = VT.getVectorElementType();
99 if (ElemTy != MVT::i64 && ElemTy != MVT::f64)
100 setOperationAction(ISD::SETCC, VT, Custom);
101 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
102 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
103 if (ElemTy == MVT::i32) {
104 setOperationAction(ISD::SINT_TO_FP, VT, Custom);
105 setOperationAction(ISD::UINT_TO_FP, VT, Custom);
106 setOperationAction(ISD::FP_TO_SINT, VT, Custom);
107 setOperationAction(ISD::FP_TO_UINT, VT, Custom);
108 } else {
109 setOperationAction(ISD::SINT_TO_FP, VT, Expand);
110 setOperationAction(ISD::UINT_TO_FP, VT, Expand);
111 setOperationAction(ISD::FP_TO_SINT, VT, Expand);
112 setOperationAction(ISD::FP_TO_UINT, VT, Expand);
113 }
114 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
115 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
116 setOperationAction(ISD::CONCAT_VECTORS, VT, Legal);
117 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
118 setOperationAction(ISD::SELECT, VT, Expand);
119 setOperationAction(ISD::SELECT_CC, VT, Expand);
120 setOperationAction(ISD::VSELECT, VT, Expand);
121 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
122 if (VT.isInteger()) {
123 setOperationAction(ISD::SHL, VT, Custom);
124 setOperationAction(ISD::SRA, VT, Custom);
125 setOperationAction(ISD::SRL, VT, Custom);
126 }
127
128 // Promote all bit-wise operations.
129 if (VT.isInteger() && VT != PromotedBitwiseVT) {
130 setOperationAction(ISD::AND, VT, Promote);
131 AddPromotedToType (ISD::AND, VT, PromotedBitwiseVT);
132 setOperationAction(ISD::OR, VT, Promote);
133 AddPromotedToType (ISD::OR, VT, PromotedBitwiseVT);
134 setOperationAction(ISD::XOR, VT, Promote);
135 AddPromotedToType (ISD::XOR, VT, PromotedBitwiseVT);
136 }
137
138 // Neon does not support vector divide/remainder operations.
139 setOperationAction(ISD::SDIV, VT, Expand);
140 setOperationAction(ISD::UDIV, VT, Expand);
141 setOperationAction(ISD::FDIV, VT, Expand);
142 setOperationAction(ISD::SREM, VT, Expand);
143 setOperationAction(ISD::UREM, VT, Expand);
144 setOperationAction(ISD::FREM, VT, Expand);
145
146 if (!VT.isFloatingPoint() &&
147 VT != MVT::v2i64 && VT != MVT::v1i64)
148 for (unsigned Opcode : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX})
149 setOperationAction(Opcode, VT, Legal);
150 }
151
addDRTypeForNEON(MVT VT)152 void ARMTargetLowering::addDRTypeForNEON(MVT VT) {
153 addRegisterClass(VT, &ARM::DPRRegClass);
154 addTypeForNEON(VT, MVT::f64, MVT::v2i32);
155 }
156
addQRTypeForNEON(MVT VT)157 void ARMTargetLowering::addQRTypeForNEON(MVT VT) {
158 addRegisterClass(VT, &ARM::DPairRegClass);
159 addTypeForNEON(VT, MVT::v2f64, MVT::v4i32);
160 }
161
ARMTargetLowering(const TargetMachine & TM,const ARMSubtarget & STI)162 ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
163 const ARMSubtarget &STI)
164 : TargetLowering(TM), Subtarget(&STI) {
165 RegInfo = Subtarget->getRegisterInfo();
166 Itins = Subtarget->getInstrItineraryData();
167
168 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
169
170 if (Subtarget->isTargetMachO()) {
171 // Uses VFP for Thumb libfuncs if available.
172 if (Subtarget->isThumb() && Subtarget->hasVFP2() &&
173 Subtarget->hasARMOps() && !Subtarget->useSoftFloat()) {
174 static const struct {
175 const RTLIB::Libcall Op;
176 const char * const Name;
177 const ISD::CondCode Cond;
178 } LibraryCalls[] = {
179 // Single-precision floating-point arithmetic.
180 { RTLIB::ADD_F32, "__addsf3vfp", ISD::SETCC_INVALID },
181 { RTLIB::SUB_F32, "__subsf3vfp", ISD::SETCC_INVALID },
182 { RTLIB::MUL_F32, "__mulsf3vfp", ISD::SETCC_INVALID },
183 { RTLIB::DIV_F32, "__divsf3vfp", ISD::SETCC_INVALID },
184
185 // Double-precision floating-point arithmetic.
186 { RTLIB::ADD_F64, "__adddf3vfp", ISD::SETCC_INVALID },
187 { RTLIB::SUB_F64, "__subdf3vfp", ISD::SETCC_INVALID },
188 { RTLIB::MUL_F64, "__muldf3vfp", ISD::SETCC_INVALID },
189 { RTLIB::DIV_F64, "__divdf3vfp", ISD::SETCC_INVALID },
190
191 // Single-precision comparisons.
192 { RTLIB::OEQ_F32, "__eqsf2vfp", ISD::SETNE },
193 { RTLIB::UNE_F32, "__nesf2vfp", ISD::SETNE },
194 { RTLIB::OLT_F32, "__ltsf2vfp", ISD::SETNE },
195 { RTLIB::OLE_F32, "__lesf2vfp", ISD::SETNE },
196 { RTLIB::OGE_F32, "__gesf2vfp", ISD::SETNE },
197 { RTLIB::OGT_F32, "__gtsf2vfp", ISD::SETNE },
198 { RTLIB::UO_F32, "__unordsf2vfp", ISD::SETNE },
199 { RTLIB::O_F32, "__unordsf2vfp", ISD::SETEQ },
200
201 // Double-precision comparisons.
202 { RTLIB::OEQ_F64, "__eqdf2vfp", ISD::SETNE },
203 { RTLIB::UNE_F64, "__nedf2vfp", ISD::SETNE },
204 { RTLIB::OLT_F64, "__ltdf2vfp", ISD::SETNE },
205 { RTLIB::OLE_F64, "__ledf2vfp", ISD::SETNE },
206 { RTLIB::OGE_F64, "__gedf2vfp", ISD::SETNE },
207 { RTLIB::OGT_F64, "__gtdf2vfp", ISD::SETNE },
208 { RTLIB::UO_F64, "__unorddf2vfp", ISD::SETNE },
209 { RTLIB::O_F64, "__unorddf2vfp", ISD::SETEQ },
210
211 // Floating-point to integer conversions.
212 // i64 conversions are done via library routines even when generating VFP
213 // instructions, so use the same ones.
214 { RTLIB::FPTOSINT_F64_I32, "__fixdfsivfp", ISD::SETCC_INVALID },
215 { RTLIB::FPTOUINT_F64_I32, "__fixunsdfsivfp", ISD::SETCC_INVALID },
216 { RTLIB::FPTOSINT_F32_I32, "__fixsfsivfp", ISD::SETCC_INVALID },
217 { RTLIB::FPTOUINT_F32_I32, "__fixunssfsivfp", ISD::SETCC_INVALID },
218
219 // Conversions between floating types.
220 { RTLIB::FPROUND_F64_F32, "__truncdfsf2vfp", ISD::SETCC_INVALID },
221 { RTLIB::FPEXT_F32_F64, "__extendsfdf2vfp", ISD::SETCC_INVALID },
222
223 // Integer to floating-point conversions.
224 // i64 conversions are done via library routines even when generating VFP
225 // instructions, so use the same ones.
226 // FIXME: There appears to be some naming inconsistency in ARM libgcc:
227 // e.g., __floatunsidf vs. __floatunssidfvfp.
228 { RTLIB::SINTTOFP_I32_F64, "__floatsidfvfp", ISD::SETCC_INVALID },
229 { RTLIB::UINTTOFP_I32_F64, "__floatunssidfvfp", ISD::SETCC_INVALID },
230 { RTLIB::SINTTOFP_I32_F32, "__floatsisfvfp", ISD::SETCC_INVALID },
231 { RTLIB::UINTTOFP_I32_F32, "__floatunssisfvfp", ISD::SETCC_INVALID },
232 };
233
234 for (const auto &LC : LibraryCalls) {
235 setLibcallName(LC.Op, LC.Name);
236 if (LC.Cond != ISD::SETCC_INVALID)
237 setCmpLibcallCC(LC.Op, LC.Cond);
238 }
239 }
240
241 // Set the correct calling convention for ARMv7k WatchOS. It's just
242 // AAPCS_VFP for functions as simple as libcalls.
243 if (Subtarget->isTargetWatchOS()) {
244 for (int i = 0; i < RTLIB::UNKNOWN_LIBCALL; ++i)
245 setLibcallCallingConv((RTLIB::Libcall)i, CallingConv::ARM_AAPCS_VFP);
246 }
247 }
248
249 // These libcalls are not available in 32-bit.
250 setLibcallName(RTLIB::SHL_I128, nullptr);
251 setLibcallName(RTLIB::SRL_I128, nullptr);
252 setLibcallName(RTLIB::SRA_I128, nullptr);
253
254 // RTLIB
255 if (Subtarget->isAAPCS_ABI() &&
256 (Subtarget->isTargetAEABI() || Subtarget->isTargetGNUAEABI() ||
257 Subtarget->isTargetAndroid())) {
258 static const struct {
259 const RTLIB::Libcall Op;
260 const char * const Name;
261 const CallingConv::ID CC;
262 const ISD::CondCode Cond;
263 } LibraryCalls[] = {
264 // Double-precision floating-point arithmetic helper functions
265 // RTABI chapter 4.1.2, Table 2
266 { RTLIB::ADD_F64, "__aeabi_dadd", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
267 { RTLIB::DIV_F64, "__aeabi_ddiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
268 { RTLIB::MUL_F64, "__aeabi_dmul", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
269 { RTLIB::SUB_F64, "__aeabi_dsub", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
270
271 // Double-precision floating-point comparison helper functions
272 // RTABI chapter 4.1.2, Table 3
273 { RTLIB::OEQ_F64, "__aeabi_dcmpeq", CallingConv::ARM_AAPCS, ISD::SETNE },
274 { RTLIB::UNE_F64, "__aeabi_dcmpeq", CallingConv::ARM_AAPCS, ISD::SETEQ },
275 { RTLIB::OLT_F64, "__aeabi_dcmplt", CallingConv::ARM_AAPCS, ISD::SETNE },
276 { RTLIB::OLE_F64, "__aeabi_dcmple", CallingConv::ARM_AAPCS, ISD::SETNE },
277 { RTLIB::OGE_F64, "__aeabi_dcmpge", CallingConv::ARM_AAPCS, ISD::SETNE },
278 { RTLIB::OGT_F64, "__aeabi_dcmpgt", CallingConv::ARM_AAPCS, ISD::SETNE },
279 { RTLIB::UO_F64, "__aeabi_dcmpun", CallingConv::ARM_AAPCS, ISD::SETNE },
280 { RTLIB::O_F64, "__aeabi_dcmpun", CallingConv::ARM_AAPCS, ISD::SETEQ },
281
282 // Single-precision floating-point arithmetic helper functions
283 // RTABI chapter 4.1.2, Table 4
284 { RTLIB::ADD_F32, "__aeabi_fadd", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
285 { RTLIB::DIV_F32, "__aeabi_fdiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
286 { RTLIB::MUL_F32, "__aeabi_fmul", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
287 { RTLIB::SUB_F32, "__aeabi_fsub", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
288
289 // Single-precision floating-point comparison helper functions
290 // RTABI chapter 4.1.2, Table 5
291 { RTLIB::OEQ_F32, "__aeabi_fcmpeq", CallingConv::ARM_AAPCS, ISD::SETNE },
292 { RTLIB::UNE_F32, "__aeabi_fcmpeq", CallingConv::ARM_AAPCS, ISD::SETEQ },
293 { RTLIB::OLT_F32, "__aeabi_fcmplt", CallingConv::ARM_AAPCS, ISD::SETNE },
294 { RTLIB::OLE_F32, "__aeabi_fcmple", CallingConv::ARM_AAPCS, ISD::SETNE },
295 { RTLIB::OGE_F32, "__aeabi_fcmpge", CallingConv::ARM_AAPCS, ISD::SETNE },
296 { RTLIB::OGT_F32, "__aeabi_fcmpgt", CallingConv::ARM_AAPCS, ISD::SETNE },
297 { RTLIB::UO_F32, "__aeabi_fcmpun", CallingConv::ARM_AAPCS, ISD::SETNE },
298 { RTLIB::O_F32, "__aeabi_fcmpun", CallingConv::ARM_AAPCS, ISD::SETEQ },
299
300 // Floating-point to integer conversions.
301 // RTABI chapter 4.1.2, Table 6
302 { RTLIB::FPTOSINT_F64_I32, "__aeabi_d2iz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
303 { RTLIB::FPTOUINT_F64_I32, "__aeabi_d2uiz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
304 { RTLIB::FPTOSINT_F64_I64, "__aeabi_d2lz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
305 { RTLIB::FPTOUINT_F64_I64, "__aeabi_d2ulz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
306 { RTLIB::FPTOSINT_F32_I32, "__aeabi_f2iz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
307 { RTLIB::FPTOUINT_F32_I32, "__aeabi_f2uiz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
308 { RTLIB::FPTOSINT_F32_I64, "__aeabi_f2lz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
309 { RTLIB::FPTOUINT_F32_I64, "__aeabi_f2ulz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
310
311 // Conversions between floating types.
312 // RTABI chapter 4.1.2, Table 7
313 { RTLIB::FPROUND_F64_F32, "__aeabi_d2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
314 { RTLIB::FPROUND_F64_F16, "__aeabi_d2h", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
315 { RTLIB::FPEXT_F32_F64, "__aeabi_f2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
316
317 // Integer to floating-point conversions.
318 // RTABI chapter 4.1.2, Table 8
319 { RTLIB::SINTTOFP_I32_F64, "__aeabi_i2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
320 { RTLIB::UINTTOFP_I32_F64, "__aeabi_ui2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
321 { RTLIB::SINTTOFP_I64_F64, "__aeabi_l2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
322 { RTLIB::UINTTOFP_I64_F64, "__aeabi_ul2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
323 { RTLIB::SINTTOFP_I32_F32, "__aeabi_i2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
324 { RTLIB::UINTTOFP_I32_F32, "__aeabi_ui2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
325 { RTLIB::SINTTOFP_I64_F32, "__aeabi_l2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
326 { RTLIB::UINTTOFP_I64_F32, "__aeabi_ul2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
327
328 // Long long helper functions
329 // RTABI chapter 4.2, Table 9
330 { RTLIB::MUL_I64, "__aeabi_lmul", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
331 { RTLIB::SHL_I64, "__aeabi_llsl", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
332 { RTLIB::SRL_I64, "__aeabi_llsr", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
333 { RTLIB::SRA_I64, "__aeabi_lasr", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
334
335 // Integer division functions
336 // RTABI chapter 4.3.1
337 { RTLIB::SDIV_I8, "__aeabi_idiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
338 { RTLIB::SDIV_I16, "__aeabi_idiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
339 { RTLIB::SDIV_I32, "__aeabi_idiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
340 { RTLIB::SDIV_I64, "__aeabi_ldivmod", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
341 { RTLIB::UDIV_I8, "__aeabi_uidiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
342 { RTLIB::UDIV_I16, "__aeabi_uidiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
343 { RTLIB::UDIV_I32, "__aeabi_uidiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
344 { RTLIB::UDIV_I64, "__aeabi_uldivmod", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
345 };
346
347 for (const auto &LC : LibraryCalls) {
348 setLibcallName(LC.Op, LC.Name);
349 setLibcallCallingConv(LC.Op, LC.CC);
350 if (LC.Cond != ISD::SETCC_INVALID)
351 setCmpLibcallCC(LC.Op, LC.Cond);
352 }
353
354 // EABI dependent RTLIB
355 if (TM.Options.EABIVersion == EABI::EABI4 ||
356 TM.Options.EABIVersion == EABI::EABI5) {
357 static const struct {
358 const RTLIB::Libcall Op;
359 const char *const Name;
360 const CallingConv::ID CC;
361 const ISD::CondCode Cond;
362 } MemOpsLibraryCalls[] = {
363 // Memory operations
364 // RTABI chapter 4.3.4
365 { RTLIB::MEMCPY, "__aeabi_memcpy", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
366 { RTLIB::MEMMOVE, "__aeabi_memmove", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
367 { RTLIB::MEMSET, "__aeabi_memset", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
368 };
369
370 for (const auto &LC : MemOpsLibraryCalls) {
371 setLibcallName(LC.Op, LC.Name);
372 setLibcallCallingConv(LC.Op, LC.CC);
373 if (LC.Cond != ISD::SETCC_INVALID)
374 setCmpLibcallCC(LC.Op, LC.Cond);
375 }
376 }
377 }
378
379 if (Subtarget->isTargetWindows()) {
380 static const struct {
381 const RTLIB::Libcall Op;
382 const char * const Name;
383 const CallingConv::ID CC;
384 } LibraryCalls[] = {
385 { RTLIB::FPTOSINT_F32_I64, "__stoi64", CallingConv::ARM_AAPCS_VFP },
386 { RTLIB::FPTOSINT_F64_I64, "__dtoi64", CallingConv::ARM_AAPCS_VFP },
387 { RTLIB::FPTOUINT_F32_I64, "__stou64", CallingConv::ARM_AAPCS_VFP },
388 { RTLIB::FPTOUINT_F64_I64, "__dtou64", CallingConv::ARM_AAPCS_VFP },
389 { RTLIB::SINTTOFP_I64_F32, "__i64tos", CallingConv::ARM_AAPCS_VFP },
390 { RTLIB::SINTTOFP_I64_F64, "__i64tod", CallingConv::ARM_AAPCS_VFP },
391 { RTLIB::UINTTOFP_I64_F32, "__u64tos", CallingConv::ARM_AAPCS_VFP },
392 { RTLIB::UINTTOFP_I64_F64, "__u64tod", CallingConv::ARM_AAPCS_VFP },
393 { RTLIB::SDIV_I32, "__rt_sdiv", CallingConv::ARM_AAPCS_VFP },
394 { RTLIB::UDIV_I32, "__rt_udiv", CallingConv::ARM_AAPCS_VFP },
395 { RTLIB::SDIV_I64, "__rt_sdiv64", CallingConv::ARM_AAPCS_VFP },
396 { RTLIB::UDIV_I64, "__rt_udiv64", CallingConv::ARM_AAPCS_VFP },
397 };
398
399 for (const auto &LC : LibraryCalls) {
400 setLibcallName(LC.Op, LC.Name);
401 setLibcallCallingConv(LC.Op, LC.CC);
402 }
403 }
404
405 // Use divmod compiler-rt calls for iOS 5.0 and later.
406 if (Subtarget->isTargetWatchOS() ||
407 (Subtarget->isTargetIOS() &&
408 !Subtarget->getTargetTriple().isOSVersionLT(5, 0))) {
409 setLibcallName(RTLIB::SDIVREM_I32, "__divmodsi4");
410 setLibcallName(RTLIB::UDIVREM_I32, "__udivmodsi4");
411 }
412
413 // The half <-> float conversion functions are always soft-float, but are
414 // needed for some targets which use a hard-float calling convention by
415 // default.
416 if (Subtarget->isAAPCS_ABI()) {
417 setLibcallCallingConv(RTLIB::FPROUND_F32_F16, CallingConv::ARM_AAPCS);
418 setLibcallCallingConv(RTLIB::FPROUND_F64_F16, CallingConv::ARM_AAPCS);
419 setLibcallCallingConv(RTLIB::FPEXT_F16_F32, CallingConv::ARM_AAPCS);
420 } else {
421 setLibcallCallingConv(RTLIB::FPROUND_F32_F16, CallingConv::ARM_APCS);
422 setLibcallCallingConv(RTLIB::FPROUND_F64_F16, CallingConv::ARM_APCS);
423 setLibcallCallingConv(RTLIB::FPEXT_F16_F32, CallingConv::ARM_APCS);
424 }
425
426 // In EABI, these functions have an __aeabi_ prefix, but in GNUEABI they have
427 // a __gnu_ prefix (which is the default).
428 if (Subtarget->isTargetAEABI()) {
429 setLibcallName(RTLIB::FPROUND_F32_F16, "__aeabi_f2h");
430 setLibcallName(RTLIB::FPROUND_F64_F16, "__aeabi_d2h");
431 setLibcallName(RTLIB::FPEXT_F16_F32, "__aeabi_h2f");
432 }
433
434 if (Subtarget->isThumb1Only())
435 addRegisterClass(MVT::i32, &ARM::tGPRRegClass);
436 else
437 addRegisterClass(MVT::i32, &ARM::GPRRegClass);
438 if (!Subtarget->useSoftFloat() && Subtarget->hasVFP2() &&
439 !Subtarget->isThumb1Only()) {
440 addRegisterClass(MVT::f32, &ARM::SPRRegClass);
441 addRegisterClass(MVT::f64, &ARM::DPRRegClass);
442 }
443
444 for (MVT VT : MVT::vector_valuetypes()) {
445 for (MVT InnerVT : MVT::vector_valuetypes()) {
446 setTruncStoreAction(VT, InnerVT, Expand);
447 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
448 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
449 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
450 }
451
452 setOperationAction(ISD::MULHS, VT, Expand);
453 setOperationAction(ISD::SMUL_LOHI, VT, Expand);
454 setOperationAction(ISD::MULHU, VT, Expand);
455 setOperationAction(ISD::UMUL_LOHI, VT, Expand);
456
457 setOperationAction(ISD::BSWAP, VT, Expand);
458 }
459
460 setOperationAction(ISD::ConstantFP, MVT::f32, Custom);
461 setOperationAction(ISD::ConstantFP, MVT::f64, Custom);
462
463 setOperationAction(ISD::READ_REGISTER, MVT::i64, Custom);
464 setOperationAction(ISD::WRITE_REGISTER, MVT::i64, Custom);
465
466 if (Subtarget->hasNEON()) {
467 addDRTypeForNEON(MVT::v2f32);
468 addDRTypeForNEON(MVT::v8i8);
469 addDRTypeForNEON(MVT::v4i16);
470 addDRTypeForNEON(MVT::v2i32);
471 addDRTypeForNEON(MVT::v1i64);
472
473 addQRTypeForNEON(MVT::v4f32);
474 addQRTypeForNEON(MVT::v2f64);
475 addQRTypeForNEON(MVT::v16i8);
476 addQRTypeForNEON(MVT::v8i16);
477 addQRTypeForNEON(MVT::v4i32);
478 addQRTypeForNEON(MVT::v2i64);
479
480 // v2f64 is legal so that QR subregs can be extracted as f64 elements, but
481 // neither Neon nor VFP support any arithmetic operations on it.
482 // The same with v4f32. But keep in mind that vadd, vsub, vmul are natively
483 // supported for v4f32.
484 setOperationAction(ISD::FADD, MVT::v2f64, Expand);
485 setOperationAction(ISD::FSUB, MVT::v2f64, Expand);
486 setOperationAction(ISD::FMUL, MVT::v2f64, Expand);
487 // FIXME: Code duplication: FDIV and FREM are expanded always, see
488 // ARMTargetLowering::addTypeForNEON method for details.
489 setOperationAction(ISD::FDIV, MVT::v2f64, Expand);
490 setOperationAction(ISD::FREM, MVT::v2f64, Expand);
491 // FIXME: Create unittest.
492 // In another words, find a way when "copysign" appears in DAG with vector
493 // operands.
494 setOperationAction(ISD::FCOPYSIGN, MVT::v2f64, Expand);
495 // FIXME: Code duplication: SETCC has custom operation action, see
496 // ARMTargetLowering::addTypeForNEON method for details.
497 setOperationAction(ISD::SETCC, MVT::v2f64, Expand);
498 // FIXME: Create unittest for FNEG and for FABS.
499 setOperationAction(ISD::FNEG, MVT::v2f64, Expand);
500 setOperationAction(ISD::FABS, MVT::v2f64, Expand);
501 setOperationAction(ISD::FSQRT, MVT::v2f64, Expand);
502 setOperationAction(ISD::FSIN, MVT::v2f64, Expand);
503 setOperationAction(ISD::FCOS, MVT::v2f64, Expand);
504 setOperationAction(ISD::FPOWI, MVT::v2f64, Expand);
505 setOperationAction(ISD::FPOW, MVT::v2f64, Expand);
506 setOperationAction(ISD::FLOG, MVT::v2f64, Expand);
507 setOperationAction(ISD::FLOG2, MVT::v2f64, Expand);
508 setOperationAction(ISD::FLOG10, MVT::v2f64, Expand);
509 setOperationAction(ISD::FEXP, MVT::v2f64, Expand);
510 setOperationAction(ISD::FEXP2, MVT::v2f64, Expand);
511 // FIXME: Create unittest for FCEIL, FTRUNC, FRINT, FNEARBYINT, FFLOOR.
512 setOperationAction(ISD::FCEIL, MVT::v2f64, Expand);
513 setOperationAction(ISD::FTRUNC, MVT::v2f64, Expand);
514 setOperationAction(ISD::FRINT, MVT::v2f64, Expand);
515 setOperationAction(ISD::FNEARBYINT, MVT::v2f64, Expand);
516 setOperationAction(ISD::FFLOOR, MVT::v2f64, Expand);
517 setOperationAction(ISD::FMA, MVT::v2f64, Expand);
518
519 setOperationAction(ISD::FSQRT, MVT::v4f32, Expand);
520 setOperationAction(ISD::FSIN, MVT::v4f32, Expand);
521 setOperationAction(ISD::FCOS, MVT::v4f32, Expand);
522 setOperationAction(ISD::FPOWI, MVT::v4f32, Expand);
523 setOperationAction(ISD::FPOW, MVT::v4f32, Expand);
524 setOperationAction(ISD::FLOG, MVT::v4f32, Expand);
525 setOperationAction(ISD::FLOG2, MVT::v4f32, Expand);
526 setOperationAction(ISD::FLOG10, MVT::v4f32, Expand);
527 setOperationAction(ISD::FEXP, MVT::v4f32, Expand);
528 setOperationAction(ISD::FEXP2, MVT::v4f32, Expand);
529 setOperationAction(ISD::FCEIL, MVT::v4f32, Expand);
530 setOperationAction(ISD::FTRUNC, MVT::v4f32, Expand);
531 setOperationAction(ISD::FRINT, MVT::v4f32, Expand);
532 setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Expand);
533 setOperationAction(ISD::FFLOOR, MVT::v4f32, Expand);
534
535 // Mark v2f32 intrinsics.
536 setOperationAction(ISD::FSQRT, MVT::v2f32, Expand);
537 setOperationAction(ISD::FSIN, MVT::v2f32, Expand);
538 setOperationAction(ISD::FCOS, MVT::v2f32, Expand);
539 setOperationAction(ISD::FPOWI, MVT::v2f32, Expand);
540 setOperationAction(ISD::FPOW, MVT::v2f32, Expand);
541 setOperationAction(ISD::FLOG, MVT::v2f32, Expand);
542 setOperationAction(ISD::FLOG2, MVT::v2f32, Expand);
543 setOperationAction(ISD::FLOG10, MVT::v2f32, Expand);
544 setOperationAction(ISD::FEXP, MVT::v2f32, Expand);
545 setOperationAction(ISD::FEXP2, MVT::v2f32, Expand);
546 setOperationAction(ISD::FCEIL, MVT::v2f32, Expand);
547 setOperationAction(ISD::FTRUNC, MVT::v2f32, Expand);
548 setOperationAction(ISD::FRINT, MVT::v2f32, Expand);
549 setOperationAction(ISD::FNEARBYINT, MVT::v2f32, Expand);
550 setOperationAction(ISD::FFLOOR, MVT::v2f32, Expand);
551
552 // Neon does not support some operations on v1i64 and v2i64 types.
553 setOperationAction(ISD::MUL, MVT::v1i64, Expand);
554 // Custom handling for some quad-vector types to detect VMULL.
555 setOperationAction(ISD::MUL, MVT::v8i16, Custom);
556 setOperationAction(ISD::MUL, MVT::v4i32, Custom);
557 setOperationAction(ISD::MUL, MVT::v2i64, Custom);
558 // Custom handling for some vector types to avoid expensive expansions
559 setOperationAction(ISD::SDIV, MVT::v4i16, Custom);
560 setOperationAction(ISD::SDIV, MVT::v8i8, Custom);
561 setOperationAction(ISD::UDIV, MVT::v4i16, Custom);
562 setOperationAction(ISD::UDIV, MVT::v8i8, Custom);
563 setOperationAction(ISD::SETCC, MVT::v1i64, Expand);
564 setOperationAction(ISD::SETCC, MVT::v2i64, Expand);
565 // Neon does not have single instruction SINT_TO_FP and UINT_TO_FP with
566 // a destination type that is wider than the source, and nor does
567 // it have a FP_TO_[SU]INT instruction with a narrower destination than
568 // source.
569 setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom);
570 setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom);
571 setOperationAction(ISD::FP_TO_UINT, MVT::v4i16, Custom);
572 setOperationAction(ISD::FP_TO_SINT, MVT::v4i16, Custom);
573
574 setOperationAction(ISD::FP_ROUND, MVT::v2f32, Expand);
575 setOperationAction(ISD::FP_EXTEND, MVT::v2f64, Expand);
576
577 // NEON does not have single instruction CTPOP for vectors with element
578 // types wider than 8-bits. However, custom lowering can leverage the
579 // v8i8/v16i8 vcnt instruction.
580 setOperationAction(ISD::CTPOP, MVT::v2i32, Custom);
581 setOperationAction(ISD::CTPOP, MVT::v4i32, Custom);
582 setOperationAction(ISD::CTPOP, MVT::v4i16, Custom);
583 setOperationAction(ISD::CTPOP, MVT::v8i16, Custom);
584
585 // NEON does not have single instruction CTTZ for vectors.
586 setOperationAction(ISD::CTTZ, MVT::v8i8, Custom);
587 setOperationAction(ISD::CTTZ, MVT::v4i16, Custom);
588 setOperationAction(ISD::CTTZ, MVT::v2i32, Custom);
589 setOperationAction(ISD::CTTZ, MVT::v1i64, Custom);
590
591 setOperationAction(ISD::CTTZ, MVT::v16i8, Custom);
592 setOperationAction(ISD::CTTZ, MVT::v8i16, Custom);
593 setOperationAction(ISD::CTTZ, MVT::v4i32, Custom);
594 setOperationAction(ISD::CTTZ, MVT::v2i64, Custom);
595
596 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v8i8, Custom);
597 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v4i16, Custom);
598 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v2i32, Custom);
599 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v1i64, Custom);
600
601 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v16i8, Custom);
602 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v8i16, Custom);
603 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v4i32, Custom);
604 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v2i64, Custom);
605
606 // NEON only has FMA instructions as of VFP4.
607 if (!Subtarget->hasVFP4()) {
608 setOperationAction(ISD::FMA, MVT::v2f32, Expand);
609 setOperationAction(ISD::FMA, MVT::v4f32, Expand);
610 }
611
612 setTargetDAGCombine(ISD::INTRINSIC_VOID);
613 setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN);
614 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
615 setTargetDAGCombine(ISD::SHL);
616 setTargetDAGCombine(ISD::SRL);
617 setTargetDAGCombine(ISD::SRA);
618 setTargetDAGCombine(ISD::SIGN_EXTEND);
619 setTargetDAGCombine(ISD::ZERO_EXTEND);
620 setTargetDAGCombine(ISD::ANY_EXTEND);
621 setTargetDAGCombine(ISD::BUILD_VECTOR);
622 setTargetDAGCombine(ISD::VECTOR_SHUFFLE);
623 setTargetDAGCombine(ISD::INSERT_VECTOR_ELT);
624 setTargetDAGCombine(ISD::STORE);
625 setTargetDAGCombine(ISD::FP_TO_SINT);
626 setTargetDAGCombine(ISD::FP_TO_UINT);
627 setTargetDAGCombine(ISD::FDIV);
628 setTargetDAGCombine(ISD::LOAD);
629
630 // It is legal to extload from v4i8 to v4i16 or v4i32.
631 for (MVT Ty : {MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v4i16, MVT::v2i16,
632 MVT::v2i32}) {
633 for (MVT VT : MVT::integer_vector_valuetypes()) {
634 setLoadExtAction(ISD::EXTLOAD, VT, Ty, Legal);
635 setLoadExtAction(ISD::ZEXTLOAD, VT, Ty, Legal);
636 setLoadExtAction(ISD::SEXTLOAD, VT, Ty, Legal);
637 }
638 }
639 }
640
641 // ARM and Thumb2 support UMLAL/SMLAL.
642 if (!Subtarget->isThumb1Only())
643 setTargetDAGCombine(ISD::ADDC);
644
645 if (Subtarget->isFPOnlySP()) {
646 // When targeting a floating-point unit with only single-precision
647 // operations, f64 is legal for the few double-precision instructions which
648 // are present However, no double-precision operations other than moves,
649 // loads and stores are provided by the hardware.
650 setOperationAction(ISD::FADD, MVT::f64, Expand);
651 setOperationAction(ISD::FSUB, MVT::f64, Expand);
652 setOperationAction(ISD::FMUL, MVT::f64, Expand);
653 setOperationAction(ISD::FMA, MVT::f64, Expand);
654 setOperationAction(ISD::FDIV, MVT::f64, Expand);
655 setOperationAction(ISD::FREM, MVT::f64, Expand);
656 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
657 setOperationAction(ISD::FGETSIGN, MVT::f64, Expand);
658 setOperationAction(ISD::FNEG, MVT::f64, Expand);
659 setOperationAction(ISD::FABS, MVT::f64, Expand);
660 setOperationAction(ISD::FSQRT, MVT::f64, Expand);
661 setOperationAction(ISD::FSIN, MVT::f64, Expand);
662 setOperationAction(ISD::FCOS, MVT::f64, Expand);
663 setOperationAction(ISD::FPOWI, MVT::f64, Expand);
664 setOperationAction(ISD::FPOW, MVT::f64, Expand);
665 setOperationAction(ISD::FLOG, MVT::f64, Expand);
666 setOperationAction(ISD::FLOG2, MVT::f64, Expand);
667 setOperationAction(ISD::FLOG10, MVT::f64, Expand);
668 setOperationAction(ISD::FEXP, MVT::f64, Expand);
669 setOperationAction(ISD::FEXP2, MVT::f64, Expand);
670 setOperationAction(ISD::FCEIL, MVT::f64, Expand);
671 setOperationAction(ISD::FTRUNC, MVT::f64, Expand);
672 setOperationAction(ISD::FRINT, MVT::f64, Expand);
673 setOperationAction(ISD::FNEARBYINT, MVT::f64, Expand);
674 setOperationAction(ISD::FFLOOR, MVT::f64, Expand);
675 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
676 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
677 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
678 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
679 setOperationAction(ISD::FP_TO_SINT, MVT::f64, Custom);
680 setOperationAction(ISD::FP_TO_UINT, MVT::f64, Custom);
681 setOperationAction(ISD::FP_ROUND, MVT::f32, Custom);
682 setOperationAction(ISD::FP_EXTEND, MVT::f64, Custom);
683 }
684
685 computeRegisterProperties(Subtarget->getRegisterInfo());
686
687 // ARM does not have floating-point extending loads.
688 for (MVT VT : MVT::fp_valuetypes()) {
689 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand);
690 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand);
691 }
692
693 // ... or truncating stores
694 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
695 setTruncStoreAction(MVT::f32, MVT::f16, Expand);
696 setTruncStoreAction(MVT::f64, MVT::f16, Expand);
697
698 // ARM does not have i1 sign extending load.
699 for (MVT VT : MVT::integer_valuetypes())
700 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
701
702 // ARM supports all 4 flavors of integer indexed load / store.
703 if (!Subtarget->isThumb1Only()) {
704 for (unsigned im = (unsigned)ISD::PRE_INC;
705 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
706 setIndexedLoadAction(im, MVT::i1, Legal);
707 setIndexedLoadAction(im, MVT::i8, Legal);
708 setIndexedLoadAction(im, MVT::i16, Legal);
709 setIndexedLoadAction(im, MVT::i32, Legal);
710 setIndexedStoreAction(im, MVT::i1, Legal);
711 setIndexedStoreAction(im, MVT::i8, Legal);
712 setIndexedStoreAction(im, MVT::i16, Legal);
713 setIndexedStoreAction(im, MVT::i32, Legal);
714 }
715 }
716
717 setOperationAction(ISD::SADDO, MVT::i32, Custom);
718 setOperationAction(ISD::UADDO, MVT::i32, Custom);
719 setOperationAction(ISD::SSUBO, MVT::i32, Custom);
720 setOperationAction(ISD::USUBO, MVT::i32, Custom);
721
722 // i64 operation support.
723 setOperationAction(ISD::MUL, MVT::i64, Expand);
724 setOperationAction(ISD::MULHU, MVT::i32, Expand);
725 if (Subtarget->isThumb1Only()) {
726 setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand);
727 setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand);
728 }
729 if (Subtarget->isThumb1Only() || !Subtarget->hasV6Ops()
730 || (Subtarget->isThumb2() && !Subtarget->hasDSP()))
731 setOperationAction(ISD::MULHS, MVT::i32, Expand);
732
733 setOperationAction(ISD::SHL_PARTS, MVT::i32, Custom);
734 setOperationAction(ISD::SRA_PARTS, MVT::i32, Custom);
735 setOperationAction(ISD::SRL_PARTS, MVT::i32, Custom);
736 setOperationAction(ISD::SRL, MVT::i64, Custom);
737 setOperationAction(ISD::SRA, MVT::i64, Custom);
738
739 if (!Subtarget->isThumb1Only()) {
740 // FIXME: We should do this for Thumb1 as well.
741 setOperationAction(ISD::ADDC, MVT::i32, Custom);
742 setOperationAction(ISD::ADDE, MVT::i32, Custom);
743 setOperationAction(ISD::SUBC, MVT::i32, Custom);
744 setOperationAction(ISD::SUBE, MVT::i32, Custom);
745 }
746
747 if (!Subtarget->isThumb1Only())
748 setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
749
750 // ARM does not have ROTL.
751 setOperationAction(ISD::ROTL, MVT::i32, Expand);
752 for (MVT VT : MVT::vector_valuetypes()) {
753 setOperationAction(ISD::ROTL, VT, Expand);
754 setOperationAction(ISD::ROTR, VT, Expand);
755 }
756 setOperationAction(ISD::CTTZ, MVT::i32, Custom);
757 setOperationAction(ISD::CTPOP, MVT::i32, Expand);
758 if (!Subtarget->hasV5TOps() || Subtarget->isThumb1Only())
759 setOperationAction(ISD::CTLZ, MVT::i32, Expand);
760
761 // These just redirect to CTTZ and CTLZ on ARM.
762 setOperationAction(ISD::CTTZ_ZERO_UNDEF , MVT::i32 , Expand);
763 setOperationAction(ISD::CTLZ_ZERO_UNDEF , MVT::i32 , Expand);
764
765 // @llvm.readcyclecounter requires the Performance Monitors extension.
766 // Default to the 0 expansion on unsupported platforms.
767 // FIXME: Technically there are older ARM CPUs that have
768 // implementation-specific ways of obtaining this information.
769 if (Subtarget->hasPerfMon())
770 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Custom);
771
772 // Only ARMv6 has BSWAP.
773 if (!Subtarget->hasV6Ops())
774 setOperationAction(ISD::BSWAP, MVT::i32, Expand);
775
776 if (!(Subtarget->hasDivide() && Subtarget->isThumb2()) &&
777 !(Subtarget->hasDivideInARMMode() && !Subtarget->isThumb())) {
778 // These are expanded into libcalls if the cpu doesn't have HW divider.
779 setOperationAction(ISD::SDIV, MVT::i32, LibCall);
780 setOperationAction(ISD::UDIV, MVT::i32, LibCall);
781 }
782
783 setOperationAction(ISD::SREM, MVT::i32, Expand);
784 setOperationAction(ISD::UREM, MVT::i32, Expand);
785 // Register based DivRem for AEABI (RTABI 4.2)
786 if (Subtarget->isTargetAEABI() || Subtarget->isTargetAndroid()) {
787 setOperationAction(ISD::SREM, MVT::i64, Custom);
788 setOperationAction(ISD::UREM, MVT::i64, Custom);
789
790 setLibcallName(RTLIB::SDIVREM_I8, "__aeabi_idivmod");
791 setLibcallName(RTLIB::SDIVREM_I16, "__aeabi_idivmod");
792 setLibcallName(RTLIB::SDIVREM_I32, "__aeabi_idivmod");
793 setLibcallName(RTLIB::SDIVREM_I64, "__aeabi_ldivmod");
794 setLibcallName(RTLIB::UDIVREM_I8, "__aeabi_uidivmod");
795 setLibcallName(RTLIB::UDIVREM_I16, "__aeabi_uidivmod");
796 setLibcallName(RTLIB::UDIVREM_I32, "__aeabi_uidivmod");
797 setLibcallName(RTLIB::UDIVREM_I64, "__aeabi_uldivmod");
798
799 setLibcallCallingConv(RTLIB::SDIVREM_I8, CallingConv::ARM_AAPCS);
800 setLibcallCallingConv(RTLIB::SDIVREM_I16, CallingConv::ARM_AAPCS);
801 setLibcallCallingConv(RTLIB::SDIVREM_I32, CallingConv::ARM_AAPCS);
802 setLibcallCallingConv(RTLIB::SDIVREM_I64, CallingConv::ARM_AAPCS);
803 setLibcallCallingConv(RTLIB::UDIVREM_I8, CallingConv::ARM_AAPCS);
804 setLibcallCallingConv(RTLIB::UDIVREM_I16, CallingConv::ARM_AAPCS);
805 setLibcallCallingConv(RTLIB::UDIVREM_I32, CallingConv::ARM_AAPCS);
806 setLibcallCallingConv(RTLIB::UDIVREM_I64, CallingConv::ARM_AAPCS);
807
808 setOperationAction(ISD::SDIVREM, MVT::i32, Custom);
809 setOperationAction(ISD::UDIVREM, MVT::i32, Custom);
810 } else {
811 setOperationAction(ISD::SDIVREM, MVT::i32, Expand);
812 setOperationAction(ISD::UDIVREM, MVT::i32, Expand);
813 }
814
815 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom);
816 setOperationAction(ISD::ConstantPool, MVT::i32, Custom);
817 setOperationAction(ISD::GlobalTLSAddress, MVT::i32, Custom);
818 setOperationAction(ISD::BlockAddress, MVT::i32, Custom);
819
820 setOperationAction(ISD::TRAP, MVT::Other, Legal);
821
822 // Use the default implementation.
823 setOperationAction(ISD::VASTART, MVT::Other, Custom);
824 setOperationAction(ISD::VAARG, MVT::Other, Expand);
825 setOperationAction(ISD::VACOPY, MVT::Other, Expand);
826 setOperationAction(ISD::VAEND, MVT::Other, Expand);
827 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
828 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
829
830 if (Subtarget->getTargetTriple().isWindowsItaniumEnvironment())
831 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom);
832 else
833 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Expand);
834
835 // ARMv6 Thumb1 (except for CPUs that support dmb / dsb) and earlier use
836 // the default expansion. If we are targeting a single threaded system,
837 // then set them all for expand so we can lower them later into their
838 // non-atomic form.
839 if (TM.Options.ThreadModel == ThreadModel::Single)
840 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Expand);
841 else if (Subtarget->hasAnyDataBarrier() && !Subtarget->isThumb1Only()) {
842 // ATOMIC_FENCE needs custom lowering; the others should have been expanded
843 // to ldrex/strex loops already.
844 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
845
846 // On v8, we have particularly efficient implementations of atomic fences
847 // if they can be combined with nearby atomic loads and stores.
848 if (!Subtarget->hasV8Ops()) {
849 // Automatically insert fences (dmb ish) around ATOMIC_SWAP etc.
850 setInsertFencesForAtomic(true);
851 }
852 } else {
853 // If there's anything we can use as a barrier, go through custom lowering
854 // for ATOMIC_FENCE.
855 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other,
856 Subtarget->hasAnyDataBarrier() ? Custom : Expand);
857
858 // Set them all for expansion, which will force libcalls.
859 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Expand);
860 setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, Expand);
861 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, Expand);
862 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Expand);
863 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Expand);
864 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, Expand);
865 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, Expand);
866 setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i32, Expand);
867 setOperationAction(ISD::ATOMIC_LOAD_MIN, MVT::i32, Expand);
868 setOperationAction(ISD::ATOMIC_LOAD_MAX, MVT::i32, Expand);
869 setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i32, Expand);
870 setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i32, Expand);
871 // Mark ATOMIC_LOAD and ATOMIC_STORE custom so we can handle the
872 // Unordered/Monotonic case.
873 setOperationAction(ISD::ATOMIC_LOAD, MVT::i32, Custom);
874 setOperationAction(ISD::ATOMIC_STORE, MVT::i32, Custom);
875 }
876
877 setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
878
879 // Requires SXTB/SXTH, available on v6 and up in both ARM and Thumb modes.
880 if (!Subtarget->hasV6Ops()) {
881 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Expand);
882 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Expand);
883 }
884 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
885
886 if (!Subtarget->useSoftFloat() && Subtarget->hasVFP2() &&
887 !Subtarget->isThumb1Only()) {
888 // Turn f64->i64 into VMOVRRD, i64 -> f64 to VMOVDRR
889 // iff target supports vfp2.
890 setOperationAction(ISD::BITCAST, MVT::i64, Custom);
891 setOperationAction(ISD::FLT_ROUNDS_, MVT::i32, Custom);
892 }
893
894 // We want to custom lower some of our intrinsics.
895 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
896 setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);
897 setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);
898 setOperationAction(ISD::EH_SJLJ_SETUP_DISPATCH, MVT::Other, Custom);
899 if (Subtarget->useSjLjEH())
900 setLibcallName(RTLIB::UNWIND_RESUME, "_Unwind_SjLj_Resume");
901
902 setOperationAction(ISD::SETCC, MVT::i32, Expand);
903 setOperationAction(ISD::SETCC, MVT::f32, Expand);
904 setOperationAction(ISD::SETCC, MVT::f64, Expand);
905 setOperationAction(ISD::SELECT, MVT::i32, Custom);
906 setOperationAction(ISD::SELECT, MVT::f32, Custom);
907 setOperationAction(ISD::SELECT, MVT::f64, Custom);
908 setOperationAction(ISD::SELECT_CC, MVT::i32, Custom);
909 setOperationAction(ISD::SELECT_CC, MVT::f32, Custom);
910 setOperationAction(ISD::SELECT_CC, MVT::f64, Custom);
911
912 setOperationAction(ISD::BRCOND, MVT::Other, Expand);
913 setOperationAction(ISD::BR_CC, MVT::i32, Custom);
914 setOperationAction(ISD::BR_CC, MVT::f32, Custom);
915 setOperationAction(ISD::BR_CC, MVT::f64, Custom);
916 setOperationAction(ISD::BR_JT, MVT::Other, Custom);
917
918 // We don't support sin/cos/fmod/copysign/pow
919 setOperationAction(ISD::FSIN, MVT::f64, Expand);
920 setOperationAction(ISD::FSIN, MVT::f32, Expand);
921 setOperationAction(ISD::FCOS, MVT::f32, Expand);
922 setOperationAction(ISD::FCOS, MVT::f64, Expand);
923 setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
924 setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
925 setOperationAction(ISD::FREM, MVT::f64, Expand);
926 setOperationAction(ISD::FREM, MVT::f32, Expand);
927 if (!Subtarget->useSoftFloat() && Subtarget->hasVFP2() &&
928 !Subtarget->isThumb1Only()) {
929 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom);
930 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
931 }
932 setOperationAction(ISD::FPOW, MVT::f64, Expand);
933 setOperationAction(ISD::FPOW, MVT::f32, Expand);
934
935 if (!Subtarget->hasVFP4()) {
936 setOperationAction(ISD::FMA, MVT::f64, Expand);
937 setOperationAction(ISD::FMA, MVT::f32, Expand);
938 }
939
940 // Various VFP goodness
941 if (!Subtarget->useSoftFloat() && !Subtarget->isThumb1Only()) {
942 // FP-ARMv8 adds f64 <-> f16 conversion. Before that it should be expanded.
943 if (!Subtarget->hasFPARMv8() || Subtarget->isFPOnlySP()) {
944 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
945 setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand);
946 }
947
948 // fp16 is a special v7 extension that adds f16 <-> f32 conversions.
949 if (!Subtarget->hasFP16()) {
950 setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand);
951 setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand);
952 }
953 }
954
955 // Combine sin / cos into one node or libcall if possible.
956 if (Subtarget->hasSinCos()) {
957 setLibcallName(RTLIB::SINCOS_F32, "sincosf");
958 setLibcallName(RTLIB::SINCOS_F64, "sincos");
959 if (Subtarget->isTargetWatchOS()) {
960 setLibcallCallingConv(RTLIB::SINCOS_F32, CallingConv::ARM_AAPCS_VFP);
961 setLibcallCallingConv(RTLIB::SINCOS_F64, CallingConv::ARM_AAPCS_VFP);
962 }
963 if (Subtarget->isTargetIOS() || Subtarget->isTargetWatchOS()) {
964 // For iOS, we don't want to the normal expansion of a libcall to
965 // sincos. We want to issue a libcall to __sincos_stret.
966 setOperationAction(ISD::FSINCOS, MVT::f64, Custom);
967 setOperationAction(ISD::FSINCOS, MVT::f32, Custom);
968 }
969 }
970
971 // FP-ARMv8 implements a lot of rounding-like FP operations.
972 if (Subtarget->hasFPARMv8()) {
973 setOperationAction(ISD::FFLOOR, MVT::f32, Legal);
974 setOperationAction(ISD::FCEIL, MVT::f32, Legal);
975 setOperationAction(ISD::FROUND, MVT::f32, Legal);
976 setOperationAction(ISD::FTRUNC, MVT::f32, Legal);
977 setOperationAction(ISD::FNEARBYINT, MVT::f32, Legal);
978 setOperationAction(ISD::FRINT, MVT::f32, Legal);
979 setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
980 setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
981 setOperationAction(ISD::FMINNUM, MVT::v2f32, Legal);
982 setOperationAction(ISD::FMAXNUM, MVT::v2f32, Legal);
983 setOperationAction(ISD::FMINNUM, MVT::v4f32, Legal);
984 setOperationAction(ISD::FMAXNUM, MVT::v4f32, Legal);
985
986 if (!Subtarget->isFPOnlySP()) {
987 setOperationAction(ISD::FFLOOR, MVT::f64, Legal);
988 setOperationAction(ISD::FCEIL, MVT::f64, Legal);
989 setOperationAction(ISD::FROUND, MVT::f64, Legal);
990 setOperationAction(ISD::FTRUNC, MVT::f64, Legal);
991 setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal);
992 setOperationAction(ISD::FRINT, MVT::f64, Legal);
993 setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
994 setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
995 }
996 }
997
998 if (Subtarget->hasNEON()) {
999 // vmin and vmax aren't available in a scalar form, so we use
1000 // a NEON instruction with an undef lane instead.
1001 setOperationAction(ISD::FMINNAN, MVT::f32, Legal);
1002 setOperationAction(ISD::FMAXNAN, MVT::f32, Legal);
1003 setOperationAction(ISD::FMINNAN, MVT::v2f32, Legal);
1004 setOperationAction(ISD::FMAXNAN, MVT::v2f32, Legal);
1005 setOperationAction(ISD::FMINNAN, MVT::v4f32, Legal);
1006 setOperationAction(ISD::FMAXNAN, MVT::v4f32, Legal);
1007 }
1008
1009 // We have target-specific dag combine patterns for the following nodes:
1010 // ARMISD::VMOVRRD - No need to call setTargetDAGCombine
1011 setTargetDAGCombine(ISD::ADD);
1012 setTargetDAGCombine(ISD::SUB);
1013 setTargetDAGCombine(ISD::MUL);
1014 setTargetDAGCombine(ISD::AND);
1015 setTargetDAGCombine(ISD::OR);
1016 setTargetDAGCombine(ISD::XOR);
1017
1018 if (Subtarget->hasV6Ops())
1019 setTargetDAGCombine(ISD::SRL);
1020
1021 setStackPointerRegisterToSaveRestore(ARM::SP);
1022
1023 if (Subtarget->useSoftFloat() || Subtarget->isThumb1Only() ||
1024 !Subtarget->hasVFP2())
1025 setSchedulingPreference(Sched::RegPressure);
1026 else
1027 setSchedulingPreference(Sched::Hybrid);
1028
1029 //// temporary - rewrite interface to use type
1030 MaxStoresPerMemset = 8;
1031 MaxStoresPerMemsetOptSize = 4;
1032 MaxStoresPerMemcpy = 4; // For @llvm.memcpy -> sequence of stores
1033 MaxStoresPerMemcpyOptSize = 2;
1034 MaxStoresPerMemmove = 4; // For @llvm.memmove -> sequence of stores
1035 MaxStoresPerMemmoveOptSize = 2;
1036
1037 // On ARM arguments smaller than 4 bytes are extended, so all arguments
1038 // are at least 4 bytes aligned.
1039 setMinStackArgumentAlignment(4);
1040
1041 // Prefer likely predicted branches to selects on out-of-order cores.
1042 PredictableSelectIsExpensive = Subtarget->isLikeA9();
1043
1044 setMinFunctionAlignment(Subtarget->isThumb() ? 1 : 2);
1045 }
1046
useSoftFloat() const1047 bool ARMTargetLowering::useSoftFloat() const {
1048 return Subtarget->useSoftFloat();
1049 }
1050
1051 // FIXME: It might make sense to define the representative register class as the
1052 // nearest super-register that has a non-null superset. For example, DPR_VFP2 is
1053 // a super-register of SPR, and DPR is a superset if DPR_VFP2. Consequently,
1054 // SPR's representative would be DPR_VFP2. This should work well if register
1055 // pressure tracking were modified such that a register use would increment the
1056 // pressure of the register class's representative and all of it's super
1057 // classes' representatives transitively. We have not implemented this because
1058 // of the difficulty prior to coalescing of modeling operand register classes
1059 // due to the common occurrence of cross class copies and subregister insertions
1060 // and extractions.
1061 std::pair<const TargetRegisterClass *, uint8_t>
findRepresentativeClass(const TargetRegisterInfo * TRI,MVT VT) const1062 ARMTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,
1063 MVT VT) const {
1064 const TargetRegisterClass *RRC = nullptr;
1065 uint8_t Cost = 1;
1066 switch (VT.SimpleTy) {
1067 default:
1068 return TargetLowering::findRepresentativeClass(TRI, VT);
1069 // Use DPR as representative register class for all floating point
1070 // and vector types. Since there are 32 SPR registers and 32 DPR registers so
1071 // the cost is 1 for both f32 and f64.
1072 case MVT::f32: case MVT::f64: case MVT::v8i8: case MVT::v4i16:
1073 case MVT::v2i32: case MVT::v1i64: case MVT::v2f32:
1074 RRC = &ARM::DPRRegClass;
1075 // When NEON is used for SP, only half of the register file is available
1076 // because operations that define both SP and DP results will be constrained
1077 // to the VFP2 class (D0-D15). We currently model this constraint prior to
1078 // coalescing by double-counting the SP regs. See the FIXME above.
1079 if (Subtarget->useNEONForSinglePrecisionFP())
1080 Cost = 2;
1081 break;
1082 case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64:
1083 case MVT::v4f32: case MVT::v2f64:
1084 RRC = &ARM::DPRRegClass;
1085 Cost = 2;
1086 break;
1087 case MVT::v4i64:
1088 RRC = &ARM::DPRRegClass;
1089 Cost = 4;
1090 break;
1091 case MVT::v8i64:
1092 RRC = &ARM::DPRRegClass;
1093 Cost = 8;
1094 break;
1095 }
1096 return std::make_pair(RRC, Cost);
1097 }
1098
getTargetNodeName(unsigned Opcode) const1099 const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const {
1100 switch ((ARMISD::NodeType)Opcode) {
1101 case ARMISD::FIRST_NUMBER: break;
1102 case ARMISD::Wrapper: return "ARMISD::Wrapper";
1103 case ARMISD::WrapperPIC: return "ARMISD::WrapperPIC";
1104 case ARMISD::WrapperJT: return "ARMISD::WrapperJT";
1105 case ARMISD::COPY_STRUCT_BYVAL: return "ARMISD::COPY_STRUCT_BYVAL";
1106 case ARMISD::CALL: return "ARMISD::CALL";
1107 case ARMISD::CALL_PRED: return "ARMISD::CALL_PRED";
1108 case ARMISD::CALL_NOLINK: return "ARMISD::CALL_NOLINK";
1109 case ARMISD::tCALL: return "ARMISD::tCALL";
1110 case ARMISD::BRCOND: return "ARMISD::BRCOND";
1111 case ARMISD::BR_JT: return "ARMISD::BR_JT";
1112 case ARMISD::BR2_JT: return "ARMISD::BR2_JT";
1113 case ARMISD::RET_FLAG: return "ARMISD::RET_FLAG";
1114 case ARMISD::INTRET_FLAG: return "ARMISD::INTRET_FLAG";
1115 case ARMISD::PIC_ADD: return "ARMISD::PIC_ADD";
1116 case ARMISD::CMP: return "ARMISD::CMP";
1117 case ARMISD::CMN: return "ARMISD::CMN";
1118 case ARMISD::CMPZ: return "ARMISD::CMPZ";
1119 case ARMISD::CMPFP: return "ARMISD::CMPFP";
1120 case ARMISD::CMPFPw0: return "ARMISD::CMPFPw0";
1121 case ARMISD::BCC_i64: return "ARMISD::BCC_i64";
1122 case ARMISD::FMSTAT: return "ARMISD::FMSTAT";
1123
1124 case ARMISD::CMOV: return "ARMISD::CMOV";
1125
1126 case ARMISD::SRL_FLAG: return "ARMISD::SRL_FLAG";
1127 case ARMISD::SRA_FLAG: return "ARMISD::SRA_FLAG";
1128 case ARMISD::RRX: return "ARMISD::RRX";
1129
1130 case ARMISD::ADDC: return "ARMISD::ADDC";
1131 case ARMISD::ADDE: return "ARMISD::ADDE";
1132 case ARMISD::SUBC: return "ARMISD::SUBC";
1133 case ARMISD::SUBE: return "ARMISD::SUBE";
1134
1135 case ARMISD::VMOVRRD: return "ARMISD::VMOVRRD";
1136 case ARMISD::VMOVDRR: return "ARMISD::VMOVDRR";
1137
1138 case ARMISD::EH_SJLJ_SETJMP: return "ARMISD::EH_SJLJ_SETJMP";
1139 case ARMISD::EH_SJLJ_LONGJMP: return "ARMISD::EH_SJLJ_LONGJMP";
1140 case ARMISD::EH_SJLJ_SETUP_DISPATCH: return "ARMISD::EH_SJLJ_SETUP_DISPATCH";
1141
1142 case ARMISD::TC_RETURN: return "ARMISD::TC_RETURN";
1143
1144 case ARMISD::THREAD_POINTER:return "ARMISD::THREAD_POINTER";
1145
1146 case ARMISD::DYN_ALLOC: return "ARMISD::DYN_ALLOC";
1147
1148 case ARMISD::MEMBARRIER_MCR: return "ARMISD::MEMBARRIER_MCR";
1149
1150 case ARMISD::PRELOAD: return "ARMISD::PRELOAD";
1151
1152 case ARMISD::WIN__CHKSTK: return "ARMISD:::WIN__CHKSTK";
1153 case ARMISD::WIN__DBZCHK: return "ARMISD::WIN__DBZCHK";
1154
1155 case ARMISD::VCEQ: return "ARMISD::VCEQ";
1156 case ARMISD::VCEQZ: return "ARMISD::VCEQZ";
1157 case ARMISD::VCGE: return "ARMISD::VCGE";
1158 case ARMISD::VCGEZ: return "ARMISD::VCGEZ";
1159 case ARMISD::VCLEZ: return "ARMISD::VCLEZ";
1160 case ARMISD::VCGEU: return "ARMISD::VCGEU";
1161 case ARMISD::VCGT: return "ARMISD::VCGT";
1162 case ARMISD::VCGTZ: return "ARMISD::VCGTZ";
1163 case ARMISD::VCLTZ: return "ARMISD::VCLTZ";
1164 case ARMISD::VCGTU: return "ARMISD::VCGTU";
1165 case ARMISD::VTST: return "ARMISD::VTST";
1166
1167 case ARMISD::VSHL: return "ARMISD::VSHL";
1168 case ARMISD::VSHRs: return "ARMISD::VSHRs";
1169 case ARMISD::VSHRu: return "ARMISD::VSHRu";
1170 case ARMISD::VRSHRs: return "ARMISD::VRSHRs";
1171 case ARMISD::VRSHRu: return "ARMISD::VRSHRu";
1172 case ARMISD::VRSHRN: return "ARMISD::VRSHRN";
1173 case ARMISD::VQSHLs: return "ARMISD::VQSHLs";
1174 case ARMISD::VQSHLu: return "ARMISD::VQSHLu";
1175 case ARMISD::VQSHLsu: return "ARMISD::VQSHLsu";
1176 case ARMISD::VQSHRNs: return "ARMISD::VQSHRNs";
1177 case ARMISD::VQSHRNu: return "ARMISD::VQSHRNu";
1178 case ARMISD::VQSHRNsu: return "ARMISD::VQSHRNsu";
1179 case ARMISD::VQRSHRNs: return "ARMISD::VQRSHRNs";
1180 case ARMISD::VQRSHRNu: return "ARMISD::VQRSHRNu";
1181 case ARMISD::VQRSHRNsu: return "ARMISD::VQRSHRNsu";
1182 case ARMISD::VSLI: return "ARMISD::VSLI";
1183 case ARMISD::VSRI: return "ARMISD::VSRI";
1184 case ARMISD::VGETLANEu: return "ARMISD::VGETLANEu";
1185 case ARMISD::VGETLANEs: return "ARMISD::VGETLANEs";
1186 case ARMISD::VMOVIMM: return "ARMISD::VMOVIMM";
1187 case ARMISD::VMVNIMM: return "ARMISD::VMVNIMM";
1188 case ARMISD::VMOVFPIMM: return "ARMISD::VMOVFPIMM";
1189 case ARMISD::VDUP: return "ARMISD::VDUP";
1190 case ARMISD::VDUPLANE: return "ARMISD::VDUPLANE";
1191 case ARMISD::VEXT: return "ARMISD::VEXT";
1192 case ARMISD::VREV64: return "ARMISD::VREV64";
1193 case ARMISD::VREV32: return "ARMISD::VREV32";
1194 case ARMISD::VREV16: return "ARMISD::VREV16";
1195 case ARMISD::VZIP: return "ARMISD::VZIP";
1196 case ARMISD::VUZP: return "ARMISD::VUZP";
1197 case ARMISD::VTRN: return "ARMISD::VTRN";
1198 case ARMISD::VTBL1: return "ARMISD::VTBL1";
1199 case ARMISD::VTBL2: return "ARMISD::VTBL2";
1200 case ARMISD::VMULLs: return "ARMISD::VMULLs";
1201 case ARMISD::VMULLu: return "ARMISD::VMULLu";
1202 case ARMISD::UMLAL: return "ARMISD::UMLAL";
1203 case ARMISD::SMLAL: return "ARMISD::SMLAL";
1204 case ARMISD::BUILD_VECTOR: return "ARMISD::BUILD_VECTOR";
1205 case ARMISD::BFI: return "ARMISD::BFI";
1206 case ARMISD::VORRIMM: return "ARMISD::VORRIMM";
1207 case ARMISD::VBICIMM: return "ARMISD::VBICIMM";
1208 case ARMISD::VBSL: return "ARMISD::VBSL";
1209 case ARMISD::MEMCPY: return "ARMISD::MEMCPY";
1210 case ARMISD::VLD2DUP: return "ARMISD::VLD2DUP";
1211 case ARMISD::VLD3DUP: return "ARMISD::VLD3DUP";
1212 case ARMISD::VLD4DUP: return "ARMISD::VLD4DUP";
1213 case ARMISD::VLD1_UPD: return "ARMISD::VLD1_UPD";
1214 case ARMISD::VLD2_UPD: return "ARMISD::VLD2_UPD";
1215 case ARMISD::VLD3_UPD: return "ARMISD::VLD3_UPD";
1216 case ARMISD::VLD4_UPD: return "ARMISD::VLD4_UPD";
1217 case ARMISD::VLD2LN_UPD: return "ARMISD::VLD2LN_UPD";
1218 case ARMISD::VLD3LN_UPD: return "ARMISD::VLD3LN_UPD";
1219 case ARMISD::VLD4LN_UPD: return "ARMISD::VLD4LN_UPD";
1220 case ARMISD::VLD2DUP_UPD: return "ARMISD::VLD2DUP_UPD";
1221 case ARMISD::VLD3DUP_UPD: return "ARMISD::VLD3DUP_UPD";
1222 case ARMISD::VLD4DUP_UPD: return "ARMISD::VLD4DUP_UPD";
1223 case ARMISD::VST1_UPD: return "ARMISD::VST1_UPD";
1224 case ARMISD::VST2_UPD: return "ARMISD::VST2_UPD";
1225 case ARMISD::VST3_UPD: return "ARMISD::VST3_UPD";
1226 case ARMISD::VST4_UPD: return "ARMISD::VST4_UPD";
1227 case ARMISD::VST2LN_UPD: return "ARMISD::VST2LN_UPD";
1228 case ARMISD::VST3LN_UPD: return "ARMISD::VST3LN_UPD";
1229 case ARMISD::VST4LN_UPD: return "ARMISD::VST4LN_UPD";
1230 }
1231 return nullptr;
1232 }
1233
getSetCCResultType(const DataLayout & DL,LLVMContext &,EVT VT) const1234 EVT ARMTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &,
1235 EVT VT) const {
1236 if (!VT.isVector())
1237 return getPointerTy(DL);
1238 return VT.changeVectorElementTypeToInteger();
1239 }
1240
1241 /// getRegClassFor - Return the register class that should be used for the
1242 /// specified value type.
getRegClassFor(MVT VT) const1243 const TargetRegisterClass *ARMTargetLowering::getRegClassFor(MVT VT) const {
1244 // Map v4i64 to QQ registers but do not make the type legal. Similarly map
1245 // v8i64 to QQQQ registers. v4i64 and v8i64 are only used for REG_SEQUENCE to
1246 // load / store 4 to 8 consecutive D registers.
1247 if (Subtarget->hasNEON()) {
1248 if (VT == MVT::v4i64)
1249 return &ARM::QQPRRegClass;
1250 if (VT == MVT::v8i64)
1251 return &ARM::QQQQPRRegClass;
1252 }
1253 return TargetLowering::getRegClassFor(VT);
1254 }
1255
1256 // memcpy, and other memory intrinsics, typically tries to use LDM/STM if the
1257 // source/dest is aligned and the copy size is large enough. We therefore want
1258 // to align such objects passed to memory intrinsics.
shouldAlignPointerArgs(CallInst * CI,unsigned & MinSize,unsigned & PrefAlign) const1259 bool ARMTargetLowering::shouldAlignPointerArgs(CallInst *CI, unsigned &MinSize,
1260 unsigned &PrefAlign) const {
1261 if (!isa<MemIntrinsic>(CI))
1262 return false;
1263 MinSize = 8;
1264 // On ARM11 onwards (excluding M class) 8-byte aligned LDM is typically 1
1265 // cycle faster than 4-byte aligned LDM.
1266 PrefAlign = (Subtarget->hasV6Ops() && !Subtarget->isMClass() ? 8 : 4);
1267 return true;
1268 }
1269
1270 // Create a fast isel object.
1271 FastISel *
createFastISel(FunctionLoweringInfo & funcInfo,const TargetLibraryInfo * libInfo) const1272 ARMTargetLowering::createFastISel(FunctionLoweringInfo &funcInfo,
1273 const TargetLibraryInfo *libInfo) const {
1274 return ARM::createFastISel(funcInfo, libInfo);
1275 }
1276
getSchedulingPreference(SDNode * N) const1277 Sched::Preference ARMTargetLowering::getSchedulingPreference(SDNode *N) const {
1278 unsigned NumVals = N->getNumValues();
1279 if (!NumVals)
1280 return Sched::RegPressure;
1281
1282 for (unsigned i = 0; i != NumVals; ++i) {
1283 EVT VT = N->getValueType(i);
1284 if (VT == MVT::Glue || VT == MVT::Other)
1285 continue;
1286 if (VT.isFloatingPoint() || VT.isVector())
1287 return Sched::ILP;
1288 }
1289
1290 if (!N->isMachineOpcode())
1291 return Sched::RegPressure;
1292
1293 // Load are scheduled for latency even if there instruction itinerary
1294 // is not available.
1295 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
1296 const MCInstrDesc &MCID = TII->get(N->getMachineOpcode());
1297
1298 if (MCID.getNumDefs() == 0)
1299 return Sched::RegPressure;
1300 if (!Itins->isEmpty() &&
1301 Itins->getOperandCycle(MCID.getSchedClass(), 0) > 2)
1302 return Sched::ILP;
1303
1304 return Sched::RegPressure;
1305 }
1306
1307 //===----------------------------------------------------------------------===//
1308 // Lowering Code
1309 //===----------------------------------------------------------------------===//
1310
1311 /// IntCCToARMCC - Convert a DAG integer condition code to an ARM CC
IntCCToARMCC(ISD::CondCode CC)1312 static ARMCC::CondCodes IntCCToARMCC(ISD::CondCode CC) {
1313 switch (CC) {
1314 default: llvm_unreachable("Unknown condition code!");
1315 case ISD::SETNE: return ARMCC::NE;
1316 case ISD::SETEQ: return ARMCC::EQ;
1317 case ISD::SETGT: return ARMCC::GT;
1318 case ISD::SETGE: return ARMCC::GE;
1319 case ISD::SETLT: return ARMCC::LT;
1320 case ISD::SETLE: return ARMCC::LE;
1321 case ISD::SETUGT: return ARMCC::HI;
1322 case ISD::SETUGE: return ARMCC::HS;
1323 case ISD::SETULT: return ARMCC::LO;
1324 case ISD::SETULE: return ARMCC::LS;
1325 }
1326 }
1327
1328 /// FPCCToARMCC - Convert a DAG fp condition code to an ARM CC.
FPCCToARMCC(ISD::CondCode CC,ARMCC::CondCodes & CondCode,ARMCC::CondCodes & CondCode2)1329 static void FPCCToARMCC(ISD::CondCode CC, ARMCC::CondCodes &CondCode,
1330 ARMCC::CondCodes &CondCode2) {
1331 CondCode2 = ARMCC::AL;
1332 switch (CC) {
1333 default: llvm_unreachable("Unknown FP condition!");
1334 case ISD::SETEQ:
1335 case ISD::SETOEQ: CondCode = ARMCC::EQ; break;
1336 case ISD::SETGT:
1337 case ISD::SETOGT: CondCode = ARMCC::GT; break;
1338 case ISD::SETGE:
1339 case ISD::SETOGE: CondCode = ARMCC::GE; break;
1340 case ISD::SETOLT: CondCode = ARMCC::MI; break;
1341 case ISD::SETOLE: CondCode = ARMCC::LS; break;
1342 case ISD::SETONE: CondCode = ARMCC::MI; CondCode2 = ARMCC::GT; break;
1343 case ISD::SETO: CondCode = ARMCC::VC; break;
1344 case ISD::SETUO: CondCode = ARMCC::VS; break;
1345 case ISD::SETUEQ: CondCode = ARMCC::EQ; CondCode2 = ARMCC::VS; break;
1346 case ISD::SETUGT: CondCode = ARMCC::HI; break;
1347 case ISD::SETUGE: CondCode = ARMCC::PL; break;
1348 case ISD::SETLT:
1349 case ISD::SETULT: CondCode = ARMCC::LT; break;
1350 case ISD::SETLE:
1351 case ISD::SETULE: CondCode = ARMCC::LE; break;
1352 case ISD::SETNE:
1353 case ISD::SETUNE: CondCode = ARMCC::NE; break;
1354 }
1355 }
1356
1357 //===----------------------------------------------------------------------===//
1358 // Calling Convention Implementation
1359 //===----------------------------------------------------------------------===//
1360
1361 #include "ARMGenCallingConv.inc"
1362
1363 /// getEffectiveCallingConv - Get the effective calling convention, taking into
1364 /// account presence of floating point hardware and calling convention
1365 /// limitations, such as support for variadic functions.
1366 CallingConv::ID
getEffectiveCallingConv(CallingConv::ID CC,bool isVarArg) const1367 ARMTargetLowering::getEffectiveCallingConv(CallingConv::ID CC,
1368 bool isVarArg) const {
1369 switch (CC) {
1370 default:
1371 llvm_unreachable("Unsupported calling convention");
1372 case CallingConv::ARM_AAPCS:
1373 case CallingConv::ARM_APCS:
1374 case CallingConv::GHC:
1375 return CC;
1376 case CallingConv::ARM_AAPCS_VFP:
1377 return isVarArg ? CallingConv::ARM_AAPCS : CallingConv::ARM_AAPCS_VFP;
1378 case CallingConv::C:
1379 if (!Subtarget->isAAPCS_ABI())
1380 return CallingConv::ARM_APCS;
1381 else if (Subtarget->hasVFP2() && !Subtarget->isThumb1Only() &&
1382 getTargetMachine().Options.FloatABIType == FloatABI::Hard &&
1383 !isVarArg)
1384 return CallingConv::ARM_AAPCS_VFP;
1385 else
1386 return CallingConv::ARM_AAPCS;
1387 case CallingConv::Fast:
1388 if (!Subtarget->isAAPCS_ABI()) {
1389 if (Subtarget->hasVFP2() && !Subtarget->isThumb1Only() && !isVarArg)
1390 return CallingConv::Fast;
1391 return CallingConv::ARM_APCS;
1392 } else if (Subtarget->hasVFP2() && !Subtarget->isThumb1Only() && !isVarArg)
1393 return CallingConv::ARM_AAPCS_VFP;
1394 else
1395 return CallingConv::ARM_AAPCS;
1396 }
1397 }
1398
1399 /// CCAssignFnForNode - Selects the correct CCAssignFn for the given
1400 /// CallingConvention.
CCAssignFnForNode(CallingConv::ID CC,bool Return,bool isVarArg) const1401 CCAssignFn *ARMTargetLowering::CCAssignFnForNode(CallingConv::ID CC,
1402 bool Return,
1403 bool isVarArg) const {
1404 switch (getEffectiveCallingConv(CC, isVarArg)) {
1405 default:
1406 llvm_unreachable("Unsupported calling convention");
1407 case CallingConv::ARM_APCS:
1408 return (Return ? RetCC_ARM_APCS : CC_ARM_APCS);
1409 case CallingConv::ARM_AAPCS:
1410 return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS);
1411 case CallingConv::ARM_AAPCS_VFP:
1412 return (Return ? RetCC_ARM_AAPCS_VFP : CC_ARM_AAPCS_VFP);
1413 case CallingConv::Fast:
1414 return (Return ? RetFastCC_ARM_APCS : FastCC_ARM_APCS);
1415 case CallingConv::GHC:
1416 return (Return ? RetCC_ARM_APCS : CC_ARM_APCS_GHC);
1417 }
1418 }
1419
1420 /// LowerCallResult - Lower the result values of a call into the
1421 /// appropriate copies out of appropriate physical registers.
1422 SDValue
LowerCallResult(SDValue Chain,SDValue InFlag,CallingConv::ID CallConv,bool isVarArg,const SmallVectorImpl<ISD::InputArg> & Ins,SDLoc dl,SelectionDAG & DAG,SmallVectorImpl<SDValue> & InVals,bool isThisReturn,SDValue ThisVal) const1423 ARMTargetLowering::LowerCallResult(SDValue Chain, SDValue InFlag,
1424 CallingConv::ID CallConv, bool isVarArg,
1425 const SmallVectorImpl<ISD::InputArg> &Ins,
1426 SDLoc dl, SelectionDAG &DAG,
1427 SmallVectorImpl<SDValue> &InVals,
1428 bool isThisReturn, SDValue ThisVal) const {
1429
1430 // Assign locations to each value returned by this call.
1431 SmallVector<CCValAssign, 16> RVLocs;
1432 ARMCCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
1433 *DAG.getContext(), Call);
1434 CCInfo.AnalyzeCallResult(Ins,
1435 CCAssignFnForNode(CallConv, /* Return*/ true,
1436 isVarArg));
1437
1438 // Copy all of the result registers out of their specified physreg.
1439 for (unsigned i = 0; i != RVLocs.size(); ++i) {
1440 CCValAssign VA = RVLocs[i];
1441
1442 // Pass 'this' value directly from the argument to return value, to avoid
1443 // reg unit interference
1444 if (i == 0 && isThisReturn) {
1445 assert(!VA.needsCustom() && VA.getLocVT() == MVT::i32 &&
1446 "unexpected return calling convention register assignment");
1447 InVals.push_back(ThisVal);
1448 continue;
1449 }
1450
1451 SDValue Val;
1452 if (VA.needsCustom()) {
1453 // Handle f64 or half of a v2f64.
1454 SDValue Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32,
1455 InFlag);
1456 Chain = Lo.getValue(1);
1457 InFlag = Lo.getValue(2);
1458 VA = RVLocs[++i]; // skip ahead to next loc
1459 SDValue Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32,
1460 InFlag);
1461 Chain = Hi.getValue(1);
1462 InFlag = Hi.getValue(2);
1463 if (!Subtarget->isLittle())
1464 std::swap (Lo, Hi);
1465 Val = DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi);
1466
1467 if (VA.getLocVT() == MVT::v2f64) {
1468 SDValue Vec = DAG.getNode(ISD::UNDEF, dl, MVT::v2f64);
1469 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Vec, Val,
1470 DAG.getConstant(0, dl, MVT::i32));
1471
1472 VA = RVLocs[++i]; // skip ahead to next loc
1473 Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, InFlag);
1474 Chain = Lo.getValue(1);
1475 InFlag = Lo.getValue(2);
1476 VA = RVLocs[++i]; // skip ahead to next loc
1477 Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, InFlag);
1478 Chain = Hi.getValue(1);
1479 InFlag = Hi.getValue(2);
1480 if (!Subtarget->isLittle())
1481 std::swap (Lo, Hi);
1482 Val = DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi);
1483 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Vec, Val,
1484 DAG.getConstant(1, dl, MVT::i32));
1485 }
1486 } else {
1487 Val = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), VA.getLocVT(),
1488 InFlag);
1489 Chain = Val.getValue(1);
1490 InFlag = Val.getValue(2);
1491 }
1492
1493 switch (VA.getLocInfo()) {
1494 default: llvm_unreachable("Unknown loc info!");
1495 case CCValAssign::Full: break;
1496 case CCValAssign::BCvt:
1497 Val = DAG.getNode(ISD::BITCAST, dl, VA.getValVT(), Val);
1498 break;
1499 }
1500
1501 InVals.push_back(Val);
1502 }
1503
1504 return Chain;
1505 }
1506
1507 /// LowerMemOpCallTo - Store the argument to the stack.
1508 SDValue
LowerMemOpCallTo(SDValue Chain,SDValue StackPtr,SDValue Arg,SDLoc dl,SelectionDAG & DAG,const CCValAssign & VA,ISD::ArgFlagsTy Flags) const1509 ARMTargetLowering::LowerMemOpCallTo(SDValue Chain,
1510 SDValue StackPtr, SDValue Arg,
1511 SDLoc dl, SelectionDAG &DAG,
1512 const CCValAssign &VA,
1513 ISD::ArgFlagsTy Flags) const {
1514 unsigned LocMemOffset = VA.getLocMemOffset();
1515 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
1516 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()),
1517 StackPtr, PtrOff);
1518 return DAG.getStore(
1519 Chain, dl, Arg, PtrOff,
1520 MachinePointerInfo::getStack(DAG.getMachineFunction(), LocMemOffset),
1521 false, false, 0);
1522 }
1523
PassF64ArgInRegs(SDLoc dl,SelectionDAG & DAG,SDValue Chain,SDValue & Arg,RegsToPassVector & RegsToPass,CCValAssign & VA,CCValAssign & NextVA,SDValue & StackPtr,SmallVectorImpl<SDValue> & MemOpChains,ISD::ArgFlagsTy Flags) const1524 void ARMTargetLowering::PassF64ArgInRegs(SDLoc dl, SelectionDAG &DAG,
1525 SDValue Chain, SDValue &Arg,
1526 RegsToPassVector &RegsToPass,
1527 CCValAssign &VA, CCValAssign &NextVA,
1528 SDValue &StackPtr,
1529 SmallVectorImpl<SDValue> &MemOpChains,
1530 ISD::ArgFlagsTy Flags) const {
1531
1532 SDValue fmrrd = DAG.getNode(ARMISD::VMOVRRD, dl,
1533 DAG.getVTList(MVT::i32, MVT::i32), Arg);
1534 unsigned id = Subtarget->isLittle() ? 0 : 1;
1535 RegsToPass.push_back(std::make_pair(VA.getLocReg(), fmrrd.getValue(id)));
1536
1537 if (NextVA.isRegLoc())
1538 RegsToPass.push_back(std::make_pair(NextVA.getLocReg(), fmrrd.getValue(1-id)));
1539 else {
1540 assert(NextVA.isMemLoc());
1541 if (!StackPtr.getNode())
1542 StackPtr = DAG.getCopyFromReg(Chain, dl, ARM::SP,
1543 getPointerTy(DAG.getDataLayout()));
1544
1545 MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, fmrrd.getValue(1-id),
1546 dl, DAG, NextVA,
1547 Flags));
1548 }
1549 }
1550
1551 /// LowerCall - Lowering a call into a callseq_start <-
1552 /// ARMISD:CALL <- callseq_end chain. Also add input and output parameter
1553 /// nodes.
1554 SDValue
LowerCall(TargetLowering::CallLoweringInfo & CLI,SmallVectorImpl<SDValue> & InVals) const1555 ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
1556 SmallVectorImpl<SDValue> &InVals) const {
1557 SelectionDAG &DAG = CLI.DAG;
1558 SDLoc &dl = CLI.DL;
1559 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
1560 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
1561 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
1562 SDValue Chain = CLI.Chain;
1563 SDValue Callee = CLI.Callee;
1564 bool &isTailCall = CLI.IsTailCall;
1565 CallingConv::ID CallConv = CLI.CallConv;
1566 bool doesNotRet = CLI.DoesNotReturn;
1567 bool isVarArg = CLI.IsVarArg;
1568
1569 MachineFunction &MF = DAG.getMachineFunction();
1570 bool isStructRet = (Outs.empty()) ? false : Outs[0].Flags.isSRet();
1571 bool isThisReturn = false;
1572 bool isSibCall = false;
1573 auto Attr = MF.getFunction()->getFnAttribute("disable-tail-calls");
1574
1575 // Disable tail calls if they're not supported.
1576 if (!Subtarget->supportsTailCall() || Attr.getValueAsString() == "true")
1577 isTailCall = false;
1578
1579 if (isTailCall) {
1580 // Check if it's really possible to do a tail call.
1581 isTailCall = IsEligibleForTailCallOptimization(Callee, CallConv,
1582 isVarArg, isStructRet, MF.getFunction()->hasStructRetAttr(),
1583 Outs, OutVals, Ins, DAG);
1584 if (!isTailCall && CLI.CS && CLI.CS->isMustTailCall())
1585 report_fatal_error("failed to perform tail call elimination on a call "
1586 "site marked musttail");
1587 // We don't support GuaranteedTailCallOpt for ARM, only automatically
1588 // detected sibcalls.
1589 if (isTailCall) {
1590 ++NumTailCalls;
1591 isSibCall = true;
1592 }
1593 }
1594
1595 // Analyze operands of the call, assigning locations to each operand.
1596 SmallVector<CCValAssign, 16> ArgLocs;
1597 ARMCCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
1598 *DAG.getContext(), Call);
1599 CCInfo.AnalyzeCallOperands(Outs,
1600 CCAssignFnForNode(CallConv, /* Return*/ false,
1601 isVarArg));
1602
1603 // Get a count of how many bytes are to be pushed on the stack.
1604 unsigned NumBytes = CCInfo.getNextStackOffset();
1605
1606 // For tail calls, memory operands are available in our caller's stack.
1607 if (isSibCall)
1608 NumBytes = 0;
1609
1610 // Adjust the stack pointer for the new arguments...
1611 // These operations are automatically eliminated by the prolog/epilog pass
1612 if (!isSibCall)
1613 Chain = DAG.getCALLSEQ_START(Chain,
1614 DAG.getIntPtrConstant(NumBytes, dl, true), dl);
1615
1616 SDValue StackPtr =
1617 DAG.getCopyFromReg(Chain, dl, ARM::SP, getPointerTy(DAG.getDataLayout()));
1618
1619 RegsToPassVector RegsToPass;
1620 SmallVector<SDValue, 8> MemOpChains;
1621
1622 // Walk the register/memloc assignments, inserting copies/loads. In the case
1623 // of tail call optimization, arguments are handled later.
1624 for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size();
1625 i != e;
1626 ++i, ++realArgIdx) {
1627 CCValAssign &VA = ArgLocs[i];
1628 SDValue Arg = OutVals[realArgIdx];
1629 ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags;
1630 bool isByVal = Flags.isByVal();
1631
1632 // Promote the value if needed.
1633 switch (VA.getLocInfo()) {
1634 default: llvm_unreachable("Unknown loc info!");
1635 case CCValAssign::Full: break;
1636 case CCValAssign::SExt:
1637 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg);
1638 break;
1639 case CCValAssign::ZExt:
1640 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg);
1641 break;
1642 case CCValAssign::AExt:
1643 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg);
1644 break;
1645 case CCValAssign::BCvt:
1646 Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg);
1647 break;
1648 }
1649
1650 // f64 and v2f64 might be passed in i32 pairs and must be split into pieces
1651 if (VA.needsCustom()) {
1652 if (VA.getLocVT() == MVT::v2f64) {
1653 SDValue Op0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
1654 DAG.getConstant(0, dl, MVT::i32));
1655 SDValue Op1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
1656 DAG.getConstant(1, dl, MVT::i32));
1657
1658 PassF64ArgInRegs(dl, DAG, Chain, Op0, RegsToPass,
1659 VA, ArgLocs[++i], StackPtr, MemOpChains, Flags);
1660
1661 VA = ArgLocs[++i]; // skip ahead to next loc
1662 if (VA.isRegLoc()) {
1663 PassF64ArgInRegs(dl, DAG, Chain, Op1, RegsToPass,
1664 VA, ArgLocs[++i], StackPtr, MemOpChains, Flags);
1665 } else {
1666 assert(VA.isMemLoc());
1667
1668 MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, Op1,
1669 dl, DAG, VA, Flags));
1670 }
1671 } else {
1672 PassF64ArgInRegs(dl, DAG, Chain, Arg, RegsToPass, VA, ArgLocs[++i],
1673 StackPtr, MemOpChains, Flags);
1674 }
1675 } else if (VA.isRegLoc()) {
1676 if (realArgIdx == 0 && Flags.isReturned() && Outs[0].VT == MVT::i32) {
1677 assert(VA.getLocVT() == MVT::i32 &&
1678 "unexpected calling convention register assignment");
1679 assert(!Ins.empty() && Ins[0].VT == MVT::i32 &&
1680 "unexpected use of 'returned'");
1681 isThisReturn = true;
1682 }
1683 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
1684 } else if (isByVal) {
1685 assert(VA.isMemLoc());
1686 unsigned offset = 0;
1687
1688 // True if this byval aggregate will be split between registers
1689 // and memory.
1690 unsigned ByValArgsCount = CCInfo.getInRegsParamsCount();
1691 unsigned CurByValIdx = CCInfo.getInRegsParamsProcessed();
1692
1693 if (CurByValIdx < ByValArgsCount) {
1694
1695 unsigned RegBegin, RegEnd;
1696 CCInfo.getInRegsParamInfo(CurByValIdx, RegBegin, RegEnd);
1697
1698 EVT PtrVT =
1699 DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
1700 unsigned int i, j;
1701 for (i = 0, j = RegBegin; j < RegEnd; i++, j++) {
1702 SDValue Const = DAG.getConstant(4*i, dl, MVT::i32);
1703 SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const);
1704 SDValue Load = DAG.getLoad(PtrVT, dl, Chain, AddArg,
1705 MachinePointerInfo(),
1706 false, false, false,
1707 DAG.InferPtrAlignment(AddArg));
1708 MemOpChains.push_back(Load.getValue(1));
1709 RegsToPass.push_back(std::make_pair(j, Load));
1710 }
1711
1712 // If parameter size outsides register area, "offset" value
1713 // helps us to calculate stack slot for remained part properly.
1714 offset = RegEnd - RegBegin;
1715
1716 CCInfo.nextInRegsParam();
1717 }
1718
1719 if (Flags.getByValSize() > 4*offset) {
1720 auto PtrVT = getPointerTy(DAG.getDataLayout());
1721 unsigned LocMemOffset = VA.getLocMemOffset();
1722 SDValue StkPtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
1723 SDValue Dst = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, StkPtrOff);
1724 SDValue SrcOffset = DAG.getIntPtrConstant(4*offset, dl);
1725 SDValue Src = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, SrcOffset);
1726 SDValue SizeNode = DAG.getConstant(Flags.getByValSize() - 4*offset, dl,
1727 MVT::i32);
1728 SDValue AlignNode = DAG.getConstant(Flags.getByValAlign(), dl,
1729 MVT::i32);
1730
1731 SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue);
1732 SDValue Ops[] = { Chain, Dst, Src, SizeNode, AlignNode};
1733 MemOpChains.push_back(DAG.getNode(ARMISD::COPY_STRUCT_BYVAL, dl, VTs,
1734 Ops));
1735 }
1736 } else if (!isSibCall) {
1737 assert(VA.isMemLoc());
1738
1739 MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, Arg,
1740 dl, DAG, VA, Flags));
1741 }
1742 }
1743
1744 if (!MemOpChains.empty())
1745 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
1746
1747 // Build a sequence of copy-to-reg nodes chained together with token chain
1748 // and flag operands which copy the outgoing args into the appropriate regs.
1749 SDValue InFlag;
1750 // Tail call byval lowering might overwrite argument registers so in case of
1751 // tail call optimization the copies to registers are lowered later.
1752 if (!isTailCall)
1753 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
1754 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
1755 RegsToPass[i].second, InFlag);
1756 InFlag = Chain.getValue(1);
1757 }
1758
1759 // For tail calls lower the arguments to the 'real' stack slot.
1760 if (isTailCall) {
1761 // Force all the incoming stack arguments to be loaded from the stack
1762 // before any new outgoing arguments are stored to the stack, because the
1763 // outgoing stack slots may alias the incoming argument stack slots, and
1764 // the alias isn't otherwise explicit. This is slightly more conservative
1765 // than necessary, because it means that each store effectively depends
1766 // on every argument instead of just those arguments it would clobber.
1767
1768 // Do not flag preceding copytoreg stuff together with the following stuff.
1769 InFlag = SDValue();
1770 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
1771 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
1772 RegsToPass[i].second, InFlag);
1773 InFlag = Chain.getValue(1);
1774 }
1775 InFlag = SDValue();
1776 }
1777
1778 // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
1779 // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
1780 // node so that legalize doesn't hack it.
1781 bool isDirect = false;
1782 bool isARMFunc = false;
1783 bool isLocalARMFunc = false;
1784 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
1785 auto PtrVt = getPointerTy(DAG.getDataLayout());
1786
1787 if (Subtarget->genLongCalls()) {
1788 assert((Subtarget->isTargetWindows() ||
1789 getTargetMachine().getRelocationModel() == Reloc::Static) &&
1790 "long-calls with non-static relocation model!");
1791 // Handle a global address or an external symbol. If it's not one of
1792 // those, the target's already in a register, so we don't need to do
1793 // anything extra.
1794 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
1795 const GlobalValue *GV = G->getGlobal();
1796 // Create a constant pool entry for the callee address
1797 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
1798 ARMConstantPoolValue *CPV =
1799 ARMConstantPoolConstant::Create(GV, ARMPCLabelIndex, ARMCP::CPValue, 0);
1800
1801 // Get the address of the callee into a register
1802 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, 4);
1803 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
1804 Callee = DAG.getLoad(
1805 PtrVt, dl, DAG.getEntryNode(), CPAddr,
1806 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), false,
1807 false, false, 0);
1808 } else if (ExternalSymbolSDNode *S=dyn_cast<ExternalSymbolSDNode>(Callee)) {
1809 const char *Sym = S->getSymbol();
1810
1811 // Create a constant pool entry for the callee address
1812 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
1813 ARMConstantPoolValue *CPV =
1814 ARMConstantPoolSymbol::Create(*DAG.getContext(), Sym,
1815 ARMPCLabelIndex, 0);
1816 // Get the address of the callee into a register
1817 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, 4);
1818 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
1819 Callee = DAG.getLoad(
1820 PtrVt, dl, DAG.getEntryNode(), CPAddr,
1821 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), false,
1822 false, false, 0);
1823 }
1824 } else if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
1825 const GlobalValue *GV = G->getGlobal();
1826 isDirect = true;
1827 bool isDef = GV->isStrongDefinitionForLinker();
1828 bool isStub = (!isDef && Subtarget->isTargetMachO()) &&
1829 getTargetMachine().getRelocationModel() != Reloc::Static;
1830 isARMFunc = !Subtarget->isThumb() || (isStub && !Subtarget->isMClass());
1831 // ARM call to a local ARM function is predicable.
1832 isLocalARMFunc = !Subtarget->isThumb() && (isDef || !ARMInterworking);
1833 // tBX takes a register source operand.
1834 if (isStub && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) {
1835 assert(Subtarget->isTargetMachO() && "WrapperPIC use on non-MachO?");
1836 Callee = DAG.getNode(
1837 ARMISD::WrapperPIC, dl, PtrVt,
1838 DAG.getTargetGlobalAddress(GV, dl, PtrVt, 0, ARMII::MO_NONLAZY));
1839 Callee = DAG.getLoad(PtrVt, dl, DAG.getEntryNode(), Callee,
1840 MachinePointerInfo::getGOT(DAG.getMachineFunction()),
1841 false, false, true, 0);
1842 } else if (Subtarget->isTargetCOFF()) {
1843 assert(Subtarget->isTargetWindows() &&
1844 "Windows is the only supported COFF target");
1845 unsigned TargetFlags = GV->hasDLLImportStorageClass()
1846 ? ARMII::MO_DLLIMPORT
1847 : ARMII::MO_NO_FLAG;
1848 Callee =
1849 DAG.getTargetGlobalAddress(GV, dl, PtrVt, /*Offset=*/0, TargetFlags);
1850 if (GV->hasDLLImportStorageClass())
1851 Callee =
1852 DAG.getLoad(PtrVt, dl, DAG.getEntryNode(),
1853 DAG.getNode(ARMISD::Wrapper, dl, PtrVt, Callee),
1854 MachinePointerInfo::getGOT(DAG.getMachineFunction()),
1855 false, false, false, 0);
1856 } else {
1857 // On ELF targets for PIC code, direct calls should go through the PLT
1858 unsigned OpFlags = 0;
1859 if (Subtarget->isTargetELF() &&
1860 getTargetMachine().getRelocationModel() == Reloc::PIC_)
1861 OpFlags = ARMII::MO_PLT;
1862 Callee = DAG.getTargetGlobalAddress(GV, dl, PtrVt, 0, OpFlags);
1863 }
1864 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
1865 isDirect = true;
1866 bool isStub = Subtarget->isTargetMachO() &&
1867 getTargetMachine().getRelocationModel() != Reloc::Static;
1868 isARMFunc = !Subtarget->isThumb() || (isStub && !Subtarget->isMClass());
1869 // tBX takes a register source operand.
1870 const char *Sym = S->getSymbol();
1871 if (isARMFunc && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) {
1872 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
1873 ARMConstantPoolValue *CPV =
1874 ARMConstantPoolSymbol::Create(*DAG.getContext(), Sym,
1875 ARMPCLabelIndex, 4);
1876 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, 4);
1877 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
1878 Callee = DAG.getLoad(
1879 PtrVt, dl, DAG.getEntryNode(), CPAddr,
1880 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), false,
1881 false, false, 0);
1882 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
1883 Callee = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVt, Callee, PICLabel);
1884 } else {
1885 unsigned OpFlags = 0;
1886 // On ELF targets for PIC code, direct calls should go through the PLT
1887 if (Subtarget->isTargetELF() &&
1888 getTargetMachine().getRelocationModel() == Reloc::PIC_)
1889 OpFlags = ARMII::MO_PLT;
1890 Callee = DAG.getTargetExternalSymbol(Sym, PtrVt, OpFlags);
1891 }
1892 }
1893
1894 // FIXME: handle tail calls differently.
1895 unsigned CallOpc;
1896 if (Subtarget->isThumb()) {
1897 if ((!isDirect || isARMFunc) && !Subtarget->hasV5TOps())
1898 CallOpc = ARMISD::CALL_NOLINK;
1899 else
1900 CallOpc = isARMFunc ? ARMISD::CALL : ARMISD::tCALL;
1901 } else {
1902 if (!isDirect && !Subtarget->hasV5TOps())
1903 CallOpc = ARMISD::CALL_NOLINK;
1904 else if (doesNotRet && isDirect && Subtarget->hasRAS() &&
1905 // Emit regular call when code size is the priority
1906 !MF.getFunction()->optForMinSize())
1907 // "mov lr, pc; b _foo" to avoid confusing the RSP
1908 CallOpc = ARMISD::CALL_NOLINK;
1909 else
1910 CallOpc = isLocalARMFunc ? ARMISD::CALL_PRED : ARMISD::CALL;
1911 }
1912
1913 std::vector<SDValue> Ops;
1914 Ops.push_back(Chain);
1915 Ops.push_back(Callee);
1916
1917 // Add argument registers to the end of the list so that they are known live
1918 // into the call.
1919 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
1920 Ops.push_back(DAG.getRegister(RegsToPass[i].first,
1921 RegsToPass[i].second.getValueType()));
1922
1923 // Add a register mask operand representing the call-preserved registers.
1924 if (!isTailCall) {
1925 const uint32_t *Mask;
1926 const ARMBaseRegisterInfo *ARI = Subtarget->getRegisterInfo();
1927 if (isThisReturn) {
1928 // For 'this' returns, use the R0-preserving mask if applicable
1929 Mask = ARI->getThisReturnPreservedMask(MF, CallConv);
1930 if (!Mask) {
1931 // Set isThisReturn to false if the calling convention is not one that
1932 // allows 'returned' to be modeled in this way, so LowerCallResult does
1933 // not try to pass 'this' straight through
1934 isThisReturn = false;
1935 Mask = ARI->getCallPreservedMask(MF, CallConv);
1936 }
1937 } else
1938 Mask = ARI->getCallPreservedMask(MF, CallConv);
1939
1940 assert(Mask && "Missing call preserved mask for calling convention");
1941 Ops.push_back(DAG.getRegisterMask(Mask));
1942 }
1943
1944 if (InFlag.getNode())
1945 Ops.push_back(InFlag);
1946
1947 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
1948 if (isTailCall) {
1949 MF.getFrameInfo()->setHasTailCall();
1950 return DAG.getNode(ARMISD::TC_RETURN, dl, NodeTys, Ops);
1951 }
1952
1953 // Returns a chain and a flag for retval copy to use.
1954 Chain = DAG.getNode(CallOpc, dl, NodeTys, Ops);
1955 InFlag = Chain.getValue(1);
1956
1957 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true),
1958 DAG.getIntPtrConstant(0, dl, true), InFlag, dl);
1959 if (!Ins.empty())
1960 InFlag = Chain.getValue(1);
1961
1962 // Handle result values, copying them out of physregs into vregs that we
1963 // return.
1964 return LowerCallResult(Chain, InFlag, CallConv, isVarArg, Ins, dl, DAG,
1965 InVals, isThisReturn,
1966 isThisReturn ? OutVals[0] : SDValue());
1967 }
1968
1969 /// HandleByVal - Every parameter *after* a byval parameter is passed
1970 /// on the stack. Remember the next parameter register to allocate,
1971 /// and then confiscate the rest of the parameter registers to insure
1972 /// this.
HandleByVal(CCState * State,unsigned & Size,unsigned Align) const1973 void ARMTargetLowering::HandleByVal(CCState *State, unsigned &Size,
1974 unsigned Align) const {
1975 assert((State->getCallOrPrologue() == Prologue ||
1976 State->getCallOrPrologue() == Call) &&
1977 "unhandled ParmContext");
1978
1979 // Byval (as with any stack) slots are always at least 4 byte aligned.
1980 Align = std::max(Align, 4U);
1981
1982 unsigned Reg = State->AllocateReg(GPRArgRegs);
1983 if (!Reg)
1984 return;
1985
1986 unsigned AlignInRegs = Align / 4;
1987 unsigned Waste = (ARM::R4 - Reg) % AlignInRegs;
1988 for (unsigned i = 0; i < Waste; ++i)
1989 Reg = State->AllocateReg(GPRArgRegs);
1990
1991 if (!Reg)
1992 return;
1993
1994 unsigned Excess = 4 * (ARM::R4 - Reg);
1995
1996 // Special case when NSAA != SP and parameter size greater than size of
1997 // all remained GPR regs. In that case we can't split parameter, we must
1998 // send it to stack. We also must set NCRN to R4, so waste all
1999 // remained registers.
2000 const unsigned NSAAOffset = State->getNextStackOffset();
2001 if (NSAAOffset != 0 && Size > Excess) {
2002 while (State->AllocateReg(GPRArgRegs))
2003 ;
2004 return;
2005 }
2006
2007 // First register for byval parameter is the first register that wasn't
2008 // allocated before this method call, so it would be "reg".
2009 // If parameter is small enough to be saved in range [reg, r4), then
2010 // the end (first after last) register would be reg + param-size-in-regs,
2011 // else parameter would be splitted between registers and stack,
2012 // end register would be r4 in this case.
2013 unsigned ByValRegBegin = Reg;
2014 unsigned ByValRegEnd = std::min<unsigned>(Reg + Size / 4, ARM::R4);
2015 State->addInRegsParamInfo(ByValRegBegin, ByValRegEnd);
2016 // Note, first register is allocated in the beginning of function already,
2017 // allocate remained amount of registers we need.
2018 for (unsigned i = Reg + 1; i != ByValRegEnd; ++i)
2019 State->AllocateReg(GPRArgRegs);
2020 // A byval parameter that is split between registers and memory needs its
2021 // size truncated here.
2022 // In the case where the entire structure fits in registers, we set the
2023 // size in memory to zero.
2024 Size = std::max<int>(Size - Excess, 0);
2025 }
2026
2027 /// MatchingStackOffset - Return true if the given stack call argument is
2028 /// already available in the same position (relatively) of the caller's
2029 /// incoming argument stack.
2030 static
MatchingStackOffset(SDValue Arg,unsigned Offset,ISD::ArgFlagsTy Flags,MachineFrameInfo * MFI,const MachineRegisterInfo * MRI,const TargetInstrInfo * TII)2031 bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags,
2032 MachineFrameInfo *MFI, const MachineRegisterInfo *MRI,
2033 const TargetInstrInfo *TII) {
2034 unsigned Bytes = Arg.getValueType().getSizeInBits() / 8;
2035 int FI = INT_MAX;
2036 if (Arg.getOpcode() == ISD::CopyFromReg) {
2037 unsigned VR = cast<RegisterSDNode>(Arg.getOperand(1))->getReg();
2038 if (!TargetRegisterInfo::isVirtualRegister(VR))
2039 return false;
2040 MachineInstr *Def = MRI->getVRegDef(VR);
2041 if (!Def)
2042 return false;
2043 if (!Flags.isByVal()) {
2044 if (!TII->isLoadFromStackSlot(Def, FI))
2045 return false;
2046 } else {
2047 return false;
2048 }
2049 } else if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Arg)) {
2050 if (Flags.isByVal())
2051 // ByVal argument is passed in as a pointer but it's now being
2052 // dereferenced. e.g.
2053 // define @foo(%struct.X* %A) {
2054 // tail call @bar(%struct.X* byval %A)
2055 // }
2056 return false;
2057 SDValue Ptr = Ld->getBasePtr();
2058 FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(Ptr);
2059 if (!FINode)
2060 return false;
2061 FI = FINode->getIndex();
2062 } else
2063 return false;
2064
2065 assert(FI != INT_MAX);
2066 if (!MFI->isFixedObjectIndex(FI))
2067 return false;
2068 return Offset == MFI->getObjectOffset(FI) && Bytes == MFI->getObjectSize(FI);
2069 }
2070
2071 /// IsEligibleForTailCallOptimization - Check whether the call is eligible
2072 /// for tail call optimization. Targets which want to do tail call
2073 /// optimization should implement this function.
2074 bool
IsEligibleForTailCallOptimization(SDValue Callee,CallingConv::ID CalleeCC,bool isVarArg,bool isCalleeStructRet,bool isCallerStructRet,const SmallVectorImpl<ISD::OutputArg> & Outs,const SmallVectorImpl<SDValue> & OutVals,const SmallVectorImpl<ISD::InputArg> & Ins,SelectionDAG & DAG) const2075 ARMTargetLowering::IsEligibleForTailCallOptimization(SDValue Callee,
2076 CallingConv::ID CalleeCC,
2077 bool isVarArg,
2078 bool isCalleeStructRet,
2079 bool isCallerStructRet,
2080 const SmallVectorImpl<ISD::OutputArg> &Outs,
2081 const SmallVectorImpl<SDValue> &OutVals,
2082 const SmallVectorImpl<ISD::InputArg> &Ins,
2083 SelectionDAG& DAG) const {
2084 const Function *CallerF = DAG.getMachineFunction().getFunction();
2085 CallingConv::ID CallerCC = CallerF->getCallingConv();
2086 bool CCMatch = CallerCC == CalleeCC;
2087
2088 assert(Subtarget->supportsTailCall());
2089
2090 // Look for obvious safe cases to perform tail call optimization that do not
2091 // require ABI changes. This is what gcc calls sibcall.
2092
2093 // Do not sibcall optimize vararg calls unless the call site is not passing
2094 // any arguments.
2095 if (isVarArg && !Outs.empty())
2096 return false;
2097
2098 // Exception-handling functions need a special set of instructions to indicate
2099 // a return to the hardware. Tail-calling another function would probably
2100 // break this.
2101 if (CallerF->hasFnAttribute("interrupt"))
2102 return false;
2103
2104 // Also avoid sibcall optimization if either caller or callee uses struct
2105 // return semantics.
2106 if (isCalleeStructRet || isCallerStructRet)
2107 return false;
2108
2109 // Externally-defined functions with weak linkage should not be
2110 // tail-called on ARM when the OS does not support dynamic
2111 // pre-emption of symbols, as the AAELF spec requires normal calls
2112 // to undefined weak functions to be replaced with a NOP or jump to the
2113 // next instruction. The behaviour of branch instructions in this
2114 // situation (as used for tail calls) is implementation-defined, so we
2115 // cannot rely on the linker replacing the tail call with a return.
2116 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
2117 const GlobalValue *GV = G->getGlobal();
2118 const Triple &TT = getTargetMachine().getTargetTriple();
2119 if (GV->hasExternalWeakLinkage() &&
2120 (!TT.isOSWindows() || TT.isOSBinFormatELF() || TT.isOSBinFormatMachO()))
2121 return false;
2122 }
2123
2124 // If the calling conventions do not match, then we'd better make sure the
2125 // results are returned in the same way as what the caller expects.
2126 if (!CCMatch) {
2127 SmallVector<CCValAssign, 16> RVLocs1;
2128 ARMCCState CCInfo1(CalleeCC, false, DAG.getMachineFunction(), RVLocs1,
2129 *DAG.getContext(), Call);
2130 CCInfo1.AnalyzeCallResult(Ins, CCAssignFnForNode(CalleeCC, true, isVarArg));
2131
2132 SmallVector<CCValAssign, 16> RVLocs2;
2133 ARMCCState CCInfo2(CallerCC, false, DAG.getMachineFunction(), RVLocs2,
2134 *DAG.getContext(), Call);
2135 CCInfo2.AnalyzeCallResult(Ins, CCAssignFnForNode(CallerCC, true, isVarArg));
2136
2137 if (RVLocs1.size() != RVLocs2.size())
2138 return false;
2139 for (unsigned i = 0, e = RVLocs1.size(); i != e; ++i) {
2140 if (RVLocs1[i].isRegLoc() != RVLocs2[i].isRegLoc())
2141 return false;
2142 if (RVLocs1[i].getLocInfo() != RVLocs2[i].getLocInfo())
2143 return false;
2144 if (RVLocs1[i].isRegLoc()) {
2145 if (RVLocs1[i].getLocReg() != RVLocs2[i].getLocReg())
2146 return false;
2147 } else {
2148 if (RVLocs1[i].getLocMemOffset() != RVLocs2[i].getLocMemOffset())
2149 return false;
2150 }
2151 }
2152 }
2153
2154 // If Caller's vararg or byval argument has been split between registers and
2155 // stack, do not perform tail call, since part of the argument is in caller's
2156 // local frame.
2157 const ARMFunctionInfo *AFI_Caller = DAG.getMachineFunction().
2158 getInfo<ARMFunctionInfo>();
2159 if (AFI_Caller->getArgRegsSaveSize())
2160 return false;
2161
2162 // If the callee takes no arguments then go on to check the results of the
2163 // call.
2164 if (!Outs.empty()) {
2165 // Check if stack adjustment is needed. For now, do not do this if any
2166 // argument is passed on the stack.
2167 SmallVector<CCValAssign, 16> ArgLocs;
2168 ARMCCState CCInfo(CalleeCC, isVarArg, DAG.getMachineFunction(), ArgLocs,
2169 *DAG.getContext(), Call);
2170 CCInfo.AnalyzeCallOperands(Outs,
2171 CCAssignFnForNode(CalleeCC, false, isVarArg));
2172 if (CCInfo.getNextStackOffset()) {
2173 MachineFunction &MF = DAG.getMachineFunction();
2174
2175 // Check if the arguments are already laid out in the right way as
2176 // the caller's fixed stack objects.
2177 MachineFrameInfo *MFI = MF.getFrameInfo();
2178 const MachineRegisterInfo *MRI = &MF.getRegInfo();
2179 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
2180 for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size();
2181 i != e;
2182 ++i, ++realArgIdx) {
2183 CCValAssign &VA = ArgLocs[i];
2184 EVT RegVT = VA.getLocVT();
2185 SDValue Arg = OutVals[realArgIdx];
2186 ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags;
2187 if (VA.getLocInfo() == CCValAssign::Indirect)
2188 return false;
2189 if (VA.needsCustom()) {
2190 // f64 and vector types are split into multiple registers or
2191 // register/stack-slot combinations. The types will not match
2192 // the registers; give up on memory f64 refs until we figure
2193 // out what to do about this.
2194 if (!VA.isRegLoc())
2195 return false;
2196 if (!ArgLocs[++i].isRegLoc())
2197 return false;
2198 if (RegVT == MVT::v2f64) {
2199 if (!ArgLocs[++i].isRegLoc())
2200 return false;
2201 if (!ArgLocs[++i].isRegLoc())
2202 return false;
2203 }
2204 } else if (!VA.isRegLoc()) {
2205 if (!MatchingStackOffset(Arg, VA.getLocMemOffset(), Flags,
2206 MFI, MRI, TII))
2207 return false;
2208 }
2209 }
2210 }
2211 }
2212
2213 return true;
2214 }
2215
2216 bool
CanLowerReturn(CallingConv::ID CallConv,MachineFunction & MF,bool isVarArg,const SmallVectorImpl<ISD::OutputArg> & Outs,LLVMContext & Context) const2217 ARMTargetLowering::CanLowerReturn(CallingConv::ID CallConv,
2218 MachineFunction &MF, bool isVarArg,
2219 const SmallVectorImpl<ISD::OutputArg> &Outs,
2220 LLVMContext &Context) const {
2221 SmallVector<CCValAssign, 16> RVLocs;
2222 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
2223 return CCInfo.CheckReturn(Outs, CCAssignFnForNode(CallConv, /*Return=*/true,
2224 isVarArg));
2225 }
2226
LowerInterruptReturn(SmallVectorImpl<SDValue> & RetOps,SDLoc DL,SelectionDAG & DAG)2227 static SDValue LowerInterruptReturn(SmallVectorImpl<SDValue> &RetOps,
2228 SDLoc DL, SelectionDAG &DAG) {
2229 const MachineFunction &MF = DAG.getMachineFunction();
2230 const Function *F = MF.getFunction();
2231
2232 StringRef IntKind = F->getFnAttribute("interrupt").getValueAsString();
2233
2234 // See ARM ARM v7 B1.8.3. On exception entry LR is set to a possibly offset
2235 // version of the "preferred return address". These offsets affect the return
2236 // instruction if this is a return from PL1 without hypervisor extensions.
2237 // IRQ/FIQ: +4 "subs pc, lr, #4"
2238 // SWI: 0 "subs pc, lr, #0"
2239 // ABORT: +4 "subs pc, lr, #4"
2240 // UNDEF: +4/+2 "subs pc, lr, #0"
2241 // UNDEF varies depending on where the exception came from ARM or Thumb
2242 // mode. Alongside GCC, we throw our hands up in disgust and pretend it's 0.
2243
2244 int64_t LROffset;
2245 if (IntKind == "" || IntKind == "IRQ" || IntKind == "FIQ" ||
2246 IntKind == "ABORT")
2247 LROffset = 4;
2248 else if (IntKind == "SWI" || IntKind == "UNDEF")
2249 LROffset = 0;
2250 else
2251 report_fatal_error("Unsupported interrupt attribute. If present, value "
2252 "must be one of: IRQ, FIQ, SWI, ABORT or UNDEF");
2253
2254 RetOps.insert(RetOps.begin() + 1,
2255 DAG.getConstant(LROffset, DL, MVT::i32, false));
2256
2257 return DAG.getNode(ARMISD::INTRET_FLAG, DL, MVT::Other, RetOps);
2258 }
2259
2260 SDValue
LowerReturn(SDValue Chain,CallingConv::ID CallConv,bool isVarArg,const SmallVectorImpl<ISD::OutputArg> & Outs,const SmallVectorImpl<SDValue> & OutVals,SDLoc dl,SelectionDAG & DAG) const2261 ARMTargetLowering::LowerReturn(SDValue Chain,
2262 CallingConv::ID CallConv, bool isVarArg,
2263 const SmallVectorImpl<ISD::OutputArg> &Outs,
2264 const SmallVectorImpl<SDValue> &OutVals,
2265 SDLoc dl, SelectionDAG &DAG) const {
2266
2267 // CCValAssign - represent the assignment of the return value to a location.
2268 SmallVector<CCValAssign, 16> RVLocs;
2269
2270 // CCState - Info about the registers and stack slots.
2271 ARMCCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
2272 *DAG.getContext(), Call);
2273
2274 // Analyze outgoing return values.
2275 CCInfo.AnalyzeReturn(Outs, CCAssignFnForNode(CallConv, /* Return */ true,
2276 isVarArg));
2277
2278 SDValue Flag;
2279 SmallVector<SDValue, 4> RetOps;
2280 RetOps.push_back(Chain); // Operand #0 = Chain (updated below)
2281 bool isLittleEndian = Subtarget->isLittle();
2282
2283 MachineFunction &MF = DAG.getMachineFunction();
2284 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
2285 AFI->setReturnRegsCount(RVLocs.size());
2286
2287 // Copy the result values into the output registers.
2288 for (unsigned i = 0, realRVLocIdx = 0;
2289 i != RVLocs.size();
2290 ++i, ++realRVLocIdx) {
2291 CCValAssign &VA = RVLocs[i];
2292 assert(VA.isRegLoc() && "Can only return in registers!");
2293
2294 SDValue Arg = OutVals[realRVLocIdx];
2295
2296 switch (VA.getLocInfo()) {
2297 default: llvm_unreachable("Unknown loc info!");
2298 case CCValAssign::Full: break;
2299 case CCValAssign::BCvt:
2300 Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg);
2301 break;
2302 }
2303
2304 if (VA.needsCustom()) {
2305 if (VA.getLocVT() == MVT::v2f64) {
2306 // Extract the first half and return it in two registers.
2307 SDValue Half = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
2308 DAG.getConstant(0, dl, MVT::i32));
2309 SDValue HalfGPRs = DAG.getNode(ARMISD::VMOVRRD, dl,
2310 DAG.getVTList(MVT::i32, MVT::i32), Half);
2311
2312 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
2313 HalfGPRs.getValue(isLittleEndian ? 0 : 1),
2314 Flag);
2315 Flag = Chain.getValue(1);
2316 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
2317 VA = RVLocs[++i]; // skip ahead to next loc
2318 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
2319 HalfGPRs.getValue(isLittleEndian ? 1 : 0),
2320 Flag);
2321 Flag = Chain.getValue(1);
2322 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
2323 VA = RVLocs[++i]; // skip ahead to next loc
2324
2325 // Extract the 2nd half and fall through to handle it as an f64 value.
2326 Arg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
2327 DAG.getConstant(1, dl, MVT::i32));
2328 }
2329 // Legalize ret f64 -> ret 2 x i32. We always have fmrrd if f64 is
2330 // available.
2331 SDValue fmrrd = DAG.getNode(ARMISD::VMOVRRD, dl,
2332 DAG.getVTList(MVT::i32, MVT::i32), Arg);
2333 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
2334 fmrrd.getValue(isLittleEndian ? 0 : 1),
2335 Flag);
2336 Flag = Chain.getValue(1);
2337 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
2338 VA = RVLocs[++i]; // skip ahead to next loc
2339 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
2340 fmrrd.getValue(isLittleEndian ? 1 : 0),
2341 Flag);
2342 } else
2343 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), Arg, Flag);
2344
2345 // Guarantee that all emitted copies are
2346 // stuck together, avoiding something bad.
2347 Flag = Chain.getValue(1);
2348 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
2349 }
2350
2351 // Update chain and glue.
2352 RetOps[0] = Chain;
2353 if (Flag.getNode())
2354 RetOps.push_back(Flag);
2355
2356 // CPUs which aren't M-class use a special sequence to return from
2357 // exceptions (roughly, any instruction setting pc and cpsr simultaneously,
2358 // though we use "subs pc, lr, #N").
2359 //
2360 // M-class CPUs actually use a normal return sequence with a special
2361 // (hardware-provided) value in LR, so the normal code path works.
2362 if (DAG.getMachineFunction().getFunction()->hasFnAttribute("interrupt") &&
2363 !Subtarget->isMClass()) {
2364 if (Subtarget->isThumb1Only())
2365 report_fatal_error("interrupt attribute is not supported in Thumb1");
2366 return LowerInterruptReturn(RetOps, dl, DAG);
2367 }
2368
2369 return DAG.getNode(ARMISD::RET_FLAG, dl, MVT::Other, RetOps);
2370 }
2371
isUsedByReturnOnly(SDNode * N,SDValue & Chain) const2372 bool ARMTargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const {
2373 if (N->getNumValues() != 1)
2374 return false;
2375 if (!N->hasNUsesOfValue(1, 0))
2376 return false;
2377
2378 SDValue TCChain = Chain;
2379 SDNode *Copy = *N->use_begin();
2380 if (Copy->getOpcode() == ISD::CopyToReg) {
2381 // If the copy has a glue operand, we conservatively assume it isn't safe to
2382 // perform a tail call.
2383 if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue)
2384 return false;
2385 TCChain = Copy->getOperand(0);
2386 } else if (Copy->getOpcode() == ARMISD::VMOVRRD) {
2387 SDNode *VMov = Copy;
2388 // f64 returned in a pair of GPRs.
2389 SmallPtrSet<SDNode*, 2> Copies;
2390 for (SDNode::use_iterator UI = VMov->use_begin(), UE = VMov->use_end();
2391 UI != UE; ++UI) {
2392 if (UI->getOpcode() != ISD::CopyToReg)
2393 return false;
2394 Copies.insert(*UI);
2395 }
2396 if (Copies.size() > 2)
2397 return false;
2398
2399 for (SDNode::use_iterator UI = VMov->use_begin(), UE = VMov->use_end();
2400 UI != UE; ++UI) {
2401 SDValue UseChain = UI->getOperand(0);
2402 if (Copies.count(UseChain.getNode()))
2403 // Second CopyToReg
2404 Copy = *UI;
2405 else {
2406 // We are at the top of this chain.
2407 // If the copy has a glue operand, we conservatively assume it
2408 // isn't safe to perform a tail call.
2409 if (UI->getOperand(UI->getNumOperands()-1).getValueType() == MVT::Glue)
2410 return false;
2411 // First CopyToReg
2412 TCChain = UseChain;
2413 }
2414 }
2415 } else if (Copy->getOpcode() == ISD::BITCAST) {
2416 // f32 returned in a single GPR.
2417 if (!Copy->hasOneUse())
2418 return false;
2419 Copy = *Copy->use_begin();
2420 if (Copy->getOpcode() != ISD::CopyToReg || !Copy->hasNUsesOfValue(1, 0))
2421 return false;
2422 // If the copy has a glue operand, we conservatively assume it isn't safe to
2423 // perform a tail call.
2424 if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue)
2425 return false;
2426 TCChain = Copy->getOperand(0);
2427 } else {
2428 return false;
2429 }
2430
2431 bool HasRet = false;
2432 for (SDNode::use_iterator UI = Copy->use_begin(), UE = Copy->use_end();
2433 UI != UE; ++UI) {
2434 if (UI->getOpcode() != ARMISD::RET_FLAG &&
2435 UI->getOpcode() != ARMISD::INTRET_FLAG)
2436 return false;
2437 HasRet = true;
2438 }
2439
2440 if (!HasRet)
2441 return false;
2442
2443 Chain = TCChain;
2444 return true;
2445 }
2446
mayBeEmittedAsTailCall(CallInst * CI) const2447 bool ARMTargetLowering::mayBeEmittedAsTailCall(CallInst *CI) const {
2448 if (!Subtarget->supportsTailCall())
2449 return false;
2450
2451 auto Attr =
2452 CI->getParent()->getParent()->getFnAttribute("disable-tail-calls");
2453 if (!CI->isTailCall() || Attr.getValueAsString() == "true")
2454 return false;
2455
2456 return true;
2457 }
2458
2459 // Trying to write a 64 bit value so need to split into two 32 bit values first,
2460 // and pass the lower and high parts through.
LowerWRITE_REGISTER(SDValue Op,SelectionDAG & DAG)2461 static SDValue LowerWRITE_REGISTER(SDValue Op, SelectionDAG &DAG) {
2462 SDLoc DL(Op);
2463 SDValue WriteValue = Op->getOperand(2);
2464
2465 // This function is only supposed to be called for i64 type argument.
2466 assert(WriteValue.getValueType() == MVT::i64
2467 && "LowerWRITE_REGISTER called for non-i64 type argument.");
2468
2469 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, WriteValue,
2470 DAG.getConstant(0, DL, MVT::i32));
2471 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, WriteValue,
2472 DAG.getConstant(1, DL, MVT::i32));
2473 SDValue Ops[] = { Op->getOperand(0), Op->getOperand(1), Lo, Hi };
2474 return DAG.getNode(ISD::WRITE_REGISTER, DL, MVT::Other, Ops);
2475 }
2476
2477 // ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as
2478 // their target counterpart wrapped in the ARMISD::Wrapper node. Suppose N is
2479 // one of the above mentioned nodes. It has to be wrapped because otherwise
2480 // Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only
2481 // be used to form addressing mode. These wrapped nodes will be selected
2482 // into MOVi.
LowerConstantPool(SDValue Op,SelectionDAG & DAG)2483 static SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) {
2484 EVT PtrVT = Op.getValueType();
2485 // FIXME there is no actual debug info here
2486 SDLoc dl(Op);
2487 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
2488 SDValue Res;
2489 if (CP->isMachineConstantPoolEntry())
2490 Res = DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT,
2491 CP->getAlignment());
2492 else
2493 Res = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT,
2494 CP->getAlignment());
2495 return DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Res);
2496 }
2497
getJumpTableEncoding() const2498 unsigned ARMTargetLowering::getJumpTableEncoding() const {
2499 return MachineJumpTableInfo::EK_Inline;
2500 }
2501
LowerBlockAddress(SDValue Op,SelectionDAG & DAG) const2502 SDValue ARMTargetLowering::LowerBlockAddress(SDValue Op,
2503 SelectionDAG &DAG) const {
2504 MachineFunction &MF = DAG.getMachineFunction();
2505 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
2506 unsigned ARMPCLabelIndex = 0;
2507 SDLoc DL(Op);
2508 EVT PtrVT = getPointerTy(DAG.getDataLayout());
2509 const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
2510 Reloc::Model RelocM = getTargetMachine().getRelocationModel();
2511 SDValue CPAddr;
2512 if (RelocM == Reloc::Static) {
2513 CPAddr = DAG.getTargetConstantPool(BA, PtrVT, 4);
2514 } else {
2515 unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
2516 ARMPCLabelIndex = AFI->createPICLabelUId();
2517 ARMConstantPoolValue *CPV =
2518 ARMConstantPoolConstant::Create(BA, ARMPCLabelIndex,
2519 ARMCP::CPBlockAddress, PCAdj);
2520 CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, 4);
2521 }
2522 CPAddr = DAG.getNode(ARMISD::Wrapper, DL, PtrVT, CPAddr);
2523 SDValue Result =
2524 DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), CPAddr,
2525 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2526 false, false, false, 0);
2527 if (RelocM == Reloc::Static)
2528 return Result;
2529 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, DL, MVT::i32);
2530 return DAG.getNode(ARMISD::PIC_ADD, DL, PtrVT, Result, PICLabel);
2531 }
2532
2533 // Lower ISD::GlobalTLSAddress using the "general dynamic" model
2534 SDValue
LowerToTLSGeneralDynamicModel(GlobalAddressSDNode * GA,SelectionDAG & DAG) const2535 ARMTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA,
2536 SelectionDAG &DAG) const {
2537 SDLoc dl(GA);
2538 EVT PtrVT = getPointerTy(DAG.getDataLayout());
2539 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
2540 MachineFunction &MF = DAG.getMachineFunction();
2541 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
2542 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
2543 ARMConstantPoolValue *CPV =
2544 ARMConstantPoolConstant::Create(GA->getGlobal(), ARMPCLabelIndex,
2545 ARMCP::CPValue, PCAdj, ARMCP::TLSGD, true);
2546 SDValue Argument = DAG.getTargetConstantPool(CPV, PtrVT, 4);
2547 Argument = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Argument);
2548 Argument =
2549 DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Argument,
2550 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2551 false, false, false, 0);
2552 SDValue Chain = Argument.getValue(1);
2553
2554 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
2555 Argument = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Argument, PICLabel);
2556
2557 // call __tls_get_addr.
2558 ArgListTy Args;
2559 ArgListEntry Entry;
2560 Entry.Node = Argument;
2561 Entry.Ty = (Type *) Type::getInt32Ty(*DAG.getContext());
2562 Args.push_back(Entry);
2563
2564 // FIXME: is there useful debug info available here?
2565 TargetLowering::CallLoweringInfo CLI(DAG);
2566 CLI.setDebugLoc(dl).setChain(Chain)
2567 .setCallee(CallingConv::C, Type::getInt32Ty(*DAG.getContext()),
2568 DAG.getExternalSymbol("__tls_get_addr", PtrVT), std::move(Args),
2569 0);
2570
2571 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
2572 return CallResult.first;
2573 }
2574
2575 // Lower ISD::GlobalTLSAddress using the "initial exec" or
2576 // "local exec" model.
2577 SDValue
LowerToTLSExecModels(GlobalAddressSDNode * GA,SelectionDAG & DAG,TLSModel::Model model) const2578 ARMTargetLowering::LowerToTLSExecModels(GlobalAddressSDNode *GA,
2579 SelectionDAG &DAG,
2580 TLSModel::Model model) const {
2581 const GlobalValue *GV = GA->getGlobal();
2582 SDLoc dl(GA);
2583 SDValue Offset;
2584 SDValue Chain = DAG.getEntryNode();
2585 EVT PtrVT = getPointerTy(DAG.getDataLayout());
2586 // Get the Thread Pointer
2587 SDValue ThreadPointer = DAG.getNode(ARMISD::THREAD_POINTER, dl, PtrVT);
2588
2589 if (model == TLSModel::InitialExec) {
2590 MachineFunction &MF = DAG.getMachineFunction();
2591 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
2592 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
2593 // Initial exec model.
2594 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
2595 ARMConstantPoolValue *CPV =
2596 ARMConstantPoolConstant::Create(GA->getGlobal(), ARMPCLabelIndex,
2597 ARMCP::CPValue, PCAdj, ARMCP::GOTTPOFF,
2598 true);
2599 Offset = DAG.getTargetConstantPool(CPV, PtrVT, 4);
2600 Offset = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Offset);
2601 Offset = DAG.getLoad(
2602 PtrVT, dl, Chain, Offset,
2603 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), false,
2604 false, false, 0);
2605 Chain = Offset.getValue(1);
2606
2607 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
2608 Offset = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Offset, PICLabel);
2609
2610 Offset = DAG.getLoad(
2611 PtrVT, dl, Chain, Offset,
2612 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), false,
2613 false, false, 0);
2614 } else {
2615 // local exec model
2616 assert(model == TLSModel::LocalExec);
2617 ARMConstantPoolValue *CPV =
2618 ARMConstantPoolConstant::Create(GV, ARMCP::TPOFF);
2619 Offset = DAG.getTargetConstantPool(CPV, PtrVT, 4);
2620 Offset = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Offset);
2621 Offset = DAG.getLoad(
2622 PtrVT, dl, Chain, Offset,
2623 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), false,
2624 false, false, 0);
2625 }
2626
2627 // The address of the thread local variable is the add of the thread
2628 // pointer with the offset of the variable.
2629 return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset);
2630 }
2631
2632 SDValue
LowerGlobalTLSAddress(SDValue Op,SelectionDAG & DAG) const2633 ARMTargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
2634 // TODO: implement the "local dynamic" model
2635 assert(Subtarget->isTargetELF() &&
2636 "TLS not implemented for non-ELF targets");
2637 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
2638 if (DAG.getTarget().Options.EmulatedTLS)
2639 return LowerToTLSEmulatedModel(GA, DAG);
2640
2641 TLSModel::Model model = getTargetMachine().getTLSModel(GA->getGlobal());
2642
2643 switch (model) {
2644 case TLSModel::GeneralDynamic:
2645 case TLSModel::LocalDynamic:
2646 return LowerToTLSGeneralDynamicModel(GA, DAG);
2647 case TLSModel::InitialExec:
2648 case TLSModel::LocalExec:
2649 return LowerToTLSExecModels(GA, DAG, model);
2650 }
2651 llvm_unreachable("bogus TLS model");
2652 }
2653
LowerGlobalAddressELF(SDValue Op,SelectionDAG & DAG) const2654 SDValue ARMTargetLowering::LowerGlobalAddressELF(SDValue Op,
2655 SelectionDAG &DAG) const {
2656 EVT PtrVT = getPointerTy(DAG.getDataLayout());
2657 SDLoc dl(Op);
2658 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
2659 if (getTargetMachine().getRelocationModel() == Reloc::PIC_) {
2660 bool UseGOT_PREL =
2661 !(GV->hasHiddenVisibility() || GV->hasLocalLinkage());
2662
2663 MachineFunction &MF = DAG.getMachineFunction();
2664 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
2665 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
2666 EVT PtrVT = getPointerTy(DAG.getDataLayout());
2667 SDLoc dl(Op);
2668 unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
2669 ARMConstantPoolValue *CPV = ARMConstantPoolConstant::Create(
2670 GV, ARMPCLabelIndex, ARMCP::CPValue, PCAdj,
2671 UseGOT_PREL ? ARMCP::GOT_PREL : ARMCP::no_modifier,
2672 /*AddCurrentAddress=*/UseGOT_PREL);
2673 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, 4);
2674 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
2675 SDValue Result = DAG.getLoad(
2676 PtrVT, dl, DAG.getEntryNode(), CPAddr,
2677 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), false,
2678 false, false, 0);
2679 SDValue Chain = Result.getValue(1);
2680 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
2681 Result = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Result, PICLabel);
2682 if (UseGOT_PREL)
2683 Result = DAG.getLoad(PtrVT, dl, Chain, Result,
2684 MachinePointerInfo::getGOT(DAG.getMachineFunction()),
2685 false, false, false, 0);
2686 return Result;
2687 }
2688
2689 // If we have T2 ops, we can materialize the address directly via movt/movw
2690 // pair. This is always cheaper.
2691 if (Subtarget->useMovt(DAG.getMachineFunction())) {
2692 ++NumMovwMovt;
2693 // FIXME: Once remat is capable of dealing with instructions with register
2694 // operands, expand this into two nodes.
2695 return DAG.getNode(ARMISD::Wrapper, dl, PtrVT,
2696 DAG.getTargetGlobalAddress(GV, dl, PtrVT));
2697 } else {
2698 SDValue CPAddr = DAG.getTargetConstantPool(GV, PtrVT, 4);
2699 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
2700 return DAG.getLoad(
2701 PtrVT, dl, DAG.getEntryNode(), CPAddr,
2702 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), false,
2703 false, false, 0);
2704 }
2705 }
2706
LowerGlobalAddressDarwin(SDValue Op,SelectionDAG & DAG) const2707 SDValue ARMTargetLowering::LowerGlobalAddressDarwin(SDValue Op,
2708 SelectionDAG &DAG) const {
2709 EVT PtrVT = getPointerTy(DAG.getDataLayout());
2710 SDLoc dl(Op);
2711 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
2712 Reloc::Model RelocM = getTargetMachine().getRelocationModel();
2713
2714 if (Subtarget->useMovt(DAG.getMachineFunction()))
2715 ++NumMovwMovt;
2716
2717 // FIXME: Once remat is capable of dealing with instructions with register
2718 // operands, expand this into multiple nodes
2719 unsigned Wrapper =
2720 RelocM == Reloc::PIC_ ? ARMISD::WrapperPIC : ARMISD::Wrapper;
2721
2722 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, ARMII::MO_NONLAZY);
2723 SDValue Result = DAG.getNode(Wrapper, dl, PtrVT, G);
2724
2725 if (Subtarget->GVIsIndirectSymbol(GV, RelocM))
2726 Result = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Result,
2727 MachinePointerInfo::getGOT(DAG.getMachineFunction()),
2728 false, false, false, 0);
2729 return Result;
2730 }
2731
LowerGlobalAddressWindows(SDValue Op,SelectionDAG & DAG) const2732 SDValue ARMTargetLowering::LowerGlobalAddressWindows(SDValue Op,
2733 SelectionDAG &DAG) const {
2734 assert(Subtarget->isTargetWindows() && "non-Windows COFF is not supported");
2735 assert(Subtarget->useMovt(DAG.getMachineFunction()) &&
2736 "Windows on ARM expects to use movw/movt");
2737
2738 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
2739 const ARMII::TOF TargetFlags =
2740 (GV->hasDLLImportStorageClass() ? ARMII::MO_DLLIMPORT : ARMII::MO_NO_FLAG);
2741 EVT PtrVT = getPointerTy(DAG.getDataLayout());
2742 SDValue Result;
2743 SDLoc DL(Op);
2744
2745 ++NumMovwMovt;
2746
2747 // FIXME: Once remat is capable of dealing with instructions with register
2748 // operands, expand this into two nodes.
2749 Result = DAG.getNode(ARMISD::Wrapper, DL, PtrVT,
2750 DAG.getTargetGlobalAddress(GV, DL, PtrVT, /*Offset=*/0,
2751 TargetFlags));
2752 if (GV->hasDLLImportStorageClass())
2753 Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result,
2754 MachinePointerInfo::getGOT(DAG.getMachineFunction()),
2755 false, false, false, 0);
2756 return Result;
2757 }
2758
2759 SDValue
LowerEH_SJLJ_SETJMP(SDValue Op,SelectionDAG & DAG) const2760 ARMTargetLowering::LowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const {
2761 SDLoc dl(Op);
2762 SDValue Val = DAG.getConstant(0, dl, MVT::i32);
2763 return DAG.getNode(ARMISD::EH_SJLJ_SETJMP, dl,
2764 DAG.getVTList(MVT::i32, MVT::Other), Op.getOperand(0),
2765 Op.getOperand(1), Val);
2766 }
2767
2768 SDValue
LowerEH_SJLJ_LONGJMP(SDValue Op,SelectionDAG & DAG) const2769 ARMTargetLowering::LowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const {
2770 SDLoc dl(Op);
2771 return DAG.getNode(ARMISD::EH_SJLJ_LONGJMP, dl, MVT::Other, Op.getOperand(0),
2772 Op.getOperand(1), DAG.getConstant(0, dl, MVT::i32));
2773 }
2774
LowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,SelectionDAG & DAG) const2775 SDValue ARMTargetLowering::LowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,
2776 SelectionDAG &DAG) const {
2777 SDLoc dl(Op);
2778 return DAG.getNode(ARMISD::EH_SJLJ_SETUP_DISPATCH, dl, MVT::Other,
2779 Op.getOperand(0));
2780 }
2781
2782 SDValue
LowerINTRINSIC_WO_CHAIN(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * Subtarget) const2783 ARMTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG,
2784 const ARMSubtarget *Subtarget) const {
2785 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
2786 SDLoc dl(Op);
2787 switch (IntNo) {
2788 default: return SDValue(); // Don't custom lower most intrinsics.
2789 case Intrinsic::arm_rbit: {
2790 assert(Op.getOperand(1).getValueType() == MVT::i32 &&
2791 "RBIT intrinsic must have i32 type!");
2792 return DAG.getNode(ISD::BITREVERSE, dl, MVT::i32, Op.getOperand(1));
2793 }
2794 case Intrinsic::arm_thread_pointer: {
2795 EVT PtrVT = getPointerTy(DAG.getDataLayout());
2796 return DAG.getNode(ARMISD::THREAD_POINTER, dl, PtrVT);
2797 }
2798 case Intrinsic::eh_sjlj_lsda: {
2799 MachineFunction &MF = DAG.getMachineFunction();
2800 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
2801 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
2802 EVT PtrVT = getPointerTy(DAG.getDataLayout());
2803 Reloc::Model RelocM = getTargetMachine().getRelocationModel();
2804 SDValue CPAddr;
2805 unsigned PCAdj = (RelocM != Reloc::PIC_)
2806 ? 0 : (Subtarget->isThumb() ? 4 : 8);
2807 ARMConstantPoolValue *CPV =
2808 ARMConstantPoolConstant::Create(MF.getFunction(), ARMPCLabelIndex,
2809 ARMCP::CPLSDA, PCAdj);
2810 CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, 4);
2811 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
2812 SDValue Result = DAG.getLoad(
2813 PtrVT, dl, DAG.getEntryNode(), CPAddr,
2814 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), false,
2815 false, false, 0);
2816
2817 if (RelocM == Reloc::PIC_) {
2818 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
2819 Result = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Result, PICLabel);
2820 }
2821 return Result;
2822 }
2823 case Intrinsic::arm_neon_vmulls:
2824 case Intrinsic::arm_neon_vmullu: {
2825 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmulls)
2826 ? ARMISD::VMULLs : ARMISD::VMULLu;
2827 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
2828 Op.getOperand(1), Op.getOperand(2));
2829 }
2830 case Intrinsic::arm_neon_vminnm:
2831 case Intrinsic::arm_neon_vmaxnm: {
2832 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminnm)
2833 ? ISD::FMINNUM : ISD::FMAXNUM;
2834 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
2835 Op.getOperand(1), Op.getOperand(2));
2836 }
2837 case Intrinsic::arm_neon_vminu:
2838 case Intrinsic::arm_neon_vmaxu: {
2839 if (Op.getValueType().isFloatingPoint())
2840 return SDValue();
2841 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminu)
2842 ? ISD::UMIN : ISD::UMAX;
2843 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
2844 Op.getOperand(1), Op.getOperand(2));
2845 }
2846 case Intrinsic::arm_neon_vmins:
2847 case Intrinsic::arm_neon_vmaxs: {
2848 // v{min,max}s is overloaded between signed integers and floats.
2849 if (!Op.getValueType().isFloatingPoint()) {
2850 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
2851 ? ISD::SMIN : ISD::SMAX;
2852 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
2853 Op.getOperand(1), Op.getOperand(2));
2854 }
2855 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
2856 ? ISD::FMINNAN : ISD::FMAXNAN;
2857 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
2858 Op.getOperand(1), Op.getOperand(2));
2859 }
2860 }
2861 }
2862
LowerATOMIC_FENCE(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * Subtarget)2863 static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
2864 const ARMSubtarget *Subtarget) {
2865 // FIXME: handle "fence singlethread" more efficiently.
2866 SDLoc dl(Op);
2867 if (!Subtarget->hasDataBarrier()) {
2868 // Some ARMv6 cpus can support data barriers with an mcr instruction.
2869 // Thumb1 and pre-v6 ARM mode use a libcall instead and should never get
2870 // here.
2871 assert(Subtarget->hasV6Ops() && !Subtarget->isThumb() &&
2872 "Unexpected ISD::ATOMIC_FENCE encountered. Should be libcall!");
2873 return DAG.getNode(ARMISD::MEMBARRIER_MCR, dl, MVT::Other, Op.getOperand(0),
2874 DAG.getConstant(0, dl, MVT::i32));
2875 }
2876
2877 ConstantSDNode *OrdN = cast<ConstantSDNode>(Op.getOperand(1));
2878 AtomicOrdering Ord = static_cast<AtomicOrdering>(OrdN->getZExtValue());
2879 ARM_MB::MemBOpt Domain = ARM_MB::ISH;
2880 if (Subtarget->isMClass()) {
2881 // Only a full system barrier exists in the M-class architectures.
2882 Domain = ARM_MB::SY;
2883 } else if (Subtarget->isSwift() && Ord == Release) {
2884 // Swift happens to implement ISHST barriers in a way that's compatible with
2885 // Release semantics but weaker than ISH so we'd be fools not to use
2886 // it. Beware: other processors probably don't!
2887 Domain = ARM_MB::ISHST;
2888 }
2889
2890 return DAG.getNode(ISD::INTRINSIC_VOID, dl, MVT::Other, Op.getOperand(0),
2891 DAG.getConstant(Intrinsic::arm_dmb, dl, MVT::i32),
2892 DAG.getConstant(Domain, dl, MVT::i32));
2893 }
2894
LowerPREFETCH(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * Subtarget)2895 static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG,
2896 const ARMSubtarget *Subtarget) {
2897 // ARM pre v5TE and Thumb1 does not have preload instructions.
2898 if (!(Subtarget->isThumb2() ||
2899 (!Subtarget->isThumb1Only() && Subtarget->hasV5TEOps())))
2900 // Just preserve the chain.
2901 return Op.getOperand(0);
2902
2903 SDLoc dl(Op);
2904 unsigned isRead = ~cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue() & 1;
2905 if (!isRead &&
2906 (!Subtarget->hasV7Ops() || !Subtarget->hasMPExtension()))
2907 // ARMv7 with MP extension has PLDW.
2908 return Op.getOperand(0);
2909
2910 unsigned isData = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue();
2911 if (Subtarget->isThumb()) {
2912 // Invert the bits.
2913 isRead = ~isRead & 1;
2914 isData = ~isData & 1;
2915 }
2916
2917 return DAG.getNode(ARMISD::PRELOAD, dl, MVT::Other, Op.getOperand(0),
2918 Op.getOperand(1), DAG.getConstant(isRead, dl, MVT::i32),
2919 DAG.getConstant(isData, dl, MVT::i32));
2920 }
2921
LowerVASTART(SDValue Op,SelectionDAG & DAG)2922 static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) {
2923 MachineFunction &MF = DAG.getMachineFunction();
2924 ARMFunctionInfo *FuncInfo = MF.getInfo<ARMFunctionInfo>();
2925
2926 // vastart just stores the address of the VarArgsFrameIndex slot into the
2927 // memory location argument.
2928 SDLoc dl(Op);
2929 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
2930 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
2931 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
2932 return DAG.getStore(Op.getOperand(0), dl, FR, Op.getOperand(1),
2933 MachinePointerInfo(SV), false, false, 0);
2934 }
2935
2936 SDValue
GetF64FormalArgument(CCValAssign & VA,CCValAssign & NextVA,SDValue & Root,SelectionDAG & DAG,SDLoc dl) const2937 ARMTargetLowering::GetF64FormalArgument(CCValAssign &VA, CCValAssign &NextVA,
2938 SDValue &Root, SelectionDAG &DAG,
2939 SDLoc dl) const {
2940 MachineFunction &MF = DAG.getMachineFunction();
2941 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
2942
2943 const TargetRegisterClass *RC;
2944 if (AFI->isThumb1OnlyFunction())
2945 RC = &ARM::tGPRRegClass;
2946 else
2947 RC = &ARM::GPRRegClass;
2948
2949 // Transform the arguments stored in physical registers into virtual ones.
2950 unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC);
2951 SDValue ArgValue = DAG.getCopyFromReg(Root, dl, Reg, MVT::i32);
2952
2953 SDValue ArgValue2;
2954 if (NextVA.isMemLoc()) {
2955 MachineFrameInfo *MFI = MF.getFrameInfo();
2956 int FI = MFI->CreateFixedObject(4, NextVA.getLocMemOffset(), true);
2957
2958 // Create load node to retrieve arguments from the stack.
2959 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
2960 ArgValue2 = DAG.getLoad(
2961 MVT::i32, dl, Root, FIN,
2962 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), false,
2963 false, false, 0);
2964 } else {
2965 Reg = MF.addLiveIn(NextVA.getLocReg(), RC);
2966 ArgValue2 = DAG.getCopyFromReg(Root, dl, Reg, MVT::i32);
2967 }
2968 if (!Subtarget->isLittle())
2969 std::swap (ArgValue, ArgValue2);
2970 return DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, ArgValue, ArgValue2);
2971 }
2972
2973 // The remaining GPRs hold either the beginning of variable-argument
2974 // data, or the beginning of an aggregate passed by value (usually
2975 // byval). Either way, we allocate stack slots adjacent to the data
2976 // provided by our caller, and store the unallocated registers there.
2977 // If this is a variadic function, the va_list pointer will begin with
2978 // these values; otherwise, this reassembles a (byval) structure that
2979 // was split between registers and memory.
2980 // Return: The frame index registers were stored into.
2981 int
StoreByValRegs(CCState & CCInfo,SelectionDAG & DAG,SDLoc dl,SDValue & Chain,const Value * OrigArg,unsigned InRegsParamRecordIdx,int ArgOffset,unsigned ArgSize) const2982 ARMTargetLowering::StoreByValRegs(CCState &CCInfo, SelectionDAG &DAG,
2983 SDLoc dl, SDValue &Chain,
2984 const Value *OrigArg,
2985 unsigned InRegsParamRecordIdx,
2986 int ArgOffset,
2987 unsigned ArgSize) const {
2988 // Currently, two use-cases possible:
2989 // Case #1. Non-var-args function, and we meet first byval parameter.
2990 // Setup first unallocated register as first byval register;
2991 // eat all remained registers
2992 // (these two actions are performed by HandleByVal method).
2993 // Then, here, we initialize stack frame with
2994 // "store-reg" instructions.
2995 // Case #2. Var-args function, that doesn't contain byval parameters.
2996 // The same: eat all remained unallocated registers,
2997 // initialize stack frame.
2998
2999 MachineFunction &MF = DAG.getMachineFunction();
3000 MachineFrameInfo *MFI = MF.getFrameInfo();
3001 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3002 unsigned RBegin, REnd;
3003 if (InRegsParamRecordIdx < CCInfo.getInRegsParamsCount()) {
3004 CCInfo.getInRegsParamInfo(InRegsParamRecordIdx, RBegin, REnd);
3005 } else {
3006 unsigned RBeginIdx = CCInfo.getFirstUnallocated(GPRArgRegs);
3007 RBegin = RBeginIdx == 4 ? (unsigned)ARM::R4 : GPRArgRegs[RBeginIdx];
3008 REnd = ARM::R4;
3009 }
3010
3011 if (REnd != RBegin)
3012 ArgOffset = -4 * (ARM::R4 - RBegin);
3013
3014 auto PtrVT = getPointerTy(DAG.getDataLayout());
3015 int FrameIndex = MFI->CreateFixedObject(ArgSize, ArgOffset, false);
3016 SDValue FIN = DAG.getFrameIndex(FrameIndex, PtrVT);
3017
3018 SmallVector<SDValue, 4> MemOps;
3019 const TargetRegisterClass *RC =
3020 AFI->isThumb1OnlyFunction() ? &ARM::tGPRRegClass : &ARM::GPRRegClass;
3021
3022 for (unsigned Reg = RBegin, i = 0; Reg < REnd; ++Reg, ++i) {
3023 unsigned VReg = MF.addLiveIn(Reg, RC);
3024 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i32);
3025 SDValue Store =
3026 DAG.getStore(Val.getValue(1), dl, Val, FIN,
3027 MachinePointerInfo(OrigArg, 4 * i), false, false, 0);
3028 MemOps.push_back(Store);
3029 FIN = DAG.getNode(ISD::ADD, dl, PtrVT, FIN, DAG.getConstant(4, dl, PtrVT));
3030 }
3031
3032 if (!MemOps.empty())
3033 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps);
3034 return FrameIndex;
3035 }
3036
3037 // Setup stack frame, the va_list pointer will start from.
3038 void
VarArgStyleRegisters(CCState & CCInfo,SelectionDAG & DAG,SDLoc dl,SDValue & Chain,unsigned ArgOffset,unsigned TotalArgRegsSaveSize,bool ForceMutable) const3039 ARMTargetLowering::VarArgStyleRegisters(CCState &CCInfo, SelectionDAG &DAG,
3040 SDLoc dl, SDValue &Chain,
3041 unsigned ArgOffset,
3042 unsigned TotalArgRegsSaveSize,
3043 bool ForceMutable) const {
3044 MachineFunction &MF = DAG.getMachineFunction();
3045 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3046
3047 // Try to store any remaining integer argument regs
3048 // to their spots on the stack so that they may be loaded by deferencing
3049 // the result of va_next.
3050 // If there is no regs to be stored, just point address after last
3051 // argument passed via stack.
3052 int FrameIndex = StoreByValRegs(CCInfo, DAG, dl, Chain, nullptr,
3053 CCInfo.getInRegsParamsCount(),
3054 CCInfo.getNextStackOffset(), 4);
3055 AFI->setVarArgsFrameIndex(FrameIndex);
3056 }
3057
3058 SDValue
LowerFormalArguments(SDValue Chain,CallingConv::ID CallConv,bool isVarArg,const SmallVectorImpl<ISD::InputArg> & Ins,SDLoc dl,SelectionDAG & DAG,SmallVectorImpl<SDValue> & InVals) const3059 ARMTargetLowering::LowerFormalArguments(SDValue Chain,
3060 CallingConv::ID CallConv, bool isVarArg,
3061 const SmallVectorImpl<ISD::InputArg>
3062 &Ins,
3063 SDLoc dl, SelectionDAG &DAG,
3064 SmallVectorImpl<SDValue> &InVals)
3065 const {
3066 MachineFunction &MF = DAG.getMachineFunction();
3067 MachineFrameInfo *MFI = MF.getFrameInfo();
3068
3069 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3070
3071 // Assign locations to all of the incoming arguments.
3072 SmallVector<CCValAssign, 16> ArgLocs;
3073 ARMCCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
3074 *DAG.getContext(), Prologue);
3075 CCInfo.AnalyzeFormalArguments(Ins,
3076 CCAssignFnForNode(CallConv, /* Return*/ false,
3077 isVarArg));
3078
3079 SmallVector<SDValue, 16> ArgValues;
3080 SDValue ArgValue;
3081 Function::const_arg_iterator CurOrigArg = MF.getFunction()->arg_begin();
3082 unsigned CurArgIdx = 0;
3083
3084 // Initially ArgRegsSaveSize is zero.
3085 // Then we increase this value each time we meet byval parameter.
3086 // We also increase this value in case of varargs function.
3087 AFI->setArgRegsSaveSize(0);
3088
3089 // Calculate the amount of stack space that we need to allocate to store
3090 // byval and variadic arguments that are passed in registers.
3091 // We need to know this before we allocate the first byval or variadic
3092 // argument, as they will be allocated a stack slot below the CFA (Canonical
3093 // Frame Address, the stack pointer at entry to the function).
3094 unsigned ArgRegBegin = ARM::R4;
3095 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
3096 if (CCInfo.getInRegsParamsProcessed() >= CCInfo.getInRegsParamsCount())
3097 break;
3098
3099 CCValAssign &VA = ArgLocs[i];
3100 unsigned Index = VA.getValNo();
3101 ISD::ArgFlagsTy Flags = Ins[Index].Flags;
3102 if (!Flags.isByVal())
3103 continue;
3104
3105 assert(VA.isMemLoc() && "unexpected byval pointer in reg");
3106 unsigned RBegin, REnd;
3107 CCInfo.getInRegsParamInfo(CCInfo.getInRegsParamsProcessed(), RBegin, REnd);
3108 ArgRegBegin = std::min(ArgRegBegin, RBegin);
3109
3110 CCInfo.nextInRegsParam();
3111 }
3112 CCInfo.rewindByValRegsInfo();
3113
3114 int lastInsIndex = -1;
3115 if (isVarArg && MFI->hasVAStart()) {
3116 unsigned RegIdx = CCInfo.getFirstUnallocated(GPRArgRegs);
3117 if (RegIdx != array_lengthof(GPRArgRegs))
3118 ArgRegBegin = std::min(ArgRegBegin, (unsigned)GPRArgRegs[RegIdx]);
3119 }
3120
3121 unsigned TotalArgRegsSaveSize = 4 * (ARM::R4 - ArgRegBegin);
3122 AFI->setArgRegsSaveSize(TotalArgRegsSaveSize);
3123 auto PtrVT = getPointerTy(DAG.getDataLayout());
3124
3125 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
3126 CCValAssign &VA = ArgLocs[i];
3127 if (Ins[VA.getValNo()].isOrigArg()) {
3128 std::advance(CurOrigArg,
3129 Ins[VA.getValNo()].getOrigArgIndex() - CurArgIdx);
3130 CurArgIdx = Ins[VA.getValNo()].getOrigArgIndex();
3131 }
3132 // Arguments stored in registers.
3133 if (VA.isRegLoc()) {
3134 EVT RegVT = VA.getLocVT();
3135
3136 if (VA.needsCustom()) {
3137 // f64 and vector types are split up into multiple registers or
3138 // combinations of registers and stack slots.
3139 if (VA.getLocVT() == MVT::v2f64) {
3140 SDValue ArgValue1 = GetF64FormalArgument(VA, ArgLocs[++i],
3141 Chain, DAG, dl);
3142 VA = ArgLocs[++i]; // skip ahead to next loc
3143 SDValue ArgValue2;
3144 if (VA.isMemLoc()) {
3145 int FI = MFI->CreateFixedObject(8, VA.getLocMemOffset(), true);
3146 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
3147 ArgValue2 = DAG.getLoad(
3148 MVT::f64, dl, Chain, FIN,
3149 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI),
3150 false, false, false, 0);
3151 } else {
3152 ArgValue2 = GetF64FormalArgument(VA, ArgLocs[++i],
3153 Chain, DAG, dl);
3154 }
3155 ArgValue = DAG.getNode(ISD::UNDEF, dl, MVT::v2f64);
3156 ArgValue = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64,
3157 ArgValue, ArgValue1,
3158 DAG.getIntPtrConstant(0, dl));
3159 ArgValue = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64,
3160 ArgValue, ArgValue2,
3161 DAG.getIntPtrConstant(1, dl));
3162 } else
3163 ArgValue = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
3164
3165 } else {
3166 const TargetRegisterClass *RC;
3167
3168 if (RegVT == MVT::f32)
3169 RC = &ARM::SPRRegClass;
3170 else if (RegVT == MVT::f64)
3171 RC = &ARM::DPRRegClass;
3172 else if (RegVT == MVT::v2f64)
3173 RC = &ARM::QPRRegClass;
3174 else if (RegVT == MVT::i32)
3175 RC = AFI->isThumb1OnlyFunction() ? &ARM::tGPRRegClass
3176 : &ARM::GPRRegClass;
3177 else
3178 llvm_unreachable("RegVT not supported by FORMAL_ARGUMENTS Lowering");
3179
3180 // Transform the arguments in physical registers into virtual ones.
3181 unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC);
3182 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, RegVT);
3183 }
3184
3185 // If this is an 8 or 16-bit value, it is really passed promoted
3186 // to 32 bits. Insert an assert[sz]ext to capture this, then
3187 // truncate to the right size.
3188 switch (VA.getLocInfo()) {
3189 default: llvm_unreachable("Unknown loc info!");
3190 case CCValAssign::Full: break;
3191 case CCValAssign::BCvt:
3192 ArgValue = DAG.getNode(ISD::BITCAST, dl, VA.getValVT(), ArgValue);
3193 break;
3194 case CCValAssign::SExt:
3195 ArgValue = DAG.getNode(ISD::AssertSext, dl, RegVT, ArgValue,
3196 DAG.getValueType(VA.getValVT()));
3197 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue);
3198 break;
3199 case CCValAssign::ZExt:
3200 ArgValue = DAG.getNode(ISD::AssertZext, dl, RegVT, ArgValue,
3201 DAG.getValueType(VA.getValVT()));
3202 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue);
3203 break;
3204 }
3205
3206 InVals.push_back(ArgValue);
3207
3208 } else { // VA.isRegLoc()
3209
3210 // sanity check
3211 assert(VA.isMemLoc());
3212 assert(VA.getValVT() != MVT::i64 && "i64 should already be lowered");
3213
3214 int index = VA.getValNo();
3215
3216 // Some Ins[] entries become multiple ArgLoc[] entries.
3217 // Process them only once.
3218 if (index != lastInsIndex)
3219 {
3220 ISD::ArgFlagsTy Flags = Ins[index].Flags;
3221 // FIXME: For now, all byval parameter objects are marked mutable.
3222 // This can be changed with more analysis.
3223 // In case of tail call optimization mark all arguments mutable.
3224 // Since they could be overwritten by lowering of arguments in case of
3225 // a tail call.
3226 if (Flags.isByVal()) {
3227 assert(Ins[index].isOrigArg() &&
3228 "Byval arguments cannot be implicit");
3229 unsigned CurByValIndex = CCInfo.getInRegsParamsProcessed();
3230
3231 int FrameIndex = StoreByValRegs(
3232 CCInfo, DAG, dl, Chain, &*CurOrigArg, CurByValIndex,
3233 VA.getLocMemOffset(), Flags.getByValSize());
3234 InVals.push_back(DAG.getFrameIndex(FrameIndex, PtrVT));
3235 CCInfo.nextInRegsParam();
3236 } else {
3237 unsigned FIOffset = VA.getLocMemOffset();
3238 int FI = MFI->CreateFixedObject(VA.getLocVT().getSizeInBits()/8,
3239 FIOffset, true);
3240
3241 // Create load nodes to retrieve arguments from the stack.
3242 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
3243 InVals.push_back(DAG.getLoad(
3244 VA.getValVT(), dl, Chain, FIN,
3245 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI),
3246 false, false, false, 0));
3247 }
3248 lastInsIndex = index;
3249 }
3250 }
3251 }
3252
3253 // varargs
3254 if (isVarArg && MFI->hasVAStart())
3255 VarArgStyleRegisters(CCInfo, DAG, dl, Chain,
3256 CCInfo.getNextStackOffset(),
3257 TotalArgRegsSaveSize);
3258
3259 AFI->setArgumentStackSize(CCInfo.getNextStackOffset());
3260
3261 return Chain;
3262 }
3263
3264 /// isFloatingPointZero - Return true if this is +0.0.
isFloatingPointZero(SDValue Op)3265 static bool isFloatingPointZero(SDValue Op) {
3266 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Op))
3267 return CFP->getValueAPF().isPosZero();
3268 else if (ISD::isEXTLoad(Op.getNode()) || ISD::isNON_EXTLoad(Op.getNode())) {
3269 // Maybe this has already been legalized into the constant pool?
3270 if (Op.getOperand(1).getOpcode() == ARMISD::Wrapper) {
3271 SDValue WrapperOp = Op.getOperand(1).getOperand(0);
3272 if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(WrapperOp))
3273 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(CP->getConstVal()))
3274 return CFP->getValueAPF().isPosZero();
3275 }
3276 } else if (Op->getOpcode() == ISD::BITCAST &&
3277 Op->getValueType(0) == MVT::f64) {
3278 // Handle (ISD::BITCAST (ARMISD::VMOVIMM (ISD::TargetConstant 0)) MVT::f64)
3279 // created by LowerConstantFP().
3280 SDValue BitcastOp = Op->getOperand(0);
3281 if (BitcastOp->getOpcode() == ARMISD::VMOVIMM &&
3282 isNullConstant(BitcastOp->getOperand(0)))
3283 return true;
3284 }
3285 return false;
3286 }
3287
3288 /// Returns appropriate ARM CMP (cmp) and corresponding condition code for
3289 /// the given operands.
3290 SDValue
getARMCmp(SDValue LHS,SDValue RHS,ISD::CondCode CC,SDValue & ARMcc,SelectionDAG & DAG,SDLoc dl) const3291 ARMTargetLowering::getARMCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
3292 SDValue &ARMcc, SelectionDAG &DAG,
3293 SDLoc dl) const {
3294 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS.getNode())) {
3295 unsigned C = RHSC->getZExtValue();
3296 if (!isLegalICmpImmediate(C)) {
3297 // Constant does not fit, try adjusting it by one?
3298 switch (CC) {
3299 default: break;
3300 case ISD::SETLT:
3301 case ISD::SETGE:
3302 if (C != 0x80000000 && isLegalICmpImmediate(C-1)) {
3303 CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT;
3304 RHS = DAG.getConstant(C - 1, dl, MVT::i32);
3305 }
3306 break;
3307 case ISD::SETULT:
3308 case ISD::SETUGE:
3309 if (C != 0 && isLegalICmpImmediate(C-1)) {
3310 CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT;
3311 RHS = DAG.getConstant(C - 1, dl, MVT::i32);
3312 }
3313 break;
3314 case ISD::SETLE:
3315 case ISD::SETGT:
3316 if (C != 0x7fffffff && isLegalICmpImmediate(C+1)) {
3317 CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE;
3318 RHS = DAG.getConstant(C + 1, dl, MVT::i32);
3319 }
3320 break;
3321 case ISD::SETULE:
3322 case ISD::SETUGT:
3323 if (C != 0xffffffff && isLegalICmpImmediate(C+1)) {
3324 CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
3325 RHS = DAG.getConstant(C + 1, dl, MVT::i32);
3326 }
3327 break;
3328 }
3329 }
3330 }
3331
3332 ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
3333 ARMISD::NodeType CompareType;
3334 switch (CondCode) {
3335 default:
3336 CompareType = ARMISD::CMP;
3337 break;
3338 case ARMCC::EQ:
3339 case ARMCC::NE:
3340 // Uses only Z Flag
3341 CompareType = ARMISD::CMPZ;
3342 break;
3343 }
3344 ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
3345 return DAG.getNode(CompareType, dl, MVT::Glue, LHS, RHS);
3346 }
3347
3348 /// Returns a appropriate VFP CMP (fcmp{s|d}+fmstat) for the given operands.
3349 SDValue
getVFPCmp(SDValue LHS,SDValue RHS,SelectionDAG & DAG,SDLoc dl) const3350 ARMTargetLowering::getVFPCmp(SDValue LHS, SDValue RHS, SelectionDAG &DAG,
3351 SDLoc dl) const {
3352 assert(!Subtarget->isFPOnlySP() || RHS.getValueType() != MVT::f64);
3353 SDValue Cmp;
3354 if (!isFloatingPointZero(RHS))
3355 Cmp = DAG.getNode(ARMISD::CMPFP, dl, MVT::Glue, LHS, RHS);
3356 else
3357 Cmp = DAG.getNode(ARMISD::CMPFPw0, dl, MVT::Glue, LHS);
3358 return DAG.getNode(ARMISD::FMSTAT, dl, MVT::Glue, Cmp);
3359 }
3360
3361 /// duplicateCmp - Glue values can have only one use, so this function
3362 /// duplicates a comparison node.
3363 SDValue
duplicateCmp(SDValue Cmp,SelectionDAG & DAG) const3364 ARMTargetLowering::duplicateCmp(SDValue Cmp, SelectionDAG &DAG) const {
3365 unsigned Opc = Cmp.getOpcode();
3366 SDLoc DL(Cmp);
3367 if (Opc == ARMISD::CMP || Opc == ARMISD::CMPZ)
3368 return DAG.getNode(Opc, DL, MVT::Glue, Cmp.getOperand(0),Cmp.getOperand(1));
3369
3370 assert(Opc == ARMISD::FMSTAT && "unexpected comparison operation");
3371 Cmp = Cmp.getOperand(0);
3372 Opc = Cmp.getOpcode();
3373 if (Opc == ARMISD::CMPFP)
3374 Cmp = DAG.getNode(Opc, DL, MVT::Glue, Cmp.getOperand(0),Cmp.getOperand(1));
3375 else {
3376 assert(Opc == ARMISD::CMPFPw0 && "unexpected operand of FMSTAT");
3377 Cmp = DAG.getNode(Opc, DL, MVT::Glue, Cmp.getOperand(0));
3378 }
3379 return DAG.getNode(ARMISD::FMSTAT, DL, MVT::Glue, Cmp);
3380 }
3381
3382 std::pair<SDValue, SDValue>
getARMXALUOOp(SDValue Op,SelectionDAG & DAG,SDValue & ARMcc) const3383 ARMTargetLowering::getARMXALUOOp(SDValue Op, SelectionDAG &DAG,
3384 SDValue &ARMcc) const {
3385 assert(Op.getValueType() == MVT::i32 && "Unsupported value type");
3386
3387 SDValue Value, OverflowCmp;
3388 SDValue LHS = Op.getOperand(0);
3389 SDValue RHS = Op.getOperand(1);
3390 SDLoc dl(Op);
3391
3392 // FIXME: We are currently always generating CMPs because we don't support
3393 // generating CMN through the backend. This is not as good as the natural
3394 // CMP case because it causes a register dependency and cannot be folded
3395 // later.
3396
3397 switch (Op.getOpcode()) {
3398 default:
3399 llvm_unreachable("Unknown overflow instruction!");
3400 case ISD::SADDO:
3401 ARMcc = DAG.getConstant(ARMCC::VC, dl, MVT::i32);
3402 Value = DAG.getNode(ISD::ADD, dl, Op.getValueType(), LHS, RHS);
3403 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, Value, LHS);
3404 break;
3405 case ISD::UADDO:
3406 ARMcc = DAG.getConstant(ARMCC::HS, dl, MVT::i32);
3407 Value = DAG.getNode(ISD::ADD, dl, Op.getValueType(), LHS, RHS);
3408 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, Value, LHS);
3409 break;
3410 case ISD::SSUBO:
3411 ARMcc = DAG.getConstant(ARMCC::VC, dl, MVT::i32);
3412 Value = DAG.getNode(ISD::SUB, dl, Op.getValueType(), LHS, RHS);
3413 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, LHS, RHS);
3414 break;
3415 case ISD::USUBO:
3416 ARMcc = DAG.getConstant(ARMCC::HS, dl, MVT::i32);
3417 Value = DAG.getNode(ISD::SUB, dl, Op.getValueType(), LHS, RHS);
3418 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, LHS, RHS);
3419 break;
3420 } // switch (...)
3421
3422 return std::make_pair(Value, OverflowCmp);
3423 }
3424
3425
3426 SDValue
LowerXALUO(SDValue Op,SelectionDAG & DAG) const3427 ARMTargetLowering::LowerXALUO(SDValue Op, SelectionDAG &DAG) const {
3428 // Let legalize expand this if it isn't a legal type yet.
3429 if (!DAG.getTargetLoweringInfo().isTypeLegal(Op.getValueType()))
3430 return SDValue();
3431
3432 SDValue Value, OverflowCmp;
3433 SDValue ARMcc;
3434 std::tie(Value, OverflowCmp) = getARMXALUOOp(Op, DAG, ARMcc);
3435 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
3436 SDLoc dl(Op);
3437 // We use 0 and 1 as false and true values.
3438 SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
3439 SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
3440 EVT VT = Op.getValueType();
3441
3442 SDValue Overflow = DAG.getNode(ARMISD::CMOV, dl, VT, TVal, FVal,
3443 ARMcc, CCR, OverflowCmp);
3444
3445 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
3446 return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow);
3447 }
3448
3449
LowerSELECT(SDValue Op,SelectionDAG & DAG) const3450 SDValue ARMTargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
3451 SDValue Cond = Op.getOperand(0);
3452 SDValue SelectTrue = Op.getOperand(1);
3453 SDValue SelectFalse = Op.getOperand(2);
3454 SDLoc dl(Op);
3455 unsigned Opc = Cond.getOpcode();
3456
3457 if (Cond.getResNo() == 1 &&
3458 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
3459 Opc == ISD::USUBO)) {
3460 if (!DAG.getTargetLoweringInfo().isTypeLegal(Cond->getValueType(0)))
3461 return SDValue();
3462
3463 SDValue Value, OverflowCmp;
3464 SDValue ARMcc;
3465 std::tie(Value, OverflowCmp) = getARMXALUOOp(Cond, DAG, ARMcc);
3466 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
3467 EVT VT = Op.getValueType();
3468
3469 return getCMOV(dl, VT, SelectTrue, SelectFalse, ARMcc, CCR,
3470 OverflowCmp, DAG);
3471 }
3472
3473 // Convert:
3474 //
3475 // (select (cmov 1, 0, cond), t, f) -> (cmov t, f, cond)
3476 // (select (cmov 0, 1, cond), t, f) -> (cmov f, t, cond)
3477 //
3478 if (Cond.getOpcode() == ARMISD::CMOV && Cond.hasOneUse()) {
3479 const ConstantSDNode *CMOVTrue =
3480 dyn_cast<ConstantSDNode>(Cond.getOperand(0));
3481 const ConstantSDNode *CMOVFalse =
3482 dyn_cast<ConstantSDNode>(Cond.getOperand(1));
3483
3484 if (CMOVTrue && CMOVFalse) {
3485 unsigned CMOVTrueVal = CMOVTrue->getZExtValue();
3486 unsigned CMOVFalseVal = CMOVFalse->getZExtValue();
3487
3488 SDValue True;
3489 SDValue False;
3490 if (CMOVTrueVal == 1 && CMOVFalseVal == 0) {
3491 True = SelectTrue;
3492 False = SelectFalse;
3493 } else if (CMOVTrueVal == 0 && CMOVFalseVal == 1) {
3494 True = SelectFalse;
3495 False = SelectTrue;
3496 }
3497
3498 if (True.getNode() && False.getNode()) {
3499 EVT VT = Op.getValueType();
3500 SDValue ARMcc = Cond.getOperand(2);
3501 SDValue CCR = Cond.getOperand(3);
3502 SDValue Cmp = duplicateCmp(Cond.getOperand(4), DAG);
3503 assert(True.getValueType() == VT);
3504 return getCMOV(dl, VT, True, False, ARMcc, CCR, Cmp, DAG);
3505 }
3506 }
3507 }
3508
3509 // ARM's BooleanContents value is UndefinedBooleanContent. Mask out the
3510 // undefined bits before doing a full-word comparison with zero.
3511 Cond = DAG.getNode(ISD::AND, dl, Cond.getValueType(), Cond,
3512 DAG.getConstant(1, dl, Cond.getValueType()));
3513
3514 return DAG.getSelectCC(dl, Cond,
3515 DAG.getConstant(0, dl, Cond.getValueType()),
3516 SelectTrue, SelectFalse, ISD::SETNE);
3517 }
3518
checkVSELConstraints(ISD::CondCode CC,ARMCC::CondCodes & CondCode,bool & swpCmpOps,bool & swpVselOps)3519 static void checkVSELConstraints(ISD::CondCode CC, ARMCC::CondCodes &CondCode,
3520 bool &swpCmpOps, bool &swpVselOps) {
3521 // Start by selecting the GE condition code for opcodes that return true for
3522 // 'equality'
3523 if (CC == ISD::SETUGE || CC == ISD::SETOGE || CC == ISD::SETOLE ||
3524 CC == ISD::SETULE)
3525 CondCode = ARMCC::GE;
3526
3527 // and GT for opcodes that return false for 'equality'.
3528 else if (CC == ISD::SETUGT || CC == ISD::SETOGT || CC == ISD::SETOLT ||
3529 CC == ISD::SETULT)
3530 CondCode = ARMCC::GT;
3531
3532 // Since we are constrained to GE/GT, if the opcode contains 'less', we need
3533 // to swap the compare operands.
3534 if (CC == ISD::SETOLE || CC == ISD::SETULE || CC == ISD::SETOLT ||
3535 CC == ISD::SETULT)
3536 swpCmpOps = true;
3537
3538 // Both GT and GE are ordered comparisons, and return false for 'unordered'.
3539 // If we have an unordered opcode, we need to swap the operands to the VSEL
3540 // instruction (effectively negating the condition).
3541 //
3542 // This also has the effect of swapping which one of 'less' or 'greater'
3543 // returns true, so we also swap the compare operands. It also switches
3544 // whether we return true for 'equality', so we compensate by picking the
3545 // opposite condition code to our original choice.
3546 if (CC == ISD::SETULE || CC == ISD::SETULT || CC == ISD::SETUGE ||
3547 CC == ISD::SETUGT) {
3548 swpCmpOps = !swpCmpOps;
3549 swpVselOps = !swpVselOps;
3550 CondCode = CondCode == ARMCC::GT ? ARMCC::GE : ARMCC::GT;
3551 }
3552
3553 // 'ordered' is 'anything but unordered', so use the VS condition code and
3554 // swap the VSEL operands.
3555 if (CC == ISD::SETO) {
3556 CondCode = ARMCC::VS;
3557 swpVselOps = true;
3558 }
3559
3560 // 'unordered or not equal' is 'anything but equal', so use the EQ condition
3561 // code and swap the VSEL operands.
3562 if (CC == ISD::SETUNE) {
3563 CondCode = ARMCC::EQ;
3564 swpVselOps = true;
3565 }
3566 }
3567
getCMOV(SDLoc dl,EVT VT,SDValue FalseVal,SDValue TrueVal,SDValue ARMcc,SDValue CCR,SDValue Cmp,SelectionDAG & DAG) const3568 SDValue ARMTargetLowering::getCMOV(SDLoc dl, EVT VT, SDValue FalseVal,
3569 SDValue TrueVal, SDValue ARMcc, SDValue CCR,
3570 SDValue Cmp, SelectionDAG &DAG) const {
3571 if (Subtarget->isFPOnlySP() && VT == MVT::f64) {
3572 FalseVal = DAG.getNode(ARMISD::VMOVRRD, dl,
3573 DAG.getVTList(MVT::i32, MVT::i32), FalseVal);
3574 TrueVal = DAG.getNode(ARMISD::VMOVRRD, dl,
3575 DAG.getVTList(MVT::i32, MVT::i32), TrueVal);
3576
3577 SDValue TrueLow = TrueVal.getValue(0);
3578 SDValue TrueHigh = TrueVal.getValue(1);
3579 SDValue FalseLow = FalseVal.getValue(0);
3580 SDValue FalseHigh = FalseVal.getValue(1);
3581
3582 SDValue Low = DAG.getNode(ARMISD::CMOV, dl, MVT::i32, FalseLow, TrueLow,
3583 ARMcc, CCR, Cmp);
3584 SDValue High = DAG.getNode(ARMISD::CMOV, dl, MVT::i32, FalseHigh, TrueHigh,
3585 ARMcc, CCR, duplicateCmp(Cmp, DAG));
3586
3587 return DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Low, High);
3588 } else {
3589 return DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal, ARMcc, CCR,
3590 Cmp);
3591 }
3592 }
3593
LowerSELECT_CC(SDValue Op,SelectionDAG & DAG) const3594 SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
3595 EVT VT = Op.getValueType();
3596 SDValue LHS = Op.getOperand(0);
3597 SDValue RHS = Op.getOperand(1);
3598 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
3599 SDValue TrueVal = Op.getOperand(2);
3600 SDValue FalseVal = Op.getOperand(3);
3601 SDLoc dl(Op);
3602
3603 if (Subtarget->isFPOnlySP() && LHS.getValueType() == MVT::f64) {
3604 DAG.getTargetLoweringInfo().softenSetCCOperands(DAG, MVT::f64, LHS, RHS, CC,
3605 dl);
3606
3607 // If softenSetCCOperands only returned one value, we should compare it to
3608 // zero.
3609 if (!RHS.getNode()) {
3610 RHS = DAG.getConstant(0, dl, LHS.getValueType());
3611 CC = ISD::SETNE;
3612 }
3613 }
3614
3615 if (LHS.getValueType() == MVT::i32) {
3616 // Try to generate VSEL on ARMv8.
3617 // The VSEL instruction can't use all the usual ARM condition
3618 // codes: it only has two bits to select the condition code, so it's
3619 // constrained to use only GE, GT, VS and EQ.
3620 //
3621 // To implement all the various ISD::SETXXX opcodes, we sometimes need to
3622 // swap the operands of the previous compare instruction (effectively
3623 // inverting the compare condition, swapping 'less' and 'greater') and
3624 // sometimes need to swap the operands to the VSEL (which inverts the
3625 // condition in the sense of firing whenever the previous condition didn't)
3626 if (Subtarget->hasFPARMv8() && (TrueVal.getValueType() == MVT::f32 ||
3627 TrueVal.getValueType() == MVT::f64)) {
3628 ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
3629 if (CondCode == ARMCC::LT || CondCode == ARMCC::LE ||
3630 CondCode == ARMCC::VC || CondCode == ARMCC::NE) {
3631 CC = ISD::getSetCCInverse(CC, true);
3632 std::swap(TrueVal, FalseVal);
3633 }
3634 }
3635
3636 SDValue ARMcc;
3637 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
3638 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
3639 return getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG);
3640 }
3641
3642 ARMCC::CondCodes CondCode, CondCode2;
3643 FPCCToARMCC(CC, CondCode, CondCode2);
3644
3645 // Try to generate VMAXNM/VMINNM on ARMv8.
3646 if (Subtarget->hasFPARMv8() && (TrueVal.getValueType() == MVT::f32 ||
3647 TrueVal.getValueType() == MVT::f64)) {
3648 bool swpCmpOps = false;
3649 bool swpVselOps = false;
3650 checkVSELConstraints(CC, CondCode, swpCmpOps, swpVselOps);
3651
3652 if (CondCode == ARMCC::GT || CondCode == ARMCC::GE ||
3653 CondCode == ARMCC::VS || CondCode == ARMCC::EQ) {
3654 if (swpCmpOps)
3655 std::swap(LHS, RHS);
3656 if (swpVselOps)
3657 std::swap(TrueVal, FalseVal);
3658 }
3659 }
3660
3661 SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
3662 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
3663 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
3664 SDValue Result = getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG);
3665 if (CondCode2 != ARMCC::AL) {
3666 SDValue ARMcc2 = DAG.getConstant(CondCode2, dl, MVT::i32);
3667 // FIXME: Needs another CMP because flag can have but one use.
3668 SDValue Cmp2 = getVFPCmp(LHS, RHS, DAG, dl);
3669 Result = getCMOV(dl, VT, Result, TrueVal, ARMcc2, CCR, Cmp2, DAG);
3670 }
3671 return Result;
3672 }
3673
3674 /// canChangeToInt - Given the fp compare operand, return true if it is suitable
3675 /// to morph to an integer compare sequence.
canChangeToInt(SDValue Op,bool & SeenZero,const ARMSubtarget * Subtarget)3676 static bool canChangeToInt(SDValue Op, bool &SeenZero,
3677 const ARMSubtarget *Subtarget) {
3678 SDNode *N = Op.getNode();
3679 if (!N->hasOneUse())
3680 // Otherwise it requires moving the value from fp to integer registers.
3681 return false;
3682 if (!N->getNumValues())
3683 return false;
3684 EVT VT = Op.getValueType();
3685 if (VT != MVT::f32 && !Subtarget->isFPBrccSlow())
3686 // f32 case is generally profitable. f64 case only makes sense when vcmpe +
3687 // vmrs are very slow, e.g. cortex-a8.
3688 return false;
3689
3690 if (isFloatingPointZero(Op)) {
3691 SeenZero = true;
3692 return true;
3693 }
3694 return ISD::isNormalLoad(N);
3695 }
3696
bitcastf32Toi32(SDValue Op,SelectionDAG & DAG)3697 static SDValue bitcastf32Toi32(SDValue Op, SelectionDAG &DAG) {
3698 if (isFloatingPointZero(Op))
3699 return DAG.getConstant(0, SDLoc(Op), MVT::i32);
3700
3701 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Op))
3702 return DAG.getLoad(MVT::i32, SDLoc(Op),
3703 Ld->getChain(), Ld->getBasePtr(), Ld->getPointerInfo(),
3704 Ld->isVolatile(), Ld->isNonTemporal(),
3705 Ld->isInvariant(), Ld->getAlignment());
3706
3707 llvm_unreachable("Unknown VFP cmp argument!");
3708 }
3709
expandf64Toi32(SDValue Op,SelectionDAG & DAG,SDValue & RetVal1,SDValue & RetVal2)3710 static void expandf64Toi32(SDValue Op, SelectionDAG &DAG,
3711 SDValue &RetVal1, SDValue &RetVal2) {
3712 SDLoc dl(Op);
3713
3714 if (isFloatingPointZero(Op)) {
3715 RetVal1 = DAG.getConstant(0, dl, MVT::i32);
3716 RetVal2 = DAG.getConstant(0, dl, MVT::i32);
3717 return;
3718 }
3719
3720 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Op)) {
3721 SDValue Ptr = Ld->getBasePtr();
3722 RetVal1 = DAG.getLoad(MVT::i32, dl,
3723 Ld->getChain(), Ptr,
3724 Ld->getPointerInfo(),
3725 Ld->isVolatile(), Ld->isNonTemporal(),
3726 Ld->isInvariant(), Ld->getAlignment());
3727
3728 EVT PtrType = Ptr.getValueType();
3729 unsigned NewAlign = MinAlign(Ld->getAlignment(), 4);
3730 SDValue NewPtr = DAG.getNode(ISD::ADD, dl,
3731 PtrType, Ptr, DAG.getConstant(4, dl, PtrType));
3732 RetVal2 = DAG.getLoad(MVT::i32, dl,
3733 Ld->getChain(), NewPtr,
3734 Ld->getPointerInfo().getWithOffset(4),
3735 Ld->isVolatile(), Ld->isNonTemporal(),
3736 Ld->isInvariant(), NewAlign);
3737 return;
3738 }
3739
3740 llvm_unreachable("Unknown VFP cmp argument!");
3741 }
3742
3743 /// OptimizeVFPBrcond - With -enable-unsafe-fp-math, it's legal to optimize some
3744 /// f32 and even f64 comparisons to integer ones.
3745 SDValue
OptimizeVFPBrcond(SDValue Op,SelectionDAG & DAG) const3746 ARMTargetLowering::OptimizeVFPBrcond(SDValue Op, SelectionDAG &DAG) const {
3747 SDValue Chain = Op.getOperand(0);
3748 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
3749 SDValue LHS = Op.getOperand(2);
3750 SDValue RHS = Op.getOperand(3);
3751 SDValue Dest = Op.getOperand(4);
3752 SDLoc dl(Op);
3753
3754 bool LHSSeenZero = false;
3755 bool LHSOk = canChangeToInt(LHS, LHSSeenZero, Subtarget);
3756 bool RHSSeenZero = false;
3757 bool RHSOk = canChangeToInt(RHS, RHSSeenZero, Subtarget);
3758 if (LHSOk && RHSOk && (LHSSeenZero || RHSSeenZero)) {
3759 // If unsafe fp math optimization is enabled and there are no other uses of
3760 // the CMP operands, and the condition code is EQ or NE, we can optimize it
3761 // to an integer comparison.
3762 if (CC == ISD::SETOEQ)
3763 CC = ISD::SETEQ;
3764 else if (CC == ISD::SETUNE)
3765 CC = ISD::SETNE;
3766
3767 SDValue Mask = DAG.getConstant(0x7fffffff, dl, MVT::i32);
3768 SDValue ARMcc;
3769 if (LHS.getValueType() == MVT::f32) {
3770 LHS = DAG.getNode(ISD::AND, dl, MVT::i32,
3771 bitcastf32Toi32(LHS, DAG), Mask);
3772 RHS = DAG.getNode(ISD::AND, dl, MVT::i32,
3773 bitcastf32Toi32(RHS, DAG), Mask);
3774 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
3775 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
3776 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other,
3777 Chain, Dest, ARMcc, CCR, Cmp);
3778 }
3779
3780 SDValue LHS1, LHS2;
3781 SDValue RHS1, RHS2;
3782 expandf64Toi32(LHS, DAG, LHS1, LHS2);
3783 expandf64Toi32(RHS, DAG, RHS1, RHS2);
3784 LHS2 = DAG.getNode(ISD::AND, dl, MVT::i32, LHS2, Mask);
3785 RHS2 = DAG.getNode(ISD::AND, dl, MVT::i32, RHS2, Mask);
3786 ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
3787 ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
3788 SDVTList VTList = DAG.getVTList(MVT::Other, MVT::Glue);
3789 SDValue Ops[] = { Chain, ARMcc, LHS1, LHS2, RHS1, RHS2, Dest };
3790 return DAG.getNode(ARMISD::BCC_i64, dl, VTList, Ops);
3791 }
3792
3793 return SDValue();
3794 }
3795
LowerBR_CC(SDValue Op,SelectionDAG & DAG) const3796 SDValue ARMTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
3797 SDValue Chain = Op.getOperand(0);
3798 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
3799 SDValue LHS = Op.getOperand(2);
3800 SDValue RHS = Op.getOperand(3);
3801 SDValue Dest = Op.getOperand(4);
3802 SDLoc dl(Op);
3803
3804 if (Subtarget->isFPOnlySP() && LHS.getValueType() == MVT::f64) {
3805 DAG.getTargetLoweringInfo().softenSetCCOperands(DAG, MVT::f64, LHS, RHS, CC,
3806 dl);
3807
3808 // If softenSetCCOperands only returned one value, we should compare it to
3809 // zero.
3810 if (!RHS.getNode()) {
3811 RHS = DAG.getConstant(0, dl, LHS.getValueType());
3812 CC = ISD::SETNE;
3813 }
3814 }
3815
3816 if (LHS.getValueType() == MVT::i32) {
3817 SDValue ARMcc;
3818 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
3819 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
3820 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other,
3821 Chain, Dest, ARMcc, CCR, Cmp);
3822 }
3823
3824 assert(LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64);
3825
3826 if (getTargetMachine().Options.UnsafeFPMath &&
3827 (CC == ISD::SETEQ || CC == ISD::SETOEQ ||
3828 CC == ISD::SETNE || CC == ISD::SETUNE)) {
3829 SDValue Result = OptimizeVFPBrcond(Op, DAG);
3830 if (Result.getNode())
3831 return Result;
3832 }
3833
3834 ARMCC::CondCodes CondCode, CondCode2;
3835 FPCCToARMCC(CC, CondCode, CondCode2);
3836
3837 SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
3838 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
3839 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
3840 SDVTList VTList = DAG.getVTList(MVT::Other, MVT::Glue);
3841 SDValue Ops[] = { Chain, Dest, ARMcc, CCR, Cmp };
3842 SDValue Res = DAG.getNode(ARMISD::BRCOND, dl, VTList, Ops);
3843 if (CondCode2 != ARMCC::AL) {
3844 ARMcc = DAG.getConstant(CondCode2, dl, MVT::i32);
3845 SDValue Ops[] = { Res, Dest, ARMcc, CCR, Res.getValue(1) };
3846 Res = DAG.getNode(ARMISD::BRCOND, dl, VTList, Ops);
3847 }
3848 return Res;
3849 }
3850
LowerBR_JT(SDValue Op,SelectionDAG & DAG) const3851 SDValue ARMTargetLowering::LowerBR_JT(SDValue Op, SelectionDAG &DAG) const {
3852 SDValue Chain = Op.getOperand(0);
3853 SDValue Table = Op.getOperand(1);
3854 SDValue Index = Op.getOperand(2);
3855 SDLoc dl(Op);
3856
3857 EVT PTy = getPointerTy(DAG.getDataLayout());
3858 JumpTableSDNode *JT = cast<JumpTableSDNode>(Table);
3859 SDValue JTI = DAG.getTargetJumpTable(JT->getIndex(), PTy);
3860 Table = DAG.getNode(ARMISD::WrapperJT, dl, MVT::i32, JTI);
3861 Index = DAG.getNode(ISD::MUL, dl, PTy, Index, DAG.getConstant(4, dl, PTy));
3862 SDValue Addr = DAG.getNode(ISD::ADD, dl, PTy, Index, Table);
3863 if (Subtarget->isThumb2()) {
3864 // Thumb2 uses a two-level jump. That is, it jumps into the jump table
3865 // which does another jump to the destination. This also makes it easier
3866 // to translate it to TBB / TBH later.
3867 // FIXME: This might not work if the function is extremely large.
3868 return DAG.getNode(ARMISD::BR2_JT, dl, MVT::Other, Chain,
3869 Addr, Op.getOperand(2), JTI);
3870 }
3871 if (getTargetMachine().getRelocationModel() == Reloc::PIC_) {
3872 Addr =
3873 DAG.getLoad((EVT)MVT::i32, dl, Chain, Addr,
3874 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()),
3875 false, false, false, 0);
3876 Chain = Addr.getValue(1);
3877 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr, Table);
3878 return DAG.getNode(ARMISD::BR_JT, dl, MVT::Other, Chain, Addr, JTI);
3879 } else {
3880 Addr =
3881 DAG.getLoad(PTy, dl, Chain, Addr,
3882 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()),
3883 false, false, false, 0);
3884 Chain = Addr.getValue(1);
3885 return DAG.getNode(ARMISD::BR_JT, dl, MVT::Other, Chain, Addr, JTI);
3886 }
3887 }
3888
LowerVectorFP_TO_INT(SDValue Op,SelectionDAG & DAG)3889 static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) {
3890 EVT VT = Op.getValueType();
3891 SDLoc dl(Op);
3892
3893 if (Op.getValueType().getVectorElementType() == MVT::i32) {
3894 if (Op.getOperand(0).getValueType().getVectorElementType() == MVT::f32)
3895 return Op;
3896 return DAG.UnrollVectorOp(Op.getNode());
3897 }
3898
3899 assert(Op.getOperand(0).getValueType() == MVT::v4f32 &&
3900 "Invalid type for custom lowering!");
3901 if (VT != MVT::v4i16)
3902 return DAG.UnrollVectorOp(Op.getNode());
3903
3904 Op = DAG.getNode(Op.getOpcode(), dl, MVT::v4i32, Op.getOperand(0));
3905 return DAG.getNode(ISD::TRUNCATE, dl, VT, Op);
3906 }
3907
LowerFP_TO_INT(SDValue Op,SelectionDAG & DAG) const3908 SDValue ARMTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
3909 EVT VT = Op.getValueType();
3910 if (VT.isVector())
3911 return LowerVectorFP_TO_INT(Op, DAG);
3912 if (Subtarget->isFPOnlySP() && Op.getOperand(0).getValueType() == MVT::f64) {
3913 RTLIB::Libcall LC;
3914 if (Op.getOpcode() == ISD::FP_TO_SINT)
3915 LC = RTLIB::getFPTOSINT(Op.getOperand(0).getValueType(),
3916 Op.getValueType());
3917 else
3918 LC = RTLIB::getFPTOUINT(Op.getOperand(0).getValueType(),
3919 Op.getValueType());
3920 return makeLibCall(DAG, LC, Op.getValueType(), Op.getOperand(0),
3921 /*isSigned*/ false, SDLoc(Op)).first;
3922 }
3923
3924 return Op;
3925 }
3926
LowerVectorINT_TO_FP(SDValue Op,SelectionDAG & DAG)3927 static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
3928 EVT VT = Op.getValueType();
3929 SDLoc dl(Op);
3930
3931 if (Op.getOperand(0).getValueType().getVectorElementType() == MVT::i32) {
3932 if (VT.getVectorElementType() == MVT::f32)
3933 return Op;
3934 return DAG.UnrollVectorOp(Op.getNode());
3935 }
3936
3937 assert(Op.getOperand(0).getValueType() == MVT::v4i16 &&
3938 "Invalid type for custom lowering!");
3939 if (VT != MVT::v4f32)
3940 return DAG.UnrollVectorOp(Op.getNode());
3941
3942 unsigned CastOpc;
3943 unsigned Opc;
3944 switch (Op.getOpcode()) {
3945 default: llvm_unreachable("Invalid opcode!");
3946 case ISD::SINT_TO_FP:
3947 CastOpc = ISD::SIGN_EXTEND;
3948 Opc = ISD::SINT_TO_FP;
3949 break;
3950 case ISD::UINT_TO_FP:
3951 CastOpc = ISD::ZERO_EXTEND;
3952 Opc = ISD::UINT_TO_FP;
3953 break;
3954 }
3955
3956 Op = DAG.getNode(CastOpc, dl, MVT::v4i32, Op.getOperand(0));
3957 return DAG.getNode(Opc, dl, VT, Op);
3958 }
3959
LowerINT_TO_FP(SDValue Op,SelectionDAG & DAG) const3960 SDValue ARMTargetLowering::LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const {
3961 EVT VT = Op.getValueType();
3962 if (VT.isVector())
3963 return LowerVectorINT_TO_FP(Op, DAG);
3964 if (Subtarget->isFPOnlySP() && Op.getValueType() == MVT::f64) {
3965 RTLIB::Libcall LC;
3966 if (Op.getOpcode() == ISD::SINT_TO_FP)
3967 LC = RTLIB::getSINTTOFP(Op.getOperand(0).getValueType(),
3968 Op.getValueType());
3969 else
3970 LC = RTLIB::getUINTTOFP(Op.getOperand(0).getValueType(),
3971 Op.getValueType());
3972 return makeLibCall(DAG, LC, Op.getValueType(), Op.getOperand(0),
3973 /*isSigned*/ false, SDLoc(Op)).first;
3974 }
3975
3976 return Op;
3977 }
3978
LowerFCOPYSIGN(SDValue Op,SelectionDAG & DAG) const3979 SDValue ARMTargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
3980 // Implement fcopysign with a fabs and a conditional fneg.
3981 SDValue Tmp0 = Op.getOperand(0);
3982 SDValue Tmp1 = Op.getOperand(1);
3983 SDLoc dl(Op);
3984 EVT VT = Op.getValueType();
3985 EVT SrcVT = Tmp1.getValueType();
3986 bool InGPR = Tmp0.getOpcode() == ISD::BITCAST ||
3987 Tmp0.getOpcode() == ARMISD::VMOVDRR;
3988 bool UseNEON = !InGPR && Subtarget->hasNEON();
3989
3990 if (UseNEON) {
3991 // Use VBSL to copy the sign bit.
3992 unsigned EncodedVal = ARM_AM::createNEONModImm(0x6, 0x80);
3993 SDValue Mask = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v2i32,
3994 DAG.getTargetConstant(EncodedVal, dl, MVT::i32));
3995 EVT OpVT = (VT == MVT::f32) ? MVT::v2i32 : MVT::v1i64;
3996 if (VT == MVT::f64)
3997 Mask = DAG.getNode(ARMISD::VSHL, dl, OpVT,
3998 DAG.getNode(ISD::BITCAST, dl, OpVT, Mask),
3999 DAG.getConstant(32, dl, MVT::i32));
4000 else /*if (VT == MVT::f32)*/
4001 Tmp0 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f32, Tmp0);
4002 if (SrcVT == MVT::f32) {
4003 Tmp1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f32, Tmp1);
4004 if (VT == MVT::f64)
4005 Tmp1 = DAG.getNode(ARMISD::VSHL, dl, OpVT,
4006 DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp1),
4007 DAG.getConstant(32, dl, MVT::i32));
4008 } else if (VT == MVT::f32)
4009 Tmp1 = DAG.getNode(ARMISD::VSHRu, dl, MVT::v1i64,
4010 DAG.getNode(ISD::BITCAST, dl, MVT::v1i64, Tmp1),
4011 DAG.getConstant(32, dl, MVT::i32));
4012 Tmp0 = DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp0);
4013 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp1);
4014
4015 SDValue AllOnes = DAG.getTargetConstant(ARM_AM::createNEONModImm(0xe, 0xff),
4016 dl, MVT::i32);
4017 AllOnes = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v8i8, AllOnes);
4018 SDValue MaskNot = DAG.getNode(ISD::XOR, dl, OpVT, Mask,
4019 DAG.getNode(ISD::BITCAST, dl, OpVT, AllOnes));
4020
4021 SDValue Res = DAG.getNode(ISD::OR, dl, OpVT,
4022 DAG.getNode(ISD::AND, dl, OpVT, Tmp1, Mask),
4023 DAG.getNode(ISD::AND, dl, OpVT, Tmp0, MaskNot));
4024 if (VT == MVT::f32) {
4025 Res = DAG.getNode(ISD::BITCAST, dl, MVT::v2f32, Res);
4026 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, Res,
4027 DAG.getConstant(0, dl, MVT::i32));
4028 } else {
4029 Res = DAG.getNode(ISD::BITCAST, dl, MVT::f64, Res);
4030 }
4031
4032 return Res;
4033 }
4034
4035 // Bitcast operand 1 to i32.
4036 if (SrcVT == MVT::f64)
4037 Tmp1 = DAG.getNode(ARMISD::VMOVRRD, dl, DAG.getVTList(MVT::i32, MVT::i32),
4038 Tmp1).getValue(1);
4039 Tmp1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Tmp1);
4040
4041 // Or in the signbit with integer operations.
4042 SDValue Mask1 = DAG.getConstant(0x80000000, dl, MVT::i32);
4043 SDValue Mask2 = DAG.getConstant(0x7fffffff, dl, MVT::i32);
4044 Tmp1 = DAG.getNode(ISD::AND, dl, MVT::i32, Tmp1, Mask1);
4045 if (VT == MVT::f32) {
4046 Tmp0 = DAG.getNode(ISD::AND, dl, MVT::i32,
4047 DAG.getNode(ISD::BITCAST, dl, MVT::i32, Tmp0), Mask2);
4048 return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
4049 DAG.getNode(ISD::OR, dl, MVT::i32, Tmp0, Tmp1));
4050 }
4051
4052 // f64: Or the high part with signbit and then combine two parts.
4053 Tmp0 = DAG.getNode(ARMISD::VMOVRRD, dl, DAG.getVTList(MVT::i32, MVT::i32),
4054 Tmp0);
4055 SDValue Lo = Tmp0.getValue(0);
4056 SDValue Hi = DAG.getNode(ISD::AND, dl, MVT::i32, Tmp0.getValue(1), Mask2);
4057 Hi = DAG.getNode(ISD::OR, dl, MVT::i32, Hi, Tmp1);
4058 return DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi);
4059 }
4060
LowerRETURNADDR(SDValue Op,SelectionDAG & DAG) const4061 SDValue ARMTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const{
4062 MachineFunction &MF = DAG.getMachineFunction();
4063 MachineFrameInfo *MFI = MF.getFrameInfo();
4064 MFI->setReturnAddressIsTaken(true);
4065
4066 if (verifyReturnAddressArgumentIsConstant(Op, DAG))
4067 return SDValue();
4068
4069 EVT VT = Op.getValueType();
4070 SDLoc dl(Op);
4071 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
4072 if (Depth) {
4073 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
4074 SDValue Offset = DAG.getConstant(4, dl, MVT::i32);
4075 return DAG.getLoad(VT, dl, DAG.getEntryNode(),
4076 DAG.getNode(ISD::ADD, dl, VT, FrameAddr, Offset),
4077 MachinePointerInfo(), false, false, false, 0);
4078 }
4079
4080 // Return LR, which contains the return address. Mark it an implicit live-in.
4081 unsigned Reg = MF.addLiveIn(ARM::LR, getRegClassFor(MVT::i32));
4082 return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT);
4083 }
4084
LowerFRAMEADDR(SDValue Op,SelectionDAG & DAG) const4085 SDValue ARMTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
4086 const ARMBaseRegisterInfo &ARI =
4087 *static_cast<const ARMBaseRegisterInfo*>(RegInfo);
4088 MachineFunction &MF = DAG.getMachineFunction();
4089 MachineFrameInfo *MFI = MF.getFrameInfo();
4090 MFI->setFrameAddressIsTaken(true);
4091
4092 EVT VT = Op.getValueType();
4093 SDLoc dl(Op); // FIXME probably not meaningful
4094 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
4095 unsigned FrameReg = ARI.getFrameRegister(MF);
4096 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT);
4097 while (Depth--)
4098 FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr,
4099 MachinePointerInfo(),
4100 false, false, false, 0);
4101 return FrameAddr;
4102 }
4103
4104 // FIXME? Maybe this could be a TableGen attribute on some registers and
4105 // this table could be generated automatically from RegInfo.
getRegisterByName(const char * RegName,EVT VT,SelectionDAG & DAG) const4106 unsigned ARMTargetLowering::getRegisterByName(const char* RegName, EVT VT,
4107 SelectionDAG &DAG) const {
4108 unsigned Reg = StringSwitch<unsigned>(RegName)
4109 .Case("sp", ARM::SP)
4110 .Default(0);
4111 if (Reg)
4112 return Reg;
4113 report_fatal_error(Twine("Invalid register name \""
4114 + StringRef(RegName) + "\"."));
4115 }
4116
4117 // Result is 64 bit value so split into two 32 bit values and return as a
4118 // pair of values.
ExpandREAD_REGISTER(SDNode * N,SmallVectorImpl<SDValue> & Results,SelectionDAG & DAG)4119 static void ExpandREAD_REGISTER(SDNode *N, SmallVectorImpl<SDValue> &Results,
4120 SelectionDAG &DAG) {
4121 SDLoc DL(N);
4122
4123 // This function is only supposed to be called for i64 type destination.
4124 assert(N->getValueType(0) == MVT::i64
4125 && "ExpandREAD_REGISTER called for non-i64 type result.");
4126
4127 SDValue Read = DAG.getNode(ISD::READ_REGISTER, DL,
4128 DAG.getVTList(MVT::i32, MVT::i32, MVT::Other),
4129 N->getOperand(0),
4130 N->getOperand(1));
4131
4132 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Read.getValue(0),
4133 Read.getValue(1)));
4134 Results.push_back(Read.getOperand(0));
4135 }
4136
4137 /// \p BC is a bitcast that is about to be turned into a VMOVDRR.
4138 /// When \p DstVT, the destination type of \p BC, is on the vector
4139 /// register bank and the source of bitcast, \p Op, operates on the same bank,
4140 /// it might be possible to combine them, such that everything stays on the
4141 /// vector register bank.
4142 /// \p return The node that would replace \p BT, if the combine
4143 /// is possible.
CombineVMOVDRRCandidateWithVecOp(const SDNode * BC,SelectionDAG & DAG)4144 static SDValue CombineVMOVDRRCandidateWithVecOp(const SDNode *BC,
4145 SelectionDAG &DAG) {
4146 SDValue Op = BC->getOperand(0);
4147 EVT DstVT = BC->getValueType(0);
4148
4149 // The only vector instruction that can produce a scalar (remember,
4150 // since the bitcast was about to be turned into VMOVDRR, the source
4151 // type is i64) from a vector is EXTRACT_VECTOR_ELT.
4152 // Moreover, we can do this combine only if there is one use.
4153 // Finally, if the destination type is not a vector, there is not
4154 // much point on forcing everything on the vector bank.
4155 if (!DstVT.isVector() || Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
4156 !Op.hasOneUse())
4157 return SDValue();
4158
4159 // If the index is not constant, we will introduce an additional
4160 // multiply that will stick.
4161 // Give up in that case.
4162 ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Op.getOperand(1));
4163 if (!Index)
4164 return SDValue();
4165 unsigned DstNumElt = DstVT.getVectorNumElements();
4166
4167 // Compute the new index.
4168 const APInt &APIntIndex = Index->getAPIntValue();
4169 APInt NewIndex(APIntIndex.getBitWidth(), DstNumElt);
4170 NewIndex *= APIntIndex;
4171 // Check if the new constant index fits into i32.
4172 if (NewIndex.getBitWidth() > 32)
4173 return SDValue();
4174
4175 // vMTy bitcast(i64 extractelt vNi64 src, i32 index) ->
4176 // vMTy extractsubvector vNxMTy (bitcast vNi64 src), i32 index*M)
4177 SDLoc dl(Op);
4178 SDValue ExtractSrc = Op.getOperand(0);
4179 EVT VecVT = EVT::getVectorVT(
4180 *DAG.getContext(), DstVT.getScalarType(),
4181 ExtractSrc.getValueType().getVectorNumElements() * DstNumElt);
4182 SDValue BitCast = DAG.getNode(ISD::BITCAST, dl, VecVT, ExtractSrc);
4183 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DstVT, BitCast,
4184 DAG.getConstant(NewIndex.getZExtValue(), dl, MVT::i32));
4185 }
4186
4187 /// ExpandBITCAST - If the target supports VFP, this function is called to
4188 /// expand a bit convert where either the source or destination type is i64 to
4189 /// use a VMOVDRR or VMOVRRD node. This should not be done when the non-i64
4190 /// operand type is illegal (e.g., v2f32 for a target that doesn't support
4191 /// vectors), since the legalizer won't know what to do with that.
ExpandBITCAST(SDNode * N,SelectionDAG & DAG)4192 static SDValue ExpandBITCAST(SDNode *N, SelectionDAG &DAG) {
4193 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4194 SDLoc dl(N);
4195 SDValue Op = N->getOperand(0);
4196
4197 // This function is only supposed to be called for i64 types, either as the
4198 // source or destination of the bit convert.
4199 EVT SrcVT = Op.getValueType();
4200 EVT DstVT = N->getValueType(0);
4201 assert((SrcVT == MVT::i64 || DstVT == MVT::i64) &&
4202 "ExpandBITCAST called for non-i64 type");
4203
4204 // Turn i64->f64 into VMOVDRR.
4205 if (SrcVT == MVT::i64 && TLI.isTypeLegal(DstVT)) {
4206 // Do not force values to GPRs (this is what VMOVDRR does for the inputs)
4207 // if we can combine the bitcast with its source.
4208 if (SDValue Val = CombineVMOVDRRCandidateWithVecOp(N, DAG))
4209 return Val;
4210
4211 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Op,
4212 DAG.getConstant(0, dl, MVT::i32));
4213 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Op,
4214 DAG.getConstant(1, dl, MVT::i32));
4215 return DAG.getNode(ISD::BITCAST, dl, DstVT,
4216 DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi));
4217 }
4218
4219 // Turn f64->i64 into VMOVRRD.
4220 if (DstVT == MVT::i64 && TLI.isTypeLegal(SrcVT)) {
4221 SDValue Cvt;
4222 if (DAG.getDataLayout().isBigEndian() && SrcVT.isVector() &&
4223 SrcVT.getVectorNumElements() > 1)
4224 Cvt = DAG.getNode(ARMISD::VMOVRRD, dl,
4225 DAG.getVTList(MVT::i32, MVT::i32),
4226 DAG.getNode(ARMISD::VREV64, dl, SrcVT, Op));
4227 else
4228 Cvt = DAG.getNode(ARMISD::VMOVRRD, dl,
4229 DAG.getVTList(MVT::i32, MVT::i32), Op);
4230 // Merge the pieces into a single i64 value.
4231 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Cvt, Cvt.getValue(1));
4232 }
4233
4234 return SDValue();
4235 }
4236
4237 /// getZeroVector - Returns a vector of specified type with all zero elements.
4238 /// Zero vectors are used to represent vector negation and in those cases
4239 /// will be implemented with the NEON VNEG instruction. However, VNEG does
4240 /// not support i64 elements, so sometimes the zero vectors will need to be
4241 /// explicitly constructed. Regardless, use a canonical VMOV to create the
4242 /// zero vector.
getZeroVector(EVT VT,SelectionDAG & DAG,SDLoc dl)4243 static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, SDLoc dl) {
4244 assert(VT.isVector() && "Expected a vector type");
4245 // The canonical modified immediate encoding of a zero vector is....0!
4246 SDValue EncodedVal = DAG.getTargetConstant(0, dl, MVT::i32);
4247 EVT VmovVT = VT.is128BitVector() ? MVT::v4i32 : MVT::v2i32;
4248 SDValue Vmov = DAG.getNode(ARMISD::VMOVIMM, dl, VmovVT, EncodedVal);
4249 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov);
4250 }
4251
4252 /// LowerShiftRightParts - Lower SRA_PARTS, which returns two
4253 /// i32 values and take a 2 x i32 value to shift plus a shift amount.
LowerShiftRightParts(SDValue Op,SelectionDAG & DAG) const4254 SDValue ARMTargetLowering::LowerShiftRightParts(SDValue Op,
4255 SelectionDAG &DAG) const {
4256 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
4257 EVT VT = Op.getValueType();
4258 unsigned VTBits = VT.getSizeInBits();
4259 SDLoc dl(Op);
4260 SDValue ShOpLo = Op.getOperand(0);
4261 SDValue ShOpHi = Op.getOperand(1);
4262 SDValue ShAmt = Op.getOperand(2);
4263 SDValue ARMcc;
4264 unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL;
4265
4266 assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS);
4267
4268 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
4269 DAG.getConstant(VTBits, dl, MVT::i32), ShAmt);
4270 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt);
4271 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
4272 DAG.getConstant(VTBits, dl, MVT::i32));
4273 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt);
4274 SDValue FalseVal = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
4275 SDValue TrueVal = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
4276
4277 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
4278 SDValue Cmp = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
4279 ISD::SETGE, ARMcc, DAG, dl);
4280 SDValue Hi = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
4281 SDValue Lo = DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal, ARMcc,
4282 CCR, Cmp);
4283
4284 SDValue Ops[2] = { Lo, Hi };
4285 return DAG.getMergeValues(Ops, dl);
4286 }
4287
4288 /// LowerShiftLeftParts - Lower SHL_PARTS, which returns two
4289 /// i32 values and take a 2 x i32 value to shift plus a shift amount.
LowerShiftLeftParts(SDValue Op,SelectionDAG & DAG) const4290 SDValue ARMTargetLowering::LowerShiftLeftParts(SDValue Op,
4291 SelectionDAG &DAG) const {
4292 assert(Op.getNumOperands() == 3 && "Not a double-shift!");
4293 EVT VT = Op.getValueType();
4294 unsigned VTBits = VT.getSizeInBits();
4295 SDLoc dl(Op);
4296 SDValue ShOpLo = Op.getOperand(0);
4297 SDValue ShOpHi = Op.getOperand(1);
4298 SDValue ShAmt = Op.getOperand(2);
4299 SDValue ARMcc;
4300
4301 assert(Op.getOpcode() == ISD::SHL_PARTS);
4302 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
4303 DAG.getConstant(VTBits, dl, MVT::i32), ShAmt);
4304 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt);
4305 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
4306 DAG.getConstant(VTBits, dl, MVT::i32));
4307 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt);
4308 SDValue Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);
4309
4310 SDValue FalseVal = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
4311 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
4312 SDValue Cmp = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
4313 ISD::SETGE, ARMcc, DAG, dl);
4314 SDValue Lo = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
4315 SDValue Hi = DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, Tmp3, ARMcc,
4316 CCR, Cmp);
4317
4318 SDValue Ops[2] = { Lo, Hi };
4319 return DAG.getMergeValues(Ops, dl);
4320 }
4321
LowerFLT_ROUNDS_(SDValue Op,SelectionDAG & DAG) const4322 SDValue ARMTargetLowering::LowerFLT_ROUNDS_(SDValue Op,
4323 SelectionDAG &DAG) const {
4324 // The rounding mode is in bits 23:22 of the FPSCR.
4325 // The ARM rounding mode value to FLT_ROUNDS mapping is 0->1, 1->2, 2->3, 3->0
4326 // The formula we use to implement this is (((FPSCR + 1 << 22) >> 22) & 3)
4327 // so that the shift + and get folded into a bitfield extract.
4328 SDLoc dl(Op);
4329 SDValue FPSCR = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::i32,
4330 DAG.getConstant(Intrinsic::arm_get_fpscr, dl,
4331 MVT::i32));
4332 SDValue FltRounds = DAG.getNode(ISD::ADD, dl, MVT::i32, FPSCR,
4333 DAG.getConstant(1U << 22, dl, MVT::i32));
4334 SDValue RMODE = DAG.getNode(ISD::SRL, dl, MVT::i32, FltRounds,
4335 DAG.getConstant(22, dl, MVT::i32));
4336 return DAG.getNode(ISD::AND, dl, MVT::i32, RMODE,
4337 DAG.getConstant(3, dl, MVT::i32));
4338 }
4339
LowerCTTZ(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * ST)4340 static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG,
4341 const ARMSubtarget *ST) {
4342 SDLoc dl(N);
4343 EVT VT = N->getValueType(0);
4344 if (VT.isVector()) {
4345 assert(ST->hasNEON());
4346
4347 // Compute the least significant set bit: LSB = X & -X
4348 SDValue X = N->getOperand(0);
4349 SDValue NX = DAG.getNode(ISD::SUB, dl, VT, getZeroVector(VT, DAG, dl), X);
4350 SDValue LSB = DAG.getNode(ISD::AND, dl, VT, X, NX);
4351
4352 EVT ElemTy = VT.getVectorElementType();
4353
4354 if (ElemTy == MVT::i8) {
4355 // Compute with: cttz(x) = ctpop(lsb - 1)
4356 SDValue One = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
4357 DAG.getTargetConstant(1, dl, ElemTy));
4358 SDValue Bits = DAG.getNode(ISD::SUB, dl, VT, LSB, One);
4359 return DAG.getNode(ISD::CTPOP, dl, VT, Bits);
4360 }
4361
4362 if ((ElemTy == MVT::i16 || ElemTy == MVT::i32) &&
4363 (N->getOpcode() == ISD::CTTZ_ZERO_UNDEF)) {
4364 // Compute with: cttz(x) = (width - 1) - ctlz(lsb), if x != 0
4365 unsigned NumBits = ElemTy.getSizeInBits();
4366 SDValue WidthMinus1 =
4367 DAG.getNode(ARMISD::VMOVIMM, dl, VT,
4368 DAG.getTargetConstant(NumBits - 1, dl, ElemTy));
4369 SDValue CTLZ = DAG.getNode(ISD::CTLZ, dl, VT, LSB);
4370 return DAG.getNode(ISD::SUB, dl, VT, WidthMinus1, CTLZ);
4371 }
4372
4373 // Compute with: cttz(x) = ctpop(lsb - 1)
4374
4375 // Since we can only compute the number of bits in a byte with vcnt.8, we
4376 // have to gather the result with pairwise addition (vpaddl) for i16, i32,
4377 // and i64.
4378
4379 // Compute LSB - 1.
4380 SDValue Bits;
4381 if (ElemTy == MVT::i64) {
4382 // Load constant 0xffff'ffff'ffff'ffff to register.
4383 SDValue FF = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
4384 DAG.getTargetConstant(0x1eff, dl, MVT::i32));
4385 Bits = DAG.getNode(ISD::ADD, dl, VT, LSB, FF);
4386 } else {
4387 SDValue One = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
4388 DAG.getTargetConstant(1, dl, ElemTy));
4389 Bits = DAG.getNode(ISD::SUB, dl, VT, LSB, One);
4390 }
4391
4392 // Count #bits with vcnt.8.
4393 EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8;
4394 SDValue BitsVT8 = DAG.getNode(ISD::BITCAST, dl, VT8Bit, Bits);
4395 SDValue Cnt8 = DAG.getNode(ISD::CTPOP, dl, VT8Bit, BitsVT8);
4396
4397 // Gather the #bits with vpaddl (pairwise add.)
4398 EVT VT16Bit = VT.is64BitVector() ? MVT::v4i16 : MVT::v8i16;
4399 SDValue Cnt16 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT16Bit,
4400 DAG.getTargetConstant(Intrinsic::arm_neon_vpaddlu, dl, MVT::i32),
4401 Cnt8);
4402 if (ElemTy == MVT::i16)
4403 return Cnt16;
4404
4405 EVT VT32Bit = VT.is64BitVector() ? MVT::v2i32 : MVT::v4i32;
4406 SDValue Cnt32 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT32Bit,
4407 DAG.getTargetConstant(Intrinsic::arm_neon_vpaddlu, dl, MVT::i32),
4408 Cnt16);
4409 if (ElemTy == MVT::i32)
4410 return Cnt32;
4411
4412 assert(ElemTy == MVT::i64);
4413 SDValue Cnt64 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
4414 DAG.getTargetConstant(Intrinsic::arm_neon_vpaddlu, dl, MVT::i32),
4415 Cnt32);
4416 return Cnt64;
4417 }
4418
4419 if (!ST->hasV6T2Ops())
4420 return SDValue();
4421
4422 SDValue rbit = DAG.getNode(ISD::BITREVERSE, dl, VT, N->getOperand(0));
4423 return DAG.getNode(ISD::CTLZ, dl, VT, rbit);
4424 }
4425
4426 /// getCTPOP16BitCounts - Returns a v8i8/v16i8 vector containing the bit-count
4427 /// for each 16-bit element from operand, repeated. The basic idea is to
4428 /// leverage vcnt to get the 8-bit counts, gather and add the results.
4429 ///
4430 /// Trace for v4i16:
4431 /// input = [v0 v1 v2 v3 ] (vi 16-bit element)
4432 /// cast: N0 = [w0 w1 w2 w3 w4 w5 w6 w7] (v0 = [w0 w1], wi 8-bit element)
4433 /// vcnt: N1 = [b0 b1 b2 b3 b4 b5 b6 b7] (bi = bit-count of 8-bit element wi)
4434 /// vrev: N2 = [b1 b0 b3 b2 b5 b4 b7 b6]
4435 /// [b0 b1 b2 b3 b4 b5 b6 b7]
4436 /// +[b1 b0 b3 b2 b5 b4 b7 b6]
4437 /// N3=N1+N2 = [k0 k0 k1 k1 k2 k2 k3 k3] (k0 = b0+b1 = bit-count of 16-bit v0,
4438 /// vuzp: = [k0 k1 k2 k3 k0 k1 k2 k3] each ki is 8-bits)
getCTPOP16BitCounts(SDNode * N,SelectionDAG & DAG)4439 static SDValue getCTPOP16BitCounts(SDNode *N, SelectionDAG &DAG) {
4440 EVT VT = N->getValueType(0);
4441 SDLoc DL(N);
4442
4443 EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8;
4444 SDValue N0 = DAG.getNode(ISD::BITCAST, DL, VT8Bit, N->getOperand(0));
4445 SDValue N1 = DAG.getNode(ISD::CTPOP, DL, VT8Bit, N0);
4446 SDValue N2 = DAG.getNode(ARMISD::VREV16, DL, VT8Bit, N1);
4447 SDValue N3 = DAG.getNode(ISD::ADD, DL, VT8Bit, N1, N2);
4448 return DAG.getNode(ARMISD::VUZP, DL, VT8Bit, N3, N3);
4449 }
4450
4451 /// lowerCTPOP16BitElements - Returns a v4i16/v8i16 vector containing the
4452 /// bit-count for each 16-bit element from the operand. We need slightly
4453 /// different sequencing for v4i16 and v8i16 to stay within NEON's available
4454 /// 64/128-bit registers.
4455 ///
4456 /// Trace for v4i16:
4457 /// input = [v0 v1 v2 v3 ] (vi 16-bit element)
4458 /// v8i8: BitCounts = [k0 k1 k2 k3 k0 k1 k2 k3 ] (ki is the bit-count of vi)
4459 /// v8i16:Extended = [k0 k1 k2 k3 k0 k1 k2 k3 ]
4460 /// v4i16:Extracted = [k0 k1 k2 k3 ]
lowerCTPOP16BitElements(SDNode * N,SelectionDAG & DAG)4461 static SDValue lowerCTPOP16BitElements(SDNode *N, SelectionDAG &DAG) {
4462 EVT VT = N->getValueType(0);
4463 SDLoc DL(N);
4464
4465 SDValue BitCounts = getCTPOP16BitCounts(N, DAG);
4466 if (VT.is64BitVector()) {
4467 SDValue Extended = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v8i16, BitCounts);
4468 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i16, Extended,
4469 DAG.getIntPtrConstant(0, DL));
4470 } else {
4471 SDValue Extracted = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i8,
4472 BitCounts, DAG.getIntPtrConstant(0, DL));
4473 return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v8i16, Extracted);
4474 }
4475 }
4476
4477 /// lowerCTPOP32BitElements - Returns a v2i32/v4i32 vector containing the
4478 /// bit-count for each 32-bit element from the operand. The idea here is
4479 /// to split the vector into 16-bit elements, leverage the 16-bit count
4480 /// routine, and then combine the results.
4481 ///
4482 /// Trace for v2i32 (v4i32 similar with Extracted/Extended exchanged):
4483 /// input = [v0 v1 ] (vi: 32-bit elements)
4484 /// Bitcast = [w0 w1 w2 w3 ] (wi: 16-bit elements, v0 = [w0 w1])
4485 /// Counts16 = [k0 k1 k2 k3 ] (ki: 16-bit elements, bit-count of wi)
4486 /// vrev: N0 = [k1 k0 k3 k2 ]
4487 /// [k0 k1 k2 k3 ]
4488 /// N1 =+[k1 k0 k3 k2 ]
4489 /// [k0 k2 k1 k3 ]
4490 /// N2 =+[k1 k3 k0 k2 ]
4491 /// [k0 k2 k1 k3 ]
4492 /// Extended =+[k1 k3 k0 k2 ]
4493 /// [k0 k2 ]
4494 /// Extracted=+[k1 k3 ]
4495 ///
lowerCTPOP32BitElements(SDNode * N,SelectionDAG & DAG)4496 static SDValue lowerCTPOP32BitElements(SDNode *N, SelectionDAG &DAG) {
4497 EVT VT = N->getValueType(0);
4498 SDLoc DL(N);
4499
4500 EVT VT16Bit = VT.is64BitVector() ? MVT::v4i16 : MVT::v8i16;
4501
4502 SDValue Bitcast = DAG.getNode(ISD::BITCAST, DL, VT16Bit, N->getOperand(0));
4503 SDValue Counts16 = lowerCTPOP16BitElements(Bitcast.getNode(), DAG);
4504 SDValue N0 = DAG.getNode(ARMISD::VREV32, DL, VT16Bit, Counts16);
4505 SDValue N1 = DAG.getNode(ISD::ADD, DL, VT16Bit, Counts16, N0);
4506 SDValue N2 = DAG.getNode(ARMISD::VUZP, DL, VT16Bit, N1, N1);
4507
4508 if (VT.is64BitVector()) {
4509 SDValue Extended = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v4i32, N2);
4510 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i32, Extended,
4511 DAG.getIntPtrConstant(0, DL));
4512 } else {
4513 SDValue Extracted = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i16, N2,
4514 DAG.getIntPtrConstant(0, DL));
4515 return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v4i32, Extracted);
4516 }
4517 }
4518
LowerCTPOP(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * ST)4519 static SDValue LowerCTPOP(SDNode *N, SelectionDAG &DAG,
4520 const ARMSubtarget *ST) {
4521 EVT VT = N->getValueType(0);
4522
4523 assert(ST->hasNEON() && "Custom ctpop lowering requires NEON.");
4524 assert((VT == MVT::v2i32 || VT == MVT::v4i32 ||
4525 VT == MVT::v4i16 || VT == MVT::v8i16) &&
4526 "Unexpected type for custom ctpop lowering");
4527
4528 if (VT.getVectorElementType() == MVT::i32)
4529 return lowerCTPOP32BitElements(N, DAG);
4530 else
4531 return lowerCTPOP16BitElements(N, DAG);
4532 }
4533
LowerShift(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * ST)4534 static SDValue LowerShift(SDNode *N, SelectionDAG &DAG,
4535 const ARMSubtarget *ST) {
4536 EVT VT = N->getValueType(0);
4537 SDLoc dl(N);
4538
4539 if (!VT.isVector())
4540 return SDValue();
4541
4542 // Lower vector shifts on NEON to use VSHL.
4543 assert(ST->hasNEON() && "unexpected vector shift");
4544
4545 // Left shifts translate directly to the vshiftu intrinsic.
4546 if (N->getOpcode() == ISD::SHL)
4547 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
4548 DAG.getConstant(Intrinsic::arm_neon_vshiftu, dl,
4549 MVT::i32),
4550 N->getOperand(0), N->getOperand(1));
4551
4552 assert((N->getOpcode() == ISD::SRA ||
4553 N->getOpcode() == ISD::SRL) && "unexpected vector shift opcode");
4554
4555 // NEON uses the same intrinsics for both left and right shifts. For
4556 // right shifts, the shift amounts are negative, so negate the vector of
4557 // shift amounts.
4558 EVT ShiftVT = N->getOperand(1).getValueType();
4559 SDValue NegatedCount = DAG.getNode(ISD::SUB, dl, ShiftVT,
4560 getZeroVector(ShiftVT, DAG, dl),
4561 N->getOperand(1));
4562 Intrinsic::ID vshiftInt = (N->getOpcode() == ISD::SRA ?
4563 Intrinsic::arm_neon_vshifts :
4564 Intrinsic::arm_neon_vshiftu);
4565 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
4566 DAG.getConstant(vshiftInt, dl, MVT::i32),
4567 N->getOperand(0), NegatedCount);
4568 }
4569
Expand64BitShift(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * ST)4570 static SDValue Expand64BitShift(SDNode *N, SelectionDAG &DAG,
4571 const ARMSubtarget *ST) {
4572 EVT VT = N->getValueType(0);
4573 SDLoc dl(N);
4574
4575 // We can get here for a node like i32 = ISD::SHL i32, i64
4576 if (VT != MVT::i64)
4577 return SDValue();
4578
4579 assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) &&
4580 "Unknown shift to lower!");
4581
4582 // We only lower SRA, SRL of 1 here, all others use generic lowering.
4583 if (!isOneConstant(N->getOperand(1)))
4584 return SDValue();
4585
4586 // If we are in thumb mode, we don't have RRX.
4587 if (ST->isThumb1Only()) return SDValue();
4588
4589 // Okay, we have a 64-bit SRA or SRL of 1. Lower this to an RRX expr.
4590 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0),
4591 DAG.getConstant(0, dl, MVT::i32));
4592 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0),
4593 DAG.getConstant(1, dl, MVT::i32));
4594
4595 // First, build a SRA_FLAG/SRL_FLAG op, which shifts the top part by one and
4596 // captures the result into a carry flag.
4597 unsigned Opc = N->getOpcode() == ISD::SRL ? ARMISD::SRL_FLAG:ARMISD::SRA_FLAG;
4598 Hi = DAG.getNode(Opc, dl, DAG.getVTList(MVT::i32, MVT::Glue), Hi);
4599
4600 // The low part is an ARMISD::RRX operand, which shifts the carry in.
4601 Lo = DAG.getNode(ARMISD::RRX, dl, MVT::i32, Lo, Hi.getValue(1));
4602
4603 // Merge the pieces into a single i64 value.
4604 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
4605 }
4606
LowerVSETCC(SDValue Op,SelectionDAG & DAG)4607 static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG) {
4608 SDValue TmpOp0, TmpOp1;
4609 bool Invert = false;
4610 bool Swap = false;
4611 unsigned Opc = 0;
4612
4613 SDValue Op0 = Op.getOperand(0);
4614 SDValue Op1 = Op.getOperand(1);
4615 SDValue CC = Op.getOperand(2);
4616 EVT CmpVT = Op0.getValueType().changeVectorElementTypeToInteger();
4617 EVT VT = Op.getValueType();
4618 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
4619 SDLoc dl(Op);
4620
4621 if (CmpVT.getVectorElementType() == MVT::i64)
4622 // 64-bit comparisons are not legal. We've marked SETCC as non-Custom,
4623 // but it's possible that our operands are 64-bit but our result is 32-bit.
4624 // Bail in this case.
4625 return SDValue();
4626
4627 if (Op1.getValueType().isFloatingPoint()) {
4628 switch (SetCCOpcode) {
4629 default: llvm_unreachable("Illegal FP comparison");
4630 case ISD::SETUNE:
4631 case ISD::SETNE: Invert = true; // Fallthrough
4632 case ISD::SETOEQ:
4633 case ISD::SETEQ: Opc = ARMISD::VCEQ; break;
4634 case ISD::SETOLT:
4635 case ISD::SETLT: Swap = true; // Fallthrough
4636 case ISD::SETOGT:
4637 case ISD::SETGT: Opc = ARMISD::VCGT; break;
4638 case ISD::SETOLE:
4639 case ISD::SETLE: Swap = true; // Fallthrough
4640 case ISD::SETOGE:
4641 case ISD::SETGE: Opc = ARMISD::VCGE; break;
4642 case ISD::SETUGE: Swap = true; // Fallthrough
4643 case ISD::SETULE: Invert = true; Opc = ARMISD::VCGT; break;
4644 case ISD::SETUGT: Swap = true; // Fallthrough
4645 case ISD::SETULT: Invert = true; Opc = ARMISD::VCGE; break;
4646 case ISD::SETUEQ: Invert = true; // Fallthrough
4647 case ISD::SETONE:
4648 // Expand this to (OLT | OGT).
4649 TmpOp0 = Op0;
4650 TmpOp1 = Op1;
4651 Opc = ISD::OR;
4652 Op0 = DAG.getNode(ARMISD::VCGT, dl, CmpVT, TmpOp1, TmpOp0);
4653 Op1 = DAG.getNode(ARMISD::VCGT, dl, CmpVT, TmpOp0, TmpOp1);
4654 break;
4655 case ISD::SETUO: Invert = true; // Fallthrough
4656 case ISD::SETO:
4657 // Expand this to (OLT | OGE).
4658 TmpOp0 = Op0;
4659 TmpOp1 = Op1;
4660 Opc = ISD::OR;
4661 Op0 = DAG.getNode(ARMISD::VCGT, dl, CmpVT, TmpOp1, TmpOp0);
4662 Op1 = DAG.getNode(ARMISD::VCGE, dl, CmpVT, TmpOp0, TmpOp1);
4663 break;
4664 }
4665 } else {
4666 // Integer comparisons.
4667 switch (SetCCOpcode) {
4668 default: llvm_unreachable("Illegal integer comparison");
4669 case ISD::SETNE: Invert = true;
4670 case ISD::SETEQ: Opc = ARMISD::VCEQ; break;
4671 case ISD::SETLT: Swap = true;
4672 case ISD::SETGT: Opc = ARMISD::VCGT; break;
4673 case ISD::SETLE: Swap = true;
4674 case ISD::SETGE: Opc = ARMISD::VCGE; break;
4675 case ISD::SETULT: Swap = true;
4676 case ISD::SETUGT: Opc = ARMISD::VCGTU; break;
4677 case ISD::SETULE: Swap = true;
4678 case ISD::SETUGE: Opc = ARMISD::VCGEU; break;
4679 }
4680
4681 // Detect VTST (Vector Test Bits) = icmp ne (and (op0, op1), zero).
4682 if (Opc == ARMISD::VCEQ) {
4683
4684 SDValue AndOp;
4685 if (ISD::isBuildVectorAllZeros(Op1.getNode()))
4686 AndOp = Op0;
4687 else if (ISD::isBuildVectorAllZeros(Op0.getNode()))
4688 AndOp = Op1;
4689
4690 // Ignore bitconvert.
4691 if (AndOp.getNode() && AndOp.getOpcode() == ISD::BITCAST)
4692 AndOp = AndOp.getOperand(0);
4693
4694 if (AndOp.getNode() && AndOp.getOpcode() == ISD::AND) {
4695 Opc = ARMISD::VTST;
4696 Op0 = DAG.getNode(ISD::BITCAST, dl, CmpVT, AndOp.getOperand(0));
4697 Op1 = DAG.getNode(ISD::BITCAST, dl, CmpVT, AndOp.getOperand(1));
4698 Invert = !Invert;
4699 }
4700 }
4701 }
4702
4703 if (Swap)
4704 std::swap(Op0, Op1);
4705
4706 // If one of the operands is a constant vector zero, attempt to fold the
4707 // comparison to a specialized compare-against-zero form.
4708 SDValue SingleOp;
4709 if (ISD::isBuildVectorAllZeros(Op1.getNode()))
4710 SingleOp = Op0;
4711 else if (ISD::isBuildVectorAllZeros(Op0.getNode())) {
4712 if (Opc == ARMISD::VCGE)
4713 Opc = ARMISD::VCLEZ;
4714 else if (Opc == ARMISD::VCGT)
4715 Opc = ARMISD::VCLTZ;
4716 SingleOp = Op1;
4717 }
4718
4719 SDValue Result;
4720 if (SingleOp.getNode()) {
4721 switch (Opc) {
4722 case ARMISD::VCEQ:
4723 Result = DAG.getNode(ARMISD::VCEQZ, dl, CmpVT, SingleOp); break;
4724 case ARMISD::VCGE:
4725 Result = DAG.getNode(ARMISD::VCGEZ, dl, CmpVT, SingleOp); break;
4726 case ARMISD::VCLEZ:
4727 Result = DAG.getNode(ARMISD::VCLEZ, dl, CmpVT, SingleOp); break;
4728 case ARMISD::VCGT:
4729 Result = DAG.getNode(ARMISD::VCGTZ, dl, CmpVT, SingleOp); break;
4730 case ARMISD::VCLTZ:
4731 Result = DAG.getNode(ARMISD::VCLTZ, dl, CmpVT, SingleOp); break;
4732 default:
4733 Result = DAG.getNode(Opc, dl, CmpVT, Op0, Op1);
4734 }
4735 } else {
4736 Result = DAG.getNode(Opc, dl, CmpVT, Op0, Op1);
4737 }
4738
4739 Result = DAG.getSExtOrTrunc(Result, dl, VT);
4740
4741 if (Invert)
4742 Result = DAG.getNOT(dl, Result, VT);
4743
4744 return Result;
4745 }
4746
4747 /// isNEONModifiedImm - Check if the specified splat value corresponds to a
4748 /// valid vector constant for a NEON instruction with a "modified immediate"
4749 /// operand (e.g., VMOV). If so, return the encoded value.
isNEONModifiedImm(uint64_t SplatBits,uint64_t SplatUndef,unsigned SplatBitSize,SelectionDAG & DAG,SDLoc dl,EVT & VT,bool is128Bits,NEONModImmType type)4750 static SDValue isNEONModifiedImm(uint64_t SplatBits, uint64_t SplatUndef,
4751 unsigned SplatBitSize, SelectionDAG &DAG,
4752 SDLoc dl, EVT &VT, bool is128Bits,
4753 NEONModImmType type) {
4754 unsigned OpCmode, Imm;
4755
4756 // SplatBitSize is set to the smallest size that splats the vector, so a
4757 // zero vector will always have SplatBitSize == 8. However, NEON modified
4758 // immediate instructions others than VMOV do not support the 8-bit encoding
4759 // of a zero vector, and the default encoding of zero is supposed to be the
4760 // 32-bit version.
4761 if (SplatBits == 0)
4762 SplatBitSize = 32;
4763
4764 switch (SplatBitSize) {
4765 case 8:
4766 if (type != VMOVModImm)
4767 return SDValue();
4768 // Any 1-byte value is OK. Op=0, Cmode=1110.
4769 assert((SplatBits & ~0xff) == 0 && "one byte splat value is too big");
4770 OpCmode = 0xe;
4771 Imm = SplatBits;
4772 VT = is128Bits ? MVT::v16i8 : MVT::v8i8;
4773 break;
4774
4775 case 16:
4776 // NEON's 16-bit VMOV supports splat values where only one byte is nonzero.
4777 VT = is128Bits ? MVT::v8i16 : MVT::v4i16;
4778 if ((SplatBits & ~0xff) == 0) {
4779 // Value = 0x00nn: Op=x, Cmode=100x.
4780 OpCmode = 0x8;
4781 Imm = SplatBits;
4782 break;
4783 }
4784 if ((SplatBits & ~0xff00) == 0) {
4785 // Value = 0xnn00: Op=x, Cmode=101x.
4786 OpCmode = 0xa;
4787 Imm = SplatBits >> 8;
4788 break;
4789 }
4790 return SDValue();
4791
4792 case 32:
4793 // NEON's 32-bit VMOV supports splat values where:
4794 // * only one byte is nonzero, or
4795 // * the least significant byte is 0xff and the second byte is nonzero, or
4796 // * the least significant 2 bytes are 0xff and the third is nonzero.
4797 VT = is128Bits ? MVT::v4i32 : MVT::v2i32;
4798 if ((SplatBits & ~0xff) == 0) {
4799 // Value = 0x000000nn: Op=x, Cmode=000x.
4800 OpCmode = 0;
4801 Imm = SplatBits;
4802 break;
4803 }
4804 if ((SplatBits & ~0xff00) == 0) {
4805 // Value = 0x0000nn00: Op=x, Cmode=001x.
4806 OpCmode = 0x2;
4807 Imm = SplatBits >> 8;
4808 break;
4809 }
4810 if ((SplatBits & ~0xff0000) == 0) {
4811 // Value = 0x00nn0000: Op=x, Cmode=010x.
4812 OpCmode = 0x4;
4813 Imm = SplatBits >> 16;
4814 break;
4815 }
4816 if ((SplatBits & ~0xff000000) == 0) {
4817 // Value = 0xnn000000: Op=x, Cmode=011x.
4818 OpCmode = 0x6;
4819 Imm = SplatBits >> 24;
4820 break;
4821 }
4822
4823 // cmode == 0b1100 and cmode == 0b1101 are not supported for VORR or VBIC
4824 if (type == OtherModImm) return SDValue();
4825
4826 if ((SplatBits & ~0xffff) == 0 &&
4827 ((SplatBits | SplatUndef) & 0xff) == 0xff) {
4828 // Value = 0x0000nnff: Op=x, Cmode=1100.
4829 OpCmode = 0xc;
4830 Imm = SplatBits >> 8;
4831 break;
4832 }
4833
4834 if ((SplatBits & ~0xffffff) == 0 &&
4835 ((SplatBits | SplatUndef) & 0xffff) == 0xffff) {
4836 // Value = 0x00nnffff: Op=x, Cmode=1101.
4837 OpCmode = 0xd;
4838 Imm = SplatBits >> 16;
4839 break;
4840 }
4841
4842 // Note: there are a few 32-bit splat values (specifically: 00ffff00,
4843 // ff000000, ff0000ff, and ffff00ff) that are valid for VMOV.I64 but not
4844 // VMOV.I32. A (very) minor optimization would be to replicate the value
4845 // and fall through here to test for a valid 64-bit splat. But, then the
4846 // caller would also need to check and handle the change in size.
4847 return SDValue();
4848
4849 case 64: {
4850 if (type != VMOVModImm)
4851 return SDValue();
4852 // NEON has a 64-bit VMOV splat where each byte is either 0 or 0xff.
4853 uint64_t BitMask = 0xff;
4854 uint64_t Val = 0;
4855 unsigned ImmMask = 1;
4856 Imm = 0;
4857 for (int ByteNum = 0; ByteNum < 8; ++ByteNum) {
4858 if (((SplatBits | SplatUndef) & BitMask) == BitMask) {
4859 Val |= BitMask;
4860 Imm |= ImmMask;
4861 } else if ((SplatBits & BitMask) != 0) {
4862 return SDValue();
4863 }
4864 BitMask <<= 8;
4865 ImmMask <<= 1;
4866 }
4867
4868 if (DAG.getDataLayout().isBigEndian())
4869 // swap higher and lower 32 bit word
4870 Imm = ((Imm & 0xf) << 4) | ((Imm & 0xf0) >> 4);
4871
4872 // Op=1, Cmode=1110.
4873 OpCmode = 0x1e;
4874 VT = is128Bits ? MVT::v2i64 : MVT::v1i64;
4875 break;
4876 }
4877
4878 default:
4879 llvm_unreachable("unexpected size for isNEONModifiedImm");
4880 }
4881
4882 unsigned EncodedVal = ARM_AM::createNEONModImm(OpCmode, Imm);
4883 return DAG.getTargetConstant(EncodedVal, dl, MVT::i32);
4884 }
4885
LowerConstantFP(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * ST) const4886 SDValue ARMTargetLowering::LowerConstantFP(SDValue Op, SelectionDAG &DAG,
4887 const ARMSubtarget *ST) const {
4888 if (!ST->hasVFP3())
4889 return SDValue();
4890
4891 bool IsDouble = Op.getValueType() == MVT::f64;
4892 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Op);
4893
4894 // Use the default (constant pool) lowering for double constants when we have
4895 // an SP-only FPU
4896 if (IsDouble && Subtarget->isFPOnlySP())
4897 return SDValue();
4898
4899 // Try splatting with a VMOV.f32...
4900 APFloat FPVal = CFP->getValueAPF();
4901 int ImmVal = IsDouble ? ARM_AM::getFP64Imm(FPVal) : ARM_AM::getFP32Imm(FPVal);
4902
4903 if (ImmVal != -1) {
4904 if (IsDouble || !ST->useNEONForSinglePrecisionFP()) {
4905 // We have code in place to select a valid ConstantFP already, no need to
4906 // do any mangling.
4907 return Op;
4908 }
4909
4910 // It's a float and we are trying to use NEON operations where
4911 // possible. Lower it to a splat followed by an extract.
4912 SDLoc DL(Op);
4913 SDValue NewVal = DAG.getTargetConstant(ImmVal, DL, MVT::i32);
4914 SDValue VecConstant = DAG.getNode(ARMISD::VMOVFPIMM, DL, MVT::v2f32,
4915 NewVal);
4916 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecConstant,
4917 DAG.getConstant(0, DL, MVT::i32));
4918 }
4919
4920 // The rest of our options are NEON only, make sure that's allowed before
4921 // proceeding..
4922 if (!ST->hasNEON() || (!IsDouble && !ST->useNEONForSinglePrecisionFP()))
4923 return SDValue();
4924
4925 EVT VMovVT;
4926 uint64_t iVal = FPVal.bitcastToAPInt().getZExtValue();
4927
4928 // It wouldn't really be worth bothering for doubles except for one very
4929 // important value, which does happen to match: 0.0. So make sure we don't do
4930 // anything stupid.
4931 if (IsDouble && (iVal & 0xffffffff) != (iVal >> 32))
4932 return SDValue();
4933
4934 // Try a VMOV.i32 (FIXME: i8, i16, or i64 could work too).
4935 SDValue NewVal = isNEONModifiedImm(iVal & 0xffffffffU, 0, 32, DAG, SDLoc(Op),
4936 VMovVT, false, VMOVModImm);
4937 if (NewVal != SDValue()) {
4938 SDLoc DL(Op);
4939 SDValue VecConstant = DAG.getNode(ARMISD::VMOVIMM, DL, VMovVT,
4940 NewVal);
4941 if (IsDouble)
4942 return DAG.getNode(ISD::BITCAST, DL, MVT::f64, VecConstant);
4943
4944 // It's a float: cast and extract a vector element.
4945 SDValue VecFConstant = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32,
4946 VecConstant);
4947 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecFConstant,
4948 DAG.getConstant(0, DL, MVT::i32));
4949 }
4950
4951 // Finally, try a VMVN.i32
4952 NewVal = isNEONModifiedImm(~iVal & 0xffffffffU, 0, 32, DAG, SDLoc(Op), VMovVT,
4953 false, VMVNModImm);
4954 if (NewVal != SDValue()) {
4955 SDLoc DL(Op);
4956 SDValue VecConstant = DAG.getNode(ARMISD::VMVNIMM, DL, VMovVT, NewVal);
4957
4958 if (IsDouble)
4959 return DAG.getNode(ISD::BITCAST, DL, MVT::f64, VecConstant);
4960
4961 // It's a float: cast and extract a vector element.
4962 SDValue VecFConstant = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32,
4963 VecConstant);
4964 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecFConstant,
4965 DAG.getConstant(0, DL, MVT::i32));
4966 }
4967
4968 return SDValue();
4969 }
4970
4971 // check if an VEXT instruction can handle the shuffle mask when the
4972 // vector sources of the shuffle are the same.
isSingletonVEXTMask(ArrayRef<int> M,EVT VT,unsigned & Imm)4973 static bool isSingletonVEXTMask(ArrayRef<int> M, EVT VT, unsigned &Imm) {
4974 unsigned NumElts = VT.getVectorNumElements();
4975
4976 // Assume that the first shuffle index is not UNDEF. Fail if it is.
4977 if (M[0] < 0)
4978 return false;
4979
4980 Imm = M[0];
4981
4982 // If this is a VEXT shuffle, the immediate value is the index of the first
4983 // element. The other shuffle indices must be the successive elements after
4984 // the first one.
4985 unsigned ExpectedElt = Imm;
4986 for (unsigned i = 1; i < NumElts; ++i) {
4987 // Increment the expected index. If it wraps around, just follow it
4988 // back to index zero and keep going.
4989 ++ExpectedElt;
4990 if (ExpectedElt == NumElts)
4991 ExpectedElt = 0;
4992
4993 if (M[i] < 0) continue; // ignore UNDEF indices
4994 if (ExpectedElt != static_cast<unsigned>(M[i]))
4995 return false;
4996 }
4997
4998 return true;
4999 }
5000
5001
isVEXTMask(ArrayRef<int> M,EVT VT,bool & ReverseVEXT,unsigned & Imm)5002 static bool isVEXTMask(ArrayRef<int> M, EVT VT,
5003 bool &ReverseVEXT, unsigned &Imm) {
5004 unsigned NumElts = VT.getVectorNumElements();
5005 ReverseVEXT = false;
5006
5007 // Assume that the first shuffle index is not UNDEF. Fail if it is.
5008 if (M[0] < 0)
5009 return false;
5010
5011 Imm = M[0];
5012
5013 // If this is a VEXT shuffle, the immediate value is the index of the first
5014 // element. The other shuffle indices must be the successive elements after
5015 // the first one.
5016 unsigned ExpectedElt = Imm;
5017 for (unsigned i = 1; i < NumElts; ++i) {
5018 // Increment the expected index. If it wraps around, it may still be
5019 // a VEXT but the source vectors must be swapped.
5020 ExpectedElt += 1;
5021 if (ExpectedElt == NumElts * 2) {
5022 ExpectedElt = 0;
5023 ReverseVEXT = true;
5024 }
5025
5026 if (M[i] < 0) continue; // ignore UNDEF indices
5027 if (ExpectedElt != static_cast<unsigned>(M[i]))
5028 return false;
5029 }
5030
5031 // Adjust the index value if the source operands will be swapped.
5032 if (ReverseVEXT)
5033 Imm -= NumElts;
5034
5035 return true;
5036 }
5037
5038 /// isVREVMask - Check if a vector shuffle corresponds to a VREV
5039 /// instruction with the specified blocksize. (The order of the elements
5040 /// within each block of the vector is reversed.)
isVREVMask(ArrayRef<int> M,EVT VT,unsigned BlockSize)5041 static bool isVREVMask(ArrayRef<int> M, EVT VT, unsigned BlockSize) {
5042 assert((BlockSize==16 || BlockSize==32 || BlockSize==64) &&
5043 "Only possible block sizes for VREV are: 16, 32, 64");
5044
5045 unsigned EltSz = VT.getVectorElementType().getSizeInBits();
5046 if (EltSz == 64)
5047 return false;
5048
5049 unsigned NumElts = VT.getVectorNumElements();
5050 unsigned BlockElts = M[0] + 1;
5051 // If the first shuffle index is UNDEF, be optimistic.
5052 if (M[0] < 0)
5053 BlockElts = BlockSize / EltSz;
5054
5055 if (BlockSize <= EltSz || BlockSize != BlockElts * EltSz)
5056 return false;
5057
5058 for (unsigned i = 0; i < NumElts; ++i) {
5059 if (M[i] < 0) continue; // ignore UNDEF indices
5060 if ((unsigned) M[i] != (i - i%BlockElts) + (BlockElts - 1 - i%BlockElts))
5061 return false;
5062 }
5063
5064 return true;
5065 }
5066
isVTBLMask(ArrayRef<int> M,EVT VT)5067 static bool isVTBLMask(ArrayRef<int> M, EVT VT) {
5068 // We can handle <8 x i8> vector shuffles. If the index in the mask is out of
5069 // range, then 0 is placed into the resulting vector. So pretty much any mask
5070 // of 8 elements can work here.
5071 return VT == MVT::v8i8 && M.size() == 8;
5072 }
5073
5074 // Checks whether the shuffle mask represents a vector transpose (VTRN) by
5075 // checking that pairs of elements in the shuffle mask represent the same index
5076 // in each vector, incrementing the expected index by 2 at each step.
5077 // e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 4, 2, 6]
5078 // v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,e,c,g}
5079 // v2={e,f,g,h}
5080 // WhichResult gives the offset for each element in the mask based on which
5081 // of the two results it belongs to.
5082 //
5083 // The transpose can be represented either as:
5084 // result1 = shufflevector v1, v2, result1_shuffle_mask
5085 // result2 = shufflevector v1, v2, result2_shuffle_mask
5086 // where v1/v2 and the shuffle masks have the same number of elements
5087 // (here WhichResult (see below) indicates which result is being checked)
5088 //
5089 // or as:
5090 // results = shufflevector v1, v2, shuffle_mask
5091 // where both results are returned in one vector and the shuffle mask has twice
5092 // as many elements as v1/v2 (here WhichResult will always be 0 if true) here we
5093 // want to check the low half and high half of the shuffle mask as if it were
5094 // the other case
isVTRNMask(ArrayRef<int> M,EVT VT,unsigned & WhichResult)5095 static bool isVTRNMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
5096 unsigned EltSz = VT.getVectorElementType().getSizeInBits();
5097 if (EltSz == 64)
5098 return false;
5099
5100 unsigned NumElts = VT.getVectorNumElements();
5101 if (M.size() != NumElts && M.size() != NumElts*2)
5102 return false;
5103
5104 // If the mask is twice as long as the input vector then we need to check the
5105 // upper and lower parts of the mask with a matching value for WhichResult
5106 // FIXME: A mask with only even values will be rejected in case the first
5107 // element is undefined, e.g. [-1, 4, 2, 6] will be rejected, because only
5108 // M[0] is used to determine WhichResult
5109 for (unsigned i = 0; i < M.size(); i += NumElts) {
5110 if (M.size() == NumElts * 2)
5111 WhichResult = i / NumElts;
5112 else
5113 WhichResult = M[i] == 0 ? 0 : 1;
5114 for (unsigned j = 0; j < NumElts; j += 2) {
5115 if ((M[i+j] >= 0 && (unsigned) M[i+j] != j + WhichResult) ||
5116 (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != j + NumElts + WhichResult))
5117 return false;
5118 }
5119 }
5120
5121 if (M.size() == NumElts*2)
5122 WhichResult = 0;
5123
5124 return true;
5125 }
5126
5127 /// isVTRN_v_undef_Mask - Special case of isVTRNMask for canonical form of
5128 /// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
5129 /// Mask is e.g., <0, 0, 2, 2> instead of <0, 4, 2, 6>.
isVTRN_v_undef_Mask(ArrayRef<int> M,EVT VT,unsigned & WhichResult)5130 static bool isVTRN_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){
5131 unsigned EltSz = VT.getVectorElementType().getSizeInBits();
5132 if (EltSz == 64)
5133 return false;
5134
5135 unsigned NumElts = VT.getVectorNumElements();
5136 if (M.size() != NumElts && M.size() != NumElts*2)
5137 return false;
5138
5139 for (unsigned i = 0; i < M.size(); i += NumElts) {
5140 if (M.size() == NumElts * 2)
5141 WhichResult = i / NumElts;
5142 else
5143 WhichResult = M[i] == 0 ? 0 : 1;
5144 for (unsigned j = 0; j < NumElts; j += 2) {
5145 if ((M[i+j] >= 0 && (unsigned) M[i+j] != j + WhichResult) ||
5146 (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != j + WhichResult))
5147 return false;
5148 }
5149 }
5150
5151 if (M.size() == NumElts*2)
5152 WhichResult = 0;
5153
5154 return true;
5155 }
5156
5157 // Checks whether the shuffle mask represents a vector unzip (VUZP) by checking
5158 // that the mask elements are either all even and in steps of size 2 or all odd
5159 // and in steps of size 2.
5160 // e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 2, 4, 6]
5161 // v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,c,e,g}
5162 // v2={e,f,g,h}
5163 // Requires similar checks to that of isVTRNMask with
5164 // respect the how results are returned.
isVUZPMask(ArrayRef<int> M,EVT VT,unsigned & WhichResult)5165 static bool isVUZPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
5166 unsigned EltSz = VT.getVectorElementType().getSizeInBits();
5167 if (EltSz == 64)
5168 return false;
5169
5170 unsigned NumElts = VT.getVectorNumElements();
5171 if (M.size() != NumElts && M.size() != NumElts*2)
5172 return false;
5173
5174 for (unsigned i = 0; i < M.size(); i += NumElts) {
5175 WhichResult = M[i] == 0 ? 0 : 1;
5176 for (unsigned j = 0; j < NumElts; ++j) {
5177 if (M[i+j] >= 0 && (unsigned) M[i+j] != 2 * j + WhichResult)
5178 return false;
5179 }
5180 }
5181
5182 if (M.size() == NumElts*2)
5183 WhichResult = 0;
5184
5185 // VUZP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
5186 if (VT.is64BitVector() && EltSz == 32)
5187 return false;
5188
5189 return true;
5190 }
5191
5192 /// isVUZP_v_undef_Mask - Special case of isVUZPMask for canonical form of
5193 /// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
5194 /// Mask is e.g., <0, 2, 0, 2> instead of <0, 2, 4, 6>,
isVUZP_v_undef_Mask(ArrayRef<int> M,EVT VT,unsigned & WhichResult)5195 static bool isVUZP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){
5196 unsigned EltSz = VT.getVectorElementType().getSizeInBits();
5197 if (EltSz == 64)
5198 return false;
5199
5200 unsigned NumElts = VT.getVectorNumElements();
5201 if (M.size() != NumElts && M.size() != NumElts*2)
5202 return false;
5203
5204 unsigned Half = NumElts / 2;
5205 for (unsigned i = 0; i < M.size(); i += NumElts) {
5206 WhichResult = M[i] == 0 ? 0 : 1;
5207 for (unsigned j = 0; j < NumElts; j += Half) {
5208 unsigned Idx = WhichResult;
5209 for (unsigned k = 0; k < Half; ++k) {
5210 int MIdx = M[i + j + k];
5211 if (MIdx >= 0 && (unsigned) MIdx != Idx)
5212 return false;
5213 Idx += 2;
5214 }
5215 }
5216 }
5217
5218 if (M.size() == NumElts*2)
5219 WhichResult = 0;
5220
5221 // VUZP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
5222 if (VT.is64BitVector() && EltSz == 32)
5223 return false;
5224
5225 return true;
5226 }
5227
5228 // Checks whether the shuffle mask represents a vector zip (VZIP) by checking
5229 // that pairs of elements of the shufflemask represent the same index in each
5230 // vector incrementing sequentially through the vectors.
5231 // e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 4, 1, 5]
5232 // v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,e,b,f}
5233 // v2={e,f,g,h}
5234 // Requires similar checks to that of isVTRNMask with respect the how results
5235 // are returned.
isVZIPMask(ArrayRef<int> M,EVT VT,unsigned & WhichResult)5236 static bool isVZIPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
5237 unsigned EltSz = VT.getVectorElementType().getSizeInBits();
5238 if (EltSz == 64)
5239 return false;
5240
5241 unsigned NumElts = VT.getVectorNumElements();
5242 if (M.size() != NumElts && M.size() != NumElts*2)
5243 return false;
5244
5245 for (unsigned i = 0; i < M.size(); i += NumElts) {
5246 WhichResult = M[i] == 0 ? 0 : 1;
5247 unsigned Idx = WhichResult * NumElts / 2;
5248 for (unsigned j = 0; j < NumElts; j += 2) {
5249 if ((M[i+j] >= 0 && (unsigned) M[i+j] != Idx) ||
5250 (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != Idx + NumElts))
5251 return false;
5252 Idx += 1;
5253 }
5254 }
5255
5256 if (M.size() == NumElts*2)
5257 WhichResult = 0;
5258
5259 // VZIP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
5260 if (VT.is64BitVector() && EltSz == 32)
5261 return false;
5262
5263 return true;
5264 }
5265
5266 /// isVZIP_v_undef_Mask - Special case of isVZIPMask for canonical form of
5267 /// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
5268 /// Mask is e.g., <0, 0, 1, 1> instead of <0, 4, 1, 5>.
isVZIP_v_undef_Mask(ArrayRef<int> M,EVT VT,unsigned & WhichResult)5269 static bool isVZIP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){
5270 unsigned EltSz = VT.getVectorElementType().getSizeInBits();
5271 if (EltSz == 64)
5272 return false;
5273
5274 unsigned NumElts = VT.getVectorNumElements();
5275 if (M.size() != NumElts && M.size() != NumElts*2)
5276 return false;
5277
5278 for (unsigned i = 0; i < M.size(); i += NumElts) {
5279 WhichResult = M[i] == 0 ? 0 : 1;
5280 unsigned Idx = WhichResult * NumElts / 2;
5281 for (unsigned j = 0; j < NumElts; j += 2) {
5282 if ((M[i+j] >= 0 && (unsigned) M[i+j] != Idx) ||
5283 (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != Idx))
5284 return false;
5285 Idx += 1;
5286 }
5287 }
5288
5289 if (M.size() == NumElts*2)
5290 WhichResult = 0;
5291
5292 // VZIP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
5293 if (VT.is64BitVector() && EltSz == 32)
5294 return false;
5295
5296 return true;
5297 }
5298
5299 /// Check if \p ShuffleMask is a NEON two-result shuffle (VZIP, VUZP, VTRN),
5300 /// and return the corresponding ARMISD opcode if it is, or 0 if it isn't.
isNEONTwoResultShuffleMask(ArrayRef<int> ShuffleMask,EVT VT,unsigned & WhichResult,bool & isV_UNDEF)5301 static unsigned isNEONTwoResultShuffleMask(ArrayRef<int> ShuffleMask, EVT VT,
5302 unsigned &WhichResult,
5303 bool &isV_UNDEF) {
5304 isV_UNDEF = false;
5305 if (isVTRNMask(ShuffleMask, VT, WhichResult))
5306 return ARMISD::VTRN;
5307 if (isVUZPMask(ShuffleMask, VT, WhichResult))
5308 return ARMISD::VUZP;
5309 if (isVZIPMask(ShuffleMask, VT, WhichResult))
5310 return ARMISD::VZIP;
5311
5312 isV_UNDEF = true;
5313 if (isVTRN_v_undef_Mask(ShuffleMask, VT, WhichResult))
5314 return ARMISD::VTRN;
5315 if (isVUZP_v_undef_Mask(ShuffleMask, VT, WhichResult))
5316 return ARMISD::VUZP;
5317 if (isVZIP_v_undef_Mask(ShuffleMask, VT, WhichResult))
5318 return ARMISD::VZIP;
5319
5320 return 0;
5321 }
5322
5323 /// \return true if this is a reverse operation on an vector.
isReverseMask(ArrayRef<int> M,EVT VT)5324 static bool isReverseMask(ArrayRef<int> M, EVT VT) {
5325 unsigned NumElts = VT.getVectorNumElements();
5326 // Make sure the mask has the right size.
5327 if (NumElts != M.size())
5328 return false;
5329
5330 // Look for <15, ..., 3, -1, 1, 0>.
5331 for (unsigned i = 0; i != NumElts; ++i)
5332 if (M[i] >= 0 && M[i] != (int) (NumElts - 1 - i))
5333 return false;
5334
5335 return true;
5336 }
5337
5338 // If N is an integer constant that can be moved into a register in one
5339 // instruction, return an SDValue of such a constant (will become a MOV
5340 // instruction). Otherwise return null.
IsSingleInstrConstant(SDValue N,SelectionDAG & DAG,const ARMSubtarget * ST,SDLoc dl)5341 static SDValue IsSingleInstrConstant(SDValue N, SelectionDAG &DAG,
5342 const ARMSubtarget *ST, SDLoc dl) {
5343 uint64_t Val;
5344 if (!isa<ConstantSDNode>(N))
5345 return SDValue();
5346 Val = cast<ConstantSDNode>(N)->getZExtValue();
5347
5348 if (ST->isThumb1Only()) {
5349 if (Val <= 255 || ~Val <= 255)
5350 return DAG.getConstant(Val, dl, MVT::i32);
5351 } else {
5352 if (ARM_AM::getSOImmVal(Val) != -1 || ARM_AM::getSOImmVal(~Val) != -1)
5353 return DAG.getConstant(Val, dl, MVT::i32);
5354 }
5355 return SDValue();
5356 }
5357
5358 // If this is a case we can't handle, return null and let the default
5359 // expansion code take care of it.
LowerBUILD_VECTOR(SDValue Op,SelectionDAG & DAG,const ARMSubtarget * ST) const5360 SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
5361 const ARMSubtarget *ST) const {
5362 BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Op.getNode());
5363 SDLoc dl(Op);
5364 EVT VT = Op.getValueType();
5365
5366 APInt SplatBits, SplatUndef;
5367 unsigned SplatBitSize;
5368 bool HasAnyUndefs;
5369 if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
5370 if (SplatBitSize <= 64) {
5371 // Check if an immediate VMOV works.
5372 EVT VmovVT;
5373 SDValue Val = isNEONModifiedImm(SplatBits.getZExtValue(),
5374 SplatUndef.getZExtValue(), SplatBitSize,
5375 DAG, dl, VmovVT, VT.is128BitVector(),
5376 VMOVModImm);
5377 if (Val.getNode()) {
5378 SDValue Vmov = DAG.getNode(ARMISD::VMOVIMM, dl, VmovVT, Val);
5379 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov);
5380 }
5381
5382 // Try an immediate VMVN.
5383 uint64_t NegatedImm = (~SplatBits).getZExtValue();
5384 Val = isNEONModifiedImm(NegatedImm,
5385 SplatUndef.getZExtValue(), SplatBitSize,
5386 DAG, dl, VmovVT, VT.is128BitVector(),
5387 VMVNModImm);
5388 if (Val.getNode()) {
5389 SDValue Vmov = DAG.getNode(ARMISD::VMVNIMM, dl, VmovVT, Val);
5390 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov);
5391 }
5392
5393 // Use vmov.f32 to materialize other v2f32 and v4f32 splats.
5394 if ((VT == MVT::v2f32 || VT == MVT::v4f32) && SplatBitSize == 32) {
5395 int ImmVal = ARM_AM::getFP32Imm(SplatBits);
5396 if (ImmVal != -1) {
5397 SDValue Val = DAG.getTargetConstant(ImmVal, dl, MVT::i32);
5398 return DAG.getNode(ARMISD::VMOVFPIMM, dl, VT, Val);
5399 }
5400 }
5401 }
5402 }
5403
5404 // Scan through the operands to see if only one value is used.
5405 //
5406 // As an optimisation, even if more than one value is used it may be more
5407 // profitable to splat with one value then change some lanes.
5408 //
5409 // Heuristically we decide to do this if the vector has a "dominant" value,
5410 // defined as splatted to more than half of the lanes.
5411 unsigned NumElts = VT.getVectorNumElements();
5412 bool isOnlyLowElement = true;
5413 bool usesOnlyOneValue = true;
5414 bool hasDominantValue = false;
5415 bool isConstant = true;
5416
5417 // Map of the number of times a particular SDValue appears in the
5418 // element list.
5419 DenseMap<SDValue, unsigned> ValueCounts;
5420 SDValue Value;
5421 for (unsigned i = 0; i < NumElts; ++i) {
5422 SDValue V = Op.getOperand(i);
5423 if (V.getOpcode() == ISD::UNDEF)
5424 continue;
5425 if (i > 0)
5426 isOnlyLowElement = false;
5427 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
5428 isConstant = false;
5429
5430 ValueCounts.insert(std::make_pair(V, 0));
5431 unsigned &Count = ValueCounts[V];
5432
5433 // Is this value dominant? (takes up more than half of the lanes)
5434 if (++Count > (NumElts / 2)) {
5435 hasDominantValue = true;
5436 Value = V;
5437 }
5438 }
5439 if (ValueCounts.size() != 1)
5440 usesOnlyOneValue = false;
5441 if (!Value.getNode() && ValueCounts.size() > 0)
5442 Value = ValueCounts.begin()->first;
5443
5444 if (ValueCounts.size() == 0)
5445 return DAG.getUNDEF(VT);
5446
5447 // Loads are better lowered with insert_vector_elt/ARMISD::BUILD_VECTOR.
5448 // Keep going if we are hitting this case.
5449 if (isOnlyLowElement && !ISD::isNormalLoad(Value.getNode()))
5450 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value);
5451
5452 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
5453
5454 // Use VDUP for non-constant splats. For f32 constant splats, reduce to
5455 // i32 and try again.
5456 if (hasDominantValue && EltSize <= 32) {
5457 if (!isConstant) {
5458 SDValue N;
5459
5460 // If we are VDUPing a value that comes directly from a vector, that will
5461 // cause an unnecessary move to and from a GPR, where instead we could
5462 // just use VDUPLANE. We can only do this if the lane being extracted
5463 // is at a constant index, as the VDUP from lane instructions only have
5464 // constant-index forms.
5465 ConstantSDNode *constIndex;
5466 if (Value->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
5467 (constIndex = dyn_cast<ConstantSDNode>(Value->getOperand(1)))) {
5468 // We need to create a new undef vector to use for the VDUPLANE if the
5469 // size of the vector from which we get the value is different than the
5470 // size of the vector that we need to create. We will insert the element
5471 // such that the register coalescer will remove unnecessary copies.
5472 if (VT != Value->getOperand(0).getValueType()) {
5473 unsigned index = constIndex->getAPIntValue().getLimitedValue() %
5474 VT.getVectorNumElements();
5475 N = DAG.getNode(ARMISD::VDUPLANE, dl, VT,
5476 DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DAG.getUNDEF(VT),
5477 Value, DAG.getConstant(index, dl, MVT::i32)),
5478 DAG.getConstant(index, dl, MVT::i32));
5479 } else
5480 N = DAG.getNode(ARMISD::VDUPLANE, dl, VT,
5481 Value->getOperand(0), Value->getOperand(1));
5482 } else
5483 N = DAG.getNode(ARMISD::VDUP, dl, VT, Value);
5484
5485 if (!usesOnlyOneValue) {
5486 // The dominant value was splatted as 'N', but we now have to insert
5487 // all differing elements.
5488 for (unsigned I = 0; I < NumElts; ++I) {
5489 if (Op.getOperand(I) == Value)
5490 continue;
5491 SmallVector<SDValue, 3> Ops;
5492 Ops.push_back(N);
5493 Ops.push_back(Op.getOperand(I));
5494 Ops.push_back(DAG.getConstant(I, dl, MVT::i32));
5495 N = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Ops);
5496 }
5497 }
5498 return N;
5499 }
5500 if (VT.getVectorElementType().isFloatingPoint()) {
5501 SmallVector<SDValue, 8> Ops;
5502 for (unsigned i = 0; i < NumElts; ++i)
5503 Ops.push_back(DAG.getNode(ISD::BITCAST, dl, MVT::i32,
5504 Op.getOperand(i)));
5505 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElts);
5506 SDValue Val = DAG.getNode(ISD::BUILD_VECTOR, dl, VecVT, Ops);
5507 Val = LowerBUILD_VECTOR(Val, DAG, ST);
5508 if (Val.getNode())
5509 return DAG.getNode(ISD::BITCAST, dl, VT, Val);
5510 }
5511 if (usesOnlyOneValue) {
5512 SDValue Val = IsSingleInstrConstant(Value, DAG, ST, dl);
5513 if (isConstant && Val.getNode())
5514 return DAG.getNode(ARMISD::VDUP, dl, VT, Val);
5515 }
5516 }
5517
5518 // If all elements are constants and the case above didn't get hit, fall back
5519 // to the default expansion, which will generate a load from the constant
5520 // pool.
5521 if (isConstant)
5522 return SDValue();
5523
5524 // Empirical tests suggest this is rarely worth it for vectors of length <= 2.
5525 if (NumElts >= 4) {
5526 SDValue shuffle = ReconstructShuffle(Op, DAG);
5527 if (shuffle != SDValue())
5528 return shuffle;
5529 }
5530
5531 // Vectors with 32- or 64-bit elements can be built by directly assigning
5532 // the subregisters. Lower it to an ARMISD::BUILD_VECTOR so the operands
5533 // will be legalized.
5534 if (EltSize >= 32) {
5535 // Do the expansion with floating-point types, since that is what the VFP
5536 // registers are defined to use, and since i64 is not legal.
5537 EVT EltVT = EVT::getFloatingPointVT(EltSize);
5538 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts);
5539 SmallVector<SDValue, 8> Ops;
5540 for (unsigned i = 0; i < NumElts; ++i)
5541 Ops.push_back(DAG.getNode(ISD::BITCAST, dl, EltVT, Op.getOperand(i)));
5542 SDValue Val = DAG.getNode(ARMISD::BUILD_VECTOR, dl, VecVT, Ops);
5543 return DAG.getNode(ISD::BITCAST, dl, VT, Val);
5544 }
5545
5546 // If all else fails, just use a sequence of INSERT_VECTOR_ELT when we
5547 // know the default expansion would otherwise fall back on something even
5548 // worse. For a vector with one or two non-undef values, that's
5549 // scalar_to_vector for the elements followed by a shuffle (provided the
5550 // shuffle is valid for the target) and materialization element by element
5551 // on the stack followed by a load for everything else.
5552 if (!isConstant && !usesOnlyOneValue) {
5553 SDValue Vec = DAG.getUNDEF(VT);
5554 for (unsigned i = 0 ; i < NumElts; ++i) {
5555 SDValue V = Op.getOperand(i);
5556 if (V.getOpcode() == ISD::UNDEF)
5557 continue;
5558 SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i32);
5559 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Vec, V, LaneIdx);
5560 }
5561 return Vec;
5562 }
5563
5564 return SDValue();
5565 }
5566
5567 // Gather data to see if the operation can be modelled as a
5568 // shuffle in combination with VEXTs.
ReconstructShuffle(SDValue Op,SelectionDAG & DAG) const5569 SDValue ARMTargetLowering::ReconstructShuffle(SDValue Op,
5570 SelectionDAG &DAG) const {
5571 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
5572 SDLoc dl(Op);
5573 EVT VT = Op.getValueType();
5574 unsigned NumElts = VT.getVectorNumElements();
5575
5576 struct ShuffleSourceInfo {
5577 SDValue Vec;
5578 unsigned MinElt;
5579 unsigned MaxElt;
5580
5581 // We may insert some combination of BITCASTs and VEXT nodes to force Vec to
5582 // be compatible with the shuffle we intend to construct. As a result
5583 // ShuffleVec will be some sliding window into the original Vec.
5584 SDValue ShuffleVec;
5585
5586 // Code should guarantee that element i in Vec starts at element "WindowBase
5587 // + i * WindowScale in ShuffleVec".
5588 int WindowBase;
5589 int WindowScale;
5590
5591 bool operator ==(SDValue OtherVec) { return Vec == OtherVec; }
5592 ShuffleSourceInfo(SDValue Vec)
5593 : Vec(Vec), MinElt(UINT_MAX), MaxElt(0), ShuffleVec(Vec), WindowBase(0),
5594 WindowScale(1) {}
5595 };
5596
5597 // First gather all vectors used as an immediate source for this BUILD_VECTOR
5598 // node.
5599 SmallVector<ShuffleSourceInfo, 2> Sources;
5600 for (unsigned i = 0; i < NumElts; ++i) {
5601 SDValue V = Op.getOperand(i);
5602 if (V.getOpcode() == ISD::UNDEF)
5603 continue;
5604 else if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT) {
5605 // A shuffle can only come from building a vector from various
5606 // elements of other vectors.
5607 return SDValue();
5608 } else if (!isa<ConstantSDNode>(V.getOperand(1))) {
5609 // Furthermore, shuffles require a constant mask, whereas extractelts
5610 // accept variable indices.
5611 return SDValue();
5612 }
5613
5614 // Add this element source to the list if it's not already there.
5615 SDValue SourceVec = V.getOperand(0);
5616 auto Source = std::find(Sources.begin(), Sources.end(), SourceVec);
5617 if (Source == Sources.end())
5618 Source = Sources.insert(Sources.end(), ShuffleSourceInfo(SourceVec));
5619
5620 // Update the minimum and maximum lane number seen.
5621 unsigned EltNo = cast<ConstantSDNode>(V.getOperand(1))->getZExtValue();
5622 Source->MinElt = std::min(Source->MinElt, EltNo);
5623 Source->MaxElt = std::max(Source->MaxElt, EltNo);
5624 }
5625
5626 // Currently only do something sane when at most two source vectors
5627 // are involved.
5628 if (Sources.size() > 2)
5629 return SDValue();
5630
5631 // Find out the smallest element size among result and two sources, and use
5632 // it as element size to build the shuffle_vector.
5633 EVT SmallestEltTy = VT.getVectorElementType();
5634 for (auto &Source : Sources) {
5635 EVT SrcEltTy = Source.Vec.getValueType().getVectorElementType();
5636 if (SrcEltTy.bitsLT(SmallestEltTy))
5637 SmallestEltTy = SrcEltTy;
5638 }
5639 unsigned ResMultiplier =
5640 VT.getVectorElementType().getSizeInBits() / SmallestEltTy.getSizeInBits();
5641 NumElts = VT.getSizeInBits() / SmallestEltTy.getSizeInBits();
5642 EVT ShuffleVT = EVT::getVectorVT(*DAG.getContext(), SmallestEltTy, NumElts);
5643
5644 // If the source vector is too wide or too narrow, we may nevertheless be able
5645 // to construct a compatible shuffle either by concatenating it with UNDEF or
5646 // extracting a suitable range of elements.
5647 for (auto &Src : Sources) {
5648 EVT SrcVT = Src.ShuffleVec.getValueType();
5649
5650 if (SrcVT.getSizeInBits() == VT.getSizeInBits())
5651 continue;
5652
5653 // This stage of the search produces a source with the same element type as
5654 // the original, but with a total width matching the BUILD_VECTOR output.
5655 EVT EltVT = SrcVT.getVectorElementType();
5656 unsigned NumSrcElts = VT.getSizeInBits() / EltVT.getSizeInBits();
5657 EVT DestVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumSrcElts);
5658
5659 if (SrcVT.getSizeInBits() < VT.getSizeInBits()) {
5660 if (2 * SrcVT.getSizeInBits() != VT.getSizeInBits())
5661 return SDValue();
5662 // We can pad out the smaller vector for free, so if it's part of a
5663 // shuffle...
5664 Src.ShuffleVec =
5665 DAG.getNode(ISD::CONCAT_VECTORS, dl, DestVT, Src.ShuffleVec,
5666 DAG.getUNDEF(Src.ShuffleVec.getValueType()));
5667 continue;
5668 }
5669
5670 if (SrcVT.getSizeInBits() != 2 * VT.getSizeInBits())
5671 return SDValue();
5672
5673 if (Src.MaxElt - Src.MinElt >= NumSrcElts) {
5674 // Span too large for a VEXT to cope
5675 return SDValue();
5676 }
5677
5678 if (Src.MinElt >= NumSrcElts) {
5679 // The extraction can just take the second half
5680 Src.ShuffleVec =
5681 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
5682 DAG.getConstant(NumSrcElts, dl, MVT::i32));
5683 Src.WindowBase = -NumSrcElts;
5684 } else if (Src.MaxElt < NumSrcElts) {
5685 // The extraction can just take the first half
5686 Src.ShuffleVec =
5687 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
5688 DAG.getConstant(0, dl, MVT::i32));
5689 } else {
5690 // An actual VEXT is needed
5691 SDValue VEXTSrc1 =
5692 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
5693 DAG.getConstant(0, dl, MVT::i32));
5694 SDValue VEXTSrc2 =
5695 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
5696 DAG.getConstant(NumSrcElts, dl, MVT::i32));
5697
5698 Src.ShuffleVec = DAG.getNode(ARMISD::VEXT, dl, DestVT, VEXTSrc1,
5699 VEXTSrc2,
5700 DAG.getConstant(Src.MinElt, dl, MVT::i32));
5701 Src.WindowBase = -Src.MinElt;
5702 }
5703 }
5704
5705 // Another possible incompatibility occurs from the vector element types. We
5706 // can fix this by bitcasting the source vectors to the same type we intend
5707 // for the shuffle.
5708 for (auto &Src : Sources) {
5709 EVT SrcEltTy = Src.ShuffleVec.getValueType().getVectorElementType();
5710 if (SrcEltTy == SmallestEltTy)
5711 continue;
5712 assert(ShuffleVT.getVectorElementType() == SmallestEltTy);
5713 Src.ShuffleVec = DAG.getNode(ISD::BITCAST, dl, ShuffleVT, Src.ShuffleVec);
5714 Src.WindowScale = SrcEltTy.getSizeInBits() / SmallestEltTy.getSizeInBits();
5715 Src.WindowBase *= Src.WindowScale;
5716 }
5717
5718 // Final sanity check before we try to actually produce a shuffle.
5719 DEBUG(
5720 for (auto Src : Sources)
5721 assert(Src.ShuffleVec.getValueType() == ShuffleVT);
5722 );
5723
5724 // The stars all align, our next step is to produce the mask for the shuffle.
5725 SmallVector<int, 8> Mask(ShuffleVT.getVectorNumElements(), -1);
5726 int BitsPerShuffleLane = ShuffleVT.getVectorElementType().getSizeInBits();
5727 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
5728 SDValue Entry = Op.getOperand(i);
5729 if (Entry.getOpcode() == ISD::UNDEF)
5730 continue;
5731
5732 auto Src = std::find(Sources.begin(), Sources.end(), Entry.getOperand(0));
5733 int EltNo = cast<ConstantSDNode>(Entry.getOperand(1))->getSExtValue();
5734
5735 // EXTRACT_VECTOR_ELT performs an implicit any_ext; BUILD_VECTOR an implicit
5736 // trunc. So only std::min(SrcBits, DestBits) actually get defined in this
5737 // segment.
5738 EVT OrigEltTy = Entry.getOperand(0).getValueType().getVectorElementType();
5739 int BitsDefined = std::min(OrigEltTy.getSizeInBits(),
5740 VT.getVectorElementType().getSizeInBits());
5741 int LanesDefined = BitsDefined / BitsPerShuffleLane;
5742
5743 // This source is expected to fill ResMultiplier lanes of the final shuffle,
5744 // starting at the appropriate offset.
5745 int *LaneMask = &Mask[i * ResMultiplier];
5746
5747 int ExtractBase = EltNo * Src->WindowScale + Src->WindowBase;
5748 ExtractBase += NumElts * (Src - Sources.begin());
5749 for (int j = 0; j < LanesDefined; ++j)
5750 LaneMask[j] = ExtractBase + j;
5751 }
5752
5753 // Final check before we try to produce nonsense...
5754 if (!isShuffleMaskLegal(Mask, ShuffleVT))
5755 return SDValue();
5756
5757 // We can't handle more than two sources. This should have already
5758 // been checked before this point.
5759 assert(Sources.size() <= 2 && "Too many sources!");
5760
5761 SDValue ShuffleOps[] = { DAG.getUNDEF(ShuffleVT), DAG.getUNDEF(ShuffleVT) };
5762 for (unsigned i = 0; i < Sources.size(); ++i)
5763 ShuffleOps[i] = Sources[i].ShuffleVec;
5764
5765 SDValue Shuffle = DAG.getVectorShuffle(ShuffleVT, dl, ShuffleOps[0],
5766 ShuffleOps[1], &Mask[0]);
5767 return DAG.getNode(ISD::BITCAST, dl, VT, Shuffle);
5768 }
5769
5770 /// isShuffleMaskLegal - Targets can use this to indicate that they only
5771 /// support *some* VECTOR_SHUFFLE operations, those with specific masks.
5772 /// By default, if a target supports the VECTOR_SHUFFLE node, all mask values
5773 /// are assumed to be legal.
5774 bool
isShuffleMaskLegal(const SmallVectorImpl<int> & M,EVT VT) const5775 ARMTargetLowering::isShuffleMaskLegal(const SmallVectorImpl<int> &M,
5776 EVT VT) const {
5777 if (VT.getVectorNumElements() == 4 &&
5778 (VT.is128BitVector() || VT.is64BitVector())) {
5779 unsigned PFIndexes[4];
5780 for (unsigned i = 0; i != 4; ++i) {
5781 if (M[i] < 0)
5782 PFIndexes[i] = 8;
5783 else
5784 PFIndexes[i] = M[i];
5785 }
5786
5787 // Compute the index in the perfect shuffle table.
5788 unsigned PFTableIndex =
5789 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
5790 unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
5791 unsigned Cost = (PFEntry >> 30);
5792
5793 if (Cost <= 4)
5794 return true;
5795 }
5796
5797 bool ReverseVEXT, isV_UNDEF;
5798 unsigned Imm, WhichResult;
5799
5800 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
5801 return (EltSize >= 32 ||
5802 ShuffleVectorSDNode::isSplatMask(&M[0], VT) ||
5803 isVREVMask(M, VT, 64) ||
5804 isVREVMask(M, VT, 32) ||
5805 isVREVMask(M, VT, 16) ||
5806 isVEXTMask(M, VT, ReverseVEXT, Imm) ||
5807 isVTBLMask(M, VT) ||
5808 isNEONTwoResultShuffleMask(M, VT, WhichResult, isV_UNDEF) ||
5809 ((VT == MVT::v8i16 || VT == MVT::v16i8) && isReverseMask(M, VT)));
5810 }
5811
5812 /// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit
5813 /// the specified operations to build the shuffle.
GeneratePerfectShuffle(unsigned PFEntry,SDValue LHS,SDValue RHS,SelectionDAG & DAG,SDLoc dl)5814 static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
5815 SDValue RHS, SelectionDAG &DAG,
5816 SDLoc dl) {
5817 unsigned OpNum = (PFEntry >> 26) & 0x0F;
5818 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
5819 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
5820
5821 enum {
5822 OP_COPY = 0, // Copy, used for things like <u,u,u,3> to say it is <0,1,2,3>
5823 OP_VREV,
5824 OP_VDUP0,
5825 OP_VDUP1,
5826 OP_VDUP2,
5827 OP_VDUP3,
5828 OP_VEXT1,
5829 OP_VEXT2,
5830 OP_VEXT3,
5831 OP_VUZPL, // VUZP, left result
5832 OP_VUZPR, // VUZP, right result
5833 OP_VZIPL, // VZIP, left result
5834 OP_VZIPR, // VZIP, right result
5835 OP_VTRNL, // VTRN, left result
5836 OP_VTRNR // VTRN, right result
5837 };
5838
5839 if (OpNum == OP_COPY) {
5840 if (LHSID == (1*9+2)*9+3) return LHS;
5841 assert(LHSID == ((4*9+5)*9+6)*9+7 && "Illegal OP_COPY!");
5842 return RHS;
5843 }
5844
5845 SDValue OpLHS, OpRHS;
5846 OpLHS = GeneratePerfectShuffle(PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl);
5847 OpRHS = GeneratePerfectShuffle(PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl);
5848 EVT VT = OpLHS.getValueType();
5849
5850 switch (OpNum) {
5851 default: llvm_unreachable("Unknown shuffle opcode!");
5852 case OP_VREV:
5853 // VREV divides the vector in half and swaps within the half.
5854 if (VT.getVectorElementType() == MVT::i32 ||
5855 VT.getVectorElementType() == MVT::f32)
5856 return DAG.getNode(ARMISD::VREV64, dl, VT, OpLHS);
5857 // vrev <4 x i16> -> VREV32
5858 if (VT.getVectorElementType() == MVT::i16)
5859 return DAG.getNode(ARMISD::VREV32, dl, VT, OpLHS);
5860 // vrev <4 x i8> -> VREV16
5861 assert(VT.getVectorElementType() == MVT::i8);
5862 return DAG.getNode(ARMISD::VREV16, dl, VT, OpLHS);
5863 case OP_VDUP0:
5864 case OP_VDUP1:
5865 case OP_VDUP2:
5866 case OP_VDUP3:
5867 return DAG.getNode(ARMISD::VDUPLANE, dl, VT,
5868 OpLHS, DAG.getConstant(OpNum-OP_VDUP0, dl, MVT::i32));
5869 case OP_VEXT1:
5870 case OP_VEXT2:
5871 case OP_VEXT3:
5872 return DAG.getNode(ARMISD::VEXT, dl, VT,
5873 OpLHS, OpRHS,
5874 DAG.getConstant(OpNum - OP_VEXT1 + 1, dl, MVT::i32));
5875 case OP_VUZPL:
5876 case OP_VUZPR:
5877 return DAG.getNode(ARMISD::VUZP, dl, DAG.getVTList(VT, VT),
5878 OpLHS, OpRHS).getValue(OpNum-OP_VUZPL);
5879 case OP_VZIPL:
5880 case OP_VZIPR:
5881 return DAG.getNode(ARMISD::VZIP, dl, DAG.getVTList(VT, VT),
5882 OpLHS, OpRHS).getValue(OpNum-OP_VZIPL);
5883 case OP_VTRNL:
5884 case OP_VTRNR:
5885 return DAG.getNode(ARMISD::VTRN, dl, DAG.getVTList(VT, VT),
5886 OpLHS, OpRHS).getValue(OpNum-OP_VTRNL);
5887 }
5888 }
5889
LowerVECTOR_SHUFFLEv8i8(SDValue Op,ArrayRef<int> ShuffleMask,SelectionDAG & DAG)5890 static SDValue LowerVECTOR_SHUFFLEv8i8(SDValue Op,
5891 ArrayRef<int> ShuffleMask,
5892 SelectionDAG &DAG) {
5893 // Check to see if we can use the VTBL instruction.
5894 SDValue V1 = Op.getOperand(0);
5895 SDValue V2 = Op.getOperand(1);
5896 SDLoc DL(Op);
5897
5898 SmallVector<SDValue, 8> VTBLMask;
5899 for (ArrayRef<int>::iterator
5900 I = ShuffleMask.begin(), E = ShuffleMask.end(); I != E; ++I)
5901 VTBLMask.push_back(DAG.getConstant(*I, DL, MVT::i32));
5902
5903 if (V2.getNode()->getOpcode() == ISD::UNDEF)
5904 return DAG.getNode(ARMISD::VTBL1, DL, MVT::v8i8, V1,
5905 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v8i8, VTBLMask));
5906
5907 return DAG.getNode(ARMISD::VTBL2, DL, MVT::v8i8, V1, V2,
5908 DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v8i8, VTBLMask));
5909 }
5910
LowerReverse_VECTOR_SHUFFLEv16i8_v8i16(SDValue Op,SelectionDAG & DAG)5911 static SDValue LowerReverse_VECTOR_SHUFFLEv16i8_v8i16(SDValue Op,
5912 SelectionDAG &DAG) {
5913 SDLoc DL(Op);
5914 SDValue OpLHS = Op.getOperand(0);
5915 EVT VT = OpLHS.getValueType();
5916
5917 assert((VT == MVT::v8i16 || VT == MVT::v16i8) &&
5918 "Expect an v8i16/v16i8 type");
5919 OpLHS = DAG.getNode(ARMISD::VREV64, DL, VT, OpLHS);
5920 // For a v16i8 type: After the VREV, we have got <8, ...15, 8, ..., 0>. Now,
5921 // extract the first 8 bytes into the top double word and the last 8 bytes
5922 // into the bottom double word. The v8i16 case is similar.
5923 unsigned ExtractNum = (VT == MVT::v16i8) ? 8 : 4;
5924 return DAG.getNode(ARMISD::VEXT, DL, VT, OpLHS, OpLHS,
5925 DAG.getConstant(ExtractNum, DL, MVT::i32));
5926 }
5927
LowerVECTOR_SHUFFLE(SDValue Op,SelectionDAG & DAG)5928 static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) {
5929 SDValue V1 = Op.getOperand(0);
5930 SDValue V2 = Op.getOperand(1);
5931 SDLoc dl(Op);
5932 EVT VT = Op.getValueType();
5933 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
5934
5935 // Convert shuffles that are directly supported on NEON to target-specific
5936 // DAG nodes, instead of keeping them as shuffles and matching them again
5937 // during code selection. This is more efficient and avoids the possibility
5938 // of inconsistencies between legalization and selection.
5939 // FIXME: floating-point vectors should be canonicalized to integer vectors
5940 // of the same time so that they get CSEd properly.
5941 ArrayRef<int> ShuffleMask = SVN->getMask();
5942
5943 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
5944 if (EltSize <= 32) {
5945 if (ShuffleVectorSDNode::isSplatMask(&ShuffleMask[0], VT)) {
5946 int Lane = SVN->getSplatIndex();
5947 // If this is undef splat, generate it via "just" vdup, if possible.
5948 if (Lane == -1) Lane = 0;
5949
5950 // Test if V1 is a SCALAR_TO_VECTOR.
5951 if (Lane == 0 && V1.getOpcode() == ISD::SCALAR_TO_VECTOR) {
5952 return DAG.getNode(ARMISD::VDUP, dl, VT, V1.getOperand(0));
5953 }
5954 // Test if V1 is a BUILD_VECTOR which is equivalent to a SCALAR_TO_VECTOR
5955 // (and probably will turn into a SCALAR_TO_VECTOR once legalization
5956 // reaches it).
5957 if (Lane == 0 && V1.getOpcode() == ISD::BUILD_VECTOR &&
5958 !isa<ConstantSDNode>(V1.getOperand(0))) {
5959 bool IsScalarToVector = true;
5960 for (unsigned i = 1, e = V1.getNumOperands(); i != e; ++i)
5961 if (V1.getOperand(i).getOpcode() != ISD::UNDEF) {
5962 IsScalarToVector = false;
5963 break;
5964 }
5965 if (IsScalarToVector)
5966 return DAG.getNode(ARMISD::VDUP, dl, VT, V1.getOperand(0));
5967 }
5968 return DAG.getNode(ARMISD::VDUPLANE, dl, VT, V1,
5969 DAG.getConstant(Lane, dl, MVT::i32));
5970 }
5971
5972 bool ReverseVEXT;
5973 unsigned Imm;
5974 if (isVEXTMask(ShuffleMask, VT, ReverseVEXT, Imm)) {
5975 if (ReverseVEXT)
5976 std::swap(V1, V2);
5977 return DAG.getNode(ARMISD::VEXT, dl, VT, V1, V2,
5978 DAG.getConstant(Imm, dl, MVT::i32));
5979 }
5980
5981 if (isVREVMask(ShuffleMask, VT, 64))
5982 return DAG.getNode(ARMISD::VREV64, dl, VT, V1);
5983 if (isVREVMask(ShuffleMask, VT, 32))
5984 return DAG.getNode(ARMISD::VREV32, dl, VT, V1);
5985 if (isVREVMask(ShuffleMask, VT, 16))
5986 return DAG.getNode(ARMISD::VREV16, dl, VT, V1);
5987
5988 if (V2->getOpcode() == ISD::UNDEF &&
5989 isSingletonVEXTMask(ShuffleMask, VT, Imm)) {
5990 return DAG.getNode(ARMISD::VEXT, dl, VT, V1, V1,
5991 DAG.getConstant(Imm, dl, MVT::i32));
5992 }
5993
5994 // Check for Neon shuffles that modify both input vectors in place.
5995 // If both results are used, i.e., if there are two shuffles with the same
5996 // source operands and with masks corresponding to both results of one of
5997 // these operations, DAG memoization will ensure that a single node is
5998 // used for both shuffles.
5999 unsigned WhichResult;
6000 bool isV_UNDEF;
6001 if (unsigned ShuffleOpc = isNEONTwoResultShuffleMask(
6002 ShuffleMask, VT, WhichResult, isV_UNDEF)) {
6003 if (isV_UNDEF)
6004 V2 = V1;
6005 return DAG.getNode(ShuffleOpc, dl, DAG.getVTList(VT, VT), V1, V2)
6006 .getValue(WhichResult);
6007 }
6008
6009 // Also check for these shuffles through CONCAT_VECTORS: we canonicalize
6010 // shuffles that produce a result larger than their operands with:
6011 // shuffle(concat(v1, undef), concat(v2, undef))
6012 // ->
6013 // shuffle(concat(v1, v2), undef)
6014 // because we can access quad vectors (see PerformVECTOR_SHUFFLECombine).
6015 //
6016 // This is useful in the general case, but there are special cases where
6017 // native shuffles produce larger results: the two-result ops.
6018 //
6019 // Look through the concat when lowering them:
6020 // shuffle(concat(v1, v2), undef)
6021 // ->
6022 // concat(VZIP(v1, v2):0, :1)
6023 //
6024 if (V1->getOpcode() == ISD::CONCAT_VECTORS &&
6025 V2->getOpcode() == ISD::UNDEF) {
6026 SDValue SubV1 = V1->getOperand(0);
6027 SDValue SubV2 = V1->getOperand(1);
6028 EVT SubVT = SubV1.getValueType();
6029
6030 // We expect these to have been canonicalized to -1.
6031 assert(std::all_of(ShuffleMask.begin(), ShuffleMask.end(), [&](int i) {
6032 return i < (int)VT.getVectorNumElements();
6033 }) && "Unexpected shuffle index into UNDEF operand!");
6034
6035 if (unsigned ShuffleOpc = isNEONTwoResultShuffleMask(
6036 ShuffleMask, SubVT, WhichResult, isV_UNDEF)) {
6037 if (isV_UNDEF)
6038 SubV2 = SubV1;
6039 assert((WhichResult == 0) &&
6040 "In-place shuffle of concat can only have one result!");
6041 SDValue Res = DAG.getNode(ShuffleOpc, dl, DAG.getVTList(SubVT, SubVT),
6042 SubV1, SubV2);
6043 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Res.getValue(0),
6044 Res.getValue(1));
6045 }
6046 }
6047 }
6048
6049 // If the shuffle is not directly supported and it has 4 elements, use
6050 // the PerfectShuffle-generated table to synthesize it from other shuffles.
6051 unsigned NumElts = VT.getVectorNumElements();
6052 if (NumElts == 4) {
6053 unsigned PFIndexes[4];
6054 for (unsigned i = 0; i != 4; ++i) {
6055 if (ShuffleMask[i] < 0)
6056 PFIndexes[i] = 8;
6057 else
6058 PFIndexes[i] = ShuffleMask[i];
6059 }
6060
6061 // Compute the index in the perfect shuffle table.
6062 unsigned PFTableIndex =
6063 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
6064 unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
6065 unsigned Cost = (PFEntry >> 30);
6066
6067 if (Cost <= 4)
6068 return GeneratePerfectShuffle(PFEntry, V1, V2, DAG, dl);
6069 }
6070
6071 // Implement shuffles with 32- or 64-bit elements as ARMISD::BUILD_VECTORs.
6072 if (EltSize >= 32) {
6073 // Do the expansion with floating-point types, since that is what the VFP
6074 // registers are defined to use, and since i64 is not legal.
6075 EVT EltVT = EVT::getFloatingPointVT(EltSize);
6076 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts);
6077 V1 = DAG.getNode(ISD::BITCAST, dl, VecVT, V1);
6078 V2 = DAG.getNode(ISD::BITCAST, dl, VecVT, V2);
6079 SmallVector<SDValue, 8> Ops;
6080 for (unsigned i = 0; i < NumElts; ++i) {
6081 if (ShuffleMask[i] < 0)
6082 Ops.push_back(DAG.getUNDEF(EltVT));
6083 else
6084 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
6085 ShuffleMask[i] < (int)NumElts ? V1 : V2,
6086 DAG.getConstant(ShuffleMask[i] & (NumElts-1),
6087 dl, MVT::i32)));
6088 }
6089 SDValue Val = DAG.getNode(ARMISD::BUILD_VECTOR, dl, VecVT, Ops);
6090 return DAG.getNode(ISD::BITCAST, dl, VT, Val);
6091 }
6092
6093 if ((VT == MVT::v8i16 || VT == MVT::v16i8) && isReverseMask(ShuffleMask, VT))
6094 return LowerReverse_VECTOR_SHUFFLEv16i8_v8i16(Op, DAG);
6095
6096 if (VT == MVT::v8i8) {
6097 SDValue NewOp = LowerVECTOR_SHUFFLEv8i8(Op, ShuffleMask, DAG);
6098 if (NewOp.getNode())
6099 return NewOp;
6100 }
6101
6102 return SDValue();
6103 }
6104
LowerINSERT_VECTOR_ELT(SDValue Op,SelectionDAG & DAG)6105 static SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) {
6106 // INSERT_VECTOR_ELT is legal only for immediate indexes.
6107 SDValue Lane = Op.getOperand(2);
6108 if (!isa<ConstantSDNode>(Lane))
6109 return SDValue();
6110
6111 return Op;
6112 }
6113
LowerEXTRACT_VECTOR_ELT(SDValue Op,SelectionDAG & DAG)6114 static SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) {
6115 // EXTRACT_VECTOR_ELT is legal only for immediate indexes.
6116 SDValue Lane = Op.getOperand(1);
6117 if (!isa<ConstantSDNode>(Lane))
6118 return SDValue();
6119
6120 SDValue Vec = Op.getOperand(0);
6121 if (Op.getValueType() == MVT::i32 &&
6122 Vec.getValueType().getVectorElementType().getSizeInBits() < 32) {
6123 SDLoc dl(Op);
6124 return DAG.getNode(ARMISD::VGETLANEu, dl, MVT::i32, Vec, Lane);
6125 }
6126
6127 return Op;
6128 }
6129
LowerCONCAT_VECTORS(SDValue Op,SelectionDAG & DAG)6130 static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) {
6131 // The only time a CONCAT_VECTORS operation can have legal types is when
6132 // two 64-bit vectors are concatenated to a 128-bit vector.
6133 assert(Op.getValueType().is128BitVector() && Op.getNumOperands() == 2 &&
6134 "unexpected CONCAT_VECTORS");
6135 SDLoc dl(Op);
6136 SDValue Val = DAG.getUNDEF(MVT::v2f64);
6137 SDValue Op0 = Op.getOperand(0);
6138 SDValue Op1 = Op.getOperand(1);
6139 if (Op0.getOpcode() != ISD::UNDEF)
6140 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Val,
6141 DAG.getNode(ISD::BITCAST, dl, MVT::f64, Op0),
6142 DAG.getIntPtrConstant(0, dl));
6143 if (Op1.getOpcode() != ISD::UNDEF)
6144 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Val,
6145 DAG.getNode(ISD::BITCAST, dl, MVT::f64, Op1),
6146 DAG.getIntPtrConstant(1, dl));
6147 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Val);
6148 }
6149
6150 /// isExtendedBUILD_VECTOR - Check if N is a constant BUILD_VECTOR where each
6151 /// element has been zero/sign-extended, depending on the isSigned parameter,
6152 /// from an integer type half its size.
isExtendedBUILD_VECTOR(SDNode * N,SelectionDAG & DAG,bool isSigned)6153 static bool isExtendedBUILD_VECTOR(SDNode *N, SelectionDAG &DAG,
6154 bool isSigned) {
6155 // A v2i64 BUILD_VECTOR will have been legalized to a BITCAST from v4i32.
6156 EVT VT = N->getValueType(0);
6157 if (VT == MVT::v2i64 && N->getOpcode() == ISD::BITCAST) {
6158 SDNode *BVN = N->getOperand(0).getNode();
6159 if (BVN->getValueType(0) != MVT::v4i32 ||
6160 BVN->getOpcode() != ISD::BUILD_VECTOR)
6161 return false;
6162 unsigned LoElt = DAG.getDataLayout().isBigEndian() ? 1 : 0;
6163 unsigned HiElt = 1 - LoElt;
6164 ConstantSDNode *Lo0 = dyn_cast<ConstantSDNode>(BVN->getOperand(LoElt));
6165 ConstantSDNode *Hi0 = dyn_cast<ConstantSDNode>(BVN->getOperand(HiElt));
6166 ConstantSDNode *Lo1 = dyn_cast<ConstantSDNode>(BVN->getOperand(LoElt+2));
6167 ConstantSDNode *Hi1 = dyn_cast<ConstantSDNode>(BVN->getOperand(HiElt+2));
6168 if (!Lo0 || !Hi0 || !Lo1 || !Hi1)
6169 return false;
6170 if (isSigned) {
6171 if (Hi0->getSExtValue() == Lo0->getSExtValue() >> 32 &&
6172 Hi1->getSExtValue() == Lo1->getSExtValue() >> 32)
6173 return true;
6174 } else {
6175 if (Hi0->isNullValue() && Hi1->isNullValue())
6176 return true;
6177 }
6178 return false;
6179 }
6180
6181 if (N->getOpcode() != ISD::BUILD_VECTOR)
6182 return false;
6183
6184 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
6185 SDNode *Elt = N->getOperand(i).getNode();
6186 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Elt)) {
6187 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
6188 unsigned HalfSize = EltSize / 2;
6189 if (isSigned) {
6190 if (!isIntN(HalfSize, C->getSExtValue()))
6191 return false;
6192 } else {
6193 if (!isUIntN(HalfSize, C->getZExtValue()))
6194 return false;
6195 }
6196 continue;
6197 }
6198 return false;
6199 }
6200
6201 return true;
6202 }
6203
6204 /// isSignExtended - Check if a node is a vector value that is sign-extended
6205 /// or a constant BUILD_VECTOR with sign-extended elements.
isSignExtended(SDNode * N,SelectionDAG & DAG)6206 static bool isSignExtended(SDNode *N, SelectionDAG &DAG) {
6207 if (N->getOpcode() == ISD::SIGN_EXTEND || ISD::isSEXTLoad(N))
6208 return true;
6209 if (isExtendedBUILD_VECTOR(N, DAG, true))
6210 return true;
6211 return false;
6212 }
6213
6214 /// isZeroExtended - Check if a node is a vector value that is zero-extended
6215 /// or a constant BUILD_VECTOR with zero-extended elements.
isZeroExtended(SDNode * N,SelectionDAG & DAG)6216 static bool isZeroExtended(SDNode *N, SelectionDAG &DAG) {
6217 if (N->getOpcode() == ISD::ZERO_EXTEND || ISD::isZEXTLoad(N))
6218 return true;
6219 if (isExtendedBUILD_VECTOR(N, DAG, false))
6220 return true;
6221 return false;
6222 }
6223
getExtensionTo64Bits(const EVT & OrigVT)6224 static EVT getExtensionTo64Bits(const EVT &OrigVT) {
6225 if (OrigVT.getSizeInBits() >= 64)
6226 return OrigVT;
6227
6228 assert(OrigVT.isSimple() && "Expecting a simple value type");
6229
6230 MVT::SimpleValueType OrigSimpleTy = OrigVT.getSimpleVT().SimpleTy;
6231 switch (OrigSimpleTy) {
6232 default: llvm_unreachable("Unexpected Vector Type");
6233 case MVT::v2i8:
6234 case MVT::v2i16:
6235 return MVT::v2i32;
6236 case MVT::v4i8:
6237 return MVT::v4i16;
6238 }
6239 }
6240
6241 /// AddRequiredExtensionForVMULL - Add a sign/zero extension to extend the total
6242 /// value size to 64 bits. We need a 64-bit D register as an operand to VMULL.
6243 /// We insert the required extension here to get the vector to fill a D register.
AddRequiredExtensionForVMULL(SDValue N,SelectionDAG & DAG,const EVT & OrigTy,const EVT & ExtTy,unsigned ExtOpcode)6244 static SDValue AddRequiredExtensionForVMULL(SDValue N, SelectionDAG &DAG,
6245 const EVT &OrigTy,
6246 const EVT &ExtTy,
6247 unsigned ExtOpcode) {
6248 // The vector originally had a size of OrigTy. It was then extended to ExtTy.
6249 // We expect the ExtTy to be 128-bits total. If the OrigTy is less than
6250 // 64-bits we need to insert a new extension so that it will be 64-bits.
6251 assert(ExtTy.is128BitVector() && "Unexpected extension size");
6252 if (OrigTy.getSizeInBits() >= 64)
6253 return N;
6254
6255 // Must extend size to at least 64 bits to be used as an operand for VMULL.
6256 EVT NewVT = getExtensionTo64Bits(OrigTy);
6257
6258 return DAG.getNode(ExtOpcode, SDLoc(N), NewVT, N);
6259 }
6260
6261 /// SkipLoadExtensionForVMULL - return a load of the original vector size that
6262 /// does not do any sign/zero extension. If the original vector is less
6263 /// than 64 bits, an appropriate extension will be added after the load to
6264 /// reach a total size of 64 bits. We have to add the extension separately
6265 /// because ARM does not have a sign/zero extending load for vectors.
SkipLoadExtensionForVMULL(LoadSDNode * LD,SelectionDAG & DAG)6266 static SDValue SkipLoadExtensionForVMULL(LoadSDNode *LD, SelectionDAG& DAG) {
6267 EVT ExtendedTy = getExtensionTo64Bits(LD->getMemoryVT());
6268
6269 // The load already has the right type.
6270 if (ExtendedTy == LD->getMemoryVT())
6271 return DAG.getLoad(LD->getMemoryVT(), SDLoc(LD), LD->getChain(),
6272 LD->getBasePtr(), LD->getPointerInfo(), LD->isVolatile(),
6273 LD->isNonTemporal(), LD->isInvariant(),
6274 LD->getAlignment());
6275
6276 // We need to create a zextload/sextload. We cannot just create a load
6277 // followed by a zext/zext node because LowerMUL is also run during normal
6278 // operation legalization where we can't create illegal types.
6279 return DAG.getExtLoad(LD->getExtensionType(), SDLoc(LD), ExtendedTy,
6280 LD->getChain(), LD->getBasePtr(), LD->getPointerInfo(),
6281 LD->getMemoryVT(), LD->isVolatile(), LD->isInvariant(),
6282 LD->isNonTemporal(), LD->getAlignment());
6283 }
6284
6285 /// SkipExtensionForVMULL - For a node that is a SIGN_EXTEND, ZERO_EXTEND,
6286 /// extending load, or BUILD_VECTOR with extended elements, return the
6287 /// unextended value. The unextended vector should be 64 bits so that it can
6288 /// be used as an operand to a VMULL instruction. If the original vector size
6289 /// before extension is less than 64 bits we add a an extension to resize
6290 /// the vector to 64 bits.
SkipExtensionForVMULL(SDNode * N,SelectionDAG & DAG)6291 static SDValue SkipExtensionForVMULL(SDNode *N, SelectionDAG &DAG) {
6292 if (N->getOpcode() == ISD::SIGN_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND)
6293 return AddRequiredExtensionForVMULL(N->getOperand(0), DAG,
6294 N->getOperand(0)->getValueType(0),
6295 N->getValueType(0),
6296 N->getOpcode());
6297
6298 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N))
6299 return SkipLoadExtensionForVMULL(LD, DAG);
6300
6301 // Otherwise, the value must be a BUILD_VECTOR. For v2i64, it will
6302 // have been legalized as a BITCAST from v4i32.
6303 if (N->getOpcode() == ISD::BITCAST) {
6304 SDNode *BVN = N->getOperand(0).getNode();
6305 assert(BVN->getOpcode() == ISD::BUILD_VECTOR &&
6306 BVN->getValueType(0) == MVT::v4i32 && "expected v4i32 BUILD_VECTOR");
6307 unsigned LowElt = DAG.getDataLayout().isBigEndian() ? 1 : 0;
6308 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(N), MVT::v2i32,
6309 BVN->getOperand(LowElt), BVN->getOperand(LowElt+2));
6310 }
6311 // Construct a new BUILD_VECTOR with elements truncated to half the size.
6312 assert(N->getOpcode() == ISD::BUILD_VECTOR && "expected BUILD_VECTOR");
6313 EVT VT = N->getValueType(0);
6314 unsigned EltSize = VT.getVectorElementType().getSizeInBits() / 2;
6315 unsigned NumElts = VT.getVectorNumElements();
6316 MVT TruncVT = MVT::getIntegerVT(EltSize);
6317 SmallVector<SDValue, 8> Ops;
6318 SDLoc dl(N);
6319 for (unsigned i = 0; i != NumElts; ++i) {
6320 ConstantSDNode *C = cast<ConstantSDNode>(N->getOperand(i));
6321 const APInt &CInt = C->getAPIntValue();
6322 // Element types smaller than 32 bits are not legal, so use i32 elements.
6323 // The values are implicitly truncated so sext vs. zext doesn't matter.
6324 Ops.push_back(DAG.getConstant(CInt.zextOrTrunc(32), dl, MVT::i32));
6325 }
6326 return DAG.getNode(ISD::BUILD_VECTOR, dl,
6327 MVT::getVectorVT(TruncVT, NumElts), Ops);
6328 }
6329
isAddSubSExt(SDNode * N,SelectionDAG & DAG)6330 static bool isAddSubSExt(SDNode *N, SelectionDAG &DAG) {
6331 unsigned Opcode = N->getOpcode();
6332 if (Opcode == ISD::ADD || Opcode == ISD::SUB) {
6333 SDNode *N0 = N->getOperand(0).getNode();
6334 SDNode *N1 = N->getOperand(1).getNode();
6335 return N0->hasOneUse() && N1->hasOneUse() &&
6336 isSignExtended(N0, DAG) && isSignExtended(N1, DAG);
6337 }
6338 return false;
6339 }
6340
isAddSubZExt(SDNode * N,SelectionDAG & DAG)6341 static bool isAddSubZExt(SDNode *N, SelectionDAG &DAG) {
6342 unsigned Opcode = N->getOpcode();
6343 if (Opcode == ISD::ADD || Opcode == ISD::SUB) {
6344 SDNode *N0 = N->getOperand(0).getNode();
6345 SDNode *N1 = N->getOperand(1).getNode();
6346 return N0->hasOneUse() && N1->hasOneUse() &&
6347 isZeroExtended(N0, DAG) && isZeroExtended(N1, DAG);
6348 }
6349 return false;
6350 }
6351
LowerMUL(SDValue Op,SelectionDAG & DAG)6352 static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) {
6353 // Multiplications are only custom-lowered for 128-bit vectors so that
6354 // VMULL can be detected. Otherwise v2i64 multiplications are not legal.
6355 EVT VT = Op.getValueType();
6356 assert(VT.is128BitVector() && VT.isInteger() &&
6357 "unexpected type for custom-lowering ISD::MUL");
6358 SDNode *N0 = Op.getOperand(0).getNode();
6359 SDNode *N1 = Op.getOperand(1).getNode();
6360 unsigned NewOpc = 0;
6361 bool isMLA = false;
6362 bool isN0SExt = isSignExtended(N0, DAG);
6363 bool isN1SExt = isSignExtended(N1, DAG);
6364 if (isN0SExt && isN1SExt)
6365 NewOpc = ARMISD::VMULLs;
6366 else {
6367 bool isN0ZExt = isZeroExtended(N0, DAG);
6368 bool isN1ZExt = isZeroExtended(N1, DAG);
6369 if (isN0ZExt && isN1ZExt)
6370 NewOpc = ARMISD::VMULLu;
6371 else if (isN1SExt || isN1ZExt) {
6372 // Look for (s/zext A + s/zext B) * (s/zext C). We want to turn these
6373 // into (s/zext A * s/zext C) + (s/zext B * s/zext C)
6374 if (isN1SExt && isAddSubSExt(N0, DAG)) {
6375 NewOpc = ARMISD::VMULLs;
6376 isMLA = true;
6377 } else if (isN1ZExt && isAddSubZExt(N0, DAG)) {
6378 NewOpc = ARMISD::VMULLu;
6379 isMLA = true;
6380 } else if (isN0ZExt && isAddSubZExt(N1, DAG)) {
6381 std::swap(N0, N1);
6382 NewOpc = ARMISD::VMULLu;
6383 isMLA = true;
6384 }
6385 }
6386
6387 if (!NewOpc) {
6388 if (VT == MVT::v2i64)
6389 // Fall through to expand this. It is not legal.
6390 return SDValue();
6391 else
6392 // Other vector multiplications are legal.
6393 return Op;
6394 }
6395 }
6396
6397 // Legalize to a VMULL instruction.
6398 SDLoc DL(Op);
6399 SDValue Op0;
6400 SDValue Op1 = SkipExtensionForVMULL(N1, DAG);
6401 if (!isMLA) {
6402 Op0 = SkipExtensionForVMULL(N0, DAG);
6403 assert(Op0.getValueType().is64BitVector() &&
6404 Op1.getValueType().is64BitVector() &&
6405 "unexpected types for extended operands to VMULL");
6406 return DAG.getNode(NewOpc, DL, VT, Op0, Op1);
6407 }
6408
6409 // Optimizing (zext A + zext B) * C, to (VMULL A, C) + (VMULL B, C) during
6410 // isel lowering to take advantage of no-stall back to back vmul + vmla.
6411 // vmull q0, d4, d6
6412 // vmlal q0, d5, d6
6413 // is faster than
6414 // vaddl q0, d4, d5
6415 // vmovl q1, d6
6416 // vmul q0, q0, q1
6417 SDValue N00 = SkipExtensionForVMULL(N0->getOperand(0).getNode(), DAG);
6418 SDValue N01 = SkipExtensionForVMULL(N0->getOperand(1).getNode(), DAG);
6419 EVT Op1VT = Op1.getValueType();
6420 return DAG.getNode(N0->getOpcode(), DL, VT,
6421 DAG.getNode(NewOpc, DL, VT,
6422 DAG.getNode(ISD::BITCAST, DL, Op1VT, N00), Op1),
6423 DAG.getNode(NewOpc, DL, VT,
6424 DAG.getNode(ISD::BITCAST, DL, Op1VT, N01), Op1));
6425 }
6426
6427 static SDValue
LowerSDIV_v4i8(SDValue X,SDValue Y,SDLoc dl,SelectionDAG & DAG)6428 LowerSDIV_v4i8(SDValue X, SDValue Y, SDLoc dl, SelectionDAG &DAG) {
6429 // TODO: Should this propagate fast-math-flags?
6430
6431 // Convert to float
6432 // float4 xf = vcvt_f32_s32(vmovl_s16(a.lo));
6433 // float4 yf = vcvt_f32_s32(vmovl_s16(b.lo));
6434 X = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, X);
6435 Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, Y);
6436 X = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, X);
6437 Y = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, Y);
6438 // Get reciprocal estimate.
6439 // float4 recip = vrecpeq_f32(yf);
6440 Y = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
6441 DAG.getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
6442 Y);
6443 // Because char has a smaller range than uchar, we can actually get away
6444 // without any newton steps. This requires that we use a weird bias
6445 // of 0xb000, however (again, this has been exhaustively tested).
6446 // float4 result = as_float4(as_int4(xf*recip) + 0xb000);
6447 X = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, X, Y);
6448 X = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, X);
6449 Y = DAG.getConstant(0xb000, dl, MVT::i32);
6450 Y = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Y, Y, Y, Y);
6451 X = DAG.getNode(ISD::ADD, dl, MVT::v4i32, X, Y);
6452 X = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, X);
6453 // Convert back to short.
6454 X = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v4i32, X);
6455 X = DAG.getNode(ISD::TRUNCATE, dl, MVT::v4i16, X);
6456 return X;
6457 }
6458
6459 static SDValue
LowerSDIV_v4i16(SDValue N0,SDValue N1,SDLoc dl,SelectionDAG & DAG)6460 LowerSDIV_v4i16(SDValue N0, SDValue N1, SDLoc dl, SelectionDAG &DAG) {
6461 // TODO: Should this propagate fast-math-flags?
6462
6463 SDValue N2;
6464 // Convert to float.
6465 // float4 yf = vcvt_f32_s32(vmovl_s16(y));
6466 // float4 xf = vcvt_f32_s32(vmovl_s16(x));
6467 N0 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, N0);
6468 N1 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, N1);
6469 N0 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N0);
6470 N1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N1);
6471
6472 // Use reciprocal estimate and one refinement step.
6473 // float4 recip = vrecpeq_f32(yf);
6474 // recip *= vrecpsq_f32(yf, recip);
6475 N2 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
6476 DAG.getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
6477 N1);
6478 N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
6479 DAG.getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
6480 N1, N2);
6481 N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2);
6482 // Because short has a smaller range than ushort, we can actually get away
6483 // with only a single newton step. This requires that we use a weird bias
6484 // of 89, however (again, this has been exhaustively tested).
6485 // float4 result = as_float4(as_int4(xf*recip) + 0x89);
6486 N0 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N0, N2);
6487 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, N0);
6488 N1 = DAG.getConstant(0x89, dl, MVT::i32);
6489 N1 = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, N1, N1, N1, N1);
6490 N0 = DAG.getNode(ISD::ADD, dl, MVT::v4i32, N0, N1);
6491 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, N0);
6492 // Convert back to integer and return.
6493 // return vmovn_s32(vcvt_s32_f32(result));
6494 N0 = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v4i32, N0);
6495 N0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::v4i16, N0);
6496 return N0;
6497 }
6498
LowerSDIV(SDValue Op,SelectionDAG & DAG)6499 static SDValue LowerSDIV(SDValue Op, SelectionDAG &DAG) {
6500 EVT VT = Op.getValueType();
6501 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
6502 "unexpected type for custom-lowering ISD::SDIV");
6503
6504 SDLoc dl(Op);
6505 SDValue N0 = Op.getOperand(0);
6506 SDValue N1 = Op.getOperand(1);
6507 SDValue N2, N3;
6508
6509 if (VT == MVT::v8i8) {
6510 N0 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v8i16, N0);
6511 N1 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v8i16, N1);
6512
6513 N2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0,
6514 DAG.getIntPtrConstant(4, dl));
6515 N3 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1,
6516 DAG.getIntPtrConstant(4, dl));
6517 N0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0,
6518 DAG.getIntPtrConstant(0, dl));
6519 N1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1,
6520 DAG.getIntPtrConstant(0, dl));
6521
6522 N0 = LowerSDIV_v4i8(N0, N1, dl, DAG); // v4i16
6523 N2 = LowerSDIV_v4i8(N2, N3, dl, DAG); // v4i16
6524
6525 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i16, N0, N2);
6526 N0 = LowerCONCAT_VECTORS(N0, DAG);
6527
6528 N0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::v8i8, N0);
6529 return N0;
6530 }
6531 return LowerSDIV_v4i16(N0, N1, dl, DAG);
6532 }
6533
LowerUDIV(SDValue Op,SelectionDAG & DAG)6534 static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG) {
6535 // TODO: Should this propagate fast-math-flags?
6536 EVT VT = Op.getValueType();
6537 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&
6538 "unexpected type for custom-lowering ISD::UDIV");
6539
6540 SDLoc dl(Op);
6541 SDValue N0 = Op.getOperand(0);
6542 SDValue N1 = Op.getOperand(1);
6543 SDValue N2, N3;
6544
6545 if (VT == MVT::v8i8) {
6546 N0 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v8i16, N0);
6547 N1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v8i16, N1);
6548
6549 N2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0,
6550 DAG.getIntPtrConstant(4, dl));
6551 N3 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1,
6552 DAG.getIntPtrConstant(4, dl));
6553 N0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0,
6554 DAG.getIntPtrConstant(0, dl));
6555 N1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1,
6556 DAG.getIntPtrConstant(0, dl));
6557
6558 N0 = LowerSDIV_v4i16(N0, N1, dl, DAG); // v4i16
6559 N2 = LowerSDIV_v4i16(N2, N3, dl, DAG); // v4i16
6560
6561 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i16, N0, N2);
6562 N0 = LowerCONCAT_VECTORS(N0, DAG);
6563
6564 N0 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v8i8,
6565 DAG.getConstant(Intrinsic::arm_neon_vqmovnsu, dl,
6566 MVT::i32),
6567 N0);
6568 return N0;
6569 }
6570
6571 // v4i16 sdiv ... Convert to float.
6572 // float4 yf = vcvt_f32_s32(vmovl_u16(y));
6573 // float4 xf = vcvt_f32_s32(vmovl_u16(x));
6574 N0 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v4i32, N0);
6575 N1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v4i32, N1);
6576 N0 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N0);
6577 SDValue BN1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N1);
6578
6579 // Use reciprocal estimate and two refinement steps.
6580 // float4 recip = vrecpeq_f32(yf);
6581 // recip *= vrecpsq_f32(yf, recip);
6582 // recip *= vrecpsq_f32(yf, recip);
6583 N2 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
6584 DAG.getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
6585 BN1);
6586 N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
6587 DAG.getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
6588 BN1, N2);
6589 N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2);
6590 N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
6591 DAG.getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
6592 BN1, N2);
6593 N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2);
6594 // Simply multiplying by the reciprocal estimate can leave us a few ulps
6595 // too low, so we add 2 ulps (exhaustive testing shows that this is enough,
6596 // and that it will never cause us to return an answer too large).
6597 // float4 result = as_float4(as_int4(xf*recip) + 2);
6598 N0 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N0, N2);
6599 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, N0);
6600 N1 = DAG.getConstant(2, dl, MVT::i32);
6601 N1 = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, N1, N1, N1, N1);
6602 N0 = DAG.getNode(ISD::ADD, dl, MVT::v4i32, N0, N1);
6603 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, N0);
6604 // Convert back to integer and return.
6605 // return vmovn_u32(vcvt_s32_f32(result));
6606 N0 = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v4i32, N0);
6607 N0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::v4i16, N0);
6608 return N0;
6609 }
6610
LowerADDC_ADDE_SUBC_SUBE(SDValue Op,SelectionDAG & DAG)6611 static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
6612 EVT VT = Op.getNode()->getValueType(0);
6613 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
6614
6615 unsigned Opc;
6616 bool ExtraOp = false;
6617 switch (Op.getOpcode()) {
6618 default: llvm_unreachable("Invalid code");
6619 case ISD::ADDC: Opc = ARMISD::ADDC; break;
6620 case ISD::ADDE: Opc = ARMISD::ADDE; ExtraOp = true; break;
6621 case ISD::SUBC: Opc = ARMISD::SUBC; break;
6622 case ISD::SUBE: Opc = ARMISD::SUBE; ExtraOp = true; break;
6623 }
6624
6625 if (!ExtraOp)
6626 return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0),
6627 Op.getOperand(1));
6628 return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0),
6629 Op.getOperand(1), Op.getOperand(2));
6630 }
6631
LowerFSINCOS(SDValue Op,SelectionDAG & DAG) const6632 SDValue ARMTargetLowering::LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const {
6633 assert(Subtarget->isTargetDarwin());
6634
6635 // For iOS, we want to call an alternative entry point: __sincos_stret,
6636 // return values are passed via sret.
6637 SDLoc dl(Op);
6638 SDValue Arg = Op.getOperand(0);
6639 EVT ArgVT = Arg.getValueType();
6640 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
6641 auto PtrVT = getPointerTy(DAG.getDataLayout());
6642
6643 MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo();
6644 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6645
6646 // Pair of floats / doubles used to pass the result.
6647 Type *RetTy = StructType::get(ArgTy, ArgTy, nullptr);
6648 auto &DL = DAG.getDataLayout();
6649
6650 ArgListTy Args;
6651 bool ShouldUseSRet = Subtarget->isAPCS_ABI();
6652 SDValue SRet;
6653 if (ShouldUseSRet) {
6654 // Create stack object for sret.
6655 const uint64_t ByteSize = DL.getTypeAllocSize(RetTy);
6656 const unsigned StackAlign = DL.getPrefTypeAlignment(RetTy);
6657 int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false);
6658 SRet = DAG.getFrameIndex(FrameIdx, TLI.getPointerTy(DL));
6659
6660 ArgListEntry Entry;
6661 Entry.Node = SRet;
6662 Entry.Ty = RetTy->getPointerTo();
6663 Entry.isSExt = false;
6664 Entry.isZExt = false;
6665 Entry.isSRet = true;
6666 Args.push_back(Entry);
6667 RetTy = Type::getVoidTy(*DAG.getContext());
6668 }
6669
6670 ArgListEntry Entry;
6671 Entry.Node = Arg;
6672 Entry.Ty = ArgTy;
6673 Entry.isSExt = false;
6674 Entry.isZExt = false;
6675 Args.push_back(Entry);
6676
6677 const char *LibcallName =
6678 (ArgVT == MVT::f64) ? "__sincos_stret" : "__sincosf_stret";
6679 RTLIB::Libcall LC =
6680 (ArgVT == MVT::f64) ? RTLIB::SINCOS_F64 : RTLIB::SINCOS_F32;
6681 CallingConv::ID CC = getLibcallCallingConv(LC);
6682 SDValue Callee = DAG.getExternalSymbol(LibcallName, getPointerTy(DL));
6683
6684 TargetLowering::CallLoweringInfo CLI(DAG);
6685 CLI.setDebugLoc(dl)
6686 .setChain(DAG.getEntryNode())
6687 .setCallee(CC, RetTy, Callee, std::move(Args), 0)
6688 .setDiscardResult(ShouldUseSRet);
6689 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
6690
6691 if (!ShouldUseSRet)
6692 return CallResult.first;
6693
6694 SDValue LoadSin = DAG.getLoad(ArgVT, dl, CallResult.second, SRet,
6695 MachinePointerInfo(), false, false, false, 0);
6696
6697 // Address of cos field.
6698 SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, SRet,
6699 DAG.getIntPtrConstant(ArgVT.getStoreSize(), dl));
6700 SDValue LoadCos = DAG.getLoad(ArgVT, dl, LoadSin.getValue(1), Add,
6701 MachinePointerInfo(), false, false, false, 0);
6702
6703 SDVTList Tys = DAG.getVTList(ArgVT, ArgVT);
6704 return DAG.getNode(ISD::MERGE_VALUES, dl, Tys,
6705 LoadSin.getValue(0), LoadCos.getValue(0));
6706 }
6707
LowerWindowsDIVLibCall(SDValue Op,SelectionDAG & DAG,bool Signed,SDValue & Chain) const6708 SDValue ARMTargetLowering::LowerWindowsDIVLibCall(SDValue Op, SelectionDAG &DAG,
6709 bool Signed,
6710 SDValue &Chain) const {
6711 EVT VT = Op.getValueType();
6712 assert((VT == MVT::i32 || VT == MVT::i64) &&
6713 "unexpected type for custom lowering DIV");
6714 SDLoc dl(Op);
6715
6716 const auto &DL = DAG.getDataLayout();
6717 const auto &TLI = DAG.getTargetLoweringInfo();
6718
6719 const char *Name = nullptr;
6720 if (Signed)
6721 Name = (VT == MVT::i32) ? "__rt_sdiv" : "__rt_sdiv64";
6722 else
6723 Name = (VT == MVT::i32) ? "__rt_udiv" : "__rt_udiv64";
6724
6725 SDValue ES = DAG.getExternalSymbol(Name, TLI.getPointerTy(DL));
6726
6727 ARMTargetLowering::ArgListTy Args;
6728
6729 for (auto AI : {1, 0}) {
6730 ArgListEntry Arg;
6731 Arg.Node = Op.getOperand(AI);
6732 Arg.Ty = Arg.Node.getValueType().getTypeForEVT(*DAG.getContext());
6733 Args.push_back(Arg);
6734 }
6735
6736 CallLoweringInfo CLI(DAG);
6737 CLI.setDebugLoc(dl)
6738 .setChain(Chain)
6739 .setCallee(CallingConv::ARM_AAPCS_VFP, VT.getTypeForEVT(*DAG.getContext()),
6740 ES, std::move(Args), 0);
6741
6742 return LowerCallTo(CLI).first;
6743 }
6744
LowerDIV_Windows(SDValue Op,SelectionDAG & DAG,bool Signed) const6745 SDValue ARMTargetLowering::LowerDIV_Windows(SDValue Op, SelectionDAG &DAG,
6746 bool Signed) const {
6747 assert(Op.getValueType() == MVT::i32 &&
6748 "unexpected type for custom lowering DIV");
6749 SDLoc dl(Op);
6750
6751 SDValue DBZCHK = DAG.getNode(ARMISD::WIN__DBZCHK, dl, MVT::Other,
6752 DAG.getEntryNode(), Op.getOperand(1));
6753
6754 return LowerWindowsDIVLibCall(Op, DAG, Signed, DBZCHK);
6755 }
6756
ExpandDIV_Windows(SDValue Op,SelectionDAG & DAG,bool Signed,SmallVectorImpl<SDValue> & Results) const6757 void ARMTargetLowering::ExpandDIV_Windows(
6758 SDValue Op, SelectionDAG &DAG, bool Signed,
6759 SmallVectorImpl<SDValue> &Results) const {
6760 const auto &DL = DAG.getDataLayout();
6761 const auto &TLI = DAG.getTargetLoweringInfo();
6762
6763 assert(Op.getValueType() == MVT::i64 &&
6764 "unexpected type for custom lowering DIV");
6765 SDLoc dl(Op);
6766
6767 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Op.getOperand(1),
6768 DAG.getConstant(0, dl, MVT::i32));
6769 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Op.getOperand(1),
6770 DAG.getConstant(1, dl, MVT::i32));
6771 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::i32, Lo, Hi);
6772
6773 SDValue DBZCHK =
6774 DAG.getNode(ARMISD::WIN__DBZCHK, dl, MVT::Other, DAG.getEntryNode(), Or);
6775
6776 SDValue Result = LowerWindowsDIVLibCall(Op, DAG, Signed, DBZCHK);
6777
6778 SDValue Lower = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Result);
6779 SDValue Upper = DAG.getNode(ISD::SRL, dl, MVT::i64, Result,
6780 DAG.getConstant(32, dl, TLI.getPointerTy(DL)));
6781 Upper = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Upper);
6782
6783 Results.push_back(Lower);
6784 Results.push_back(Upper);
6785 }
6786
LowerAtomicLoadStore(SDValue Op,SelectionDAG & DAG)6787 static SDValue LowerAtomicLoadStore(SDValue Op, SelectionDAG &DAG) {
6788 // Monotonic load/store is legal for all targets
6789 if (cast<AtomicSDNode>(Op)->getOrdering() <= Monotonic)
6790 return Op;
6791
6792 // Acquire/Release load/store is not legal for targets without a
6793 // dmb or equivalent available.
6794 return SDValue();
6795 }
6796
ReplaceREADCYCLECOUNTER(SDNode * N,SmallVectorImpl<SDValue> & Results,SelectionDAG & DAG,const ARMSubtarget * Subtarget)6797 static void ReplaceREADCYCLECOUNTER(SDNode *N,
6798 SmallVectorImpl<SDValue> &Results,
6799 SelectionDAG &DAG,
6800 const ARMSubtarget *Subtarget) {
6801 SDLoc DL(N);
6802 // Under Power Management extensions, the cycle-count is:
6803 // mrc p15, #0, <Rt>, c9, c13, #0
6804 SDValue Ops[] = { N->getOperand(0), // Chain
6805 DAG.getConstant(Intrinsic::arm_mrc, DL, MVT::i32),
6806 DAG.getConstant(15, DL, MVT::i32),
6807 DAG.getConstant(0, DL, MVT::i32),
6808 DAG.getConstant(9, DL, MVT::i32),
6809 DAG.getConstant(13, DL, MVT::i32),
6810 DAG.getConstant(0, DL, MVT::i32)
6811 };
6812
6813 SDValue Cycles32 = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
6814 DAG.getVTList(MVT::i32, MVT::Other), Ops);
6815 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Cycles32,
6816 DAG.getConstant(0, DL, MVT::i32)));
6817 Results.push_back(Cycles32.getValue(1));
6818 }
6819
LowerOperation(SDValue Op,SelectionDAG & DAG) const6820 SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
6821 switch (Op.getOpcode()) {
6822 default: llvm_unreachable("Don't know how to custom lower this!");
6823 case ISD::WRITE_REGISTER: return LowerWRITE_REGISTER(Op, DAG);
6824 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
6825 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
6826 case ISD::GlobalAddress:
6827 switch (Subtarget->getTargetTriple().getObjectFormat()) {
6828 default: llvm_unreachable("unknown object format");
6829 case Triple::COFF:
6830 return LowerGlobalAddressWindows(Op, DAG);
6831 case Triple::ELF:
6832 return LowerGlobalAddressELF(Op, DAG);
6833 case Triple::MachO:
6834 return LowerGlobalAddressDarwin(Op, DAG);
6835 }
6836 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
6837 case ISD::SELECT: return LowerSELECT(Op, DAG);
6838 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG);
6839 case ISD::BR_CC: return LowerBR_CC(Op, DAG);
6840 case ISD::BR_JT: return LowerBR_JT(Op, DAG);
6841 case ISD::VASTART: return LowerVASTART(Op, DAG);
6842 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, DAG, Subtarget);
6843 case ISD::PREFETCH: return LowerPREFETCH(Op, DAG, Subtarget);
6844 case ISD::SINT_TO_FP:
6845 case ISD::UINT_TO_FP: return LowerINT_TO_FP(Op, DAG);
6846 case ISD::FP_TO_SINT:
6847 case ISD::FP_TO_UINT: return LowerFP_TO_INT(Op, DAG);
6848 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG);
6849 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
6850 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
6851 case ISD::EH_SJLJ_SETJMP: return LowerEH_SJLJ_SETJMP(Op, DAG);
6852 case ISD::EH_SJLJ_LONGJMP: return LowerEH_SJLJ_LONGJMP(Op, DAG);
6853 case ISD::EH_SJLJ_SETUP_DISPATCH: return LowerEH_SJLJ_SETUP_DISPATCH(Op, DAG);
6854 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG,
6855 Subtarget);
6856 case ISD::BITCAST: return ExpandBITCAST(Op.getNode(), DAG);
6857 case ISD::SHL:
6858 case ISD::SRL:
6859 case ISD::SRA: return LowerShift(Op.getNode(), DAG, Subtarget);
6860 case ISD::SREM: return LowerREM(Op.getNode(), DAG);
6861 case ISD::UREM: return LowerREM(Op.getNode(), DAG);
6862 case ISD::SHL_PARTS: return LowerShiftLeftParts(Op, DAG);
6863 case ISD::SRL_PARTS:
6864 case ISD::SRA_PARTS: return LowerShiftRightParts(Op, DAG);
6865 case ISD::CTTZ:
6866 case ISD::CTTZ_ZERO_UNDEF: return LowerCTTZ(Op.getNode(), DAG, Subtarget);
6867 case ISD::CTPOP: return LowerCTPOP(Op.getNode(), DAG, Subtarget);
6868 case ISD::SETCC: return LowerVSETCC(Op, DAG);
6869 case ISD::ConstantFP: return LowerConstantFP(Op, DAG, Subtarget);
6870 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG, Subtarget);
6871 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);
6872 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
6873 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
6874 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
6875 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG);
6876 case ISD::MUL: return LowerMUL(Op, DAG);
6877 case ISD::SDIV: return LowerSDIV(Op, DAG);
6878 case ISD::UDIV: return LowerUDIV(Op, DAG);
6879 case ISD::ADDC:
6880 case ISD::ADDE:
6881 case ISD::SUBC:
6882 case ISD::SUBE: return LowerADDC_ADDE_SUBC_SUBE(Op, DAG);
6883 case ISD::SADDO:
6884 case ISD::UADDO:
6885 case ISD::SSUBO:
6886 case ISD::USUBO:
6887 return LowerXALUO(Op, DAG);
6888 case ISD::ATOMIC_LOAD:
6889 case ISD::ATOMIC_STORE: return LowerAtomicLoadStore(Op, DAG);
6890 case ISD::FSINCOS: return LowerFSINCOS(Op, DAG);
6891 case ISD::SDIVREM:
6892 case ISD::UDIVREM: return LowerDivRem(Op, DAG);
6893 case ISD::DYNAMIC_STACKALLOC:
6894 if (Subtarget->getTargetTriple().isWindowsItaniumEnvironment())
6895 return LowerDYNAMIC_STACKALLOC(Op, DAG);
6896 llvm_unreachable("Don't know how to custom lower this!");
6897 case ISD::FP_ROUND: return LowerFP_ROUND(Op, DAG);
6898 case ISD::FP_EXTEND: return LowerFP_EXTEND(Op, DAG);
6899 case ARMISD::WIN__DBZCHK: return SDValue();
6900 }
6901 }
6902
6903 /// ReplaceNodeResults - Replace the results of node with an illegal result
6904 /// type with new values built out of custom code.
ReplaceNodeResults(SDNode * N,SmallVectorImpl<SDValue> & Results,SelectionDAG & DAG) const6905 void ARMTargetLowering::ReplaceNodeResults(SDNode *N,
6906 SmallVectorImpl<SDValue> &Results,
6907 SelectionDAG &DAG) const {
6908 SDValue Res;
6909 switch (N->getOpcode()) {
6910 default:
6911 llvm_unreachable("Don't know how to custom expand this!");
6912 case ISD::READ_REGISTER:
6913 ExpandREAD_REGISTER(N, Results, DAG);
6914 break;
6915 case ISD::BITCAST:
6916 Res = ExpandBITCAST(N, DAG);
6917 break;
6918 case ISD::SRL:
6919 case ISD::SRA:
6920 Res = Expand64BitShift(N, DAG, Subtarget);
6921 break;
6922 case ISD::SREM:
6923 case ISD::UREM:
6924 Res = LowerREM(N, DAG);
6925 break;
6926 case ISD::READCYCLECOUNTER:
6927 ReplaceREADCYCLECOUNTER(N, Results, DAG, Subtarget);
6928 return;
6929 case ISD::UDIV:
6930 case ISD::SDIV:
6931 assert(Subtarget->isTargetWindows() && "can only expand DIV on Windows");
6932 return ExpandDIV_Windows(SDValue(N, 0), DAG, N->getOpcode() == ISD::SDIV,
6933 Results);
6934 }
6935 if (Res.getNode())
6936 Results.push_back(Res);
6937 }
6938
6939 //===----------------------------------------------------------------------===//
6940 // ARM Scheduler Hooks
6941 //===----------------------------------------------------------------------===//
6942
6943 /// SetupEntryBlockForSjLj - Insert code into the entry block that creates and
6944 /// registers the function context.
6945 void ARMTargetLowering::
SetupEntryBlockForSjLj(MachineInstr * MI,MachineBasicBlock * MBB,MachineBasicBlock * DispatchBB,int FI) const6946 SetupEntryBlockForSjLj(MachineInstr *MI, MachineBasicBlock *MBB,
6947 MachineBasicBlock *DispatchBB, int FI) const {
6948 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
6949 DebugLoc dl = MI->getDebugLoc();
6950 MachineFunction *MF = MBB->getParent();
6951 MachineRegisterInfo *MRI = &MF->getRegInfo();
6952 MachineConstantPool *MCP = MF->getConstantPool();
6953 ARMFunctionInfo *AFI = MF->getInfo<ARMFunctionInfo>();
6954 const Function *F = MF->getFunction();
6955
6956 bool isThumb = Subtarget->isThumb();
6957 bool isThumb2 = Subtarget->isThumb2();
6958
6959 unsigned PCLabelId = AFI->createPICLabelUId();
6960 unsigned PCAdj = (isThumb || isThumb2) ? 4 : 8;
6961 ARMConstantPoolValue *CPV =
6962 ARMConstantPoolMBB::Create(F->getContext(), DispatchBB, PCLabelId, PCAdj);
6963 unsigned CPI = MCP->getConstantPoolIndex(CPV, 4);
6964
6965 const TargetRegisterClass *TRC = isThumb ? &ARM::tGPRRegClass
6966 : &ARM::GPRRegClass;
6967
6968 // Grab constant pool and fixed stack memory operands.
6969 MachineMemOperand *CPMMO =
6970 MF->getMachineMemOperand(MachinePointerInfo::getConstantPool(*MF),
6971 MachineMemOperand::MOLoad, 4, 4);
6972
6973 MachineMemOperand *FIMMOSt =
6974 MF->getMachineMemOperand(MachinePointerInfo::getFixedStack(*MF, FI),
6975 MachineMemOperand::MOStore, 4, 4);
6976
6977 // Load the address of the dispatch MBB into the jump buffer.
6978 if (isThumb2) {
6979 // Incoming value: jbuf
6980 // ldr.n r5, LCPI1_1
6981 // orr r5, r5, #1
6982 // add r5, pc
6983 // str r5, [$jbuf, #+4] ; &jbuf[1]
6984 unsigned NewVReg1 = MRI->createVirtualRegister(TRC);
6985 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::t2LDRpci), NewVReg1)
6986 .addConstantPoolIndex(CPI)
6987 .addMemOperand(CPMMO));
6988 // Set the low bit because of thumb mode.
6989 unsigned NewVReg2 = MRI->createVirtualRegister(TRC);
6990 AddDefaultCC(
6991 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::t2ORRri), NewVReg2)
6992 .addReg(NewVReg1, RegState::Kill)
6993 .addImm(0x01)));
6994 unsigned NewVReg3 = MRI->createVirtualRegister(TRC);
6995 BuildMI(*MBB, MI, dl, TII->get(ARM::tPICADD), NewVReg3)
6996 .addReg(NewVReg2, RegState::Kill)
6997 .addImm(PCLabelId);
6998 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::t2STRi12))
6999 .addReg(NewVReg3, RegState::Kill)
7000 .addFrameIndex(FI)
7001 .addImm(36) // &jbuf[1] :: pc
7002 .addMemOperand(FIMMOSt));
7003 } else if (isThumb) {
7004 // Incoming value: jbuf
7005 // ldr.n r1, LCPI1_4
7006 // add r1, pc
7007 // mov r2, #1
7008 // orrs r1, r2
7009 // add r2, $jbuf, #+4 ; &jbuf[1]
7010 // str r1, [r2]
7011 unsigned NewVReg1 = MRI->createVirtualRegister(TRC);
7012 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::tLDRpci), NewVReg1)
7013 .addConstantPoolIndex(CPI)
7014 .addMemOperand(CPMMO));
7015 unsigned NewVReg2 = MRI->createVirtualRegister(TRC);
7016 BuildMI(*MBB, MI, dl, TII->get(ARM::tPICADD), NewVReg2)
7017 .addReg(NewVReg1, RegState::Kill)
7018 .addImm(PCLabelId);
7019 // Set the low bit because of thumb mode.
7020 unsigned NewVReg3 = MRI->createVirtualRegister(TRC);
7021 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::tMOVi8), NewVReg3)
7022 .addReg(ARM::CPSR, RegState::Define)
7023 .addImm(1));
7024 unsigned NewVReg4 = MRI->createVirtualRegister(TRC);
7025 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::tORR), NewVReg4)
7026 .addReg(ARM::CPSR, RegState::Define)
7027 .addReg(NewVReg2, RegState::Kill)
7028 .addReg(NewVReg3, RegState::Kill));
7029 unsigned NewVReg5 = MRI->createVirtualRegister(TRC);
7030 BuildMI(*MBB, MI, dl, TII->get(ARM::tADDframe), NewVReg5)
7031 .addFrameIndex(FI)
7032 .addImm(36); // &jbuf[1] :: pc
7033 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::tSTRi))
7034 .addReg(NewVReg4, RegState::Kill)
7035 .addReg(NewVReg5, RegState::Kill)
7036 .addImm(0)
7037 .addMemOperand(FIMMOSt));
7038 } else {
7039 // Incoming value: jbuf
7040 // ldr r1, LCPI1_1
7041 // add r1, pc, r1
7042 // str r1, [$jbuf, #+4] ; &jbuf[1]
7043 unsigned NewVReg1 = MRI->createVirtualRegister(TRC);
7044 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::LDRi12), NewVReg1)
7045 .addConstantPoolIndex(CPI)
7046 .addImm(0)
7047 .addMemOperand(CPMMO));
7048 unsigned NewVReg2 = MRI->createVirtualRegister(TRC);
7049 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::PICADD), NewVReg2)
7050 .addReg(NewVReg1, RegState::Kill)
7051 .addImm(PCLabelId));
7052 AddDefaultPred(BuildMI(*MBB, MI, dl, TII->get(ARM::STRi12))
7053 .addReg(NewVReg2, RegState::Kill)
7054 .addFrameIndex(FI)
7055 .addImm(36) // &jbuf[1] :: pc
7056 .addMemOperand(FIMMOSt));
7057 }
7058 }
7059
EmitSjLjDispatchBlock(MachineInstr * MI,MachineBasicBlock * MBB) const7060 void ARMTargetLowering::EmitSjLjDispatchBlock(MachineInstr *MI,
7061 MachineBasicBlock *MBB) const {
7062 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
7063 DebugLoc dl = MI->getDebugLoc();
7064 MachineFunction *MF = MBB->getParent();
7065 MachineRegisterInfo *MRI = &MF->getRegInfo();
7066 MachineFrameInfo *MFI = MF->getFrameInfo();
7067 int FI = MFI->getFunctionContextIndex();
7068
7069 const TargetRegisterClass *TRC = Subtarget->isThumb() ? &ARM::tGPRRegClass
7070 : &ARM::GPRnopcRegClass;
7071
7072 // Get a mapping of the call site numbers to all of the landing pads they're
7073 // associated with.
7074 DenseMap<unsigned, SmallVector<MachineBasicBlock*, 2> > CallSiteNumToLPad;
7075 unsigned MaxCSNum = 0;
7076 MachineModuleInfo &MMI = MF->getMMI();
7077 for (MachineFunction::iterator BB = MF->begin(), E = MF->end(); BB != E;
7078 ++BB) {
7079 if (!BB->isEHPad()) continue;
7080
7081 // FIXME: We should assert that the EH_LABEL is the first MI in the landing
7082 // pad.
7083 for (MachineBasicBlock::iterator
7084 II = BB->begin(), IE = BB->end(); II != IE; ++II) {
7085 if (!II->isEHLabel()) continue;
7086
7087 MCSymbol *Sym = II->getOperand(0).getMCSymbol();
7088 if (!MMI.hasCallSiteLandingPad(Sym)) continue;
7089
7090 SmallVectorImpl<unsigned> &CallSiteIdxs = MMI.getCallSiteLandingPad(Sym);
7091 for (SmallVectorImpl<unsigned>::iterator
7092 CSI = CallSiteIdxs.begin(), CSE = CallSiteIdxs.end();
7093 CSI != CSE; ++CSI) {
7094 CallSiteNumToLPad[*CSI].push_back(&*BB);
7095 MaxCSNum = std::max(MaxCSNum, *CSI);
7096 }
7097 break;
7098 }
7099 }
7100
7101 // Get an ordered list of the machine basic blocks for the jump table.
7102 std::vector<MachineBasicBlock*> LPadList;
7103 SmallPtrSet<MachineBasicBlock*, 64> InvokeBBs;
7104 LPadList.reserve(CallSiteNumToLPad.size());
7105 for (unsigned I = 1; I <= MaxCSNum; ++I) {
7106 SmallVectorImpl<MachineBasicBlock*> &MBBList = CallSiteNumToLPad[I];
7107 for (SmallVectorImpl<MachineBasicBlock*>::iterator
7108 II = MBBList.begin(), IE = MBBList.end(); II != IE; ++II) {
7109 LPadList.push_back(*II);
7110 InvokeBBs.insert((*II)->pred_begin(), (*II)->pred_end());
7111 }
7112 }
7113
7114 assert(!LPadList.empty() &&
7115 "No landing pad destinations for the dispatch jump table!");
7116
7117 // Create the jump table and associated information.
7118 MachineJumpTableInfo *JTI =
7119 MF->getOrCreateJumpTableInfo(MachineJumpTableInfo::EK_Inline);
7120 unsigned MJTI = JTI->createJumpTableIndex(LPadList);
7121 Reloc::Model RelocM = getTargetMachine().getRelocationModel();
7122
7123 // Create the MBBs for the dispatch code.
7124
7125 // Shove the dispatch's address into the return slot in the function context.
7126 MachineBasicBlock *DispatchBB = MF->CreateMachineBasicBlock();
7127 DispatchBB->setIsEHPad();
7128
7129 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
7130 unsigned trap_opcode;
7131 if (Subtarget->isThumb())
7132 trap_opcode = ARM::tTRAP;
7133 else
7134 trap_opcode = Subtarget->useNaClTrap() ? ARM::TRAPNaCl : ARM::TRAP;
7135
7136 BuildMI(TrapBB, dl, TII->get(trap_opcode));
7137 DispatchBB->addSuccessor(TrapBB);
7138
7139 MachineBasicBlock *DispContBB = MF->CreateMachineBasicBlock();
7140 DispatchBB->addSuccessor(DispContBB);
7141
7142 // Insert and MBBs.
7143 MF->insert(MF->end(), DispatchBB);
7144 MF->insert(MF->end(), DispContBB);
7145 MF->insert(MF->end(), TrapBB);
7146
7147 // Insert code into the entry block that creates and registers the function
7148 // context.
7149 SetupEntryBlockForSjLj(MI, MBB, DispatchBB, FI);
7150
7151 MachineMemOperand *FIMMOLd = MF->getMachineMemOperand(
7152 MachinePointerInfo::getFixedStack(*MF, FI),
7153 MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile, 4, 4);
7154
7155 MachineInstrBuilder MIB;
7156 MIB = BuildMI(DispatchBB, dl, TII->get(ARM::Int_eh_sjlj_dispatchsetup));
7157
7158 const ARMBaseInstrInfo *AII = static_cast<const ARMBaseInstrInfo*>(TII);
7159 const ARMBaseRegisterInfo &RI = AII->getRegisterInfo();
7160
7161 // Add a register mask with no preserved registers. This results in all
7162 // registers being marked as clobbered.
7163 MIB.addRegMask(RI.getNoPreservedMask());
7164
7165 unsigned NumLPads = LPadList.size();
7166 if (Subtarget->isThumb2()) {
7167 unsigned NewVReg1 = MRI->createVirtualRegister(TRC);
7168 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::t2LDRi12), NewVReg1)
7169 .addFrameIndex(FI)
7170 .addImm(4)
7171 .addMemOperand(FIMMOLd));
7172
7173 if (NumLPads < 256) {
7174 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::t2CMPri))
7175 .addReg(NewVReg1)
7176 .addImm(LPadList.size()));
7177 } else {
7178 unsigned VReg1 = MRI->createVirtualRegister(TRC);
7179 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::t2MOVi16), VReg1)
7180 .addImm(NumLPads & 0xFFFF));
7181
7182 unsigned VReg2 = VReg1;
7183 if ((NumLPads & 0xFFFF0000) != 0) {
7184 VReg2 = MRI->createVirtualRegister(TRC);
7185 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::t2MOVTi16), VReg2)
7186 .addReg(VReg1)
7187 .addImm(NumLPads >> 16));
7188 }
7189
7190 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::t2CMPrr))
7191 .addReg(NewVReg1)
7192 .addReg(VReg2));
7193 }
7194
7195 BuildMI(DispatchBB, dl, TII->get(ARM::t2Bcc))
7196 .addMBB(TrapBB)
7197 .addImm(ARMCC::HI)
7198 .addReg(ARM::CPSR);
7199
7200 unsigned NewVReg3 = MRI->createVirtualRegister(TRC);
7201 AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::t2LEApcrelJT),NewVReg3)
7202 .addJumpTableIndex(MJTI));
7203
7204 unsigned NewVReg4 = MRI->createVirtualRegister(TRC);
7205 AddDefaultCC(
7206 AddDefaultPred(
7207 BuildMI(DispContBB, dl, TII->get(ARM::t2ADDrs), NewVReg4)
7208 .addReg(NewVReg3, RegState::Kill)
7209 .addReg(NewVReg1)
7210 .addImm(ARM_AM::getSORegOpc(ARM_AM::lsl, 2))));
7211
7212 BuildMI(DispContBB, dl, TII->get(ARM::t2BR_JT))
7213 .addReg(NewVReg4, RegState::Kill)
7214 .addReg(NewVReg1)
7215 .addJumpTableIndex(MJTI);
7216 } else if (Subtarget->isThumb()) {
7217 unsigned NewVReg1 = MRI->createVirtualRegister(TRC);
7218 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::tLDRspi), NewVReg1)
7219 .addFrameIndex(FI)
7220 .addImm(1)
7221 .addMemOperand(FIMMOLd));
7222
7223 if (NumLPads < 256) {
7224 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::tCMPi8))
7225 .addReg(NewVReg1)
7226 .addImm(NumLPads));
7227 } else {
7228 MachineConstantPool *ConstantPool = MF->getConstantPool();
7229 Type *Int32Ty = Type::getInt32Ty(MF->getFunction()->getContext());
7230 const Constant *C = ConstantInt::get(Int32Ty, NumLPads);
7231
7232 // MachineConstantPool wants an explicit alignment.
7233 unsigned Align = MF->getDataLayout().getPrefTypeAlignment(Int32Ty);
7234 if (Align == 0)
7235 Align = MF->getDataLayout().getTypeAllocSize(C->getType());
7236 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Align);
7237
7238 unsigned VReg1 = MRI->createVirtualRegister(TRC);
7239 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::tLDRpci))
7240 .addReg(VReg1, RegState::Define)
7241 .addConstantPoolIndex(Idx));
7242 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::tCMPr))
7243 .addReg(NewVReg1)
7244 .addReg(VReg1));
7245 }
7246
7247 BuildMI(DispatchBB, dl, TII->get(ARM::tBcc))
7248 .addMBB(TrapBB)
7249 .addImm(ARMCC::HI)
7250 .addReg(ARM::CPSR);
7251
7252 unsigned NewVReg2 = MRI->createVirtualRegister(TRC);
7253 AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::tLSLri), NewVReg2)
7254 .addReg(ARM::CPSR, RegState::Define)
7255 .addReg(NewVReg1)
7256 .addImm(2));
7257
7258 unsigned NewVReg3 = MRI->createVirtualRegister(TRC);
7259 AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::tLEApcrelJT), NewVReg3)
7260 .addJumpTableIndex(MJTI));
7261
7262 unsigned NewVReg4 = MRI->createVirtualRegister(TRC);
7263 AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::tADDrr), NewVReg4)
7264 .addReg(ARM::CPSR, RegState::Define)
7265 .addReg(NewVReg2, RegState::Kill)
7266 .addReg(NewVReg3));
7267
7268 MachineMemOperand *JTMMOLd = MF->getMachineMemOperand(
7269 MachinePointerInfo::getJumpTable(*MF), MachineMemOperand::MOLoad, 4, 4);
7270
7271 unsigned NewVReg5 = MRI->createVirtualRegister(TRC);
7272 AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::tLDRi), NewVReg5)
7273 .addReg(NewVReg4, RegState::Kill)
7274 .addImm(0)
7275 .addMemOperand(JTMMOLd));
7276
7277 unsigned NewVReg6 = NewVReg5;
7278 if (RelocM == Reloc::PIC_) {
7279 NewVReg6 = MRI->createVirtualRegister(TRC);
7280 AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::tADDrr), NewVReg6)
7281 .addReg(ARM::CPSR, RegState::Define)
7282 .addReg(NewVReg5, RegState::Kill)
7283 .addReg(NewVReg3));
7284 }
7285
7286 BuildMI(DispContBB, dl, TII->get(ARM::tBR_JTr))
7287 .addReg(NewVReg6, RegState::Kill)
7288 .addJumpTableIndex(MJTI);
7289 } else {
7290 unsigned NewVReg1 = MRI->createVirtualRegister(TRC);
7291 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::LDRi12), NewVReg1)
7292 .addFrameIndex(FI)
7293 .addImm(4)
7294 .addMemOperand(FIMMOLd));
7295
7296 if (NumLPads < 256) {
7297 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::CMPri))
7298 .addReg(NewVReg1)
7299 .addImm(NumLPads));
7300 } else if (Subtarget->hasV6T2Ops() && isUInt<16>(NumLPads)) {
7301 unsigned VReg1 = MRI->createVirtualRegister(TRC);
7302 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::MOVi16), VReg1)
7303 .addImm(NumLPads & 0xFFFF));
7304
7305 unsigned VReg2 = VReg1;
7306 if ((NumLPads & 0xFFFF0000) != 0) {
7307 VReg2 = MRI->createVirtualRegister(TRC);
7308 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::MOVTi16), VReg2)
7309 .addReg(VReg1)
7310 .addImm(NumLPads >> 16));
7311 }
7312
7313 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::CMPrr))
7314 .addReg(NewVReg1)
7315 .addReg(VReg2));
7316 } else {
7317 MachineConstantPool *ConstantPool = MF->getConstantPool();
7318 Type *Int32Ty = Type::getInt32Ty(MF->getFunction()->getContext());
7319 const Constant *C = ConstantInt::get(Int32Ty, NumLPads);
7320
7321 // MachineConstantPool wants an explicit alignment.
7322 unsigned Align = MF->getDataLayout().getPrefTypeAlignment(Int32Ty);
7323 if (Align == 0)
7324 Align = MF->getDataLayout().getTypeAllocSize(C->getType());
7325 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Align);
7326
7327 unsigned VReg1 = MRI->createVirtualRegister(TRC);
7328 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::LDRcp))
7329 .addReg(VReg1, RegState::Define)
7330 .addConstantPoolIndex(Idx)
7331 .addImm(0));
7332 AddDefaultPred(BuildMI(DispatchBB, dl, TII->get(ARM::CMPrr))
7333 .addReg(NewVReg1)
7334 .addReg(VReg1, RegState::Kill));
7335 }
7336
7337 BuildMI(DispatchBB, dl, TII->get(ARM::Bcc))
7338 .addMBB(TrapBB)
7339 .addImm(ARMCC::HI)
7340 .addReg(ARM::CPSR);
7341
7342 unsigned NewVReg3 = MRI->createVirtualRegister(TRC);
7343 AddDefaultCC(
7344 AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::MOVsi), NewVReg3)
7345 .addReg(NewVReg1)
7346 .addImm(ARM_AM::getSORegOpc(ARM_AM::lsl, 2))));
7347 unsigned NewVReg4 = MRI->createVirtualRegister(TRC);
7348 AddDefaultPred(BuildMI(DispContBB, dl, TII->get(ARM::LEApcrelJT), NewVReg4)
7349 .addJumpTableIndex(MJTI));
7350
7351 MachineMemOperand *JTMMOLd = MF->getMachineMemOperand(
7352 MachinePointerInfo::getJumpTable(*MF), MachineMemOperand::MOLoad, 4, 4);
7353 unsigned NewVReg5 = MRI->createVirtualRegister(TRC);
7354 AddDefaultPred(
7355 BuildMI(DispContBB, dl, TII->get(ARM::LDRrs), NewVReg5)
7356 .addReg(NewVReg3, RegState::Kill)
7357 .addReg(NewVReg4)
7358 .addImm(0)
7359 .addMemOperand(JTMMOLd));
7360
7361 if (RelocM == Reloc::PIC_) {
7362 BuildMI(DispContBB, dl, TII->get(ARM::BR_JTadd))
7363 .addReg(NewVReg5, RegState::Kill)
7364 .addReg(NewVReg4)
7365 .addJumpTableIndex(MJTI);
7366 } else {
7367 BuildMI(DispContBB, dl, TII->get(ARM::BR_JTr))
7368 .addReg(NewVReg5, RegState::Kill)
7369 .addJumpTableIndex(MJTI);
7370 }
7371 }
7372
7373 // Add the jump table entries as successors to the MBB.
7374 SmallPtrSet<MachineBasicBlock*, 8> SeenMBBs;
7375 for (std::vector<MachineBasicBlock*>::iterator
7376 I = LPadList.begin(), E = LPadList.end(); I != E; ++I) {
7377 MachineBasicBlock *CurMBB = *I;
7378 if (SeenMBBs.insert(CurMBB).second)
7379 DispContBB->addSuccessor(CurMBB);
7380 }
7381
7382 // N.B. the order the invoke BBs are processed in doesn't matter here.
7383 const MCPhysReg *SavedRegs = RI.getCalleeSavedRegs(MF);
7384 SmallVector<MachineBasicBlock*, 64> MBBLPads;
7385 for (MachineBasicBlock *BB : InvokeBBs) {
7386
7387 // Remove the landing pad successor from the invoke block and replace it
7388 // with the new dispatch block.
7389 SmallVector<MachineBasicBlock*, 4> Successors(BB->succ_begin(),
7390 BB->succ_end());
7391 while (!Successors.empty()) {
7392 MachineBasicBlock *SMBB = Successors.pop_back_val();
7393 if (SMBB->isEHPad()) {
7394 BB->removeSuccessor(SMBB);
7395 MBBLPads.push_back(SMBB);
7396 }
7397 }
7398
7399 BB->addSuccessor(DispatchBB, BranchProbability::getZero());
7400 BB->normalizeSuccProbs();
7401
7402 // Find the invoke call and mark all of the callee-saved registers as
7403 // 'implicit defined' so that they're spilled. This prevents code from
7404 // moving instructions to before the EH block, where they will never be
7405 // executed.
7406 for (MachineBasicBlock::reverse_iterator
7407 II = BB->rbegin(), IE = BB->rend(); II != IE; ++II) {
7408 if (!II->isCall()) continue;
7409
7410 DenseMap<unsigned, bool> DefRegs;
7411 for (MachineInstr::mop_iterator
7412 OI = II->operands_begin(), OE = II->operands_end();
7413 OI != OE; ++OI) {
7414 if (!OI->isReg()) continue;
7415 DefRegs[OI->getReg()] = true;
7416 }
7417
7418 MachineInstrBuilder MIB(*MF, &*II);
7419
7420 for (unsigned i = 0; SavedRegs[i] != 0; ++i) {
7421 unsigned Reg = SavedRegs[i];
7422 if (Subtarget->isThumb2() &&
7423 !ARM::tGPRRegClass.contains(Reg) &&
7424 !ARM::hGPRRegClass.contains(Reg))
7425 continue;
7426 if (Subtarget->isThumb1Only() && !ARM::tGPRRegClass.contains(Reg))
7427 continue;
7428 if (!Subtarget->isThumb() && !ARM::GPRRegClass.contains(Reg))
7429 continue;
7430 if (!DefRegs[Reg])
7431 MIB.addReg(Reg, RegState::ImplicitDefine | RegState::Dead);
7432 }
7433
7434 break;
7435 }
7436 }
7437
7438 // Mark all former landing pads as non-landing pads. The dispatch is the only
7439 // landing pad now.
7440 for (SmallVectorImpl<MachineBasicBlock*>::iterator
7441 I = MBBLPads.begin(), E = MBBLPads.end(); I != E; ++I)
7442 (*I)->setIsEHPad(false);
7443
7444 // The instruction is gone now.
7445 MI->eraseFromParent();
7446 }
7447
7448 static
OtherSucc(MachineBasicBlock * MBB,MachineBasicBlock * Succ)7449 MachineBasicBlock *OtherSucc(MachineBasicBlock *MBB, MachineBasicBlock *Succ) {
7450 for (MachineBasicBlock::succ_iterator I = MBB->succ_begin(),
7451 E = MBB->succ_end(); I != E; ++I)
7452 if (*I != Succ)
7453 return *I;
7454 llvm_unreachable("Expecting a BB with two successors!");
7455 }
7456
7457 /// Return the load opcode for a given load size. If load size >= 8,
7458 /// neon opcode will be returned.
getLdOpcode(unsigned LdSize,bool IsThumb1,bool IsThumb2)7459 static unsigned getLdOpcode(unsigned LdSize, bool IsThumb1, bool IsThumb2) {
7460 if (LdSize >= 8)
7461 return LdSize == 16 ? ARM::VLD1q32wb_fixed
7462 : LdSize == 8 ? ARM::VLD1d32wb_fixed : 0;
7463 if (IsThumb1)
7464 return LdSize == 4 ? ARM::tLDRi
7465 : LdSize == 2 ? ARM::tLDRHi
7466 : LdSize == 1 ? ARM::tLDRBi : 0;
7467 if (IsThumb2)
7468 return LdSize == 4 ? ARM::t2LDR_POST
7469 : LdSize == 2 ? ARM::t2LDRH_POST
7470 : LdSize == 1 ? ARM::t2LDRB_POST : 0;
7471 return LdSize == 4 ? ARM::LDR_POST_IMM
7472 : LdSize == 2 ? ARM::LDRH_POST
7473 : LdSize == 1 ? ARM::LDRB_POST_IMM : 0;
7474 }
7475
7476 /// Return the store opcode for a given store size. If store size >= 8,
7477 /// neon opcode will be returned.
getStOpcode(unsigned StSize,bool IsThumb1,bool IsThumb2)7478 static unsigned getStOpcode(unsigned StSize, bool IsThumb1, bool IsThumb2) {
7479 if (StSize >= 8)
7480 return StSize == 16 ? ARM::VST1q32wb_fixed
7481 : StSize == 8 ? ARM::VST1d32wb_fixed : 0;
7482 if (IsThumb1)
7483 return StSize == 4 ? ARM::tSTRi
7484 : StSize == 2 ? ARM::tSTRHi
7485 : StSize == 1 ? ARM::tSTRBi : 0;
7486 if (IsThumb2)
7487 return StSize == 4 ? ARM::t2STR_POST
7488 : StSize == 2 ? ARM::t2STRH_POST
7489 : StSize == 1 ? ARM::t2STRB_POST : 0;
7490 return StSize == 4 ? ARM::STR_POST_IMM
7491 : StSize == 2 ? ARM::STRH_POST
7492 : StSize == 1 ? ARM::STRB_POST_IMM : 0;
7493 }
7494
7495 /// Emit a post-increment load operation with given size. The instructions
7496 /// will be added to BB at Pos.
emitPostLd(MachineBasicBlock * BB,MachineInstr * Pos,const TargetInstrInfo * TII,DebugLoc dl,unsigned LdSize,unsigned Data,unsigned AddrIn,unsigned AddrOut,bool IsThumb1,bool IsThumb2)7497 static void emitPostLd(MachineBasicBlock *BB, MachineInstr *Pos,
7498 const TargetInstrInfo *TII, DebugLoc dl,
7499 unsigned LdSize, unsigned Data, unsigned AddrIn,
7500 unsigned AddrOut, bool IsThumb1, bool IsThumb2) {
7501 unsigned LdOpc = getLdOpcode(LdSize, IsThumb1, IsThumb2);
7502 assert(LdOpc != 0 && "Should have a load opcode");
7503 if (LdSize >= 8) {
7504 AddDefaultPred(BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data)
7505 .addReg(AddrOut, RegState::Define).addReg(AddrIn)
7506 .addImm(0));
7507 } else if (IsThumb1) {
7508 // load + update AddrIn
7509 AddDefaultPred(BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data)
7510 .addReg(AddrIn).addImm(0));
7511 MachineInstrBuilder MIB =
7512 BuildMI(*BB, Pos, dl, TII->get(ARM::tADDi8), AddrOut);
7513 MIB = AddDefaultT1CC(MIB);
7514 MIB.addReg(AddrIn).addImm(LdSize);
7515 AddDefaultPred(MIB);
7516 } else if (IsThumb2) {
7517 AddDefaultPred(BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data)
7518 .addReg(AddrOut, RegState::Define).addReg(AddrIn)
7519 .addImm(LdSize));
7520 } else { // arm
7521 AddDefaultPred(BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data)
7522 .addReg(AddrOut, RegState::Define).addReg(AddrIn)
7523 .addReg(0).addImm(LdSize));
7524 }
7525 }
7526
7527 /// Emit a post-increment store operation with given size. The instructions
7528 /// will be added to BB at Pos.
emitPostSt(MachineBasicBlock * BB,MachineInstr * Pos,const TargetInstrInfo * TII,DebugLoc dl,unsigned StSize,unsigned Data,unsigned AddrIn,unsigned AddrOut,bool IsThumb1,bool IsThumb2)7529 static void emitPostSt(MachineBasicBlock *BB, MachineInstr *Pos,
7530 const TargetInstrInfo *TII, DebugLoc dl,
7531 unsigned StSize, unsigned Data, unsigned AddrIn,
7532 unsigned AddrOut, bool IsThumb1, bool IsThumb2) {
7533 unsigned StOpc = getStOpcode(StSize, IsThumb1, IsThumb2);
7534 assert(StOpc != 0 && "Should have a store opcode");
7535 if (StSize >= 8) {
7536 AddDefaultPred(BuildMI(*BB, Pos, dl, TII->get(StOpc), AddrOut)
7537 .addReg(AddrIn).addImm(0).addReg(Data));
7538 } else if (IsThumb1) {
7539 // store + update AddrIn
7540 AddDefaultPred(BuildMI(*BB, Pos, dl, TII->get(StOpc)).addReg(Data)
7541 .addReg(AddrIn).addImm(0));
7542 MachineInstrBuilder MIB =
7543 BuildMI(*BB, Pos, dl, TII->get(ARM::tADDi8), AddrOut);
7544 MIB = AddDefaultT1CC(MIB);
7545 MIB.addReg(AddrIn).addImm(StSize);
7546 AddDefaultPred(MIB);
7547 } else if (IsThumb2) {
7548 AddDefaultPred(BuildMI(*BB, Pos, dl, TII->get(StOpc), AddrOut)
7549 .addReg(Data).addReg(AddrIn).addImm(StSize));
7550 } else { // arm
7551 AddDefaultPred(BuildMI(*BB, Pos, dl, TII->get(StOpc), AddrOut)
7552 .addReg(Data).addReg(AddrIn).addReg(0)
7553 .addImm(StSize));
7554 }
7555 }
7556
7557 MachineBasicBlock *
EmitStructByval(MachineInstr * MI,MachineBasicBlock * BB) const7558 ARMTargetLowering::EmitStructByval(MachineInstr *MI,
7559 MachineBasicBlock *BB) const {
7560 // This pseudo instruction has 3 operands: dst, src, size
7561 // We expand it to a loop if size > Subtarget->getMaxInlineSizeThreshold().
7562 // Otherwise, we will generate unrolled scalar copies.
7563 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
7564 const BasicBlock *LLVM_BB = BB->getBasicBlock();
7565 MachineFunction::iterator It = ++BB->getIterator();
7566
7567 unsigned dest = MI->getOperand(0).getReg();
7568 unsigned src = MI->getOperand(1).getReg();
7569 unsigned SizeVal = MI->getOperand(2).getImm();
7570 unsigned Align = MI->getOperand(3).getImm();
7571 DebugLoc dl = MI->getDebugLoc();
7572
7573 MachineFunction *MF = BB->getParent();
7574 MachineRegisterInfo &MRI = MF->getRegInfo();
7575 unsigned UnitSize = 0;
7576 const TargetRegisterClass *TRC = nullptr;
7577 const TargetRegisterClass *VecTRC = nullptr;
7578
7579 bool IsThumb1 = Subtarget->isThumb1Only();
7580 bool IsThumb2 = Subtarget->isThumb2();
7581
7582 if (Align & 1) {
7583 UnitSize = 1;
7584 } else if (Align & 2) {
7585 UnitSize = 2;
7586 } else {
7587 // Check whether we can use NEON instructions.
7588 if (!MF->getFunction()->hasFnAttribute(Attribute::NoImplicitFloat) &&
7589 Subtarget->hasNEON()) {
7590 if ((Align % 16 == 0) && SizeVal >= 16)
7591 UnitSize = 16;
7592 else if ((Align % 8 == 0) && SizeVal >= 8)
7593 UnitSize = 8;
7594 }
7595 // Can't use NEON instructions.
7596 if (UnitSize == 0)
7597 UnitSize = 4;
7598 }
7599
7600 // Select the correct opcode and register class for unit size load/store
7601 bool IsNeon = UnitSize >= 8;
7602 TRC = (IsThumb1 || IsThumb2) ? &ARM::tGPRRegClass : &ARM::GPRRegClass;
7603 if (IsNeon)
7604 VecTRC = UnitSize == 16 ? &ARM::DPairRegClass
7605 : UnitSize == 8 ? &ARM::DPRRegClass
7606 : nullptr;
7607
7608 unsigned BytesLeft = SizeVal % UnitSize;
7609 unsigned LoopSize = SizeVal - BytesLeft;
7610
7611 if (SizeVal <= Subtarget->getMaxInlineSizeThreshold()) {
7612 // Use LDR and STR to copy.
7613 // [scratch, srcOut] = LDR_POST(srcIn, UnitSize)
7614 // [destOut] = STR_POST(scratch, destIn, UnitSize)
7615 unsigned srcIn = src;
7616 unsigned destIn = dest;
7617 for (unsigned i = 0; i < LoopSize; i+=UnitSize) {
7618 unsigned srcOut = MRI.createVirtualRegister(TRC);
7619 unsigned destOut = MRI.createVirtualRegister(TRC);
7620 unsigned scratch = MRI.createVirtualRegister(IsNeon ? VecTRC : TRC);
7621 emitPostLd(BB, MI, TII, dl, UnitSize, scratch, srcIn, srcOut,
7622 IsThumb1, IsThumb2);
7623 emitPostSt(BB, MI, TII, dl, UnitSize, scratch, destIn, destOut,
7624 IsThumb1, IsThumb2);
7625 srcIn = srcOut;
7626 destIn = destOut;
7627 }
7628
7629 // Handle the leftover bytes with LDRB and STRB.
7630 // [scratch, srcOut] = LDRB_POST(srcIn, 1)
7631 // [destOut] = STRB_POST(scratch, destIn, 1)
7632 for (unsigned i = 0; i < BytesLeft; i++) {
7633 unsigned srcOut = MRI.createVirtualRegister(TRC);
7634 unsigned destOut = MRI.createVirtualRegister(TRC);
7635 unsigned scratch = MRI.createVirtualRegister(TRC);
7636 emitPostLd(BB, MI, TII, dl, 1, scratch, srcIn, srcOut,
7637 IsThumb1, IsThumb2);
7638 emitPostSt(BB, MI, TII, dl, 1, scratch, destIn, destOut,
7639 IsThumb1, IsThumb2);
7640 srcIn = srcOut;
7641 destIn = destOut;
7642 }
7643 MI->eraseFromParent(); // The instruction is gone now.
7644 return BB;
7645 }
7646
7647 // Expand the pseudo op to a loop.
7648 // thisMBB:
7649 // ...
7650 // movw varEnd, # --> with thumb2
7651 // movt varEnd, #
7652 // ldrcp varEnd, idx --> without thumb2
7653 // fallthrough --> loopMBB
7654 // loopMBB:
7655 // PHI varPhi, varEnd, varLoop
7656 // PHI srcPhi, src, srcLoop
7657 // PHI destPhi, dst, destLoop
7658 // [scratch, srcLoop] = LDR_POST(srcPhi, UnitSize)
7659 // [destLoop] = STR_POST(scratch, destPhi, UnitSize)
7660 // subs varLoop, varPhi, #UnitSize
7661 // bne loopMBB
7662 // fallthrough --> exitMBB
7663 // exitMBB:
7664 // epilogue to handle left-over bytes
7665 // [scratch, srcOut] = LDRB_POST(srcLoop, 1)
7666 // [destOut] = STRB_POST(scratch, destLoop, 1)
7667 MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB);
7668 MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
7669 MF->insert(It, loopMBB);
7670 MF->insert(It, exitMBB);
7671
7672 // Transfer the remainder of BB and its successor edges to exitMBB.
7673 exitMBB->splice(exitMBB->begin(), BB,
7674 std::next(MachineBasicBlock::iterator(MI)), BB->end());
7675 exitMBB->transferSuccessorsAndUpdatePHIs(BB);
7676
7677 // Load an immediate to varEnd.
7678 unsigned varEnd = MRI.createVirtualRegister(TRC);
7679 if (Subtarget->useMovt(*MF)) {
7680 unsigned Vtmp = varEnd;
7681 if ((LoopSize & 0xFFFF0000) != 0)
7682 Vtmp = MRI.createVirtualRegister(TRC);
7683 AddDefaultPred(BuildMI(BB, dl,
7684 TII->get(IsThumb2 ? ARM::t2MOVi16 : ARM::MOVi16),
7685 Vtmp).addImm(LoopSize & 0xFFFF));
7686
7687 if ((LoopSize & 0xFFFF0000) != 0)
7688 AddDefaultPred(BuildMI(BB, dl,
7689 TII->get(IsThumb2 ? ARM::t2MOVTi16 : ARM::MOVTi16),
7690 varEnd)
7691 .addReg(Vtmp)
7692 .addImm(LoopSize >> 16));
7693 } else {
7694 MachineConstantPool *ConstantPool = MF->getConstantPool();
7695 Type *Int32Ty = Type::getInt32Ty(MF->getFunction()->getContext());
7696 const Constant *C = ConstantInt::get(Int32Ty, LoopSize);
7697
7698 // MachineConstantPool wants an explicit alignment.
7699 unsigned Align = MF->getDataLayout().getPrefTypeAlignment(Int32Ty);
7700 if (Align == 0)
7701 Align = MF->getDataLayout().getTypeAllocSize(C->getType());
7702 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Align);
7703
7704 if (IsThumb1)
7705 AddDefaultPred(BuildMI(*BB, MI, dl, TII->get(ARM::tLDRpci)).addReg(
7706 varEnd, RegState::Define).addConstantPoolIndex(Idx));
7707 else
7708 AddDefaultPred(BuildMI(*BB, MI, dl, TII->get(ARM::LDRcp)).addReg(
7709 varEnd, RegState::Define).addConstantPoolIndex(Idx).addImm(0));
7710 }
7711 BB->addSuccessor(loopMBB);
7712
7713 // Generate the loop body:
7714 // varPhi = PHI(varLoop, varEnd)
7715 // srcPhi = PHI(srcLoop, src)
7716 // destPhi = PHI(destLoop, dst)
7717 MachineBasicBlock *entryBB = BB;
7718 BB = loopMBB;
7719 unsigned varLoop = MRI.createVirtualRegister(TRC);
7720 unsigned varPhi = MRI.createVirtualRegister(TRC);
7721 unsigned srcLoop = MRI.createVirtualRegister(TRC);
7722 unsigned srcPhi = MRI.createVirtualRegister(TRC);
7723 unsigned destLoop = MRI.createVirtualRegister(TRC);
7724 unsigned destPhi = MRI.createVirtualRegister(TRC);
7725
7726 BuildMI(*BB, BB->begin(), dl, TII->get(ARM::PHI), varPhi)
7727 .addReg(varLoop).addMBB(loopMBB)
7728 .addReg(varEnd).addMBB(entryBB);
7729 BuildMI(BB, dl, TII->get(ARM::PHI), srcPhi)
7730 .addReg(srcLoop).addMBB(loopMBB)
7731 .addReg(src).addMBB(entryBB);
7732 BuildMI(BB, dl, TII->get(ARM::PHI), destPhi)
7733 .addReg(destLoop).addMBB(loopMBB)
7734 .addReg(dest).addMBB(entryBB);
7735
7736 // [scratch, srcLoop] = LDR_POST(srcPhi, UnitSize)
7737 // [destLoop] = STR_POST(scratch, destPhi, UnitSiz)
7738 unsigned scratch = MRI.createVirtualRegister(IsNeon ? VecTRC : TRC);
7739 emitPostLd(BB, BB->end(), TII, dl, UnitSize, scratch, srcPhi, srcLoop,
7740 IsThumb1, IsThumb2);
7741 emitPostSt(BB, BB->end(), TII, dl, UnitSize, scratch, destPhi, destLoop,
7742 IsThumb1, IsThumb2);
7743
7744 // Decrement loop variable by UnitSize.
7745 if (IsThumb1) {
7746 MachineInstrBuilder MIB =
7747 BuildMI(*BB, BB->end(), dl, TII->get(ARM::tSUBi8), varLoop);
7748 MIB = AddDefaultT1CC(MIB);
7749 MIB.addReg(varPhi).addImm(UnitSize);
7750 AddDefaultPred(MIB);
7751 } else {
7752 MachineInstrBuilder MIB =
7753 BuildMI(*BB, BB->end(), dl,
7754 TII->get(IsThumb2 ? ARM::t2SUBri : ARM::SUBri), varLoop);
7755 AddDefaultCC(AddDefaultPred(MIB.addReg(varPhi).addImm(UnitSize)));
7756 MIB->getOperand(5).setReg(ARM::CPSR);
7757 MIB->getOperand(5).setIsDef(true);
7758 }
7759 BuildMI(*BB, BB->end(), dl,
7760 TII->get(IsThumb1 ? ARM::tBcc : IsThumb2 ? ARM::t2Bcc : ARM::Bcc))
7761 .addMBB(loopMBB).addImm(ARMCC::NE).addReg(ARM::CPSR);
7762
7763 // loopMBB can loop back to loopMBB or fall through to exitMBB.
7764 BB->addSuccessor(loopMBB);
7765 BB->addSuccessor(exitMBB);
7766
7767 // Add epilogue to handle BytesLeft.
7768 BB = exitMBB;
7769 MachineInstr *StartOfExit = exitMBB->begin();
7770
7771 // [scratch, srcOut] = LDRB_POST(srcLoop, 1)
7772 // [destOut] = STRB_POST(scratch, destLoop, 1)
7773 unsigned srcIn = srcLoop;
7774 unsigned destIn = destLoop;
7775 for (unsigned i = 0; i < BytesLeft; i++) {
7776 unsigned srcOut = MRI.createVirtualRegister(TRC);
7777 unsigned destOut = MRI.createVirtualRegister(TRC);
7778 unsigned scratch = MRI.createVirtualRegister(TRC);
7779 emitPostLd(BB, StartOfExit, TII, dl, 1, scratch, srcIn, srcOut,
7780 IsThumb1, IsThumb2);
7781 emitPostSt(BB, StartOfExit, TII, dl, 1, scratch, destIn, destOut,
7782 IsThumb1, IsThumb2);
7783 srcIn = srcOut;
7784 destIn = destOut;
7785 }
7786
7787 MI->eraseFromParent(); // The instruction is gone now.
7788 return BB;
7789 }
7790
7791 MachineBasicBlock *
EmitLowered__chkstk(MachineInstr * MI,MachineBasicBlock * MBB) const7792 ARMTargetLowering::EmitLowered__chkstk(MachineInstr *MI,
7793 MachineBasicBlock *MBB) const {
7794 const TargetMachine &TM = getTargetMachine();
7795 const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
7796 DebugLoc DL = MI->getDebugLoc();
7797
7798 assert(Subtarget->isTargetWindows() &&
7799 "__chkstk is only supported on Windows");
7800 assert(Subtarget->isThumb2() && "Windows on ARM requires Thumb-2 mode");
7801
7802 // __chkstk takes the number of words to allocate on the stack in R4, and
7803 // returns the stack adjustment in number of bytes in R4. This will not
7804 // clober any other registers (other than the obvious lr).
7805 //
7806 // Although, technically, IP should be considered a register which may be
7807 // clobbered, the call itself will not touch it. Windows on ARM is a pure
7808 // thumb-2 environment, so there is no interworking required. As a result, we
7809 // do not expect a veneer to be emitted by the linker, clobbering IP.
7810 //
7811 // Each module receives its own copy of __chkstk, so no import thunk is
7812 // required, again, ensuring that IP is not clobbered.
7813 //
7814 // Finally, although some linkers may theoretically provide a trampoline for
7815 // out of range calls (which is quite common due to a 32M range limitation of
7816 // branches for Thumb), we can generate the long-call version via
7817 // -mcmodel=large, alleviating the need for the trampoline which may clobber
7818 // IP.
7819
7820 switch (TM.getCodeModel()) {
7821 case CodeModel::Small:
7822 case CodeModel::Medium:
7823 case CodeModel::Default:
7824 case CodeModel::Kernel:
7825 BuildMI(*MBB, MI, DL, TII.get(ARM::tBL))
7826 .addImm((unsigned)ARMCC::AL).addReg(0)
7827 .addExternalSymbol("__chkstk")
7828 .addReg(ARM::R4, RegState::Implicit | RegState::Kill)
7829 .addReg(ARM::R4, RegState::Implicit | RegState::Define)
7830 .addReg(ARM::R12, RegState::Implicit | RegState::Define | RegState::Dead);
7831 break;
7832 case CodeModel::Large:
7833 case CodeModel::JITDefault: {
7834 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
7835 unsigned Reg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
7836
7837 BuildMI(*MBB, MI, DL, TII.get(ARM::t2MOVi32imm), Reg)
7838 .addExternalSymbol("__chkstk");
7839 BuildMI(*MBB, MI, DL, TII.get(ARM::tBLXr))
7840 .addImm((unsigned)ARMCC::AL).addReg(0)
7841 .addReg(Reg, RegState::Kill)
7842 .addReg(ARM::R4, RegState::Implicit | RegState::Kill)
7843 .addReg(ARM::R4, RegState::Implicit | RegState::Define)
7844 .addReg(ARM::R12, RegState::Implicit | RegState::Define | RegState::Dead);
7845 break;
7846 }
7847 }
7848
7849 AddDefaultCC(AddDefaultPred(BuildMI(*MBB, MI, DL, TII.get(ARM::t2SUBrr),
7850 ARM::SP)
7851 .addReg(ARM::SP).addReg(ARM::R4)));
7852
7853 MI->eraseFromParent();
7854 return MBB;
7855 }
7856
7857 MachineBasicBlock *
EmitLowered__dbzchk(MachineInstr * MI,MachineBasicBlock * MBB) const7858 ARMTargetLowering::EmitLowered__dbzchk(MachineInstr *MI,
7859 MachineBasicBlock *MBB) const {
7860 DebugLoc DL = MI->getDebugLoc();
7861 MachineFunction *MF = MBB->getParent();
7862 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
7863
7864 MachineBasicBlock *ContBB = MF->CreateMachineBasicBlock();
7865 MF->push_back(ContBB);
7866 ContBB->splice(ContBB->begin(), MBB,
7867 std::next(MachineBasicBlock::iterator(MI)), MBB->end());
7868 MBB->addSuccessor(ContBB);
7869
7870 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
7871 MF->push_back(TrapBB);
7872 BuildMI(TrapBB, DL, TII->get(ARM::t2UDF)).addImm(249);
7873 MBB->addSuccessor(TrapBB);
7874
7875 BuildMI(*MBB, MI, DL, TII->get(ARM::tCBZ))
7876 .addReg(MI->getOperand(0).getReg())
7877 .addMBB(TrapBB);
7878
7879 MI->eraseFromParent();
7880 return ContBB;
7881 }
7882
7883 MachineBasicBlock *
EmitInstrWithCustomInserter(MachineInstr * MI,MachineBasicBlock * BB) const7884 ARMTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
7885 MachineBasicBlock *BB) const {
7886 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
7887 DebugLoc dl = MI->getDebugLoc();
7888 bool isThumb2 = Subtarget->isThumb2();
7889 switch (MI->getOpcode()) {
7890 default: {
7891 MI->dump();
7892 llvm_unreachable("Unexpected instr type to insert");
7893 }
7894 // The Thumb2 pre-indexed stores have the same MI operands, they just
7895 // define them differently in the .td files from the isel patterns, so
7896 // they need pseudos.
7897 case ARM::t2STR_preidx:
7898 MI->setDesc(TII->get(ARM::t2STR_PRE));
7899 return BB;
7900 case ARM::t2STRB_preidx:
7901 MI->setDesc(TII->get(ARM::t2STRB_PRE));
7902 return BB;
7903 case ARM::t2STRH_preidx:
7904 MI->setDesc(TII->get(ARM::t2STRH_PRE));
7905 return BB;
7906
7907 case ARM::STRi_preidx:
7908 case ARM::STRBi_preidx: {
7909 unsigned NewOpc = MI->getOpcode() == ARM::STRi_preidx ?
7910 ARM::STR_PRE_IMM : ARM::STRB_PRE_IMM;
7911 // Decode the offset.
7912 unsigned Offset = MI->getOperand(4).getImm();
7913 bool isSub = ARM_AM::getAM2Op(Offset) == ARM_AM::sub;
7914 Offset = ARM_AM::getAM2Offset(Offset);
7915 if (isSub)
7916 Offset = -Offset;
7917
7918 MachineMemOperand *MMO = *MI->memoperands_begin();
7919 BuildMI(*BB, MI, dl, TII->get(NewOpc))
7920 .addOperand(MI->getOperand(0)) // Rn_wb
7921 .addOperand(MI->getOperand(1)) // Rt
7922 .addOperand(MI->getOperand(2)) // Rn
7923 .addImm(Offset) // offset (skip GPR==zero_reg)
7924 .addOperand(MI->getOperand(5)) // pred
7925 .addOperand(MI->getOperand(6))
7926 .addMemOperand(MMO);
7927 MI->eraseFromParent();
7928 return BB;
7929 }
7930 case ARM::STRr_preidx:
7931 case ARM::STRBr_preidx:
7932 case ARM::STRH_preidx: {
7933 unsigned NewOpc;
7934 switch (MI->getOpcode()) {
7935 default: llvm_unreachable("unexpected opcode!");
7936 case ARM::STRr_preidx: NewOpc = ARM::STR_PRE_REG; break;
7937 case ARM::STRBr_preidx: NewOpc = ARM::STRB_PRE_REG; break;
7938 case ARM::STRH_preidx: NewOpc = ARM::STRH_PRE; break;
7939 }
7940 MachineInstrBuilder MIB = BuildMI(*BB, MI, dl, TII->get(NewOpc));
7941 for (unsigned i = 0; i < MI->getNumOperands(); ++i)
7942 MIB.addOperand(MI->getOperand(i));
7943 MI->eraseFromParent();
7944 return BB;
7945 }
7946
7947 case ARM::tMOVCCr_pseudo: {
7948 // To "insert" a SELECT_CC instruction, we actually have to insert the
7949 // diamond control-flow pattern. The incoming instruction knows the
7950 // destination vreg to set, the condition code register to branch on, the
7951 // true/false values to select between, and a branch opcode to use.
7952 const BasicBlock *LLVM_BB = BB->getBasicBlock();
7953 MachineFunction::iterator It = ++BB->getIterator();
7954
7955 // thisMBB:
7956 // ...
7957 // TrueVal = ...
7958 // cmpTY ccX, r1, r2
7959 // bCC copy1MBB
7960 // fallthrough --> copy0MBB
7961 MachineBasicBlock *thisMBB = BB;
7962 MachineFunction *F = BB->getParent();
7963 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB);
7964 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB);
7965 F->insert(It, copy0MBB);
7966 F->insert(It, sinkMBB);
7967
7968 // Transfer the remainder of BB and its successor edges to sinkMBB.
7969 sinkMBB->splice(sinkMBB->begin(), BB,
7970 std::next(MachineBasicBlock::iterator(MI)), BB->end());
7971 sinkMBB->transferSuccessorsAndUpdatePHIs(BB);
7972
7973 BB->addSuccessor(copy0MBB);
7974 BB->addSuccessor(sinkMBB);
7975
7976 BuildMI(BB, dl, TII->get(ARM::tBcc)).addMBB(sinkMBB)
7977 .addImm(MI->getOperand(3).getImm()).addReg(MI->getOperand(4).getReg());
7978
7979 // copy0MBB:
7980 // %FalseValue = ...
7981 // # fallthrough to sinkMBB
7982 BB = copy0MBB;
7983
7984 // Update machine-CFG edges
7985 BB->addSuccessor(sinkMBB);
7986
7987 // sinkMBB:
7988 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ]
7989 // ...
7990 BB = sinkMBB;
7991 BuildMI(*BB, BB->begin(), dl,
7992 TII->get(ARM::PHI), MI->getOperand(0).getReg())
7993 .addReg(MI->getOperand(1).getReg()).addMBB(copy0MBB)
7994 .addReg(MI->getOperand(2).getReg()).addMBB(thisMBB);
7995
7996 MI->eraseFromParent(); // The pseudo instruction is gone now.
7997 return BB;
7998 }
7999
8000 case ARM::BCCi64:
8001 case ARM::BCCZi64: {
8002 // If there is an unconditional branch to the other successor, remove it.
8003 BB->erase(std::next(MachineBasicBlock::iterator(MI)), BB->end());
8004
8005 // Compare both parts that make up the double comparison separately for
8006 // equality.
8007 bool RHSisZero = MI->getOpcode() == ARM::BCCZi64;
8008
8009 unsigned LHS1 = MI->getOperand(1).getReg();
8010 unsigned LHS2 = MI->getOperand(2).getReg();
8011 if (RHSisZero) {
8012 AddDefaultPred(BuildMI(BB, dl,
8013 TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
8014 .addReg(LHS1).addImm(0));
8015 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
8016 .addReg(LHS2).addImm(0)
8017 .addImm(ARMCC::EQ).addReg(ARM::CPSR);
8018 } else {
8019 unsigned RHS1 = MI->getOperand(3).getReg();
8020 unsigned RHS2 = MI->getOperand(4).getReg();
8021 AddDefaultPred(BuildMI(BB, dl,
8022 TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
8023 .addReg(LHS1).addReg(RHS1));
8024 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
8025 .addReg(LHS2).addReg(RHS2)
8026 .addImm(ARMCC::EQ).addReg(ARM::CPSR);
8027 }
8028
8029 MachineBasicBlock *destMBB = MI->getOperand(RHSisZero ? 3 : 5).getMBB();
8030 MachineBasicBlock *exitMBB = OtherSucc(BB, destMBB);
8031 if (MI->getOperand(0).getImm() == ARMCC::NE)
8032 std::swap(destMBB, exitMBB);
8033
8034 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc))
8035 .addMBB(destMBB).addImm(ARMCC::EQ).addReg(ARM::CPSR);
8036 if (isThumb2)
8037 AddDefaultPred(BuildMI(BB, dl, TII->get(ARM::t2B)).addMBB(exitMBB));
8038 else
8039 BuildMI(BB, dl, TII->get(ARM::B)) .addMBB(exitMBB);
8040
8041 MI->eraseFromParent(); // The pseudo instruction is gone now.
8042 return BB;
8043 }
8044
8045 case ARM::Int_eh_sjlj_setjmp:
8046 case ARM::Int_eh_sjlj_setjmp_nofp:
8047 case ARM::tInt_eh_sjlj_setjmp:
8048 case ARM::t2Int_eh_sjlj_setjmp:
8049 case ARM::t2Int_eh_sjlj_setjmp_nofp:
8050 return BB;
8051
8052 case ARM::Int_eh_sjlj_setup_dispatch:
8053 EmitSjLjDispatchBlock(MI, BB);
8054 return BB;
8055
8056 case ARM::ABS:
8057 case ARM::t2ABS: {
8058 // To insert an ABS instruction, we have to insert the
8059 // diamond control-flow pattern. The incoming instruction knows the
8060 // source vreg to test against 0, the destination vreg to set,
8061 // the condition code register to branch on, the
8062 // true/false values to select between, and a branch opcode to use.
8063 // It transforms
8064 // V1 = ABS V0
8065 // into
8066 // V2 = MOVS V0
8067 // BCC (branch to SinkBB if V0 >= 0)
8068 // RSBBB: V3 = RSBri V2, 0 (compute ABS if V2 < 0)
8069 // SinkBB: V1 = PHI(V2, V3)
8070 const BasicBlock *LLVM_BB = BB->getBasicBlock();
8071 MachineFunction::iterator BBI = ++BB->getIterator();
8072 MachineFunction *Fn = BB->getParent();
8073 MachineBasicBlock *RSBBB = Fn->CreateMachineBasicBlock(LLVM_BB);
8074 MachineBasicBlock *SinkBB = Fn->CreateMachineBasicBlock(LLVM_BB);
8075 Fn->insert(BBI, RSBBB);
8076 Fn->insert(BBI, SinkBB);
8077
8078 unsigned int ABSSrcReg = MI->getOperand(1).getReg();
8079 unsigned int ABSDstReg = MI->getOperand(0).getReg();
8080 bool ABSSrcKIll = MI->getOperand(1).isKill();
8081 bool isThumb2 = Subtarget->isThumb2();
8082 MachineRegisterInfo &MRI = Fn->getRegInfo();
8083 // In Thumb mode S must not be specified if source register is the SP or
8084 // PC and if destination register is the SP, so restrict register class
8085 unsigned NewRsbDstReg =
8086 MRI.createVirtualRegister(isThumb2 ? &ARM::rGPRRegClass : &ARM::GPRRegClass);
8087
8088 // Transfer the remainder of BB and its successor edges to sinkMBB.
8089 SinkBB->splice(SinkBB->begin(), BB,
8090 std::next(MachineBasicBlock::iterator(MI)), BB->end());
8091 SinkBB->transferSuccessorsAndUpdatePHIs(BB);
8092
8093 BB->addSuccessor(RSBBB);
8094 BB->addSuccessor(SinkBB);
8095
8096 // fall through to SinkMBB
8097 RSBBB->addSuccessor(SinkBB);
8098
8099 // insert a cmp at the end of BB
8100 AddDefaultPred(BuildMI(BB, dl,
8101 TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
8102 .addReg(ABSSrcReg).addImm(0));
8103
8104 // insert a bcc with opposite CC to ARMCC::MI at the end of BB
8105 BuildMI(BB, dl,
8106 TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc)).addMBB(SinkBB)
8107 .addImm(ARMCC::getOppositeCondition(ARMCC::MI)).addReg(ARM::CPSR);
8108
8109 // insert rsbri in RSBBB
8110 // Note: BCC and rsbri will be converted into predicated rsbmi
8111 // by if-conversion pass
8112 BuildMI(*RSBBB, RSBBB->begin(), dl,
8113 TII->get(isThumb2 ? ARM::t2RSBri : ARM::RSBri), NewRsbDstReg)
8114 .addReg(ABSSrcReg, ABSSrcKIll ? RegState::Kill : 0)
8115 .addImm(0).addImm((unsigned)ARMCC::AL).addReg(0).addReg(0);
8116
8117 // insert PHI in SinkBB,
8118 // reuse ABSDstReg to not change uses of ABS instruction
8119 BuildMI(*SinkBB, SinkBB->begin(), dl,
8120 TII->get(ARM::PHI), ABSDstReg)
8121 .addReg(NewRsbDstReg).addMBB(RSBBB)
8122 .addReg(ABSSrcReg).addMBB(BB);
8123
8124 // remove ABS instruction
8125 MI->eraseFromParent();
8126
8127 // return last added BB
8128 return SinkBB;
8129 }
8130 case ARM::COPY_STRUCT_BYVAL_I32:
8131 ++NumLoopByVals;
8132 return EmitStructByval(MI, BB);
8133 case ARM::WIN__CHKSTK:
8134 return EmitLowered__chkstk(MI, BB);
8135 case ARM::WIN__DBZCHK:
8136 return EmitLowered__dbzchk(MI, BB);
8137 }
8138 }
8139
8140 /// \brief Attaches vregs to MEMCPY that it will use as scratch registers
8141 /// when it is expanded into LDM/STM. This is done as a post-isel lowering
8142 /// instead of as a custom inserter because we need the use list from the SDNode.
attachMEMCPYScratchRegs(const ARMSubtarget * Subtarget,MachineInstr * MI,const SDNode * Node)8143 static void attachMEMCPYScratchRegs(const ARMSubtarget *Subtarget,
8144 MachineInstr *MI, const SDNode *Node) {
8145 bool isThumb1 = Subtarget->isThumb1Only();
8146
8147 DebugLoc DL = MI->getDebugLoc();
8148 MachineFunction *MF = MI->getParent()->getParent();
8149 MachineRegisterInfo &MRI = MF->getRegInfo();
8150 MachineInstrBuilder MIB(*MF, MI);
8151
8152 // If the new dst/src is unused mark it as dead.
8153 if (!Node->hasAnyUseOfValue(0)) {
8154 MI->getOperand(0).setIsDead(true);
8155 }
8156 if (!Node->hasAnyUseOfValue(1)) {
8157 MI->getOperand(1).setIsDead(true);
8158 }
8159
8160 // The MEMCPY both defines and kills the scratch registers.
8161 for (unsigned I = 0; I != MI->getOperand(4).getImm(); ++I) {
8162 unsigned TmpReg = MRI.createVirtualRegister(isThumb1 ? &ARM::tGPRRegClass
8163 : &ARM::GPRRegClass);
8164 MIB.addReg(TmpReg, RegState::Define|RegState::Dead);
8165 }
8166 }
8167
AdjustInstrPostInstrSelection(MachineInstr * MI,SDNode * Node) const8168 void ARMTargetLowering::AdjustInstrPostInstrSelection(MachineInstr *MI,
8169 SDNode *Node) const {
8170 if (MI->getOpcode() == ARM::MEMCPY) {
8171 attachMEMCPYScratchRegs(Subtarget, MI, Node);
8172 return;
8173 }
8174
8175 const MCInstrDesc *MCID = &MI->getDesc();
8176 // Adjust potentially 's' setting instructions after isel, i.e. ADC, SBC, RSB,
8177 // RSC. Coming out of isel, they have an implicit CPSR def, but the optional
8178 // operand is still set to noreg. If needed, set the optional operand's
8179 // register to CPSR, and remove the redundant implicit def.
8180 //
8181 // e.g. ADCS (..., CPSR<imp-def>) -> ADC (... opt:CPSR<def>).
8182
8183 // Rename pseudo opcodes.
8184 unsigned NewOpc = convertAddSubFlagsOpcode(MI->getOpcode());
8185 if (NewOpc) {
8186 const ARMBaseInstrInfo *TII = Subtarget->getInstrInfo();
8187 MCID = &TII->get(NewOpc);
8188
8189 assert(MCID->getNumOperands() == MI->getDesc().getNumOperands() + 1 &&
8190 "converted opcode should be the same except for cc_out");
8191
8192 MI->setDesc(*MCID);
8193
8194 // Add the optional cc_out operand
8195 MI->addOperand(MachineOperand::CreateReg(0, /*isDef=*/true));
8196 }
8197 unsigned ccOutIdx = MCID->getNumOperands() - 1;
8198
8199 // Any ARM instruction that sets the 's' bit should specify an optional
8200 // "cc_out" operand in the last operand position.
8201 if (!MI->hasOptionalDef() || !MCID->OpInfo[ccOutIdx].isOptionalDef()) {
8202 assert(!NewOpc && "Optional cc_out operand required");
8203 return;
8204 }
8205 // Look for an implicit def of CPSR added by MachineInstr ctor. Remove it
8206 // since we already have an optional CPSR def.
8207 bool definesCPSR = false;
8208 bool deadCPSR = false;
8209 for (unsigned i = MCID->getNumOperands(), e = MI->getNumOperands();
8210 i != e; ++i) {
8211 const MachineOperand &MO = MI->getOperand(i);
8212 if (MO.isReg() && MO.isDef() && MO.getReg() == ARM::CPSR) {
8213 definesCPSR = true;
8214 if (MO.isDead())
8215 deadCPSR = true;
8216 MI->RemoveOperand(i);
8217 break;
8218 }
8219 }
8220 if (!definesCPSR) {
8221 assert(!NewOpc && "Optional cc_out operand required");
8222 return;
8223 }
8224 assert(deadCPSR == !Node->hasAnyUseOfValue(1) && "inconsistent dead flag");
8225 if (deadCPSR) {
8226 assert(!MI->getOperand(ccOutIdx).getReg() &&
8227 "expect uninitialized optional cc_out operand");
8228 return;
8229 }
8230
8231 // If this instruction was defined with an optional CPSR def and its dag node
8232 // had a live implicit CPSR def, then activate the optional CPSR def.
8233 MachineOperand &MO = MI->getOperand(ccOutIdx);
8234 MO.setReg(ARM::CPSR);
8235 MO.setIsDef(true);
8236 }
8237
8238 //===----------------------------------------------------------------------===//
8239 // ARM Optimization Hooks
8240 //===----------------------------------------------------------------------===//
8241
8242 // Helper function that checks if N is a null or all ones constant.
isZeroOrAllOnes(SDValue N,bool AllOnes)8243 static inline bool isZeroOrAllOnes(SDValue N, bool AllOnes) {
8244 return AllOnes ? isAllOnesConstant(N) : isNullConstant(N);
8245 }
8246
8247 // Return true if N is conditionally 0 or all ones.
8248 // Detects these expressions where cc is an i1 value:
8249 //
8250 // (select cc 0, y) [AllOnes=0]
8251 // (select cc y, 0) [AllOnes=0]
8252 // (zext cc) [AllOnes=0]
8253 // (sext cc) [AllOnes=0/1]
8254 // (select cc -1, y) [AllOnes=1]
8255 // (select cc y, -1) [AllOnes=1]
8256 //
8257 // Invert is set when N is the null/all ones constant when CC is false.
8258 // OtherOp is set to the alternative value of N.
isConditionalZeroOrAllOnes(SDNode * N,bool AllOnes,SDValue & CC,bool & Invert,SDValue & OtherOp,SelectionDAG & DAG)8259 static bool isConditionalZeroOrAllOnes(SDNode *N, bool AllOnes,
8260 SDValue &CC, bool &Invert,
8261 SDValue &OtherOp,
8262 SelectionDAG &DAG) {
8263 switch (N->getOpcode()) {
8264 default: return false;
8265 case ISD::SELECT: {
8266 CC = N->getOperand(0);
8267 SDValue N1 = N->getOperand(1);
8268 SDValue N2 = N->getOperand(2);
8269 if (isZeroOrAllOnes(N1, AllOnes)) {
8270 Invert = false;
8271 OtherOp = N2;
8272 return true;
8273 }
8274 if (isZeroOrAllOnes(N2, AllOnes)) {
8275 Invert = true;
8276 OtherOp = N1;
8277 return true;
8278 }
8279 return false;
8280 }
8281 case ISD::ZERO_EXTEND:
8282 // (zext cc) can never be the all ones value.
8283 if (AllOnes)
8284 return false;
8285 // Fall through.
8286 case ISD::SIGN_EXTEND: {
8287 SDLoc dl(N);
8288 EVT VT = N->getValueType(0);
8289 CC = N->getOperand(0);
8290 if (CC.getValueType() != MVT::i1)
8291 return false;
8292 Invert = !AllOnes;
8293 if (AllOnes)
8294 // When looking for an AllOnes constant, N is an sext, and the 'other'
8295 // value is 0.
8296 OtherOp = DAG.getConstant(0, dl, VT);
8297 else if (N->getOpcode() == ISD::ZERO_EXTEND)
8298 // When looking for a 0 constant, N can be zext or sext.
8299 OtherOp = DAG.getConstant(1, dl, VT);
8300 else
8301 OtherOp = DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl,
8302 VT);
8303 return true;
8304 }
8305 }
8306 }
8307
8308 // Combine a constant select operand into its use:
8309 //
8310 // (add (select cc, 0, c), x) -> (select cc, x, (add, x, c))
8311 // (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c))
8312 // (and (select cc, -1, c), x) -> (select cc, x, (and, x, c)) [AllOnes=1]
8313 // (or (select cc, 0, c), x) -> (select cc, x, (or, x, c))
8314 // (xor (select cc, 0, c), x) -> (select cc, x, (xor, x, c))
8315 //
8316 // The transform is rejected if the select doesn't have a constant operand that
8317 // is null, or all ones when AllOnes is set.
8318 //
8319 // Also recognize sext/zext from i1:
8320 //
8321 // (add (zext cc), x) -> (select cc (add x, 1), x)
8322 // (add (sext cc), x) -> (select cc (add x, -1), x)
8323 //
8324 // These transformations eventually create predicated instructions.
8325 //
8326 // @param N The node to transform.
8327 // @param Slct The N operand that is a select.
8328 // @param OtherOp The other N operand (x above).
8329 // @param DCI Context.
8330 // @param AllOnes Require the select constant to be all ones instead of null.
8331 // @returns The new node, or SDValue() on failure.
8332 static
combineSelectAndUse(SDNode * N,SDValue Slct,SDValue OtherOp,TargetLowering::DAGCombinerInfo & DCI,bool AllOnes=false)8333 SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
8334 TargetLowering::DAGCombinerInfo &DCI,
8335 bool AllOnes = false) {
8336 SelectionDAG &DAG = DCI.DAG;
8337 EVT VT = N->getValueType(0);
8338 SDValue NonConstantVal;
8339 SDValue CCOp;
8340 bool SwapSelectOps;
8341 if (!isConditionalZeroOrAllOnes(Slct.getNode(), AllOnes, CCOp, SwapSelectOps,
8342 NonConstantVal, DAG))
8343 return SDValue();
8344
8345 // Slct is now know to be the desired identity constant when CC is true.
8346 SDValue TrueVal = OtherOp;
8347 SDValue FalseVal = DAG.getNode(N->getOpcode(), SDLoc(N), VT,
8348 OtherOp, NonConstantVal);
8349 // Unless SwapSelectOps says CC should be false.
8350 if (SwapSelectOps)
8351 std::swap(TrueVal, FalseVal);
8352
8353 return DAG.getNode(ISD::SELECT, SDLoc(N), VT,
8354 CCOp, TrueVal, FalseVal);
8355 }
8356
8357 // Attempt combineSelectAndUse on each operand of a commutative operator N.
8358 static
combineSelectAndUseCommutative(SDNode * N,bool AllOnes,TargetLowering::DAGCombinerInfo & DCI)8359 SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes,
8360 TargetLowering::DAGCombinerInfo &DCI) {
8361 SDValue N0 = N->getOperand(0);
8362 SDValue N1 = N->getOperand(1);
8363 if (N0.getNode()->hasOneUse()) {
8364 SDValue Result = combineSelectAndUse(N, N0, N1, DCI, AllOnes);
8365 if (Result.getNode())
8366 return Result;
8367 }
8368 if (N1.getNode()->hasOneUse()) {
8369 SDValue Result = combineSelectAndUse(N, N1, N0, DCI, AllOnes);
8370 if (Result.getNode())
8371 return Result;
8372 }
8373 return SDValue();
8374 }
8375
8376 // AddCombineToVPADDL- For pair-wise add on neon, use the vpaddl instruction
8377 // (only after legalization).
AddCombineToVPADDL(SDNode * N,SDValue N0,SDValue N1,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)8378 static SDValue AddCombineToVPADDL(SDNode *N, SDValue N0, SDValue N1,
8379 TargetLowering::DAGCombinerInfo &DCI,
8380 const ARMSubtarget *Subtarget) {
8381
8382 // Only perform optimization if after legalize, and if NEON is available. We
8383 // also expected both operands to be BUILD_VECTORs.
8384 if (DCI.isBeforeLegalize() || !Subtarget->hasNEON()
8385 || N0.getOpcode() != ISD::BUILD_VECTOR
8386 || N1.getOpcode() != ISD::BUILD_VECTOR)
8387 return SDValue();
8388
8389 // Check output type since VPADDL operand elements can only be 8, 16, or 32.
8390 EVT VT = N->getValueType(0);
8391 if (!VT.isInteger() || VT.getVectorElementType() == MVT::i64)
8392 return SDValue();
8393
8394 // Check that the vector operands are of the right form.
8395 // N0 and N1 are BUILD_VECTOR nodes with N number of EXTRACT_VECTOR
8396 // operands, where N is the size of the formed vector.
8397 // Each EXTRACT_VECTOR should have the same input vector and odd or even
8398 // index such that we have a pair wise add pattern.
8399
8400 // Grab the vector that all EXTRACT_VECTOR nodes should be referencing.
8401 if (N0->getOperand(0)->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
8402 return SDValue();
8403 SDValue Vec = N0->getOperand(0)->getOperand(0);
8404 SDNode *V = Vec.getNode();
8405 unsigned nextIndex = 0;
8406
8407 // For each operands to the ADD which are BUILD_VECTORs,
8408 // check to see if each of their operands are an EXTRACT_VECTOR with
8409 // the same vector and appropriate index.
8410 for (unsigned i = 0, e = N0->getNumOperands(); i != e; ++i) {
8411 if (N0->getOperand(i)->getOpcode() == ISD::EXTRACT_VECTOR_ELT
8412 && N1->getOperand(i)->getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
8413
8414 SDValue ExtVec0 = N0->getOperand(i);
8415 SDValue ExtVec1 = N1->getOperand(i);
8416
8417 // First operand is the vector, verify its the same.
8418 if (V != ExtVec0->getOperand(0).getNode() ||
8419 V != ExtVec1->getOperand(0).getNode())
8420 return SDValue();
8421
8422 // Second is the constant, verify its correct.
8423 ConstantSDNode *C0 = dyn_cast<ConstantSDNode>(ExtVec0->getOperand(1));
8424 ConstantSDNode *C1 = dyn_cast<ConstantSDNode>(ExtVec1->getOperand(1));
8425
8426 // For the constant, we want to see all the even or all the odd.
8427 if (!C0 || !C1 || C0->getZExtValue() != nextIndex
8428 || C1->getZExtValue() != nextIndex+1)
8429 return SDValue();
8430
8431 // Increment index.
8432 nextIndex+=2;
8433 } else
8434 return SDValue();
8435 }
8436
8437 // Create VPADDL node.
8438 SelectionDAG &DAG = DCI.DAG;
8439 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8440
8441 SDLoc dl(N);
8442
8443 // Build operand list.
8444 SmallVector<SDValue, 8> Ops;
8445 Ops.push_back(DAG.getConstant(Intrinsic::arm_neon_vpaddls, dl,
8446 TLI.getPointerTy(DAG.getDataLayout())));
8447
8448 // Input is the vector.
8449 Ops.push_back(Vec);
8450
8451 // Get widened type and narrowed type.
8452 MVT widenType;
8453 unsigned numElem = VT.getVectorNumElements();
8454
8455 EVT inputLaneType = Vec.getValueType().getVectorElementType();
8456 switch (inputLaneType.getSimpleVT().SimpleTy) {
8457 case MVT::i8: widenType = MVT::getVectorVT(MVT::i16, numElem); break;
8458 case MVT::i16: widenType = MVT::getVectorVT(MVT::i32, numElem); break;
8459 case MVT::i32: widenType = MVT::getVectorVT(MVT::i64, numElem); break;
8460 default:
8461 llvm_unreachable("Invalid vector element type for padd optimization.");
8462 }
8463
8464 SDValue tmp = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, widenType, Ops);
8465 unsigned ExtOp = VT.bitsGT(tmp.getValueType()) ? ISD::ANY_EXTEND : ISD::TRUNCATE;
8466 return DAG.getNode(ExtOp, dl, VT, tmp);
8467 }
8468
findMUL_LOHI(SDValue V)8469 static SDValue findMUL_LOHI(SDValue V) {
8470 if (V->getOpcode() == ISD::UMUL_LOHI ||
8471 V->getOpcode() == ISD::SMUL_LOHI)
8472 return V;
8473 return SDValue();
8474 }
8475
AddCombineTo64bitMLAL(SDNode * AddcNode,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)8476 static SDValue AddCombineTo64bitMLAL(SDNode *AddcNode,
8477 TargetLowering::DAGCombinerInfo &DCI,
8478 const ARMSubtarget *Subtarget) {
8479
8480 if (Subtarget->isThumb1Only()) return SDValue();
8481
8482 // Only perform the checks after legalize when the pattern is available.
8483 if (DCI.isBeforeLegalize()) return SDValue();
8484
8485 // Look for multiply add opportunities.
8486 // The pattern is a ISD::UMUL_LOHI followed by two add nodes, where
8487 // each add nodes consumes a value from ISD::UMUL_LOHI and there is
8488 // a glue link from the first add to the second add.
8489 // If we find this pattern, we can replace the U/SMUL_LOHI, ADDC, and ADDE by
8490 // a S/UMLAL instruction.
8491 // UMUL_LOHI
8492 // / :lo \ :hi
8493 // / \ [no multiline comment]
8494 // loAdd -> ADDE |
8495 // \ :glue /
8496 // \ /
8497 // ADDC <- hiAdd
8498 //
8499 assert(AddcNode->getOpcode() == ISD::ADDC && "Expect an ADDC");
8500 SDValue AddcOp0 = AddcNode->getOperand(0);
8501 SDValue AddcOp1 = AddcNode->getOperand(1);
8502
8503 // Check if the two operands are from the same mul_lohi node.
8504 if (AddcOp0.getNode() == AddcOp1.getNode())
8505 return SDValue();
8506
8507 assert(AddcNode->getNumValues() == 2 &&
8508 AddcNode->getValueType(0) == MVT::i32 &&
8509 "Expect ADDC with two result values. First: i32");
8510
8511 // Check that we have a glued ADDC node.
8512 if (AddcNode->getValueType(1) != MVT::Glue)
8513 return SDValue();
8514
8515 // Check that the ADDC adds the low result of the S/UMUL_LOHI.
8516 if (AddcOp0->getOpcode() != ISD::UMUL_LOHI &&
8517 AddcOp0->getOpcode() != ISD::SMUL_LOHI &&
8518 AddcOp1->getOpcode() != ISD::UMUL_LOHI &&
8519 AddcOp1->getOpcode() != ISD::SMUL_LOHI)
8520 return SDValue();
8521
8522 // Look for the glued ADDE.
8523 SDNode* AddeNode = AddcNode->getGluedUser();
8524 if (!AddeNode)
8525 return SDValue();
8526
8527 // Make sure it is really an ADDE.
8528 if (AddeNode->getOpcode() != ISD::ADDE)
8529 return SDValue();
8530
8531 assert(AddeNode->getNumOperands() == 3 &&
8532 AddeNode->getOperand(2).getValueType() == MVT::Glue &&
8533 "ADDE node has the wrong inputs");
8534
8535 // Check for the triangle shape.
8536 SDValue AddeOp0 = AddeNode->getOperand(0);
8537 SDValue AddeOp1 = AddeNode->getOperand(1);
8538
8539 // Make sure that the ADDE operands are not coming from the same node.
8540 if (AddeOp0.getNode() == AddeOp1.getNode())
8541 return SDValue();
8542
8543 // Find the MUL_LOHI node walking up ADDE's operands.
8544 bool IsLeftOperandMUL = false;
8545 SDValue MULOp = findMUL_LOHI(AddeOp0);
8546 if (MULOp == SDValue())
8547 MULOp = findMUL_LOHI(AddeOp1);
8548 else
8549 IsLeftOperandMUL = true;
8550 if (MULOp == SDValue())
8551 return SDValue();
8552
8553 // Figure out the right opcode.
8554 unsigned Opc = MULOp->getOpcode();
8555 unsigned FinalOpc = (Opc == ISD::SMUL_LOHI) ? ARMISD::SMLAL : ARMISD::UMLAL;
8556
8557 // Figure out the high and low input values to the MLAL node.
8558 SDValue* HiAdd = nullptr;
8559 SDValue* LoMul = nullptr;
8560 SDValue* LowAdd = nullptr;
8561
8562 // Ensure that ADDE is from high result of ISD::SMUL_LOHI.
8563 if ((AddeOp0 != MULOp.getValue(1)) && (AddeOp1 != MULOp.getValue(1)))
8564 return SDValue();
8565
8566 if (IsLeftOperandMUL)
8567 HiAdd = &AddeOp1;
8568 else
8569 HiAdd = &AddeOp0;
8570
8571
8572 // Ensure that LoMul and LowAdd are taken from correct ISD::SMUL_LOHI node
8573 // whose low result is fed to the ADDC we are checking.
8574
8575 if (AddcOp0 == MULOp.getValue(0)) {
8576 LoMul = &AddcOp0;
8577 LowAdd = &AddcOp1;
8578 }
8579 if (AddcOp1 == MULOp.getValue(0)) {
8580 LoMul = &AddcOp1;
8581 LowAdd = &AddcOp0;
8582 }
8583
8584 if (!LoMul)
8585 return SDValue();
8586
8587 // Create the merged node.
8588 SelectionDAG &DAG = DCI.DAG;
8589
8590 // Build operand list.
8591 SmallVector<SDValue, 8> Ops;
8592 Ops.push_back(LoMul->getOperand(0));
8593 Ops.push_back(LoMul->getOperand(1));
8594 Ops.push_back(*LowAdd);
8595 Ops.push_back(*HiAdd);
8596
8597 SDValue MLALNode = DAG.getNode(FinalOpc, SDLoc(AddcNode),
8598 DAG.getVTList(MVT::i32, MVT::i32), Ops);
8599
8600 // Replace the ADDs' nodes uses by the MLA node's values.
8601 SDValue HiMLALResult(MLALNode.getNode(), 1);
8602 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeNode, 0), HiMLALResult);
8603
8604 SDValue LoMLALResult(MLALNode.getNode(), 0);
8605 DAG.ReplaceAllUsesOfValueWith(SDValue(AddcNode, 0), LoMLALResult);
8606
8607 // Return original node to notify the driver to stop replacing.
8608 SDValue resNode(AddcNode, 0);
8609 return resNode;
8610 }
8611
8612 /// PerformADDCCombine - Target-specific dag combine transform from
8613 /// ISD::ADDC, ISD::ADDE, and ISD::MUL_LOHI to MLAL.
PerformADDCCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)8614 static SDValue PerformADDCCombine(SDNode *N,
8615 TargetLowering::DAGCombinerInfo &DCI,
8616 const ARMSubtarget *Subtarget) {
8617
8618 return AddCombineTo64bitMLAL(N, DCI, Subtarget);
8619
8620 }
8621
8622 /// PerformADDCombineWithOperands - Try DAG combinations for an ADD with
8623 /// operands N0 and N1. This is a helper for PerformADDCombine that is
8624 /// called with the default operands, and if that fails, with commuted
8625 /// operands.
PerformADDCombineWithOperands(SDNode * N,SDValue N0,SDValue N1,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)8626 static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
8627 TargetLowering::DAGCombinerInfo &DCI,
8628 const ARMSubtarget *Subtarget){
8629
8630 // Attempt to create vpaddl for this add.
8631 SDValue Result = AddCombineToVPADDL(N, N0, N1, DCI, Subtarget);
8632 if (Result.getNode())
8633 return Result;
8634
8635 // fold (add (select cc, 0, c), x) -> (select cc, x, (add, x, c))
8636 if (N0.getNode()->hasOneUse()) {
8637 SDValue Result = combineSelectAndUse(N, N0, N1, DCI);
8638 if (Result.getNode()) return Result;
8639 }
8640 return SDValue();
8641 }
8642
8643 /// PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
8644 ///
PerformADDCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)8645 static SDValue PerformADDCombine(SDNode *N,
8646 TargetLowering::DAGCombinerInfo &DCI,
8647 const ARMSubtarget *Subtarget) {
8648 SDValue N0 = N->getOperand(0);
8649 SDValue N1 = N->getOperand(1);
8650
8651 // First try with the default operand order.
8652 SDValue Result = PerformADDCombineWithOperands(N, N0, N1, DCI, Subtarget);
8653 if (Result.getNode())
8654 return Result;
8655
8656 // If that didn't work, try again with the operands commuted.
8657 return PerformADDCombineWithOperands(N, N1, N0, DCI, Subtarget);
8658 }
8659
8660 /// PerformSUBCombine - Target-specific dag combine xforms for ISD::SUB.
8661 ///
PerformSUBCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI)8662 static SDValue PerformSUBCombine(SDNode *N,
8663 TargetLowering::DAGCombinerInfo &DCI) {
8664 SDValue N0 = N->getOperand(0);
8665 SDValue N1 = N->getOperand(1);
8666
8667 // fold (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c))
8668 if (N1.getNode()->hasOneUse()) {
8669 SDValue Result = combineSelectAndUse(N, N1, N0, DCI);
8670 if (Result.getNode()) return Result;
8671 }
8672
8673 return SDValue();
8674 }
8675
8676 /// PerformVMULCombine
8677 /// Distribute (A + B) * C to (A * C) + (B * C) to take advantage of the
8678 /// special multiplier accumulator forwarding.
8679 /// vmul d3, d0, d2
8680 /// vmla d3, d1, d2
8681 /// is faster than
8682 /// vadd d3, d0, d1
8683 /// vmul d3, d3, d2
8684 // However, for (A + B) * (A + B),
8685 // vadd d2, d0, d1
8686 // vmul d3, d0, d2
8687 // vmla d3, d1, d2
8688 // is slower than
8689 // vadd d2, d0, d1
8690 // vmul d3, d2, d2
PerformVMULCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)8691 static SDValue PerformVMULCombine(SDNode *N,
8692 TargetLowering::DAGCombinerInfo &DCI,
8693 const ARMSubtarget *Subtarget) {
8694 if (!Subtarget->hasVMLxForwarding())
8695 return SDValue();
8696
8697 SelectionDAG &DAG = DCI.DAG;
8698 SDValue N0 = N->getOperand(0);
8699 SDValue N1 = N->getOperand(1);
8700 unsigned Opcode = N0.getOpcode();
8701 if (Opcode != ISD::ADD && Opcode != ISD::SUB &&
8702 Opcode != ISD::FADD && Opcode != ISD::FSUB) {
8703 Opcode = N1.getOpcode();
8704 if (Opcode != ISD::ADD && Opcode != ISD::SUB &&
8705 Opcode != ISD::FADD && Opcode != ISD::FSUB)
8706 return SDValue();
8707 std::swap(N0, N1);
8708 }
8709
8710 if (N0 == N1)
8711 return SDValue();
8712
8713 EVT VT = N->getValueType(0);
8714 SDLoc DL(N);
8715 SDValue N00 = N0->getOperand(0);
8716 SDValue N01 = N0->getOperand(1);
8717 return DAG.getNode(Opcode, DL, VT,
8718 DAG.getNode(ISD::MUL, DL, VT, N00, N1),
8719 DAG.getNode(ISD::MUL, DL, VT, N01, N1));
8720 }
8721
PerformMULCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)8722 static SDValue PerformMULCombine(SDNode *N,
8723 TargetLowering::DAGCombinerInfo &DCI,
8724 const ARMSubtarget *Subtarget) {
8725 SelectionDAG &DAG = DCI.DAG;
8726
8727 if (Subtarget->isThumb1Only())
8728 return SDValue();
8729
8730 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
8731 return SDValue();
8732
8733 EVT VT = N->getValueType(0);
8734 if (VT.is64BitVector() || VT.is128BitVector())
8735 return PerformVMULCombine(N, DCI, Subtarget);
8736 if (VT != MVT::i32)
8737 return SDValue();
8738
8739 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
8740 if (!C)
8741 return SDValue();
8742
8743 int64_t MulAmt = C->getSExtValue();
8744 unsigned ShiftAmt = countTrailingZeros<uint64_t>(MulAmt);
8745
8746 ShiftAmt = ShiftAmt & (32 - 1);
8747 SDValue V = N->getOperand(0);
8748 SDLoc DL(N);
8749
8750 SDValue Res;
8751 MulAmt >>= ShiftAmt;
8752
8753 if (MulAmt >= 0) {
8754 if (isPowerOf2_32(MulAmt - 1)) {
8755 // (mul x, 2^N + 1) => (add (shl x, N), x)
8756 Res = DAG.getNode(ISD::ADD, DL, VT,
8757 V,
8758 DAG.getNode(ISD::SHL, DL, VT,
8759 V,
8760 DAG.getConstant(Log2_32(MulAmt - 1), DL,
8761 MVT::i32)));
8762 } else if (isPowerOf2_32(MulAmt + 1)) {
8763 // (mul x, 2^N - 1) => (sub (shl x, N), x)
8764 Res = DAG.getNode(ISD::SUB, DL, VT,
8765 DAG.getNode(ISD::SHL, DL, VT,
8766 V,
8767 DAG.getConstant(Log2_32(MulAmt + 1), DL,
8768 MVT::i32)),
8769 V);
8770 } else
8771 return SDValue();
8772 } else {
8773 uint64_t MulAmtAbs = -MulAmt;
8774 if (isPowerOf2_32(MulAmtAbs + 1)) {
8775 // (mul x, -(2^N - 1)) => (sub x, (shl x, N))
8776 Res = DAG.getNode(ISD::SUB, DL, VT,
8777 V,
8778 DAG.getNode(ISD::SHL, DL, VT,
8779 V,
8780 DAG.getConstant(Log2_32(MulAmtAbs + 1), DL,
8781 MVT::i32)));
8782 } else if (isPowerOf2_32(MulAmtAbs - 1)) {
8783 // (mul x, -(2^N + 1)) => - (add (shl x, N), x)
8784 Res = DAG.getNode(ISD::ADD, DL, VT,
8785 V,
8786 DAG.getNode(ISD::SHL, DL, VT,
8787 V,
8788 DAG.getConstant(Log2_32(MulAmtAbs - 1), DL,
8789 MVT::i32)));
8790 Res = DAG.getNode(ISD::SUB, DL, VT,
8791 DAG.getConstant(0, DL, MVT::i32), Res);
8792
8793 } else
8794 return SDValue();
8795 }
8796
8797 if (ShiftAmt != 0)
8798 Res = DAG.getNode(ISD::SHL, DL, VT,
8799 Res, DAG.getConstant(ShiftAmt, DL, MVT::i32));
8800
8801 // Do not add new nodes to DAG combiner worklist.
8802 DCI.CombineTo(N, Res, false);
8803 return SDValue();
8804 }
8805
PerformANDCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)8806 static SDValue PerformANDCombine(SDNode *N,
8807 TargetLowering::DAGCombinerInfo &DCI,
8808 const ARMSubtarget *Subtarget) {
8809
8810 // Attempt to use immediate-form VBIC
8811 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(N->getOperand(1));
8812 SDLoc dl(N);
8813 EVT VT = N->getValueType(0);
8814 SelectionDAG &DAG = DCI.DAG;
8815
8816 if(!DAG.getTargetLoweringInfo().isTypeLegal(VT))
8817 return SDValue();
8818
8819 APInt SplatBits, SplatUndef;
8820 unsigned SplatBitSize;
8821 bool HasAnyUndefs;
8822 if (BVN &&
8823 BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
8824 if (SplatBitSize <= 64) {
8825 EVT VbicVT;
8826 SDValue Val = isNEONModifiedImm((~SplatBits).getZExtValue(),
8827 SplatUndef.getZExtValue(), SplatBitSize,
8828 DAG, dl, VbicVT, VT.is128BitVector(),
8829 OtherModImm);
8830 if (Val.getNode()) {
8831 SDValue Input =
8832 DAG.getNode(ISD::BITCAST, dl, VbicVT, N->getOperand(0));
8833 SDValue Vbic = DAG.getNode(ARMISD::VBICIMM, dl, VbicVT, Input, Val);
8834 return DAG.getNode(ISD::BITCAST, dl, VT, Vbic);
8835 }
8836 }
8837 }
8838
8839 if (!Subtarget->isThumb1Only()) {
8840 // fold (and (select cc, -1, c), x) -> (select cc, x, (and, x, c))
8841 SDValue Result = combineSelectAndUseCommutative(N, true, DCI);
8842 if (Result.getNode())
8843 return Result;
8844 }
8845
8846 return SDValue();
8847 }
8848
8849 /// PerformORCombine - Target-specific dag combine xforms for ISD::OR
PerformORCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)8850 static SDValue PerformORCombine(SDNode *N,
8851 TargetLowering::DAGCombinerInfo &DCI,
8852 const ARMSubtarget *Subtarget) {
8853 // Attempt to use immediate-form VORR
8854 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(N->getOperand(1));
8855 SDLoc dl(N);
8856 EVT VT = N->getValueType(0);
8857 SelectionDAG &DAG = DCI.DAG;
8858
8859 if(!DAG.getTargetLoweringInfo().isTypeLegal(VT))
8860 return SDValue();
8861
8862 APInt SplatBits, SplatUndef;
8863 unsigned SplatBitSize;
8864 bool HasAnyUndefs;
8865 if (BVN && Subtarget->hasNEON() &&
8866 BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
8867 if (SplatBitSize <= 64) {
8868 EVT VorrVT;
8869 SDValue Val = isNEONModifiedImm(SplatBits.getZExtValue(),
8870 SplatUndef.getZExtValue(), SplatBitSize,
8871 DAG, dl, VorrVT, VT.is128BitVector(),
8872 OtherModImm);
8873 if (Val.getNode()) {
8874 SDValue Input =
8875 DAG.getNode(ISD::BITCAST, dl, VorrVT, N->getOperand(0));
8876 SDValue Vorr = DAG.getNode(ARMISD::VORRIMM, dl, VorrVT, Input, Val);
8877 return DAG.getNode(ISD::BITCAST, dl, VT, Vorr);
8878 }
8879 }
8880 }
8881
8882 if (!Subtarget->isThumb1Only()) {
8883 // fold (or (select cc, 0, c), x) -> (select cc, x, (or, x, c))
8884 SDValue Result = combineSelectAndUseCommutative(N, false, DCI);
8885 if (Result.getNode())
8886 return Result;
8887 }
8888
8889 // The code below optimizes (or (and X, Y), Z).
8890 // The AND operand needs to have a single user to make these optimizations
8891 // profitable.
8892 SDValue N0 = N->getOperand(0);
8893 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
8894 return SDValue();
8895 SDValue N1 = N->getOperand(1);
8896
8897 // (or (and B, A), (and C, ~A)) => (VBSL A, B, C) when A is a constant.
8898 if (Subtarget->hasNEON() && N1.getOpcode() == ISD::AND && VT.isVector() &&
8899 DAG.getTargetLoweringInfo().isTypeLegal(VT)) {
8900 APInt SplatUndef;
8901 unsigned SplatBitSize;
8902 bool HasAnyUndefs;
8903
8904 APInt SplatBits0, SplatBits1;
8905 BuildVectorSDNode *BVN0 = dyn_cast<BuildVectorSDNode>(N0->getOperand(1));
8906 BuildVectorSDNode *BVN1 = dyn_cast<BuildVectorSDNode>(N1->getOperand(1));
8907 // Ensure that the second operand of both ands are constants
8908 if (BVN0 && BVN0->isConstantSplat(SplatBits0, SplatUndef, SplatBitSize,
8909 HasAnyUndefs) && !HasAnyUndefs) {
8910 if (BVN1 && BVN1->isConstantSplat(SplatBits1, SplatUndef, SplatBitSize,
8911 HasAnyUndefs) && !HasAnyUndefs) {
8912 // Ensure that the bit width of the constants are the same and that
8913 // the splat arguments are logical inverses as per the pattern we
8914 // are trying to simplify.
8915 if (SplatBits0.getBitWidth() == SplatBits1.getBitWidth() &&
8916 SplatBits0 == ~SplatBits1) {
8917 // Canonicalize the vector type to make instruction selection
8918 // simpler.
8919 EVT CanonicalVT = VT.is128BitVector() ? MVT::v4i32 : MVT::v2i32;
8920 SDValue Result = DAG.getNode(ARMISD::VBSL, dl, CanonicalVT,
8921 N0->getOperand(1),
8922 N0->getOperand(0),
8923 N1->getOperand(0));
8924 return DAG.getNode(ISD::BITCAST, dl, VT, Result);
8925 }
8926 }
8927 }
8928 }
8929
8930 // Try to use the ARM/Thumb2 BFI (bitfield insert) instruction when
8931 // reasonable.
8932
8933 // BFI is only available on V6T2+
8934 if (Subtarget->isThumb1Only() || !Subtarget->hasV6T2Ops())
8935 return SDValue();
8936
8937 SDLoc DL(N);
8938 // 1) or (and A, mask), val => ARMbfi A, val, mask
8939 // iff (val & mask) == val
8940 //
8941 // 2) or (and A, mask), (and B, mask2) => ARMbfi A, (lsr B, amt), mask
8942 // 2a) iff isBitFieldInvertedMask(mask) && isBitFieldInvertedMask(~mask2)
8943 // && mask == ~mask2
8944 // 2b) iff isBitFieldInvertedMask(~mask) && isBitFieldInvertedMask(mask2)
8945 // && ~mask == mask2
8946 // (i.e., copy a bitfield value into another bitfield of the same width)
8947
8948 if (VT != MVT::i32)
8949 return SDValue();
8950
8951 SDValue N00 = N0.getOperand(0);
8952
8953 // The value and the mask need to be constants so we can verify this is
8954 // actually a bitfield set. If the mask is 0xffff, we can do better
8955 // via a movt instruction, so don't use BFI in that case.
8956 SDValue MaskOp = N0.getOperand(1);
8957 ConstantSDNode *MaskC = dyn_cast<ConstantSDNode>(MaskOp);
8958 if (!MaskC)
8959 return SDValue();
8960 unsigned Mask = MaskC->getZExtValue();
8961 if (Mask == 0xffff)
8962 return SDValue();
8963 SDValue Res;
8964 // Case (1): or (and A, mask), val => ARMbfi A, val, mask
8965 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
8966 if (N1C) {
8967 unsigned Val = N1C->getZExtValue();
8968 if ((Val & ~Mask) != Val)
8969 return SDValue();
8970
8971 if (ARM::isBitFieldInvertedMask(Mask)) {
8972 Val >>= countTrailingZeros(~Mask);
8973
8974 Res = DAG.getNode(ARMISD::BFI, DL, VT, N00,
8975 DAG.getConstant(Val, DL, MVT::i32),
8976 DAG.getConstant(Mask, DL, MVT::i32));
8977
8978 // Do not add new nodes to DAG combiner worklist.
8979 DCI.CombineTo(N, Res, false);
8980 return SDValue();
8981 }
8982 } else if (N1.getOpcode() == ISD::AND) {
8983 // case (2) or (and A, mask), (and B, mask2) => ARMbfi A, (lsr B, amt), mask
8984 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
8985 if (!N11C)
8986 return SDValue();
8987 unsigned Mask2 = N11C->getZExtValue();
8988
8989 // Mask and ~Mask2 (or reverse) must be equivalent for the BFI pattern
8990 // as is to match.
8991 if (ARM::isBitFieldInvertedMask(Mask) &&
8992 (Mask == ~Mask2)) {
8993 // The pack halfword instruction works better for masks that fit it,
8994 // so use that when it's available.
8995 if (Subtarget->hasT2ExtractPack() &&
8996 (Mask == 0xffff || Mask == 0xffff0000))
8997 return SDValue();
8998 // 2a
8999 unsigned amt = countTrailingZeros(Mask2);
9000 Res = DAG.getNode(ISD::SRL, DL, VT, N1.getOperand(0),
9001 DAG.getConstant(amt, DL, MVT::i32));
9002 Res = DAG.getNode(ARMISD::BFI, DL, VT, N00, Res,
9003 DAG.getConstant(Mask, DL, MVT::i32));
9004 // Do not add new nodes to DAG combiner worklist.
9005 DCI.CombineTo(N, Res, false);
9006 return SDValue();
9007 } else if (ARM::isBitFieldInvertedMask(~Mask) &&
9008 (~Mask == Mask2)) {
9009 // The pack halfword instruction works better for masks that fit it,
9010 // so use that when it's available.
9011 if (Subtarget->hasT2ExtractPack() &&
9012 (Mask2 == 0xffff || Mask2 == 0xffff0000))
9013 return SDValue();
9014 // 2b
9015 unsigned lsb = countTrailingZeros(Mask);
9016 Res = DAG.getNode(ISD::SRL, DL, VT, N00,
9017 DAG.getConstant(lsb, DL, MVT::i32));
9018 Res = DAG.getNode(ARMISD::BFI, DL, VT, N1.getOperand(0), Res,
9019 DAG.getConstant(Mask2, DL, MVT::i32));
9020 // Do not add new nodes to DAG combiner worklist.
9021 DCI.CombineTo(N, Res, false);
9022 return SDValue();
9023 }
9024 }
9025
9026 if (DAG.MaskedValueIsZero(N1, MaskC->getAPIntValue()) &&
9027 N00.getOpcode() == ISD::SHL && isa<ConstantSDNode>(N00.getOperand(1)) &&
9028 ARM::isBitFieldInvertedMask(~Mask)) {
9029 // Case (3): or (and (shl A, #shamt), mask), B => ARMbfi B, A, ~mask
9030 // where lsb(mask) == #shamt and masked bits of B are known zero.
9031 SDValue ShAmt = N00.getOperand(1);
9032 unsigned ShAmtC = cast<ConstantSDNode>(ShAmt)->getZExtValue();
9033 unsigned LSB = countTrailingZeros(Mask);
9034 if (ShAmtC != LSB)
9035 return SDValue();
9036
9037 Res = DAG.getNode(ARMISD::BFI, DL, VT, N1, N00.getOperand(0),
9038 DAG.getConstant(~Mask, DL, MVT::i32));
9039
9040 // Do not add new nodes to DAG combiner worklist.
9041 DCI.CombineTo(N, Res, false);
9042 }
9043
9044 return SDValue();
9045 }
9046
PerformXORCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)9047 static SDValue PerformXORCombine(SDNode *N,
9048 TargetLowering::DAGCombinerInfo &DCI,
9049 const ARMSubtarget *Subtarget) {
9050 EVT VT = N->getValueType(0);
9051 SelectionDAG &DAG = DCI.DAG;
9052
9053 if(!DAG.getTargetLoweringInfo().isTypeLegal(VT))
9054 return SDValue();
9055
9056 if (!Subtarget->isThumb1Only()) {
9057 // fold (xor (select cc, 0, c), x) -> (select cc, x, (xor, x, c))
9058 SDValue Result = combineSelectAndUseCommutative(N, false, DCI);
9059 if (Result.getNode())
9060 return Result;
9061 }
9062
9063 return SDValue();
9064 }
9065
9066 // ParseBFI - given a BFI instruction in N, extract the "from" value (Rn) and return it,
9067 // and fill in FromMask and ToMask with (consecutive) bits in "from" to be extracted and
9068 // their position in "to" (Rd).
ParseBFI(SDNode * N,APInt & ToMask,APInt & FromMask)9069 static SDValue ParseBFI(SDNode *N, APInt &ToMask, APInt &FromMask) {
9070 assert(N->getOpcode() == ARMISD::BFI);
9071
9072 SDValue From = N->getOperand(1);
9073 ToMask = ~cast<ConstantSDNode>(N->getOperand(2))->getAPIntValue();
9074 FromMask = APInt::getLowBitsSet(ToMask.getBitWidth(), ToMask.countPopulation());
9075
9076 // If the Base came from a SHR #C, we can deduce that it is really testing bit
9077 // #C in the base of the SHR.
9078 if (From->getOpcode() == ISD::SRL &&
9079 isa<ConstantSDNode>(From->getOperand(1))) {
9080 APInt Shift = cast<ConstantSDNode>(From->getOperand(1))->getAPIntValue();
9081 assert(Shift.getLimitedValue() < 32 && "Shift too large!");
9082 FromMask <<= Shift.getLimitedValue(31);
9083 From = From->getOperand(0);
9084 }
9085
9086 return From;
9087 }
9088
9089 // If A and B contain one contiguous set of bits, does A | B == A . B?
9090 //
9091 // Neither A nor B must be zero.
BitsProperlyConcatenate(const APInt & A,const APInt & B)9092 static bool BitsProperlyConcatenate(const APInt &A, const APInt &B) {
9093 unsigned LastActiveBitInA = A.countTrailingZeros();
9094 unsigned FirstActiveBitInB = B.getBitWidth() - B.countLeadingZeros() - 1;
9095 return LastActiveBitInA - 1 == FirstActiveBitInB;
9096 }
9097
FindBFIToCombineWith(SDNode * N)9098 static SDValue FindBFIToCombineWith(SDNode *N) {
9099 // We have a BFI in N. Follow a possible chain of BFIs and find a BFI it can combine with,
9100 // if one exists.
9101 APInt ToMask, FromMask;
9102 SDValue From = ParseBFI(N, ToMask, FromMask);
9103 SDValue To = N->getOperand(0);
9104
9105 // Now check for a compatible BFI to merge with. We can pass through BFIs that
9106 // aren't compatible, but not if they set the same bit in their destination as
9107 // we do (or that of any BFI we're going to combine with).
9108 SDValue V = To;
9109 APInt CombinedToMask = ToMask;
9110 while (V.getOpcode() == ARMISD::BFI) {
9111 APInt NewToMask, NewFromMask;
9112 SDValue NewFrom = ParseBFI(V.getNode(), NewToMask, NewFromMask);
9113 if (NewFrom != From) {
9114 // This BFI has a different base. Keep going.
9115 CombinedToMask |= NewToMask;
9116 V = V.getOperand(0);
9117 continue;
9118 }
9119
9120 // Do the written bits conflict with any we've seen so far?
9121 if ((NewToMask & CombinedToMask).getBoolValue())
9122 // Conflicting bits - bail out because going further is unsafe.
9123 return SDValue();
9124
9125 // Are the new bits contiguous when combined with the old bits?
9126 if (BitsProperlyConcatenate(ToMask, NewToMask) &&
9127 BitsProperlyConcatenate(FromMask, NewFromMask))
9128 return V;
9129 if (BitsProperlyConcatenate(NewToMask, ToMask) &&
9130 BitsProperlyConcatenate(NewFromMask, FromMask))
9131 return V;
9132
9133 // We've seen a write to some bits, so track it.
9134 CombinedToMask |= NewToMask;
9135 // Keep going...
9136 V = V.getOperand(0);
9137 }
9138
9139 return SDValue();
9140 }
9141
PerformBFICombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI)9142 static SDValue PerformBFICombine(SDNode *N,
9143 TargetLowering::DAGCombinerInfo &DCI) {
9144 SDValue N1 = N->getOperand(1);
9145 if (N1.getOpcode() == ISD::AND) {
9146 // (bfi A, (and B, Mask1), Mask2) -> (bfi A, B, Mask2) iff
9147 // the bits being cleared by the AND are not demanded by the BFI.
9148 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
9149 if (!N11C)
9150 return SDValue();
9151 unsigned InvMask = cast<ConstantSDNode>(N->getOperand(2))->getZExtValue();
9152 unsigned LSB = countTrailingZeros(~InvMask);
9153 unsigned Width = (32 - countLeadingZeros(~InvMask)) - LSB;
9154 assert(Width <
9155 static_cast<unsigned>(std::numeric_limits<unsigned>::digits) &&
9156 "undefined behavior");
9157 unsigned Mask = (1u << Width) - 1;
9158 unsigned Mask2 = N11C->getZExtValue();
9159 if ((Mask & (~Mask2)) == 0)
9160 return DCI.DAG.getNode(ARMISD::BFI, SDLoc(N), N->getValueType(0),
9161 N->getOperand(0), N1.getOperand(0),
9162 N->getOperand(2));
9163 } else if (N->getOperand(0).getOpcode() == ARMISD::BFI) {
9164 // We have a BFI of a BFI. Walk up the BFI chain to see how long it goes.
9165 // Keep track of any consecutive bits set that all come from the same base
9166 // value. We can combine these together into a single BFI.
9167 SDValue CombineBFI = FindBFIToCombineWith(N);
9168 if (CombineBFI == SDValue())
9169 return SDValue();
9170
9171 // We've found a BFI.
9172 APInt ToMask1, FromMask1;
9173 SDValue From1 = ParseBFI(N, ToMask1, FromMask1);
9174
9175 APInt ToMask2, FromMask2;
9176 SDValue From2 = ParseBFI(CombineBFI.getNode(), ToMask2, FromMask2);
9177 assert(From1 == From2);
9178 (void)From2;
9179
9180 // First, unlink CombineBFI.
9181 DCI.DAG.ReplaceAllUsesWith(CombineBFI, CombineBFI.getOperand(0));
9182 // Then create a new BFI, combining the two together.
9183 APInt NewFromMask = FromMask1 | FromMask2;
9184 APInt NewToMask = ToMask1 | ToMask2;
9185
9186 EVT VT = N->getValueType(0);
9187 SDLoc dl(N);
9188
9189 if (NewFromMask[0] == 0)
9190 From1 = DCI.DAG.getNode(
9191 ISD::SRL, dl, VT, From1,
9192 DCI.DAG.getConstant(NewFromMask.countTrailingZeros(), dl, VT));
9193 return DCI.DAG.getNode(ARMISD::BFI, dl, VT, N->getOperand(0), From1,
9194 DCI.DAG.getConstant(~NewToMask, dl, VT));
9195 }
9196 return SDValue();
9197 }
9198
9199 /// PerformVMOVRRDCombine - Target-specific dag combine xforms for
9200 /// ARMISD::VMOVRRD.
PerformVMOVRRDCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)9201 static SDValue PerformVMOVRRDCombine(SDNode *N,
9202 TargetLowering::DAGCombinerInfo &DCI,
9203 const ARMSubtarget *Subtarget) {
9204 // vmovrrd(vmovdrr x, y) -> x,y
9205 SDValue InDouble = N->getOperand(0);
9206 if (InDouble.getOpcode() == ARMISD::VMOVDRR && !Subtarget->isFPOnlySP())
9207 return DCI.CombineTo(N, InDouble.getOperand(0), InDouble.getOperand(1));
9208
9209 // vmovrrd(load f64) -> (load i32), (load i32)
9210 SDNode *InNode = InDouble.getNode();
9211 if (ISD::isNormalLoad(InNode) && InNode->hasOneUse() &&
9212 InNode->getValueType(0) == MVT::f64 &&
9213 InNode->getOperand(1).getOpcode() == ISD::FrameIndex &&
9214 !cast<LoadSDNode>(InNode)->isVolatile()) {
9215 // TODO: Should this be done for non-FrameIndex operands?
9216 LoadSDNode *LD = cast<LoadSDNode>(InNode);
9217
9218 SelectionDAG &DAG = DCI.DAG;
9219 SDLoc DL(LD);
9220 SDValue BasePtr = LD->getBasePtr();
9221 SDValue NewLD1 = DAG.getLoad(MVT::i32, DL, LD->getChain(), BasePtr,
9222 LD->getPointerInfo(), LD->isVolatile(),
9223 LD->isNonTemporal(), LD->isInvariant(),
9224 LD->getAlignment());
9225
9226 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr,
9227 DAG.getConstant(4, DL, MVT::i32));
9228 SDValue NewLD2 = DAG.getLoad(MVT::i32, DL, NewLD1.getValue(1), OffsetPtr,
9229 LD->getPointerInfo(), LD->isVolatile(),
9230 LD->isNonTemporal(), LD->isInvariant(),
9231 std::min(4U, LD->getAlignment() / 2));
9232
9233 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), NewLD2.getValue(1));
9234 if (DCI.DAG.getDataLayout().isBigEndian())
9235 std::swap (NewLD1, NewLD2);
9236 SDValue Result = DCI.CombineTo(N, NewLD1, NewLD2);
9237 return Result;
9238 }
9239
9240 return SDValue();
9241 }
9242
9243 /// PerformVMOVDRRCombine - Target-specific dag combine xforms for
9244 /// ARMISD::VMOVDRR. This is also used for BUILD_VECTORs with 2 operands.
PerformVMOVDRRCombine(SDNode * N,SelectionDAG & DAG)9245 static SDValue PerformVMOVDRRCombine(SDNode *N, SelectionDAG &DAG) {
9246 // N=vmovrrd(X); vmovdrr(N:0, N:1) -> bit_convert(X)
9247 SDValue Op0 = N->getOperand(0);
9248 SDValue Op1 = N->getOperand(1);
9249 if (Op0.getOpcode() == ISD::BITCAST)
9250 Op0 = Op0.getOperand(0);
9251 if (Op1.getOpcode() == ISD::BITCAST)
9252 Op1 = Op1.getOperand(0);
9253 if (Op0.getOpcode() == ARMISD::VMOVRRD &&
9254 Op0.getNode() == Op1.getNode() &&
9255 Op0.getResNo() == 0 && Op1.getResNo() == 1)
9256 return DAG.getNode(ISD::BITCAST, SDLoc(N),
9257 N->getValueType(0), Op0.getOperand(0));
9258 return SDValue();
9259 }
9260
9261 /// hasNormalLoadOperand - Check if any of the operands of a BUILD_VECTOR node
9262 /// are normal, non-volatile loads. If so, it is profitable to bitcast an
9263 /// i64 vector to have f64 elements, since the value can then be loaded
9264 /// directly into a VFP register.
hasNormalLoadOperand(SDNode * N)9265 static bool hasNormalLoadOperand(SDNode *N) {
9266 unsigned NumElts = N->getValueType(0).getVectorNumElements();
9267 for (unsigned i = 0; i < NumElts; ++i) {
9268 SDNode *Elt = N->getOperand(i).getNode();
9269 if (ISD::isNormalLoad(Elt) && !cast<LoadSDNode>(Elt)->isVolatile())
9270 return true;
9271 }
9272 return false;
9273 }
9274
9275 /// PerformBUILD_VECTORCombine - Target-specific dag combine xforms for
9276 /// ISD::BUILD_VECTOR.
PerformBUILD_VECTORCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI,const ARMSubtarget * Subtarget)9277 static SDValue PerformBUILD_VECTORCombine(SDNode *N,
9278 TargetLowering::DAGCombinerInfo &DCI,
9279 const ARMSubtarget *Subtarget) {
9280 // build_vector(N=ARMISD::VMOVRRD(X), N:1) -> bit_convert(X):
9281 // VMOVRRD is introduced when legalizing i64 types. It forces the i64 value
9282 // into a pair of GPRs, which is fine when the value is used as a scalar,
9283 // but if the i64 value is converted to a vector, we need to undo the VMOVRRD.
9284 SelectionDAG &DAG = DCI.DAG;
9285 if (N->getNumOperands() == 2) {
9286 SDValue RV = PerformVMOVDRRCombine(N, DAG);
9287 if (RV.getNode())
9288 return RV;
9289 }
9290
9291 // Load i64 elements as f64 values so that type legalization does not split
9292 // them up into i32 values.
9293 EVT VT = N->getValueType(0);
9294 if (VT.getVectorElementType() != MVT::i64 || !hasNormalLoadOperand(N))
9295 return SDValue();
9296 SDLoc dl(N);
9297 SmallVector<SDValue, 8> Ops;
9298 unsigned NumElts = VT.getVectorNumElements();
9299 for (unsigned i = 0; i < NumElts; ++i) {
9300 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::f64, N->getOperand(i));
9301 Ops.push_back(V);
9302 // Make the DAGCombiner fold the bitcast.
9303 DCI.AddToWorklist(V.getNode());
9304 }
9305 EVT FloatVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64, NumElts);
9306 SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, dl, FloatVT, Ops);
9307 return DAG.getNode(ISD::BITCAST, dl, VT, BV);
9308 }
9309
9310 /// \brief Target-specific dag combine xforms for ARMISD::BUILD_VECTOR.
9311 static SDValue
PerformARMBUILD_VECTORCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI)9312 PerformARMBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
9313 // ARMISD::BUILD_VECTOR is introduced when legalizing ISD::BUILD_VECTOR.
9314 // At that time, we may have inserted bitcasts from integer to float.
9315 // If these bitcasts have survived DAGCombine, change the lowering of this
9316 // BUILD_VECTOR in something more vector friendly, i.e., that does not
9317 // force to use floating point types.
9318
9319 // Make sure we can change the type of the vector.
9320 // This is possible iff:
9321 // 1. The vector is only used in a bitcast to a integer type. I.e.,
9322 // 1.1. Vector is used only once.
9323 // 1.2. Use is a bit convert to an integer type.
9324 // 2. The size of its operands are 32-bits (64-bits are not legal).
9325 EVT VT = N->getValueType(0);
9326 EVT EltVT = VT.getVectorElementType();
9327
9328 // Check 1.1. and 2.
9329 if (EltVT.getSizeInBits() != 32 || !N->hasOneUse())
9330 return SDValue();
9331
9332 // By construction, the input type must be float.
9333 assert(EltVT == MVT::f32 && "Unexpected type!");
9334
9335 // Check 1.2.
9336 SDNode *Use = *N->use_begin();
9337 if (Use->getOpcode() != ISD::BITCAST ||
9338 Use->getValueType(0).isFloatingPoint())
9339 return SDValue();
9340
9341 // Check profitability.
9342 // Model is, if more than half of the relevant operands are bitcast from
9343 // i32, turn the build_vector into a sequence of insert_vector_elt.
9344 // Relevant operands are everything that is not statically
9345 // (i.e., at compile time) bitcasted.
9346 unsigned NumOfBitCastedElts = 0;
9347 unsigned NumElts = VT.getVectorNumElements();
9348 unsigned NumOfRelevantElts = NumElts;
9349 for (unsigned Idx = 0; Idx < NumElts; ++Idx) {
9350 SDValue Elt = N->getOperand(Idx);
9351 if (Elt->getOpcode() == ISD::BITCAST) {
9352 // Assume only bit cast to i32 will go away.
9353 if (Elt->getOperand(0).getValueType() == MVT::i32)
9354 ++NumOfBitCastedElts;
9355 } else if (Elt.getOpcode() == ISD::UNDEF || isa<ConstantSDNode>(Elt))
9356 // Constants are statically casted, thus do not count them as
9357 // relevant operands.
9358 --NumOfRelevantElts;
9359 }
9360
9361 // Check if more than half of the elements require a non-free bitcast.
9362 if (NumOfBitCastedElts <= NumOfRelevantElts / 2)
9363 return SDValue();
9364
9365 SelectionDAG &DAG = DCI.DAG;
9366 // Create the new vector type.
9367 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElts);
9368 // Check if the type is legal.
9369 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9370 if (!TLI.isTypeLegal(VecVT))
9371 return SDValue();
9372
9373 // Combine:
9374 // ARMISD::BUILD_VECTOR E1, E2, ..., EN.
9375 // => BITCAST INSERT_VECTOR_ELT
9376 // (INSERT_VECTOR_ELT (...), (BITCAST EN-1), N-1),
9377 // (BITCAST EN), N.
9378 SDValue Vec = DAG.getUNDEF(VecVT);
9379 SDLoc dl(N);
9380 for (unsigned Idx = 0 ; Idx < NumElts; ++Idx) {
9381 SDValue V = N->getOperand(Idx);
9382 if (V.getOpcode() == ISD::UNDEF)
9383 continue;
9384 if (V.getOpcode() == ISD::BITCAST &&
9385 V->getOperand(0).getValueType() == MVT::i32)
9386 // Fold obvious case.
9387 V = V.getOperand(0);
9388 else {
9389 V = DAG.getNode(ISD::BITCAST, SDLoc(V), MVT::i32, V);
9390 // Make the DAGCombiner fold the bitcasts.
9391 DCI.AddToWorklist(V.getNode());
9392 }
9393 SDValue LaneIdx = DAG.getConstant(Idx, dl, MVT::i32);
9394 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VecVT, Vec, V, LaneIdx);
9395 }
9396 Vec = DAG.getNode(ISD::BITCAST, dl, VT, Vec);
9397 // Make the DAGCombiner fold the bitcasts.
9398 DCI.AddToWorklist(Vec.getNode());
9399 return Vec;
9400 }
9401
9402 /// PerformInsertEltCombine - Target-specific dag combine xforms for
9403 /// ISD::INSERT_VECTOR_ELT.
PerformInsertEltCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI)9404 static SDValue PerformInsertEltCombine(SDNode *N,
9405 TargetLowering::DAGCombinerInfo &DCI) {
9406 // Bitcast an i64 load inserted into a vector to f64.
9407 // Otherwise, the i64 value will be legalized to a pair of i32 values.
9408 EVT VT = N->getValueType(0);
9409 SDNode *Elt = N->getOperand(1).getNode();
9410 if (VT.getVectorElementType() != MVT::i64 ||
9411 !ISD::isNormalLoad(Elt) || cast<LoadSDNode>(Elt)->isVolatile())
9412 return SDValue();
9413
9414 SelectionDAG &DAG = DCI.DAG;
9415 SDLoc dl(N);
9416 EVT FloatVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64,
9417 VT.getVectorNumElements());
9418 SDValue Vec = DAG.getNode(ISD::BITCAST, dl, FloatVT, N->getOperand(0));
9419 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::f64, N->getOperand(1));
9420 // Make the DAGCombiner fold the bitcasts.
9421 DCI.AddToWorklist(Vec.getNode());
9422 DCI.AddToWorklist(V.getNode());
9423 SDValue InsElt = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, FloatVT,
9424 Vec, V, N->getOperand(2));
9425 return DAG.getNode(ISD::BITCAST, dl, VT, InsElt);
9426 }
9427
9428 /// PerformVECTOR_SHUFFLECombine - Target-specific dag combine xforms for
9429 /// ISD::VECTOR_SHUFFLE.
PerformVECTOR_SHUFFLECombine(SDNode * N,SelectionDAG & DAG)9430 static SDValue PerformVECTOR_SHUFFLECombine(SDNode *N, SelectionDAG &DAG) {
9431 // The LLVM shufflevector instruction does not require the shuffle mask
9432 // length to match the operand vector length, but ISD::VECTOR_SHUFFLE does
9433 // have that requirement. When translating to ISD::VECTOR_SHUFFLE, if the
9434 // operands do not match the mask length, they are extended by concatenating
9435 // them with undef vectors. That is probably the right thing for other
9436 // targets, but for NEON it is better to concatenate two double-register
9437 // size vector operands into a single quad-register size vector. Do that
9438 // transformation here:
9439 // shuffle(concat(v1, undef), concat(v2, undef)) ->
9440 // shuffle(concat(v1, v2), undef)
9441 SDValue Op0 = N->getOperand(0);
9442 SDValue Op1 = N->getOperand(1);
9443 if (Op0.getOpcode() != ISD::CONCAT_VECTORS ||
9444 Op1.getOpcode() != ISD::CONCAT_VECTORS ||
9445 Op0.getNumOperands() != 2 ||
9446 Op1.getNumOperands() != 2)
9447 return SDValue();
9448 SDValue Concat0Op1 = Op0.getOperand(1);
9449 SDValue Concat1Op1 = Op1.getOperand(1);
9450 if (Concat0Op1.getOpcode() != ISD::UNDEF ||
9451 Concat1Op1.getOpcode() != ISD::UNDEF)
9452 return SDValue();
9453 // Skip the transformation if any of the types are illegal.
9454 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9455 EVT VT = N->getValueType(0);
9456 if (!TLI.isTypeLegal(VT) ||
9457 !TLI.isTypeLegal(Concat0Op1.getValueType()) ||
9458 !TLI.isTypeLegal(Concat1Op1.getValueType()))
9459 return SDValue();
9460
9461 SDValue NewConcat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT,
9462 Op0.getOperand(0), Op1.getOperand(0));
9463 // Translate the shuffle mask.
9464 SmallVector<int, 16> NewMask;
9465 unsigned NumElts = VT.getVectorNumElements();
9466 unsigned HalfElts = NumElts/2;
9467 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
9468 for (unsigned n = 0; n < NumElts; ++n) {
9469 int MaskElt = SVN->getMaskElt(n);
9470 int NewElt = -1;
9471 if (MaskElt < (int)HalfElts)
9472 NewElt = MaskElt;
9473 else if (MaskElt >= (int)NumElts && MaskElt < (int)(NumElts + HalfElts))
9474 NewElt = HalfElts + MaskElt - NumElts;
9475 NewMask.push_back(NewElt);
9476 }
9477 return DAG.getVectorShuffle(VT, SDLoc(N), NewConcat,
9478 DAG.getUNDEF(VT), NewMask.data());
9479 }
9480
9481 /// CombineBaseUpdate - Target-specific DAG combine function for VLDDUP,
9482 /// NEON load/store intrinsics, and generic vector load/stores, to merge
9483 /// base address updates.
9484 /// For generic load/stores, the memory type is assumed to be a vector.
9485 /// The caller is assumed to have checked legality.
CombineBaseUpdate(SDNode * N,TargetLowering::DAGCombinerInfo & DCI)9486 static SDValue CombineBaseUpdate(SDNode *N,
9487 TargetLowering::DAGCombinerInfo &DCI) {
9488 SelectionDAG &DAG = DCI.DAG;
9489 const bool isIntrinsic = (N->getOpcode() == ISD::INTRINSIC_VOID ||
9490 N->getOpcode() == ISD::INTRINSIC_W_CHAIN);
9491 const bool isStore = N->getOpcode() == ISD::STORE;
9492 const unsigned AddrOpIdx = ((isIntrinsic || isStore) ? 2 : 1);
9493 SDValue Addr = N->getOperand(AddrOpIdx);
9494 MemSDNode *MemN = cast<MemSDNode>(N);
9495 SDLoc dl(N);
9496
9497 // Search for a use of the address operand that is an increment.
9498 for (SDNode::use_iterator UI = Addr.getNode()->use_begin(),
9499 UE = Addr.getNode()->use_end(); UI != UE; ++UI) {
9500 SDNode *User = *UI;
9501 if (User->getOpcode() != ISD::ADD ||
9502 UI.getUse().getResNo() != Addr.getResNo())
9503 continue;
9504
9505 // Check that the add is independent of the load/store. Otherwise, folding
9506 // it would create a cycle.
9507 if (User->isPredecessorOf(N) || N->isPredecessorOf(User))
9508 continue;
9509
9510 // Find the new opcode for the updating load/store.
9511 bool isLoadOp = true;
9512 bool isLaneOp = false;
9513 unsigned NewOpc = 0;
9514 unsigned NumVecs = 0;
9515 if (isIntrinsic) {
9516 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
9517 switch (IntNo) {
9518 default: llvm_unreachable("unexpected intrinsic for Neon base update");
9519 case Intrinsic::arm_neon_vld1: NewOpc = ARMISD::VLD1_UPD;
9520 NumVecs = 1; break;
9521 case Intrinsic::arm_neon_vld2: NewOpc = ARMISD::VLD2_UPD;
9522 NumVecs = 2; break;
9523 case Intrinsic::arm_neon_vld3: NewOpc = ARMISD::VLD3_UPD;
9524 NumVecs = 3; break;
9525 case Intrinsic::arm_neon_vld4: NewOpc = ARMISD::VLD4_UPD;
9526 NumVecs = 4; break;
9527 case Intrinsic::arm_neon_vld2lane: NewOpc = ARMISD::VLD2LN_UPD;
9528 NumVecs = 2; isLaneOp = true; break;
9529 case Intrinsic::arm_neon_vld3lane: NewOpc = ARMISD::VLD3LN_UPD;
9530 NumVecs = 3; isLaneOp = true; break;
9531 case Intrinsic::arm_neon_vld4lane: NewOpc = ARMISD::VLD4LN_UPD;
9532 NumVecs = 4; isLaneOp = true; break;
9533 case Intrinsic::arm_neon_vst1: NewOpc = ARMISD::VST1_UPD;
9534 NumVecs = 1; isLoadOp = false; break;
9535 case Intrinsic::arm_neon_vst2: NewOpc = ARMISD::VST2_UPD;
9536 NumVecs = 2; isLoadOp = false; break;
9537 case Intrinsic::arm_neon_vst3: NewOpc = ARMISD::VST3_UPD;
9538 NumVecs = 3; isLoadOp = false; break;
9539 case Intrinsic::arm_neon_vst4: NewOpc = ARMISD::VST4_UPD;
9540 NumVecs = 4; isLoadOp = false; break;
9541 case Intrinsic::arm_neon_vst2lane: NewOpc = ARMISD::VST2LN_UPD;
9542 NumVecs = 2; isLoadOp = false; isLaneOp = true; break;
9543 case Intrinsic::arm_neon_vst3lane: NewOpc = ARMISD::VST3LN_UPD;
9544 NumVecs = 3; isLoadOp = false; isLaneOp = true; break;
9545 case Intrinsic::arm_neon_vst4lane: NewOpc = ARMISD::VST4LN_UPD;
9546 NumVecs = 4; isLoadOp = false; isLaneOp = true; break;
9547 }
9548 } else {
9549 isLaneOp = true;
9550 switch (N->getOpcode()) {
9551 default: llvm_unreachable("unexpected opcode for Neon base update");
9552 case ARMISD::VLD2DUP: NewOpc = ARMISD::VLD2DUP_UPD; NumVecs = 2; break;
9553 case ARMISD::VLD3DUP: NewOpc = ARMISD::VLD3DUP_UPD; NumVecs = 3; break;
9554 case ARMISD::VLD4DUP: NewOpc = ARMISD::VLD4DUP_UPD; NumVecs = 4; break;
9555 case ISD::LOAD: NewOpc = ARMISD::VLD1_UPD;
9556 NumVecs = 1; isLaneOp = false; break;
9557 case ISD::STORE: NewOpc = ARMISD::VST1_UPD;
9558 NumVecs = 1; isLaneOp = false; isLoadOp = false; break;
9559 }
9560 }
9561
9562 // Find the size of memory referenced by the load/store.
9563 EVT VecTy;
9564 if (isLoadOp) {
9565 VecTy = N->getValueType(0);
9566 } else if (isIntrinsic) {
9567 VecTy = N->getOperand(AddrOpIdx+1).getValueType();
9568 } else {
9569 assert(isStore && "Node has to be a load, a store, or an intrinsic!");
9570 VecTy = N->getOperand(1).getValueType();
9571 }
9572
9573 unsigned NumBytes = NumVecs * VecTy.getSizeInBits() / 8;
9574 if (isLaneOp)
9575 NumBytes /= VecTy.getVectorNumElements();
9576
9577 // If the increment is a constant, it must match the memory ref size.
9578 SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0);
9579 if (ConstantSDNode *CInc = dyn_cast<ConstantSDNode>(Inc.getNode())) {
9580 uint64_t IncVal = CInc->getZExtValue();
9581 if (IncVal != NumBytes)
9582 continue;
9583 } else if (NumBytes >= 3 * 16) {
9584 // VLD3/4 and VST3/4 for 128-bit vectors are implemented with two
9585 // separate instructions that make it harder to use a non-constant update.
9586 continue;
9587 }
9588
9589 // OK, we found an ADD we can fold into the base update.
9590 // Now, create a _UPD node, taking care of not breaking alignment.
9591
9592 EVT AlignedVecTy = VecTy;
9593 unsigned Alignment = MemN->getAlignment();
9594
9595 // If this is a less-than-standard-aligned load/store, change the type to
9596 // match the standard alignment.
9597 // The alignment is overlooked when selecting _UPD variants; and it's
9598 // easier to introduce bitcasts here than fix that.
9599 // There are 3 ways to get to this base-update combine:
9600 // - intrinsics: they are assumed to be properly aligned (to the standard
9601 // alignment of the memory type), so we don't need to do anything.
9602 // - ARMISD::VLDx nodes: they are only generated from the aforementioned
9603 // intrinsics, so, likewise, there's nothing to do.
9604 // - generic load/store instructions: the alignment is specified as an
9605 // explicit operand, rather than implicitly as the standard alignment
9606 // of the memory type (like the intrisics). We need to change the
9607 // memory type to match the explicit alignment. That way, we don't
9608 // generate non-standard-aligned ARMISD::VLDx nodes.
9609 if (isa<LSBaseSDNode>(N)) {
9610 if (Alignment == 0)
9611 Alignment = 1;
9612 if (Alignment < VecTy.getScalarSizeInBits() / 8) {
9613 MVT EltTy = MVT::getIntegerVT(Alignment * 8);
9614 assert(NumVecs == 1 && "Unexpected multi-element generic load/store.");
9615 assert(!isLaneOp && "Unexpected generic load/store lane.");
9616 unsigned NumElts = NumBytes / (EltTy.getSizeInBits() / 8);
9617 AlignedVecTy = MVT::getVectorVT(EltTy, NumElts);
9618 }
9619 // Don't set an explicit alignment on regular load/stores that we want
9620 // to transform to VLD/VST 1_UPD nodes.
9621 // This matches the behavior of regular load/stores, which only get an
9622 // explicit alignment if the MMO alignment is larger than the standard
9623 // alignment of the memory type.
9624 // Intrinsics, however, always get an explicit alignment, set to the
9625 // alignment of the MMO.
9626 Alignment = 1;
9627 }
9628
9629 // Create the new updating load/store node.
9630 // First, create an SDVTList for the new updating node's results.
9631 EVT Tys[6];
9632 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
9633 unsigned n;
9634 for (n = 0; n < NumResultVecs; ++n)
9635 Tys[n] = AlignedVecTy;
9636 Tys[n++] = MVT::i32;
9637 Tys[n] = MVT::Other;
9638 SDVTList SDTys = DAG.getVTList(makeArrayRef(Tys, NumResultVecs+2));
9639
9640 // Then, gather the new node's operands.
9641 SmallVector<SDValue, 8> Ops;
9642 Ops.push_back(N->getOperand(0)); // incoming chain
9643 Ops.push_back(N->getOperand(AddrOpIdx));
9644 Ops.push_back(Inc);
9645
9646 if (StoreSDNode *StN = dyn_cast<StoreSDNode>(N)) {
9647 // Try to match the intrinsic's signature
9648 Ops.push_back(StN->getValue());
9649 } else {
9650 // Loads (and of course intrinsics) match the intrinsics' signature,
9651 // so just add all but the alignment operand.
9652 for (unsigned i = AddrOpIdx + 1; i < N->getNumOperands() - 1; ++i)
9653 Ops.push_back(N->getOperand(i));
9654 }
9655
9656 // For all node types, the alignment operand is always the last one.
9657 Ops.push_back(DAG.getConstant(Alignment, dl, MVT::i32));
9658
9659 // If this is a non-standard-aligned STORE, the penultimate operand is the
9660 // stored value. Bitcast it to the aligned type.
9661 if (AlignedVecTy != VecTy && N->getOpcode() == ISD::STORE) {
9662 SDValue &StVal = Ops[Ops.size()-2];
9663 StVal = DAG.getNode(ISD::BITCAST, dl, AlignedVecTy, StVal);
9664 }
9665
9666 SDValue UpdN = DAG.getMemIntrinsicNode(NewOpc, dl, SDTys,
9667 Ops, AlignedVecTy,
9668 MemN->getMemOperand());
9669
9670 // Update the uses.
9671 SmallVector<SDValue, 5> NewResults;
9672 for (unsigned i = 0; i < NumResultVecs; ++i)
9673 NewResults.push_back(SDValue(UpdN.getNode(), i));
9674
9675 // If this is an non-standard-aligned LOAD, the first result is the loaded
9676 // value. Bitcast it to the expected result type.
9677 if (AlignedVecTy != VecTy && N->getOpcode() == ISD::LOAD) {
9678 SDValue &LdVal = NewResults[0];
9679 LdVal = DAG.getNode(ISD::BITCAST, dl, VecTy, LdVal);
9680 }
9681
9682 NewResults.push_back(SDValue(UpdN.getNode(), NumResultVecs+1)); // chain
9683 DCI.CombineTo(N, NewResults);
9684 DCI.CombineTo(User, SDValue(UpdN.getNode(), NumResultVecs));
9685
9686 break;
9687 }
9688 return SDValue();
9689 }
9690
PerformVLDCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI)9691 static SDValue PerformVLDCombine(SDNode *N,
9692 TargetLowering::DAGCombinerInfo &DCI) {
9693 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
9694 return SDValue();
9695
9696 return CombineBaseUpdate(N, DCI);
9697 }
9698
9699 /// CombineVLDDUP - For a VDUPLANE node N, check if its source operand is a
9700 /// vldN-lane (N > 1) intrinsic, and if all the other uses of that intrinsic
9701 /// are also VDUPLANEs. If so, combine them to a vldN-dup operation and
9702 /// return true.
CombineVLDDUP(SDNode * N,TargetLowering::DAGCombinerInfo & DCI)9703 static bool CombineVLDDUP(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
9704 SelectionDAG &DAG = DCI.DAG;
9705 EVT VT = N->getValueType(0);
9706 // vldN-dup instructions only support 64-bit vectors for N > 1.
9707 if (!VT.is64BitVector())
9708 return false;
9709
9710 // Check if the VDUPLANE operand is a vldN-dup intrinsic.
9711 SDNode *VLD = N->getOperand(0).getNode();
9712 if (VLD->getOpcode() != ISD::INTRINSIC_W_CHAIN)
9713 return false;
9714 unsigned NumVecs = 0;
9715 unsigned NewOpc = 0;
9716 unsigned IntNo = cast<ConstantSDNode>(VLD->getOperand(1))->getZExtValue();
9717 if (IntNo == Intrinsic::arm_neon_vld2lane) {
9718 NumVecs = 2;
9719 NewOpc = ARMISD::VLD2DUP;
9720 } else if (IntNo == Intrinsic::arm_neon_vld3lane) {
9721 NumVecs = 3;
9722 NewOpc = ARMISD::VLD3DUP;
9723 } else if (IntNo == Intrinsic::arm_neon_vld4lane) {
9724 NumVecs = 4;
9725 NewOpc = ARMISD::VLD4DUP;
9726 } else {
9727 return false;
9728 }
9729
9730 // First check that all the vldN-lane uses are VDUPLANEs and that the lane
9731 // numbers match the load.
9732 unsigned VLDLaneNo =
9733 cast<ConstantSDNode>(VLD->getOperand(NumVecs+3))->getZExtValue();
9734 for (SDNode::use_iterator UI = VLD->use_begin(), UE = VLD->use_end();
9735 UI != UE; ++UI) {
9736 // Ignore uses of the chain result.
9737 if (UI.getUse().getResNo() == NumVecs)
9738 continue;
9739 SDNode *User = *UI;
9740 if (User->getOpcode() != ARMISD::VDUPLANE ||
9741 VLDLaneNo != cast<ConstantSDNode>(User->getOperand(1))->getZExtValue())
9742 return false;
9743 }
9744
9745 // Create the vldN-dup node.
9746 EVT Tys[5];
9747 unsigned n;
9748 for (n = 0; n < NumVecs; ++n)
9749 Tys[n] = VT;
9750 Tys[n] = MVT::Other;
9751 SDVTList SDTys = DAG.getVTList(makeArrayRef(Tys, NumVecs+1));
9752 SDValue Ops[] = { VLD->getOperand(0), VLD->getOperand(2) };
9753 MemIntrinsicSDNode *VLDMemInt = cast<MemIntrinsicSDNode>(VLD);
9754 SDValue VLDDup = DAG.getMemIntrinsicNode(NewOpc, SDLoc(VLD), SDTys,
9755 Ops, VLDMemInt->getMemoryVT(),
9756 VLDMemInt->getMemOperand());
9757
9758 // Update the uses.
9759 for (SDNode::use_iterator UI = VLD->use_begin(), UE = VLD->use_end();
9760 UI != UE; ++UI) {
9761 unsigned ResNo = UI.getUse().getResNo();
9762 // Ignore uses of the chain result.
9763 if (ResNo == NumVecs)
9764 continue;
9765 SDNode *User = *UI;
9766 DCI.CombineTo(User, SDValue(VLDDup.getNode(), ResNo));
9767 }
9768
9769 // Now the vldN-lane intrinsic is dead except for its chain result.
9770 // Update uses of the chain.
9771 std::vector<SDValue> VLDDupResults;
9772 for (unsigned n = 0; n < NumVecs; ++n)
9773 VLDDupResults.push_back(SDValue(VLDDup.getNode(), n));
9774 VLDDupResults.push_back(SDValue(VLDDup.getNode(), NumVecs));
9775 DCI.CombineTo(VLD, VLDDupResults);
9776
9777 return true;
9778 }
9779
9780 /// PerformVDUPLANECombine - Target-specific dag combine xforms for
9781 /// ARMISD::VDUPLANE.
PerformVDUPLANECombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI)9782 static SDValue PerformVDUPLANECombine(SDNode *N,
9783 TargetLowering::DAGCombinerInfo &DCI) {
9784 SDValue Op = N->getOperand(0);
9785
9786 // If the source is a vldN-lane (N > 1) intrinsic, and all the other uses
9787 // of that intrinsic are also VDUPLANEs, combine them to a vldN-dup operation.
9788 if (CombineVLDDUP(N, DCI))
9789 return SDValue(N, 0);
9790
9791 // If the source is already a VMOVIMM or VMVNIMM splat, the VDUPLANE is
9792 // redundant. Ignore bit_converts for now; element sizes are checked below.
9793 while (Op.getOpcode() == ISD::BITCAST)
9794 Op = Op.getOperand(0);
9795 if (Op.getOpcode() != ARMISD::VMOVIMM && Op.getOpcode() != ARMISD::VMVNIMM)
9796 return SDValue();
9797
9798 // Make sure the VMOV element size is not bigger than the VDUPLANE elements.
9799 unsigned EltSize = Op.getValueType().getVectorElementType().getSizeInBits();
9800 // The canonical VMOV for a zero vector uses a 32-bit element size.
9801 unsigned Imm = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
9802 unsigned EltBits;
9803 if (ARM_AM::decodeNEONModImm(Imm, EltBits) == 0)
9804 EltSize = 8;
9805 EVT VT = N->getValueType(0);
9806 if (EltSize > VT.getVectorElementType().getSizeInBits())
9807 return SDValue();
9808
9809 return DCI.DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Op);
9810 }
9811
PerformLOADCombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI)9812 static SDValue PerformLOADCombine(SDNode *N,
9813 TargetLowering::DAGCombinerInfo &DCI) {
9814 EVT VT = N->getValueType(0);
9815
9816 // If this is a legal vector load, try to combine it into a VLD1_UPD.
9817 if (ISD::isNormalLoad(N) && VT.isVector() &&
9818 DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT))
9819 return CombineBaseUpdate(N, DCI);
9820
9821 return SDValue();
9822 }
9823
9824 /// PerformSTORECombine - Target-specific dag combine xforms for
9825 /// ISD::STORE.
PerformSTORECombine(SDNode * N,TargetLowering::DAGCombinerInfo & DCI)9826 static SDValue PerformSTORECombine(SDNode *N,
9827 TargetLowering::DAGCombinerInfo &DCI) {
9828 StoreSDNode *St = cast<StoreSDNode>(N);
9829 if (St->isVolatile())
9830 return SDValue();
9831
9832 // Optimize trunc store (of multiple scalars) to shuffle and store. First,
9833 // pack all of the elements in one place. Next, store to memory in fewer
9834 // chunks.
9835 SDValue StVal = St->getValue();
9836 EVT VT = StVal.getValueType();
9837 if (St->isTruncatingStore() && VT.isVector()) {
9838 SelectionDAG &DAG = DCI.DAG;
9839 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9840 EVT StVT = St->getMemoryVT();
9841 unsigned NumElems = VT.getVectorNumElements();
9842 assert(StVT != VT && "Cannot truncate to the same type");
9843 unsigned FromEltSz = VT.getVectorElementType().getSizeInBits();
9844 unsigned ToEltSz = StVT.getVectorElementType().getSizeInBits();
9845
9846 // From, To sizes and ElemCount must be pow of two
9847 if (!isPowerOf2_32(NumElems * FromEltSz * ToEltSz)) return SDValue();
9848
9849 // We are going to use the original vector elt for storing.
9850 // Accumulated smaller vector elements must be a multiple of the store size.
9851 if (0 != (NumElems * FromEltSz) % ToEltSz) return SDValue();
9852
9853 unsigned SizeRatio = FromEltSz / ToEltSz;
9854 assert(SizeRatio * NumElems * ToEltSz == VT.getSizeInBits());
9855
9856 // Create a type on which we perform the shuffle.
9857 EVT WideVecVT = EVT::getVectorVT(*DAG.getContext(), StVT.getScalarType(),
9858 NumElems*SizeRatio);
9859 assert(WideVecVT.getSizeInBits() == VT.getSizeInBits());
9860
9861 SDLoc DL(St);
9862 SDValue WideVec = DAG.getNode(ISD::BITCAST, DL, WideVecVT, StVal);
9863 SmallVector<int, 8> ShuffleVec(NumElems * SizeRatio, -1);
9864 for (unsigned i = 0; i < NumElems; ++i)
9865 ShuffleVec[i] = DAG.getDataLayout().isBigEndian()
9866 ? (i + 1) * SizeRatio - 1
9867 : i * SizeRatio;
9868
9869 // Can't shuffle using an illegal type.
9870 if (!TLI.isTypeLegal(WideVecVT)) return SDValue();
9871
9872 SDValue Shuff = DAG.getVectorShuffle(WideVecVT, DL, WideVec,
9873 DAG.getUNDEF(WideVec.getValueType()),
9874 ShuffleVec.data());
9875 // At this point all of the data is stored at the bottom of the
9876 // register. We now need to save it to mem.
9877
9878 // Find the largest store unit
9879 MVT StoreType = MVT::i8;
9880 for (MVT Tp : MVT::integer_valuetypes()) {
9881 if (TLI.isTypeLegal(Tp) && Tp.getSizeInBits() <= NumElems * ToEltSz)
9882 StoreType = Tp;
9883 }
9884 // Didn't find a legal store type.
9885 if (!TLI.isTypeLegal(StoreType))
9886 return SDValue();
9887
9888 // Bitcast the original vector into a vector of store-size units
9889 EVT StoreVecVT = EVT::getVectorVT(*DAG.getContext(),
9890 StoreType, VT.getSizeInBits()/EVT(StoreType).getSizeInBits());
9891 assert(StoreVecVT.getSizeInBits() == VT.getSizeInBits());
9892 SDValue ShuffWide = DAG.getNode(ISD::BITCAST, DL, StoreVecVT, Shuff);
9893 SmallVector<SDValue, 8> Chains;
9894 SDValue Increment = DAG.getConstant(StoreType.getSizeInBits() / 8, DL,
9895 TLI.getPointerTy(DAG.getDataLayout()));
9896 SDValue BasePtr = St->getBasePtr();
9897
9898 // Perform one or more big stores into memory.
9899 unsigned E = (ToEltSz*NumElems)/StoreType.getSizeInBits();
9900 for (unsigned I = 0; I < E; I++) {
9901 SDValue SubVec = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
9902 StoreType, ShuffWide,
9903 DAG.getIntPtrConstant(I, DL));
9904 SDValue Ch = DAG.getStore(St->getChain(), DL, SubVec, BasePtr,
9905 St->getPointerInfo(), St->isVolatile(),
9906 St->isNonTemporal(), St->getAlignment());
9907 BasePtr = DAG.getNode(ISD::ADD, DL, BasePtr.getValueType(), BasePtr,
9908 Increment);
9909 Chains.push_back(Ch);
9910 }
9911 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
9912 }
9913
9914 if (!ISD::isNormalStore(St))
9915 return SDValue();
9916
9917 // Split a store of a VMOVDRR into two integer stores to avoid mixing NEON and
9918 // ARM stores of arguments in the same cache line.
9919 if (StVal.getNode()->getOpcode() == ARMISD::VMOVDRR &&
9920 StVal.getNode()->hasOneUse()) {
9921 SelectionDAG &DAG = DCI.DAG;
9922 bool isBigEndian = DAG.getDataLayout().isBigEndian();
9923 SDLoc DL(St);
9924 SDValue BasePtr = St->getBasePtr();
9925 SDValue NewST1 = DAG.getStore(St->getChain(), DL,
9926 StVal.getNode()->getOperand(isBigEndian ? 1 : 0 ),
9927 BasePtr, St->getPointerInfo(), St->isVolatile(),
9928 St->isNonTemporal(), St->getAlignment());
9929
9930 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr,
9931 DAG.getConstant(4, DL, MVT::i32));
9932 return DAG.getStore(NewST1.getValue(0), DL,
9933 StVal.getNode()->getOperand(isBigEndian ? 0 : 1),
9934 OffsetPtr, St->getPointerInfo(), St->isVolatile(),
9935 St->isNonTemporal(),
9936 std::min(4U, St->getAlignment() / 2));
9937 }
9938
9939 if (StVal.getValueType() == MVT::i64 &&
9940 StVal.getNode()->getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
9941
9942 // Bitcast an i64 store extracted from a vector to f64.
9943 // Otherwise, the i64 value will be legalized to a pair of i32 values.
9944 SelectionDAG &DAG = DCI.DAG;
9945 SDLoc dl(StVal);
9946 SDValue IntVec = StVal.getOperand(0);
9947 EVT FloatVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64,
9948 IntVec.getValueType().getVectorNumElements());
9949 SDValue Vec = DAG.getNode(ISD::BITCAST, dl, FloatVT, IntVec);
9950 SDValue ExtElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
9951 Vec, StVal.getOperand(1));
9952 dl = SDLoc(N);
9953 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::i64, ExtElt);
9954 // Make the DAGCombiner fold the bitcasts.
9955 DCI.AddToWorklist(Vec.getNode());
9956 DCI.AddToWorklist(ExtElt.getNode());
9957 DCI.AddToWorklist(V.getNode());
9958 return DAG.getStore(St->getChain(), dl, V, St->getBasePtr(),
9959 St->getPointerInfo(), St->isVolatile(),
9960 St->isNonTemporal(), St->getAlignment(),
9961 St->getAAInfo());
9962 }
9963
9964 // If this is a legal vector store, try to combine it into a VST1_UPD.
9965 if (ISD::isNormalStore(N) && VT.isVector() &&
9966 DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT))
9967 return CombineBaseUpdate(N, DCI);
9968
9969 return SDValue();
9970 }
9971
9972 /// PerformVCVTCombine - VCVT (floating-point to fixed-point, Advanced SIMD)
9973 /// can replace combinations of VMUL and VCVT (floating-point to integer)
9974 /// when the VMUL has a constant operand that is a power of 2.
9975 ///
9976 /// Example (assume d17 = <float 8.000000e+00, float 8.000000e+00>):
9977 /// vmul.f32 d16, d17, d16
9978 /// vcvt.s32.f32 d16, d16
9979 /// becomes:
9980 /// vcvt.s32.f32 d16, d16, #3
PerformVCVTCombine(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * Subtarget)9981 static SDValue PerformVCVTCombine(SDNode *N, SelectionDAG &DAG,
9982 const ARMSubtarget *Subtarget) {
9983 if (!Subtarget->hasNEON())
9984 return SDValue();
9985
9986 SDValue Op = N->getOperand(0);
9987 if (!Op.getValueType().isVector() || Op.getOpcode() != ISD::FMUL)
9988 return SDValue();
9989
9990 SDValue ConstVec = Op->getOperand(1);
9991 if (!isa<BuildVectorSDNode>(ConstVec))
9992 return SDValue();
9993
9994 MVT FloatTy = Op.getSimpleValueType().getVectorElementType();
9995 uint32_t FloatBits = FloatTy.getSizeInBits();
9996 MVT IntTy = N->getSimpleValueType(0).getVectorElementType();
9997 uint32_t IntBits = IntTy.getSizeInBits();
9998 unsigned NumLanes = Op.getValueType().getVectorNumElements();
9999 if (FloatBits != 32 || IntBits > 32 || NumLanes > 4) {
10000 // These instructions only exist converting from f32 to i32. We can handle
10001 // smaller integers by generating an extra truncate, but larger ones would
10002 // be lossy. We also can't handle more then 4 lanes, since these intructions
10003 // only support v2i32/v4i32 types.
10004 return SDValue();
10005 }
10006
10007 BitVector UndefElements;
10008 BuildVectorSDNode *BV = cast<BuildVectorSDNode>(ConstVec);
10009 int32_t C = BV->getConstantFPSplatPow2ToLog2Int(&UndefElements, 33);
10010 if (C == -1 || C == 0 || C > 32)
10011 return SDValue();
10012
10013 SDLoc dl(N);
10014 bool isSigned = N->getOpcode() == ISD::FP_TO_SINT;
10015 unsigned IntrinsicOpcode = isSigned ? Intrinsic::arm_neon_vcvtfp2fxs :
10016 Intrinsic::arm_neon_vcvtfp2fxu;
10017 SDValue FixConv = DAG.getNode(
10018 ISD::INTRINSIC_WO_CHAIN, dl, NumLanes == 2 ? MVT::v2i32 : MVT::v4i32,
10019 DAG.getConstant(IntrinsicOpcode, dl, MVT::i32), Op->getOperand(0),
10020 DAG.getConstant(C, dl, MVT::i32));
10021
10022 if (IntBits < FloatBits)
10023 FixConv = DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), FixConv);
10024
10025 return FixConv;
10026 }
10027
10028 /// PerformVDIVCombine - VCVT (fixed-point to floating-point, Advanced SIMD)
10029 /// can replace combinations of VCVT (integer to floating-point) and VDIV
10030 /// when the VDIV has a constant operand that is a power of 2.
10031 ///
10032 /// Example (assume d17 = <float 8.000000e+00, float 8.000000e+00>):
10033 /// vcvt.f32.s32 d16, d16
10034 /// vdiv.f32 d16, d17, d16
10035 /// becomes:
10036 /// vcvt.f32.s32 d16, d16, #3
PerformVDIVCombine(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * Subtarget)10037 static SDValue PerformVDIVCombine(SDNode *N, SelectionDAG &DAG,
10038 const ARMSubtarget *Subtarget) {
10039 if (!Subtarget->hasNEON())
10040 return SDValue();
10041
10042 SDValue Op = N->getOperand(0);
10043 unsigned OpOpcode = Op.getNode()->getOpcode();
10044 if (!N->getValueType(0).isVector() ||
10045 (OpOpcode != ISD::SINT_TO_FP && OpOpcode != ISD::UINT_TO_FP))
10046 return SDValue();
10047
10048 SDValue ConstVec = N->getOperand(1);
10049 if (!isa<BuildVectorSDNode>(ConstVec))
10050 return SDValue();
10051
10052 MVT FloatTy = N->getSimpleValueType(0).getVectorElementType();
10053 uint32_t FloatBits = FloatTy.getSizeInBits();
10054 MVT IntTy = Op.getOperand(0).getSimpleValueType().getVectorElementType();
10055 uint32_t IntBits = IntTy.getSizeInBits();
10056 unsigned NumLanes = Op.getValueType().getVectorNumElements();
10057 if (FloatBits != 32 || IntBits > 32 || NumLanes > 4) {
10058 // These instructions only exist converting from i32 to f32. We can handle
10059 // smaller integers by generating an extra extend, but larger ones would
10060 // be lossy. We also can't handle more then 4 lanes, since these intructions
10061 // only support v2i32/v4i32 types.
10062 return SDValue();
10063 }
10064
10065 BitVector UndefElements;
10066 BuildVectorSDNode *BV = cast<BuildVectorSDNode>(ConstVec);
10067 int32_t C = BV->getConstantFPSplatPow2ToLog2Int(&UndefElements, 33);
10068 if (C == -1 || C == 0 || C > 32)
10069 return SDValue();
10070
10071 SDLoc dl(N);
10072 bool isSigned = OpOpcode == ISD::SINT_TO_FP;
10073 SDValue ConvInput = Op.getOperand(0);
10074 if (IntBits < FloatBits)
10075 ConvInput = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
10076 dl, NumLanes == 2 ? MVT::v2i32 : MVT::v4i32,
10077 ConvInput);
10078
10079 unsigned IntrinsicOpcode = isSigned ? Intrinsic::arm_neon_vcvtfxs2fp :
10080 Intrinsic::arm_neon_vcvtfxu2fp;
10081 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl,
10082 Op.getValueType(),
10083 DAG.getConstant(IntrinsicOpcode, dl, MVT::i32),
10084 ConvInput, DAG.getConstant(C, dl, MVT::i32));
10085 }
10086
10087 /// Getvshiftimm - Check if this is a valid build_vector for the immediate
10088 /// operand of a vector shift operation, where all the elements of the
10089 /// build_vector must have the same constant integer value.
getVShiftImm(SDValue Op,unsigned ElementBits,int64_t & Cnt)10090 static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt) {
10091 // Ignore bit_converts.
10092 while (Op.getOpcode() == ISD::BITCAST)
10093 Op = Op.getOperand(0);
10094 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Op.getNode());
10095 APInt SplatBits, SplatUndef;
10096 unsigned SplatBitSize;
10097 bool HasAnyUndefs;
10098 if (! BVN || ! BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize,
10099 HasAnyUndefs, ElementBits) ||
10100 SplatBitSize > ElementBits)
10101 return false;
10102 Cnt = SplatBits.getSExtValue();
10103 return true;
10104 }
10105
10106 /// isVShiftLImm - Check if this is a valid build_vector for the immediate
10107 /// operand of a vector shift left operation. That value must be in the range:
10108 /// 0 <= Value < ElementBits for a left shift; or
10109 /// 0 <= Value <= ElementBits for a long left shift.
isVShiftLImm(SDValue Op,EVT VT,bool isLong,int64_t & Cnt)10110 static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) {
10111 assert(VT.isVector() && "vector shift count is not a vector type");
10112 int64_t ElementBits = VT.getVectorElementType().getSizeInBits();
10113 if (! getVShiftImm(Op, ElementBits, Cnt))
10114 return false;
10115 return (Cnt >= 0 && (isLong ? Cnt-1 : Cnt) < ElementBits);
10116 }
10117
10118 /// isVShiftRImm - Check if this is a valid build_vector for the immediate
10119 /// operand of a vector shift right operation. For a shift opcode, the value
10120 /// is positive, but for an intrinsic the value count must be negative. The
10121 /// absolute value must be in the range:
10122 /// 1 <= |Value| <= ElementBits for a right shift; or
10123 /// 1 <= |Value| <= ElementBits/2 for a narrow right shift.
isVShiftRImm(SDValue Op,EVT VT,bool isNarrow,bool isIntrinsic,int64_t & Cnt)10124 static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, bool isIntrinsic,
10125 int64_t &Cnt) {
10126 assert(VT.isVector() && "vector shift count is not a vector type");
10127 int64_t ElementBits = VT.getVectorElementType().getSizeInBits();
10128 if (! getVShiftImm(Op, ElementBits, Cnt))
10129 return false;
10130 if (!isIntrinsic)
10131 return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits/2 : ElementBits));
10132 if (Cnt >= -(isNarrow ? ElementBits/2 : ElementBits) && Cnt <= -1) {
10133 Cnt = -Cnt;
10134 return true;
10135 }
10136 return false;
10137 }
10138
10139 /// PerformIntrinsicCombine - ARM-specific DAG combining for intrinsics.
PerformIntrinsicCombine(SDNode * N,SelectionDAG & DAG)10140 static SDValue PerformIntrinsicCombine(SDNode *N, SelectionDAG &DAG) {
10141 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
10142 switch (IntNo) {
10143 default:
10144 // Don't do anything for most intrinsics.
10145 break;
10146
10147 // Vector shifts: check for immediate versions and lower them.
10148 // Note: This is done during DAG combining instead of DAG legalizing because
10149 // the build_vectors for 64-bit vector element shift counts are generally
10150 // not legal, and it is hard to see their values after they get legalized to
10151 // loads from a constant pool.
10152 case Intrinsic::arm_neon_vshifts:
10153 case Intrinsic::arm_neon_vshiftu:
10154 case Intrinsic::arm_neon_vrshifts:
10155 case Intrinsic::arm_neon_vrshiftu:
10156 case Intrinsic::arm_neon_vrshiftn:
10157 case Intrinsic::arm_neon_vqshifts:
10158 case Intrinsic::arm_neon_vqshiftu:
10159 case Intrinsic::arm_neon_vqshiftsu:
10160 case Intrinsic::arm_neon_vqshiftns:
10161 case Intrinsic::arm_neon_vqshiftnu:
10162 case Intrinsic::arm_neon_vqshiftnsu:
10163 case Intrinsic::arm_neon_vqrshiftns:
10164 case Intrinsic::arm_neon_vqrshiftnu:
10165 case Intrinsic::arm_neon_vqrshiftnsu: {
10166 EVT VT = N->getOperand(1).getValueType();
10167 int64_t Cnt;
10168 unsigned VShiftOpc = 0;
10169
10170 switch (IntNo) {
10171 case Intrinsic::arm_neon_vshifts:
10172 case Intrinsic::arm_neon_vshiftu:
10173 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt)) {
10174 VShiftOpc = ARMISD::VSHL;
10175 break;
10176 }
10177 if (isVShiftRImm(N->getOperand(2), VT, false, true, Cnt)) {
10178 VShiftOpc = (IntNo == Intrinsic::arm_neon_vshifts ?
10179 ARMISD::VSHRs : ARMISD::VSHRu);
10180 break;
10181 }
10182 return SDValue();
10183
10184 case Intrinsic::arm_neon_vrshifts:
10185 case Intrinsic::arm_neon_vrshiftu:
10186 if (isVShiftRImm(N->getOperand(2), VT, false, true, Cnt))
10187 break;
10188 return SDValue();
10189
10190 case Intrinsic::arm_neon_vqshifts:
10191 case Intrinsic::arm_neon_vqshiftu:
10192 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt))
10193 break;
10194 return SDValue();
10195
10196 case Intrinsic::arm_neon_vqshiftsu:
10197 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt))
10198 break;
10199 llvm_unreachable("invalid shift count for vqshlu intrinsic");
10200
10201 case Intrinsic::arm_neon_vrshiftn:
10202 case Intrinsic::arm_neon_vqshiftns:
10203 case Intrinsic::arm_neon_vqshiftnu:
10204 case Intrinsic::arm_neon_vqshiftnsu:
10205 case Intrinsic::arm_neon_vqrshiftns:
10206 case Intrinsic::arm_neon_vqrshiftnu:
10207 case Intrinsic::arm_neon_vqrshiftnsu:
10208 // Narrowing shifts require an immediate right shift.
10209 if (isVShiftRImm(N->getOperand(2), VT, true, true, Cnt))
10210 break;
10211 llvm_unreachable("invalid shift count for narrowing vector shift "
10212 "intrinsic");
10213
10214 default:
10215 llvm_unreachable("unhandled vector shift");
10216 }
10217
10218 switch (IntNo) {
10219 case Intrinsic::arm_neon_vshifts:
10220 case Intrinsic::arm_neon_vshiftu:
10221 // Opcode already set above.
10222 break;
10223 case Intrinsic::arm_neon_vrshifts:
10224 VShiftOpc = ARMISD::VRSHRs; break;
10225 case Intrinsic::arm_neon_vrshiftu:
10226 VShiftOpc = ARMISD::VRSHRu; break;
10227 case Intrinsic::arm_neon_vrshiftn:
10228 VShiftOpc = ARMISD::VRSHRN; break;
10229 case Intrinsic::arm_neon_vqshifts:
10230 VShiftOpc = ARMISD::VQSHLs; break;
10231 case Intrinsic::arm_neon_vqshiftu:
10232 VShiftOpc = ARMISD::VQSHLu; break;
10233 case Intrinsic::arm_neon_vqshiftsu:
10234 VShiftOpc = ARMISD::VQSHLsu; break;
10235 case Intrinsic::arm_neon_vqshiftns:
10236 VShiftOpc = ARMISD::VQSHRNs; break;
10237 case Intrinsic::arm_neon_vqshiftnu:
10238 VShiftOpc = ARMISD::VQSHRNu; break;
10239 case Intrinsic::arm_neon_vqshiftnsu:
10240 VShiftOpc = ARMISD::VQSHRNsu; break;
10241 case Intrinsic::arm_neon_vqrshiftns:
10242 VShiftOpc = ARMISD::VQRSHRNs; break;
10243 case Intrinsic::arm_neon_vqrshiftnu:
10244 VShiftOpc = ARMISD::VQRSHRNu; break;
10245 case Intrinsic::arm_neon_vqrshiftnsu:
10246 VShiftOpc = ARMISD::VQRSHRNsu; break;
10247 }
10248
10249 SDLoc dl(N);
10250 return DAG.getNode(VShiftOpc, dl, N->getValueType(0),
10251 N->getOperand(1), DAG.getConstant(Cnt, dl, MVT::i32));
10252 }
10253
10254 case Intrinsic::arm_neon_vshiftins: {
10255 EVT VT = N->getOperand(1).getValueType();
10256 int64_t Cnt;
10257 unsigned VShiftOpc = 0;
10258
10259 if (isVShiftLImm(N->getOperand(3), VT, false, Cnt))
10260 VShiftOpc = ARMISD::VSLI;
10261 else if (isVShiftRImm(N->getOperand(3), VT, false, true, Cnt))
10262 VShiftOpc = ARMISD::VSRI;
10263 else {
10264 llvm_unreachable("invalid shift count for vsli/vsri intrinsic");
10265 }
10266
10267 SDLoc dl(N);
10268 return DAG.getNode(VShiftOpc, dl, N->getValueType(0),
10269 N->getOperand(1), N->getOperand(2),
10270 DAG.getConstant(Cnt, dl, MVT::i32));
10271 }
10272
10273 case Intrinsic::arm_neon_vqrshifts:
10274 case Intrinsic::arm_neon_vqrshiftu:
10275 // No immediate versions of these to check for.
10276 break;
10277 }
10278
10279 return SDValue();
10280 }
10281
10282 /// PerformShiftCombine - Checks for immediate versions of vector shifts and
10283 /// lowers them. As with the vector shift intrinsics, this is done during DAG
10284 /// combining instead of DAG legalizing because the build_vectors for 64-bit
10285 /// vector element shift counts are generally not legal, and it is hard to see
10286 /// their values after they get legalized to loads from a constant pool.
PerformShiftCombine(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * ST)10287 static SDValue PerformShiftCombine(SDNode *N, SelectionDAG &DAG,
10288 const ARMSubtarget *ST) {
10289 EVT VT = N->getValueType(0);
10290 if (N->getOpcode() == ISD::SRL && VT == MVT::i32 && ST->hasV6Ops()) {
10291 // Canonicalize (srl (bswap x), 16) to (rotr (bswap x), 16) if the high
10292 // 16-bits of x is zero. This optimizes rev + lsr 16 to rev16.
10293 SDValue N1 = N->getOperand(1);
10294 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
10295 SDValue N0 = N->getOperand(0);
10296 if (C->getZExtValue() == 16 && N0.getOpcode() == ISD::BSWAP &&
10297 DAG.MaskedValueIsZero(N0.getOperand(0),
10298 APInt::getHighBitsSet(32, 16)))
10299 return DAG.getNode(ISD::ROTR, SDLoc(N), VT, N0, N1);
10300 }
10301 }
10302
10303 // Nothing to be done for scalar shifts.
10304 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10305 if (!VT.isVector() || !TLI.isTypeLegal(VT))
10306 return SDValue();
10307
10308 assert(ST->hasNEON() && "unexpected vector shift");
10309 int64_t Cnt;
10310
10311 switch (N->getOpcode()) {
10312 default: llvm_unreachable("unexpected shift opcode");
10313
10314 case ISD::SHL:
10315 if (isVShiftLImm(N->getOperand(1), VT, false, Cnt)) {
10316 SDLoc dl(N);
10317 return DAG.getNode(ARMISD::VSHL, dl, VT, N->getOperand(0),
10318 DAG.getConstant(Cnt, dl, MVT::i32));
10319 }
10320 break;
10321
10322 case ISD::SRA:
10323 case ISD::SRL:
10324 if (isVShiftRImm(N->getOperand(1), VT, false, false, Cnt)) {
10325 unsigned VShiftOpc = (N->getOpcode() == ISD::SRA ?
10326 ARMISD::VSHRs : ARMISD::VSHRu);
10327 SDLoc dl(N);
10328 return DAG.getNode(VShiftOpc, dl, VT, N->getOperand(0),
10329 DAG.getConstant(Cnt, dl, MVT::i32));
10330 }
10331 }
10332 return SDValue();
10333 }
10334
10335 /// PerformExtendCombine - Target-specific DAG combining for ISD::SIGN_EXTEND,
10336 /// ISD::ZERO_EXTEND, and ISD::ANY_EXTEND.
PerformExtendCombine(SDNode * N,SelectionDAG & DAG,const ARMSubtarget * ST)10337 static SDValue PerformExtendCombine(SDNode *N, SelectionDAG &DAG,
10338 const ARMSubtarget *ST) {
10339 SDValue N0 = N->getOperand(0);
10340
10341 // Check for sign- and zero-extensions of vector extract operations of 8-
10342 // and 16-bit vector elements. NEON supports these directly. They are
10343 // handled during DAG combining because type legalization will promote them
10344 // to 32-bit types and it is messy to recognize the operations after that.
10345 if (ST->hasNEON() && N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
10346 SDValue Vec = N0.getOperand(0);
10347 SDValue Lane = N0.getOperand(1);
10348 EVT VT = N->getValueType(0);
10349 EVT EltVT = N0.getValueType();
10350 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10351
10352 if (VT == MVT::i32 &&
10353 (EltVT == MVT::i8 || EltVT == MVT::i16) &&
10354 TLI.isTypeLegal(Vec.getValueType()) &&
10355 isa<ConstantSDNode>(Lane)) {
10356
10357 unsigned Opc = 0;
10358 switch (N->getOpcode()) {
10359 default: llvm_unreachable("unexpected opcode");
10360 case ISD::SIGN_EXTEND:
10361 Opc = ARMISD::VGETLANEs;
10362 break;
10363 case ISD::ZERO_EXTEND:
10364 case ISD::ANY_EXTEND:
10365 Opc = ARMISD::VGETLANEu;
10366 break;
10367 }
10368 return DAG.getNode(Opc, SDLoc(N), VT, Vec, Lane);
10369 }
10370 }
10371
10372 return SDValue();
10373 }
10374
computeKnownBits(SelectionDAG & DAG,SDValue Op,APInt & KnownZero,APInt & KnownOne)10375 static void computeKnownBits(SelectionDAG &DAG, SDValue Op, APInt &KnownZero,
10376 APInt &KnownOne) {
10377 if (Op.getOpcode() == ARMISD::BFI) {
10378 // Conservatively, we can recurse down the first operand
10379 // and just mask out all affected bits.
10380 computeKnownBits(DAG, Op.getOperand(0), KnownZero, KnownOne);
10381
10382 // The operand to BFI is already a mask suitable for removing the bits it
10383 // sets.
10384 ConstantSDNode *CI = cast<ConstantSDNode>(Op.getOperand(2));
10385 APInt Mask = CI->getAPIntValue();
10386 KnownZero &= Mask;
10387 KnownOne &= Mask;
10388 return;
10389 }
10390 if (Op.getOpcode() == ARMISD::CMOV) {
10391 APInt KZ2(KnownZero.getBitWidth(), 0);
10392 APInt KO2(KnownOne.getBitWidth(), 0);
10393 computeKnownBits(DAG, Op.getOperand(1), KnownZero, KnownOne);
10394 computeKnownBits(DAG, Op.getOperand(2), KZ2, KO2);
10395
10396 KnownZero &= KZ2;
10397 KnownOne &= KO2;
10398 return;
10399 }
10400 return DAG.computeKnownBits(Op, KnownZero, KnownOne);
10401 }
10402
PerformCMOVToBFICombine(SDNode * CMOV,SelectionDAG & DAG) const10403 SDValue ARMTargetLowering::PerformCMOVToBFICombine(SDNode *CMOV, SelectionDAG &DAG) const {
10404 // If we have a CMOV, OR and AND combination such as:
10405 // if (x & CN)
10406 // y |= CM;
10407 //
10408 // And:
10409 // * CN is a single bit;
10410 // * All bits covered by CM are known zero in y
10411 //
10412 // Then we can convert this into a sequence of BFI instructions. This will
10413 // always be a win if CM is a single bit, will always be no worse than the
10414 // TST&OR sequence if CM is two bits, and for thumb will be no worse if CM is
10415 // three bits (due to the extra IT instruction).
10416
10417 SDValue Op0 = CMOV->getOperand(0);
10418 SDValue Op1 = CMOV->getOperand(1);
10419 auto CCNode = cast<ConstantSDNode>(CMOV->getOperand(2));
10420 auto CC = CCNode->getAPIntValue().getLimitedValue();
10421 SDValue CmpZ = CMOV->getOperand(4);
10422
10423 // The compare must be against zero.
10424 if (!isNullConstant(CmpZ->getOperand(1)))
10425 return SDValue();
10426
10427 assert(CmpZ->getOpcode() == ARMISD::CMPZ);
10428 SDValue And = CmpZ->getOperand(0);
10429 if (And->getOpcode() != ISD::AND)
10430 return SDValue();
10431 ConstantSDNode *AndC = dyn_cast<ConstantSDNode>(And->getOperand(1));
10432 if (!AndC || !AndC->getAPIntValue().isPowerOf2())
10433 return SDValue();
10434 SDValue X = And->getOperand(0);
10435
10436 if (CC == ARMCC::EQ) {
10437 // We're performing an "equal to zero" compare. Swap the operands so we
10438 // canonicalize on a "not equal to zero" compare.
10439 std::swap(Op0, Op1);
10440 } else {
10441 assert(CC == ARMCC::NE && "How can a CMPZ node not be EQ or NE?");
10442 }
10443
10444 if (Op1->getOpcode() != ISD::OR)
10445 return SDValue();
10446
10447 ConstantSDNode *OrC = dyn_cast<ConstantSDNode>(Op1->getOperand(1));
10448 if (!OrC)
10449 return SDValue();
10450 SDValue Y = Op1->getOperand(0);
10451
10452 if (Op0 != Y)
10453 return SDValue();
10454
10455 // Now, is it profitable to continue?
10456 APInt OrCI = OrC->getAPIntValue();
10457 unsigned Heuristic = Subtarget->isThumb() ? 3 : 2;
10458 if (OrCI.countPopulation() > Heuristic)
10459 return SDValue();
10460
10461 // Lastly, can we determine that the bits defined by OrCI
10462 // are zero in Y?
10463 APInt KnownZero, KnownOne;
10464 computeKnownBits(DAG, Y, KnownZero, KnownOne);
10465 if ((OrCI & KnownZero) != OrCI)
10466 return SDValue();
10467
10468 // OK, we can do the combine.
10469 SDValue V = Y;
10470 SDLoc dl(X);
10471 EVT VT = X.getValueType();
10472 unsigned BitInX = AndC->getAPIntValue().logBase2();
10473
10474 if (BitInX != 0) {
10475 // We must shift X first.
10476 X = DAG.getNode(ISD::SRL, dl, VT, X,
10477 DAG.getConstant(BitInX, dl, VT));
10478 }
10479
10480 for (unsigned BitInY = 0, NumActiveBits = OrCI.getActiveBits();
10481 BitInY < NumActiveBits; ++BitInY) {
10482 if (OrCI[BitInY] == 0)
10483 continue;
10484 APInt Mask(VT.getSizeInBits(), 0);
10485 Mask.setBit(BitInY);
10486 V = DAG.getNode(ARMISD::BFI, dl, VT, V, X,
10487 // Confusingly, the operand is an *inverted* mask.
10488 DAG.getConstant(~Mask, dl, VT));
10489 }
10490
10491 return V;
10492 }
10493
10494 /// PerformCMOVCombine - Target-specific DAG combining for ARMISD::CMOV.
10495 SDValue
PerformCMOVCombine(SDNode * N,SelectionDAG & DAG) const10496 ARMTargetLowering::PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const {
10497 SDValue Cmp = N->getOperand(4);
10498 if (Cmp.getOpcode() != ARMISD::CMPZ)
10499 // Only looking at EQ and NE cases.
10500 return SDValue();
10501
10502 EVT VT = N->getValueType(0);
10503 SDLoc dl(N);
10504 SDValue LHS = Cmp.getOperand(0);
10505 SDValue RHS = Cmp.getOperand(1);
10506 SDValue FalseVal = N->getOperand(0);
10507 SDValue TrueVal = N->getOperand(1);
10508 SDValue ARMcc = N->getOperand(2);
10509 ARMCC::CondCodes CC =
10510 (ARMCC::CondCodes)cast<ConstantSDNode>(ARMcc)->getZExtValue();
10511
10512 // BFI is only available on V6T2+.
10513 if (!Subtarget->isThumb1Only() && Subtarget->hasV6T2Ops()) {
10514 SDValue R = PerformCMOVToBFICombine(N, DAG);
10515 if (R)
10516 return R;
10517 }
10518
10519 // Simplify
10520 // mov r1, r0
10521 // cmp r1, x
10522 // mov r0, y
10523 // moveq r0, x
10524 // to
10525 // cmp r0, x
10526 // movne r0, y
10527 //
10528 // mov r1, r0
10529 // cmp r1, x
10530 // mov r0, x
10531 // movne r0, y
10532 // to
10533 // cmp r0, x
10534 // movne r0, y
10535 /// FIXME: Turn this into a target neutral optimization?
10536 SDValue Res;
10537 if (CC == ARMCC::NE && FalseVal == RHS && FalseVal != LHS) {
10538 Res = DAG.getNode(ARMISD::CMOV, dl, VT, LHS, TrueVal, ARMcc,
10539 N->getOperand(3), Cmp);
10540 } else if (CC == ARMCC::EQ && TrueVal == RHS) {
10541 SDValue ARMcc;
10542 SDValue NewCmp = getARMCmp(LHS, RHS, ISD::SETNE, ARMcc, DAG, dl);
10543 Res = DAG.getNode(ARMISD::CMOV, dl, VT, LHS, FalseVal, ARMcc,
10544 N->getOperand(3), NewCmp);
10545 }
10546
10547 if (Res.getNode()) {
10548 APInt KnownZero, KnownOne;
10549 DAG.computeKnownBits(SDValue(N,0), KnownZero, KnownOne);
10550 // Capture demanded bits information that would be otherwise lost.
10551 if (KnownZero == 0xfffffffe)
10552 Res = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Res,
10553 DAG.getValueType(MVT::i1));
10554 else if (KnownZero == 0xffffff00)
10555 Res = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Res,
10556 DAG.getValueType(MVT::i8));
10557 else if (KnownZero == 0xffff0000)
10558 Res = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Res,
10559 DAG.getValueType(MVT::i16));
10560 }
10561
10562 return Res;
10563 }
10564
PerformDAGCombine(SDNode * N,DAGCombinerInfo & DCI) const10565 SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N,
10566 DAGCombinerInfo &DCI) const {
10567 switch (N->getOpcode()) {
10568 default: break;
10569 case ISD::ADDC: return PerformADDCCombine(N, DCI, Subtarget);
10570 case ISD::ADD: return PerformADDCombine(N, DCI, Subtarget);
10571 case ISD::SUB: return PerformSUBCombine(N, DCI);
10572 case ISD::MUL: return PerformMULCombine(N, DCI, Subtarget);
10573 case ISD::OR: return PerformORCombine(N, DCI, Subtarget);
10574 case ISD::XOR: return PerformXORCombine(N, DCI, Subtarget);
10575 case ISD::AND: return PerformANDCombine(N, DCI, Subtarget);
10576 case ARMISD::BFI: return PerformBFICombine(N, DCI);
10577 case ARMISD::VMOVRRD: return PerformVMOVRRDCombine(N, DCI, Subtarget);
10578 case ARMISD::VMOVDRR: return PerformVMOVDRRCombine(N, DCI.DAG);
10579 case ISD::STORE: return PerformSTORECombine(N, DCI);
10580 case ISD::BUILD_VECTOR: return PerformBUILD_VECTORCombine(N, DCI, Subtarget);
10581 case ISD::INSERT_VECTOR_ELT: return PerformInsertEltCombine(N, DCI);
10582 case ISD::VECTOR_SHUFFLE: return PerformVECTOR_SHUFFLECombine(N, DCI.DAG);
10583 case ARMISD::VDUPLANE: return PerformVDUPLANECombine(N, DCI);
10584 case ISD::FP_TO_SINT:
10585 case ISD::FP_TO_UINT:
10586 return PerformVCVTCombine(N, DCI.DAG, Subtarget);
10587 case ISD::FDIV:
10588 return PerformVDIVCombine(N, DCI.DAG, Subtarget);
10589 case ISD::INTRINSIC_WO_CHAIN: return PerformIntrinsicCombine(N, DCI.DAG);
10590 case ISD::SHL:
10591 case ISD::SRA:
10592 case ISD::SRL: return PerformShiftCombine(N, DCI.DAG, Subtarget);
10593 case ISD::SIGN_EXTEND:
10594 case ISD::ZERO_EXTEND:
10595 case ISD::ANY_EXTEND: return PerformExtendCombine(N, DCI.DAG, Subtarget);
10596 case ARMISD::CMOV: return PerformCMOVCombine(N, DCI.DAG);
10597 case ISD::LOAD: return PerformLOADCombine(N, DCI);
10598 case ARMISD::VLD2DUP:
10599 case ARMISD::VLD3DUP:
10600 case ARMISD::VLD4DUP:
10601 return PerformVLDCombine(N, DCI);
10602 case ARMISD::BUILD_VECTOR:
10603 return PerformARMBUILD_VECTORCombine(N, DCI);
10604 case ISD::INTRINSIC_VOID:
10605 case ISD::INTRINSIC_W_CHAIN:
10606 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) {
10607 case Intrinsic::arm_neon_vld1:
10608 case Intrinsic::arm_neon_vld2:
10609 case Intrinsic::arm_neon_vld3:
10610 case Intrinsic::arm_neon_vld4:
10611 case Intrinsic::arm_neon_vld2lane:
10612 case Intrinsic::arm_neon_vld3lane:
10613 case Intrinsic::arm_neon_vld4lane:
10614 case Intrinsic::arm_neon_vst1:
10615 case Intrinsic::arm_neon_vst2:
10616 case Intrinsic::arm_neon_vst3:
10617 case Intrinsic::arm_neon_vst4:
10618 case Intrinsic::arm_neon_vst2lane:
10619 case Intrinsic::arm_neon_vst3lane:
10620 case Intrinsic::arm_neon_vst4lane:
10621 return PerformVLDCombine(N, DCI);
10622 default: break;
10623 }
10624 break;
10625 }
10626 return SDValue();
10627 }
10628
isDesirableToTransformToIntegerOp(unsigned Opc,EVT VT) const10629 bool ARMTargetLowering::isDesirableToTransformToIntegerOp(unsigned Opc,
10630 EVT VT) const {
10631 return (VT == MVT::f32) && (Opc == ISD::LOAD || Opc == ISD::STORE);
10632 }
10633
allowsMisalignedMemoryAccesses(EVT VT,unsigned,unsigned,bool * Fast) const10634 bool ARMTargetLowering::allowsMisalignedMemoryAccesses(EVT VT,
10635 unsigned,
10636 unsigned,
10637 bool *Fast) const {
10638 // The AllowsUnaliged flag models the SCTLR.A setting in ARM cpus
10639 bool AllowsUnaligned = Subtarget->allowsUnalignedMem();
10640
10641 switch (VT.getSimpleVT().SimpleTy) {
10642 default:
10643 return false;
10644 case MVT::i8:
10645 case MVT::i16:
10646 case MVT::i32: {
10647 // Unaligned access can use (for example) LRDB, LRDH, LDR
10648 if (AllowsUnaligned) {
10649 if (Fast)
10650 *Fast = Subtarget->hasV7Ops();
10651 return true;
10652 }
10653 return false;
10654 }
10655 case MVT::f64:
10656 case MVT::v2f64: {
10657 // For any little-endian targets with neon, we can support unaligned ld/st
10658 // of D and Q (e.g. {D0,D1}) registers by using vld1.i8/vst1.i8.
10659 // A big-endian target may also explicitly support unaligned accesses
10660 if (Subtarget->hasNEON() && (AllowsUnaligned || Subtarget->isLittle())) {
10661 if (Fast)
10662 *Fast = true;
10663 return true;
10664 }
10665 return false;
10666 }
10667 }
10668 }
10669
memOpAlign(unsigned DstAlign,unsigned SrcAlign,unsigned AlignCheck)10670 static bool memOpAlign(unsigned DstAlign, unsigned SrcAlign,
10671 unsigned AlignCheck) {
10672 return ((SrcAlign == 0 || SrcAlign % AlignCheck == 0) &&
10673 (DstAlign == 0 || DstAlign % AlignCheck == 0));
10674 }
10675
getOptimalMemOpType(uint64_t Size,unsigned DstAlign,unsigned SrcAlign,bool IsMemset,bool ZeroMemset,bool MemcpyStrSrc,MachineFunction & MF) const10676 EVT ARMTargetLowering::getOptimalMemOpType(uint64_t Size,
10677 unsigned DstAlign, unsigned SrcAlign,
10678 bool IsMemset, bool ZeroMemset,
10679 bool MemcpyStrSrc,
10680 MachineFunction &MF) const {
10681 const Function *F = MF.getFunction();
10682
10683 // See if we can use NEON instructions for this...
10684 if ((!IsMemset || ZeroMemset) && Subtarget->hasNEON() &&
10685 !F->hasFnAttribute(Attribute::NoImplicitFloat)) {
10686 bool Fast;
10687 if (Size >= 16 &&
10688 (memOpAlign(SrcAlign, DstAlign, 16) ||
10689 (allowsMisalignedMemoryAccesses(MVT::v2f64, 0, 1, &Fast) && Fast))) {
10690 return MVT::v2f64;
10691 } else if (Size >= 8 &&
10692 (memOpAlign(SrcAlign, DstAlign, 8) ||
10693 (allowsMisalignedMemoryAccesses(MVT::f64, 0, 1, &Fast) &&
10694 Fast))) {
10695 return MVT::f64;
10696 }
10697 }
10698
10699 // Lowering to i32/i16 if the size permits.
10700 if (Size >= 4)
10701 return MVT::i32;
10702 else if (Size >= 2)
10703 return MVT::i16;
10704
10705 // Let the target-independent logic figure it out.
10706 return MVT::Other;
10707 }
10708
isZExtFree(SDValue Val,EVT VT2) const10709 bool ARMTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
10710 if (Val.getOpcode() != ISD::LOAD)
10711 return false;
10712
10713 EVT VT1 = Val.getValueType();
10714 if (!VT1.isSimple() || !VT1.isInteger() ||
10715 !VT2.isSimple() || !VT2.isInteger())
10716 return false;
10717
10718 switch (VT1.getSimpleVT().SimpleTy) {
10719 default: break;
10720 case MVT::i1:
10721 case MVT::i8:
10722 case MVT::i16:
10723 // 8-bit and 16-bit loads implicitly zero-extend to 32-bits.
10724 return true;
10725 }
10726
10727 return false;
10728 }
10729
isVectorLoadExtDesirable(SDValue ExtVal) const10730 bool ARMTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
10731 EVT VT = ExtVal.getValueType();
10732
10733 if (!isTypeLegal(VT))
10734 return false;
10735
10736 // Don't create a loadext if we can fold the extension into a wide/long
10737 // instruction.
10738 // If there's more than one user instruction, the loadext is desirable no
10739 // matter what. There can be two uses by the same instruction.
10740 if (ExtVal->use_empty() ||
10741 !ExtVal->use_begin()->isOnlyUserOf(ExtVal.getNode()))
10742 return true;
10743
10744 SDNode *U = *ExtVal->use_begin();
10745 if ((U->getOpcode() == ISD::ADD || U->getOpcode() == ISD::SUB ||
10746 U->getOpcode() == ISD::SHL || U->getOpcode() == ARMISD::VSHL))
10747 return false;
10748
10749 return true;
10750 }
10751
allowTruncateForTailCall(Type * Ty1,Type * Ty2) const10752 bool ARMTargetLowering::allowTruncateForTailCall(Type *Ty1, Type *Ty2) const {
10753 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
10754 return false;
10755
10756 if (!isTypeLegal(EVT::getEVT(Ty1)))
10757 return false;
10758
10759 assert(Ty1->getPrimitiveSizeInBits() <= 64 && "i128 is probably not a noop");
10760
10761 // Assuming the caller doesn't have a zeroext or signext return parameter,
10762 // truncation all the way down to i1 is valid.
10763 return true;
10764 }
10765
10766
isLegalT1AddressImmediate(int64_t V,EVT VT)10767 static bool isLegalT1AddressImmediate(int64_t V, EVT VT) {
10768 if (V < 0)
10769 return false;
10770
10771 unsigned Scale = 1;
10772 switch (VT.getSimpleVT().SimpleTy) {
10773 default: return false;
10774 case MVT::i1:
10775 case MVT::i8:
10776 // Scale == 1;
10777 break;
10778 case MVT::i16:
10779 // Scale == 2;
10780 Scale = 2;
10781 break;
10782 case MVT::i32:
10783 // Scale == 4;
10784 Scale = 4;
10785 break;
10786 }
10787
10788 if ((V & (Scale - 1)) != 0)
10789 return false;
10790 V /= Scale;
10791 return V == (V & ((1LL << 5) - 1));
10792 }
10793
isLegalT2AddressImmediate(int64_t V,EVT VT,const ARMSubtarget * Subtarget)10794 static bool isLegalT2AddressImmediate(int64_t V, EVT VT,
10795 const ARMSubtarget *Subtarget) {
10796 bool isNeg = false;
10797 if (V < 0) {
10798 isNeg = true;
10799 V = - V;
10800 }
10801
10802 switch (VT.getSimpleVT().SimpleTy) {
10803 default: return false;
10804 case MVT::i1:
10805 case MVT::i8:
10806 case MVT::i16:
10807 case MVT::i32:
10808 // + imm12 or - imm8
10809 if (isNeg)
10810 return V == (V & ((1LL << 8) - 1));
10811 return V == (V & ((1LL << 12) - 1));
10812 case MVT::f32:
10813 case MVT::f64:
10814 // Same as ARM mode. FIXME: NEON?
10815 if (!Subtarget->hasVFP2())
10816 return false;
10817 if ((V & 3) != 0)
10818 return false;
10819 V >>= 2;
10820 return V == (V & ((1LL << 8) - 1));
10821 }
10822 }
10823
10824 /// isLegalAddressImmediate - Return true if the integer value can be used
10825 /// as the offset of the target addressing mode for load / store of the
10826 /// given type.
isLegalAddressImmediate(int64_t V,EVT VT,const ARMSubtarget * Subtarget)10827 static bool isLegalAddressImmediate(int64_t V, EVT VT,
10828 const ARMSubtarget *Subtarget) {
10829 if (V == 0)
10830 return true;
10831
10832 if (!VT.isSimple())
10833 return false;
10834
10835 if (Subtarget->isThumb1Only())
10836 return isLegalT1AddressImmediate(V, VT);
10837 else if (Subtarget->isThumb2())
10838 return isLegalT2AddressImmediate(V, VT, Subtarget);
10839
10840 // ARM mode.
10841 if (V < 0)
10842 V = - V;
10843 switch (VT.getSimpleVT().SimpleTy) {
10844 default: return false;
10845 case MVT::i1:
10846 case MVT::i8:
10847 case MVT::i32:
10848 // +- imm12
10849 return V == (V & ((1LL << 12) - 1));
10850 case MVT::i16:
10851 // +- imm8
10852 return V == (V & ((1LL << 8) - 1));
10853 case MVT::f32:
10854 case MVT::f64:
10855 if (!Subtarget->hasVFP2()) // FIXME: NEON?
10856 return false;
10857 if ((V & 3) != 0)
10858 return false;
10859 V >>= 2;
10860 return V == (V & ((1LL << 8) - 1));
10861 }
10862 }
10863
isLegalT2ScaledAddressingMode(const AddrMode & AM,EVT VT) const10864 bool ARMTargetLowering::isLegalT2ScaledAddressingMode(const AddrMode &AM,
10865 EVT VT) const {
10866 int Scale = AM.Scale;
10867 if (Scale < 0)
10868 return false;
10869
10870 switch (VT.getSimpleVT().SimpleTy) {
10871 default: return false;
10872 case MVT::i1:
10873 case MVT::i8:
10874 case MVT::i16:
10875 case MVT::i32:
10876 if (Scale == 1)
10877 return true;
10878 // r + r << imm
10879 Scale = Scale & ~1;
10880 return Scale == 2 || Scale == 4 || Scale == 8;
10881 case MVT::i64:
10882 // r + r
10883 if (((unsigned)AM.HasBaseReg + Scale) <= 2)
10884 return true;
10885 return false;
10886 case MVT::isVoid:
10887 // Note, we allow "void" uses (basically, uses that aren't loads or
10888 // stores), because arm allows folding a scale into many arithmetic
10889 // operations. This should be made more precise and revisited later.
10890
10891 // Allow r << imm, but the imm has to be a multiple of two.
10892 if (Scale & 1) return false;
10893 return isPowerOf2_32(Scale);
10894 }
10895 }
10896
10897 /// isLegalAddressingMode - Return true if the addressing mode represented
10898 /// by AM is legal for this target, for a load/store of the specified type.
isLegalAddressingMode(const DataLayout & DL,const AddrMode & AM,Type * Ty,unsigned AS) const10899 bool ARMTargetLowering::isLegalAddressingMode(const DataLayout &DL,
10900 const AddrMode &AM, Type *Ty,
10901 unsigned AS) const {
10902 EVT VT = getValueType(DL, Ty, true);
10903 if (!isLegalAddressImmediate(AM.BaseOffs, VT, Subtarget))
10904 return false;
10905
10906 // Can never fold addr of global into load/store.
10907 if (AM.BaseGV)
10908 return false;
10909
10910 switch (AM.Scale) {
10911 case 0: // no scale reg, must be "r+i" or "r", or "i".
10912 break;
10913 case 1:
10914 if (Subtarget->isThumb1Only())
10915 return false;
10916 // FALL THROUGH.
10917 default:
10918 // ARM doesn't support any R+R*scale+imm addr modes.
10919 if (AM.BaseOffs)
10920 return false;
10921
10922 if (!VT.isSimple())
10923 return false;
10924
10925 if (Subtarget->isThumb2())
10926 return isLegalT2ScaledAddressingMode(AM, VT);
10927
10928 int Scale = AM.Scale;
10929 switch (VT.getSimpleVT().SimpleTy) {
10930 default: return false;
10931 case MVT::i1:
10932 case MVT::i8:
10933 case MVT::i32:
10934 if (Scale < 0) Scale = -Scale;
10935 if (Scale == 1)
10936 return true;
10937 // r + r << imm
10938 return isPowerOf2_32(Scale & ~1);
10939 case MVT::i16:
10940 case MVT::i64:
10941 // r + r
10942 if (((unsigned)AM.HasBaseReg + Scale) <= 2)
10943 return true;
10944 return false;
10945
10946 case MVT::isVoid:
10947 // Note, we allow "void" uses (basically, uses that aren't loads or
10948 // stores), because arm allows folding a scale into many arithmetic
10949 // operations. This should be made more precise and revisited later.
10950
10951 // Allow r << imm, but the imm has to be a multiple of two.
10952 if (Scale & 1) return false;
10953 return isPowerOf2_32(Scale);
10954 }
10955 }
10956 return true;
10957 }
10958
10959 /// isLegalICmpImmediate - Return true if the specified immediate is legal
10960 /// icmp immediate, that is the target has icmp instructions which can compare
10961 /// a register against the immediate without having to materialize the
10962 /// immediate into a register.
isLegalICmpImmediate(int64_t Imm) const10963 bool ARMTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
10964 // Thumb2 and ARM modes can use cmn for negative immediates.
10965 if (!Subtarget->isThumb())
10966 return ARM_AM::getSOImmVal(std::abs(Imm)) != -1;
10967 if (Subtarget->isThumb2())
10968 return ARM_AM::getT2SOImmVal(std::abs(Imm)) != -1;
10969 // Thumb1 doesn't have cmn, and only 8-bit immediates.
10970 return Imm >= 0 && Imm <= 255;
10971 }
10972
10973 /// isLegalAddImmediate - Return true if the specified immediate is a legal add
10974 /// *or sub* immediate, that is the target has add or sub instructions which can
10975 /// add a register with the immediate without having to materialize the
10976 /// immediate into a register.
isLegalAddImmediate(int64_t Imm) const10977 bool ARMTargetLowering::isLegalAddImmediate(int64_t Imm) const {
10978 // Same encoding for add/sub, just flip the sign.
10979 int64_t AbsImm = std::abs(Imm);
10980 if (!Subtarget->isThumb())
10981 return ARM_AM::getSOImmVal(AbsImm) != -1;
10982 if (Subtarget->isThumb2())
10983 return ARM_AM::getT2SOImmVal(AbsImm) != -1;
10984 // Thumb1 only has 8-bit unsigned immediate.
10985 return AbsImm >= 0 && AbsImm <= 255;
10986 }
10987
getARMIndexedAddressParts(SDNode * Ptr,EVT VT,bool isSEXTLoad,SDValue & Base,SDValue & Offset,bool & isInc,SelectionDAG & DAG)10988 static bool getARMIndexedAddressParts(SDNode *Ptr, EVT VT,
10989 bool isSEXTLoad, SDValue &Base,
10990 SDValue &Offset, bool &isInc,
10991 SelectionDAG &DAG) {
10992 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB)
10993 return false;
10994
10995 if (VT == MVT::i16 || ((VT == MVT::i8 || VT == MVT::i1) && isSEXTLoad)) {
10996 // AddressingMode 3
10997 Base = Ptr->getOperand(0);
10998 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ptr->getOperand(1))) {
10999 int RHSC = (int)RHS->getZExtValue();
11000 if (RHSC < 0 && RHSC > -256) {
11001 assert(Ptr->getOpcode() == ISD::ADD);
11002 isInc = false;
11003 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0));
11004 return true;
11005 }
11006 }
11007 isInc = (Ptr->getOpcode() == ISD::ADD);
11008 Offset = Ptr->getOperand(1);
11009 return true;
11010 } else if (VT == MVT::i32 || VT == MVT::i8 || VT == MVT::i1) {
11011 // AddressingMode 2
11012 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ptr->getOperand(1))) {
11013 int RHSC = (int)RHS->getZExtValue();
11014 if (RHSC < 0 && RHSC > -0x1000) {
11015 assert(Ptr->getOpcode() == ISD::ADD);
11016 isInc = false;
11017 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0));
11018 Base = Ptr->getOperand(0);
11019 return true;
11020 }
11021 }
11022
11023 if (Ptr->getOpcode() == ISD::ADD) {
11024 isInc = true;
11025 ARM_AM::ShiftOpc ShOpcVal=
11026 ARM_AM::getShiftOpcForNode(Ptr->getOperand(0).getOpcode());
11027 if (ShOpcVal != ARM_AM::no_shift) {
11028 Base = Ptr->getOperand(1);
11029 Offset = Ptr->getOperand(0);
11030 } else {
11031 Base = Ptr->getOperand(0);
11032 Offset = Ptr->getOperand(1);
11033 }
11034 return true;
11035 }
11036
11037 isInc = (Ptr->getOpcode() == ISD::ADD);
11038 Base = Ptr->getOperand(0);
11039 Offset = Ptr->getOperand(1);
11040 return true;
11041 }
11042
11043 // FIXME: Use VLDM / VSTM to emulate indexed FP load / store.
11044 return false;
11045 }
11046
getT2IndexedAddressParts(SDNode * Ptr,EVT VT,bool isSEXTLoad,SDValue & Base,SDValue & Offset,bool & isInc,SelectionDAG & DAG)11047 static bool getT2IndexedAddressParts(SDNode *Ptr, EVT VT,
11048 bool isSEXTLoad, SDValue &Base,
11049 SDValue &Offset, bool &isInc,
11050 SelectionDAG &DAG) {
11051 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB)
11052 return false;
11053
11054 Base = Ptr->getOperand(0);
11055 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ptr->getOperand(1))) {
11056 int RHSC = (int)RHS->getZExtValue();
11057 if (RHSC < 0 && RHSC > -0x100) { // 8 bits.
11058 assert(Ptr->getOpcode() == ISD::ADD);
11059 isInc = false;
11060 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0));
11061 return true;
11062 } else if (RHSC > 0 && RHSC < 0x100) { // 8 bit, no zero.
11063 isInc = Ptr->getOpcode() == ISD::ADD;
11064 Offset = DAG.getConstant(RHSC, SDLoc(Ptr), RHS->getValueType(0));
11065 return true;
11066 }
11067 }
11068
11069 return false;
11070 }
11071
11072 /// getPreIndexedAddressParts - returns true by value, base pointer and
11073 /// offset pointer and addressing mode by reference if the node's address
11074 /// can be legally represented as pre-indexed load / store address.
11075 bool
getPreIndexedAddressParts(SDNode * N,SDValue & Base,SDValue & Offset,ISD::MemIndexedMode & AM,SelectionDAG & DAG) const11076 ARMTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
11077 SDValue &Offset,
11078 ISD::MemIndexedMode &AM,
11079 SelectionDAG &DAG) const {
11080 if (Subtarget->isThumb1Only())
11081 return false;
11082
11083 EVT VT;
11084 SDValue Ptr;
11085 bool isSEXTLoad = false;
11086 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
11087 Ptr = LD->getBasePtr();
11088 VT = LD->getMemoryVT();
11089 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
11090 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
11091 Ptr = ST->getBasePtr();
11092 VT = ST->getMemoryVT();
11093 } else
11094 return false;
11095
11096 bool isInc;
11097 bool isLegal = false;
11098 if (Subtarget->isThumb2())
11099 isLegal = getT2IndexedAddressParts(Ptr.getNode(), VT, isSEXTLoad, Base,
11100 Offset, isInc, DAG);
11101 else
11102 isLegal = getARMIndexedAddressParts(Ptr.getNode(), VT, isSEXTLoad, Base,
11103 Offset, isInc, DAG);
11104 if (!isLegal)
11105 return false;
11106
11107 AM = isInc ? ISD::PRE_INC : ISD::PRE_DEC;
11108 return true;
11109 }
11110
11111 /// getPostIndexedAddressParts - returns true by value, base pointer and
11112 /// offset pointer and addressing mode by reference if this node can be
11113 /// combined with a load / store to form a post-indexed load / store.
getPostIndexedAddressParts(SDNode * N,SDNode * Op,SDValue & Base,SDValue & Offset,ISD::MemIndexedMode & AM,SelectionDAG & DAG) const11114 bool ARMTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op,
11115 SDValue &Base,
11116 SDValue &Offset,
11117 ISD::MemIndexedMode &AM,
11118 SelectionDAG &DAG) const {
11119 if (Subtarget->isThumb1Only())
11120 return false;
11121
11122 EVT VT;
11123 SDValue Ptr;
11124 bool isSEXTLoad = false;
11125 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
11126 VT = LD->getMemoryVT();
11127 Ptr = LD->getBasePtr();
11128 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
11129 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
11130 VT = ST->getMemoryVT();
11131 Ptr = ST->getBasePtr();
11132 } else
11133 return false;
11134
11135 bool isInc;
11136 bool isLegal = false;
11137 if (Subtarget->isThumb2())
11138 isLegal = getT2IndexedAddressParts(Op, VT, isSEXTLoad, Base, Offset,
11139 isInc, DAG);
11140 else
11141 isLegal = getARMIndexedAddressParts(Op, VT, isSEXTLoad, Base, Offset,
11142 isInc, DAG);
11143 if (!isLegal)
11144 return false;
11145
11146 if (Ptr != Base) {
11147 // Swap base ptr and offset to catch more post-index load / store when
11148 // it's legal. In Thumb2 mode, offset must be an immediate.
11149 if (Ptr == Offset && Op->getOpcode() == ISD::ADD &&
11150 !Subtarget->isThumb2())
11151 std::swap(Base, Offset);
11152
11153 // Post-indexed load / store update the base pointer.
11154 if (Ptr != Base)
11155 return false;
11156 }
11157
11158 AM = isInc ? ISD::POST_INC : ISD::POST_DEC;
11159 return true;
11160 }
11161
computeKnownBitsForTargetNode(const SDValue Op,APInt & KnownZero,APInt & KnownOne,const SelectionDAG & DAG,unsigned Depth) const11162 void ARMTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
11163 APInt &KnownZero,
11164 APInt &KnownOne,
11165 const SelectionDAG &DAG,
11166 unsigned Depth) const {
11167 unsigned BitWidth = KnownOne.getBitWidth();
11168 KnownZero = KnownOne = APInt(BitWidth, 0);
11169 switch (Op.getOpcode()) {
11170 default: break;
11171 case ARMISD::ADDC:
11172 case ARMISD::ADDE:
11173 case ARMISD::SUBC:
11174 case ARMISD::SUBE:
11175 // These nodes' second result is a boolean
11176 if (Op.getResNo() == 0)
11177 break;
11178 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
11179 break;
11180 case ARMISD::CMOV: {
11181 // Bits are known zero/one if known on the LHS and RHS.
11182 DAG.computeKnownBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1);
11183 if (KnownZero == 0 && KnownOne == 0) return;
11184
11185 APInt KnownZeroRHS, KnownOneRHS;
11186 DAG.computeKnownBits(Op.getOperand(1), KnownZeroRHS, KnownOneRHS, Depth+1);
11187 KnownZero &= KnownZeroRHS;
11188 KnownOne &= KnownOneRHS;
11189 return;
11190 }
11191 case ISD::INTRINSIC_W_CHAIN: {
11192 ConstantSDNode *CN = cast<ConstantSDNode>(Op->getOperand(1));
11193 Intrinsic::ID IntID = static_cast<Intrinsic::ID>(CN->getZExtValue());
11194 switch (IntID) {
11195 default: return;
11196 case Intrinsic::arm_ldaex:
11197 case Intrinsic::arm_ldrex: {
11198 EVT VT = cast<MemIntrinsicSDNode>(Op)->getMemoryVT();
11199 unsigned MemBits = VT.getScalarType().getSizeInBits();
11200 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
11201 return;
11202 }
11203 }
11204 }
11205 }
11206 }
11207
11208 //===----------------------------------------------------------------------===//
11209 // ARM Inline Assembly Support
11210 //===----------------------------------------------------------------------===//
11211
ExpandInlineAsm(CallInst * CI) const11212 bool ARMTargetLowering::ExpandInlineAsm(CallInst *CI) const {
11213 // Looking for "rev" which is V6+.
11214 if (!Subtarget->hasV6Ops())
11215 return false;
11216
11217 InlineAsm *IA = cast<InlineAsm>(CI->getCalledValue());
11218 std::string AsmStr = IA->getAsmString();
11219 SmallVector<StringRef, 4> AsmPieces;
11220 SplitString(AsmStr, AsmPieces, ";\n");
11221
11222 switch (AsmPieces.size()) {
11223 default: return false;
11224 case 1:
11225 AsmStr = AsmPieces[0];
11226 AsmPieces.clear();
11227 SplitString(AsmStr, AsmPieces, " \t,");
11228
11229 // rev $0, $1
11230 if (AsmPieces.size() == 3 &&
11231 AsmPieces[0] == "rev" && AsmPieces[1] == "$0" && AsmPieces[2] == "$1" &&
11232 IA->getConstraintString().compare(0, 4, "=l,l") == 0) {
11233 IntegerType *Ty = dyn_cast<IntegerType>(CI->getType());
11234 if (Ty && Ty->getBitWidth() == 32)
11235 return IntrinsicLowering::LowerToByteSwap(CI);
11236 }
11237 break;
11238 }
11239
11240 return false;
11241 }
11242
11243 /// getConstraintType - Given a constraint letter, return the type of
11244 /// constraint it is for this target.
11245 ARMTargetLowering::ConstraintType
getConstraintType(StringRef Constraint) const11246 ARMTargetLowering::getConstraintType(StringRef Constraint) const {
11247 if (Constraint.size() == 1) {
11248 switch (Constraint[0]) {
11249 default: break;
11250 case 'l': return C_RegisterClass;
11251 case 'w': return C_RegisterClass;
11252 case 'h': return C_RegisterClass;
11253 case 'x': return C_RegisterClass;
11254 case 't': return C_RegisterClass;
11255 case 'j': return C_Other; // Constant for movw.
11256 // An address with a single base register. Due to the way we
11257 // currently handle addresses it is the same as an 'r' memory constraint.
11258 case 'Q': return C_Memory;
11259 }
11260 } else if (Constraint.size() == 2) {
11261 switch (Constraint[0]) {
11262 default: break;
11263 // All 'U+' constraints are addresses.
11264 case 'U': return C_Memory;
11265 }
11266 }
11267 return TargetLowering::getConstraintType(Constraint);
11268 }
11269
11270 /// Examine constraint type and operand type and determine a weight value.
11271 /// This object must already have been set up with the operand type
11272 /// and the current alternative constraint selected.
11273 TargetLowering::ConstraintWeight
getSingleConstraintMatchWeight(AsmOperandInfo & info,const char * constraint) const11274 ARMTargetLowering::getSingleConstraintMatchWeight(
11275 AsmOperandInfo &info, const char *constraint) const {
11276 ConstraintWeight weight = CW_Invalid;
11277 Value *CallOperandVal = info.CallOperandVal;
11278 // If we don't have a value, we can't do a match,
11279 // but allow it at the lowest weight.
11280 if (!CallOperandVal)
11281 return CW_Default;
11282 Type *type = CallOperandVal->getType();
11283 // Look at the constraint type.
11284 switch (*constraint) {
11285 default:
11286 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint);
11287 break;
11288 case 'l':
11289 if (type->isIntegerTy()) {
11290 if (Subtarget->isThumb())
11291 weight = CW_SpecificReg;
11292 else
11293 weight = CW_Register;
11294 }
11295 break;
11296 case 'w':
11297 if (type->isFloatingPointTy())
11298 weight = CW_Register;
11299 break;
11300 }
11301 return weight;
11302 }
11303
11304 typedef std::pair<unsigned, const TargetRegisterClass*> RCPair;
getRegForInlineAsmConstraint(const TargetRegisterInfo * TRI,StringRef Constraint,MVT VT) const11305 RCPair ARMTargetLowering::getRegForInlineAsmConstraint(
11306 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
11307 if (Constraint.size() == 1) {
11308 // GCC ARM Constraint Letters
11309 switch (Constraint[0]) {
11310 case 'l': // Low regs or general regs.
11311 if (Subtarget->isThumb())
11312 return RCPair(0U, &ARM::tGPRRegClass);
11313 return RCPair(0U, &ARM::GPRRegClass);
11314 case 'h': // High regs or no regs.
11315 if (Subtarget->isThumb())
11316 return RCPair(0U, &ARM::hGPRRegClass);
11317 break;
11318 case 'r':
11319 if (Subtarget->isThumb1Only())
11320 return RCPair(0U, &ARM::tGPRRegClass);
11321 return RCPair(0U, &ARM::GPRRegClass);
11322 case 'w':
11323 if (VT == MVT::Other)
11324 break;
11325 if (VT == MVT::f32)
11326 return RCPair(0U, &ARM::SPRRegClass);
11327 if (VT.getSizeInBits() == 64)
11328 return RCPair(0U, &ARM::DPRRegClass);
11329 if (VT.getSizeInBits() == 128)
11330 return RCPair(0U, &ARM::QPRRegClass);
11331 break;
11332 case 'x':
11333 if (VT == MVT::Other)
11334 break;
11335 if (VT == MVT::f32)
11336 return RCPair(0U, &ARM::SPR_8RegClass);
11337 if (VT.getSizeInBits() == 64)
11338 return RCPair(0U, &ARM::DPR_8RegClass);
11339 if (VT.getSizeInBits() == 128)
11340 return RCPair(0U, &ARM::QPR_8RegClass);
11341 break;
11342 case 't':
11343 if (VT == MVT::f32)
11344 return RCPair(0U, &ARM::SPRRegClass);
11345 break;
11346 }
11347 }
11348 if (StringRef("{cc}").equals_lower(Constraint))
11349 return std::make_pair(unsigned(ARM::CPSR), &ARM::CCRRegClass);
11350
11351 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
11352 }
11353
11354 /// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
11355 /// vector. If it is invalid, don't add anything to Ops.
LowerAsmOperandForConstraint(SDValue Op,std::string & Constraint,std::vector<SDValue> & Ops,SelectionDAG & DAG) const11356 void ARMTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
11357 std::string &Constraint,
11358 std::vector<SDValue>&Ops,
11359 SelectionDAG &DAG) const {
11360 SDValue Result;
11361
11362 // Currently only support length 1 constraints.
11363 if (Constraint.length() != 1) return;
11364
11365 char ConstraintLetter = Constraint[0];
11366 switch (ConstraintLetter) {
11367 default: break;
11368 case 'j':
11369 case 'I': case 'J': case 'K': case 'L':
11370 case 'M': case 'N': case 'O':
11371 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op);
11372 if (!C)
11373 return;
11374
11375 int64_t CVal64 = C->getSExtValue();
11376 int CVal = (int) CVal64;
11377 // None of these constraints allow values larger than 32 bits. Check
11378 // that the value fits in an int.
11379 if (CVal != CVal64)
11380 return;
11381
11382 switch (ConstraintLetter) {
11383 case 'j':
11384 // Constant suitable for movw, must be between 0 and
11385 // 65535.
11386 if (Subtarget->hasV6T2Ops())
11387 if (CVal >= 0 && CVal <= 65535)
11388 break;
11389 return;
11390 case 'I':
11391 if (Subtarget->isThumb1Only()) {
11392 // This must be a constant between 0 and 255, for ADD
11393 // immediates.
11394 if (CVal >= 0 && CVal <= 255)
11395 break;
11396 } else if (Subtarget->isThumb2()) {
11397 // A constant that can be used as an immediate value in a
11398 // data-processing instruction.
11399 if (ARM_AM::getT2SOImmVal(CVal) != -1)
11400 break;
11401 } else {
11402 // A constant that can be used as an immediate value in a
11403 // data-processing instruction.
11404 if (ARM_AM::getSOImmVal(CVal) != -1)
11405 break;
11406 }
11407 return;
11408
11409 case 'J':
11410 if (Subtarget->isThumb()) { // FIXME thumb2
11411 // This must be a constant between -255 and -1, for negated ADD
11412 // immediates. This can be used in GCC with an "n" modifier that
11413 // prints the negated value, for use with SUB instructions. It is
11414 // not useful otherwise but is implemented for compatibility.
11415 if (CVal >= -255 && CVal <= -1)
11416 break;
11417 } else {
11418 // This must be a constant between -4095 and 4095. It is not clear
11419 // what this constraint is intended for. Implemented for
11420 // compatibility with GCC.
11421 if (CVal >= -4095 && CVal <= 4095)
11422 break;
11423 }
11424 return;
11425
11426 case 'K':
11427 if (Subtarget->isThumb1Only()) {
11428 // A 32-bit value where only one byte has a nonzero value. Exclude
11429 // zero to match GCC. This constraint is used by GCC internally for
11430 // constants that can be loaded with a move/shift combination.
11431 // It is not useful otherwise but is implemented for compatibility.
11432 if (CVal != 0 && ARM_AM::isThumbImmShiftedVal(CVal))
11433 break;
11434 } else if (Subtarget->isThumb2()) {
11435 // A constant whose bitwise inverse can be used as an immediate
11436 // value in a data-processing instruction. This can be used in GCC
11437 // with a "B" modifier that prints the inverted value, for use with
11438 // BIC and MVN instructions. It is not useful otherwise but is
11439 // implemented for compatibility.
11440 if (ARM_AM::getT2SOImmVal(~CVal) != -1)
11441 break;
11442 } else {
11443 // A constant whose bitwise inverse can be used as an immediate
11444 // value in a data-processing instruction. This can be used in GCC
11445 // with a "B" modifier that prints the inverted value, for use with
11446 // BIC and MVN instructions. It is not useful otherwise but is
11447 // implemented for compatibility.
11448 if (ARM_AM::getSOImmVal(~CVal) != -1)
11449 break;
11450 }
11451 return;
11452
11453 case 'L':
11454 if (Subtarget->isThumb1Only()) {
11455 // This must be a constant between -7 and 7,
11456 // for 3-operand ADD/SUB immediate instructions.
11457 if (CVal >= -7 && CVal < 7)
11458 break;
11459 } else if (Subtarget->isThumb2()) {
11460 // A constant whose negation can be used as an immediate value in a
11461 // data-processing instruction. This can be used in GCC with an "n"
11462 // modifier that prints the negated value, for use with SUB
11463 // instructions. It is not useful otherwise but is implemented for
11464 // compatibility.
11465 if (ARM_AM::getT2SOImmVal(-CVal) != -1)
11466 break;
11467 } else {
11468 // A constant whose negation can be used as an immediate value in a
11469 // data-processing instruction. This can be used in GCC with an "n"
11470 // modifier that prints the negated value, for use with SUB
11471 // instructions. It is not useful otherwise but is implemented for
11472 // compatibility.
11473 if (ARM_AM::getSOImmVal(-CVal) != -1)
11474 break;
11475 }
11476 return;
11477
11478 case 'M':
11479 if (Subtarget->isThumb()) { // FIXME thumb2
11480 // This must be a multiple of 4 between 0 and 1020, for
11481 // ADD sp + immediate.
11482 if ((CVal >= 0 && CVal <= 1020) && ((CVal & 3) == 0))
11483 break;
11484 } else {
11485 // A power of two or a constant between 0 and 32. This is used in
11486 // GCC for the shift amount on shifted register operands, but it is
11487 // useful in general for any shift amounts.
11488 if ((CVal >= 0 && CVal <= 32) || ((CVal & (CVal - 1)) == 0))
11489 break;
11490 }
11491 return;
11492
11493 case 'N':
11494 if (Subtarget->isThumb()) { // FIXME thumb2
11495 // This must be a constant between 0 and 31, for shift amounts.
11496 if (CVal >= 0 && CVal <= 31)
11497 break;
11498 }
11499 return;
11500
11501 case 'O':
11502 if (Subtarget->isThumb()) { // FIXME thumb2
11503 // This must be a multiple of 4 between -508 and 508, for
11504 // ADD/SUB sp = sp + immediate.
11505 if ((CVal >= -508 && CVal <= 508) && ((CVal & 3) == 0))
11506 break;
11507 }
11508 return;
11509 }
11510 Result = DAG.getTargetConstant(CVal, SDLoc(Op), Op.getValueType());
11511 break;
11512 }
11513
11514 if (Result.getNode()) {
11515 Ops.push_back(Result);
11516 return;
11517 }
11518 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
11519 }
11520
getDivRemLibcall(const SDNode * N,MVT::SimpleValueType SVT)11521 static RTLIB::Libcall getDivRemLibcall(
11522 const SDNode *N, MVT::SimpleValueType SVT) {
11523 assert((N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM ||
11524 N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) &&
11525 "Unhandled Opcode in getDivRemLibcall");
11526 bool isSigned = N->getOpcode() == ISD::SDIVREM ||
11527 N->getOpcode() == ISD::SREM;
11528 RTLIB::Libcall LC;
11529 switch (SVT) {
11530 default: llvm_unreachable("Unexpected request for libcall!");
11531 case MVT::i8: LC = isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
11532 case MVT::i16: LC = isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
11533 case MVT::i32: LC = isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
11534 case MVT::i64: LC = isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
11535 }
11536 return LC;
11537 }
11538
getDivRemArgList(const SDNode * N,LLVMContext * Context)11539 static TargetLowering::ArgListTy getDivRemArgList(
11540 const SDNode *N, LLVMContext *Context) {
11541 assert((N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM ||
11542 N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) &&
11543 "Unhandled Opcode in getDivRemArgList");
11544 bool isSigned = N->getOpcode() == ISD::SDIVREM ||
11545 N->getOpcode() == ISD::SREM;
11546 TargetLowering::ArgListTy Args;
11547 TargetLowering::ArgListEntry Entry;
11548 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
11549 EVT ArgVT = N->getOperand(i).getValueType();
11550 Type *ArgTy = ArgVT.getTypeForEVT(*Context);
11551 Entry.Node = N->getOperand(i);
11552 Entry.Ty = ArgTy;
11553 Entry.isSExt = isSigned;
11554 Entry.isZExt = !isSigned;
11555 Args.push_back(Entry);
11556 }
11557 return Args;
11558 }
11559
LowerDivRem(SDValue Op,SelectionDAG & DAG) const11560 SDValue ARMTargetLowering::LowerDivRem(SDValue Op, SelectionDAG &DAG) const {
11561 assert((Subtarget->isTargetAEABI() || Subtarget->isTargetAndroid()) &&
11562 "Register-based DivRem lowering only");
11563 unsigned Opcode = Op->getOpcode();
11564 assert((Opcode == ISD::SDIVREM || Opcode == ISD::UDIVREM) &&
11565 "Invalid opcode for Div/Rem lowering");
11566 bool isSigned = (Opcode == ISD::SDIVREM);
11567 EVT VT = Op->getValueType(0);
11568 Type *Ty = VT.getTypeForEVT(*DAG.getContext());
11569
11570 RTLIB::Libcall LC = getDivRemLibcall(Op.getNode(),
11571 VT.getSimpleVT().SimpleTy);
11572 SDValue InChain = DAG.getEntryNode();
11573
11574 TargetLowering::ArgListTy Args = getDivRemArgList(Op.getNode(),
11575 DAG.getContext());
11576
11577 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
11578 getPointerTy(DAG.getDataLayout()));
11579
11580 Type *RetTy = (Type*)StructType::get(Ty, Ty, nullptr);
11581
11582 SDLoc dl(Op);
11583 TargetLowering::CallLoweringInfo CLI(DAG);
11584 CLI.setDebugLoc(dl).setChain(InChain)
11585 .setCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args), 0)
11586 .setInRegister().setSExtResult(isSigned).setZExtResult(!isSigned);
11587
11588 std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
11589 return CallInfo.first;
11590 }
11591
11592 // Lowers REM using divmod helpers
11593 // see RTABI section 4.2/4.3
LowerREM(SDNode * N,SelectionDAG & DAG) const11594 SDValue ARMTargetLowering::LowerREM(SDNode *N, SelectionDAG &DAG) const {
11595 // Build return types (div and rem)
11596 std::vector<Type*> RetTyParams;
11597 Type *RetTyElement;
11598
11599 switch (N->getValueType(0).getSimpleVT().SimpleTy) {
11600 default: llvm_unreachable("Unexpected request for libcall!");
11601 case MVT::i8: RetTyElement = Type::getInt8Ty(*DAG.getContext()); break;
11602 case MVT::i16: RetTyElement = Type::getInt16Ty(*DAG.getContext()); break;
11603 case MVT::i32: RetTyElement = Type::getInt32Ty(*DAG.getContext()); break;
11604 case MVT::i64: RetTyElement = Type::getInt64Ty(*DAG.getContext()); break;
11605 }
11606
11607 RetTyParams.push_back(RetTyElement);
11608 RetTyParams.push_back(RetTyElement);
11609 ArrayRef<Type*> ret = ArrayRef<Type*>(RetTyParams);
11610 Type *RetTy = StructType::get(*DAG.getContext(), ret);
11611
11612 RTLIB::Libcall LC = getDivRemLibcall(N, N->getValueType(0).getSimpleVT().
11613 SimpleTy);
11614 SDValue InChain = DAG.getEntryNode();
11615 TargetLowering::ArgListTy Args = getDivRemArgList(N, DAG.getContext());
11616 bool isSigned = N->getOpcode() == ISD::SREM;
11617 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
11618 getPointerTy(DAG.getDataLayout()));
11619
11620 // Lower call
11621 CallLoweringInfo CLI(DAG);
11622 CLI.setChain(InChain)
11623 .setCallee(CallingConv::ARM_AAPCS, RetTy, Callee, std::move(Args), 0)
11624 .setSExtResult(isSigned).setZExtResult(!isSigned).setDebugLoc(SDLoc(N));
11625 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
11626
11627 // Return second (rem) result operand (first contains div)
11628 SDNode *ResNode = CallResult.first.getNode();
11629 assert(ResNode->getNumOperands() == 2 && "divmod should return two operands");
11630 return ResNode->getOperand(1);
11631 }
11632
11633 SDValue
LowerDYNAMIC_STACKALLOC(SDValue Op,SelectionDAG & DAG) const11634 ARMTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const {
11635 assert(Subtarget->isTargetWindows() && "unsupported target platform");
11636 SDLoc DL(Op);
11637
11638 // Get the inputs.
11639 SDValue Chain = Op.getOperand(0);
11640 SDValue Size = Op.getOperand(1);
11641
11642 SDValue Words = DAG.getNode(ISD::SRL, DL, MVT::i32, Size,
11643 DAG.getConstant(2, DL, MVT::i32));
11644
11645 SDValue Flag;
11646 Chain = DAG.getCopyToReg(Chain, DL, ARM::R4, Words, Flag);
11647 Flag = Chain.getValue(1);
11648
11649 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
11650 Chain = DAG.getNode(ARMISD::WIN__CHKSTK, DL, NodeTys, Chain, Flag);
11651
11652 SDValue NewSP = DAG.getCopyFromReg(Chain, DL, ARM::SP, MVT::i32);
11653 Chain = NewSP.getValue(1);
11654
11655 SDValue Ops[2] = { NewSP, Chain };
11656 return DAG.getMergeValues(Ops, DL);
11657 }
11658
LowerFP_EXTEND(SDValue Op,SelectionDAG & DAG) const11659 SDValue ARMTargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
11660 assert(Op.getValueType() == MVT::f64 && Subtarget->isFPOnlySP() &&
11661 "Unexpected type for custom-lowering FP_EXTEND");
11662
11663 RTLIB::Libcall LC;
11664 LC = RTLIB::getFPEXT(Op.getOperand(0).getValueType(), Op.getValueType());
11665
11666 SDValue SrcVal = Op.getOperand(0);
11667 return makeLibCall(DAG, LC, Op.getValueType(), SrcVal, /*isSigned*/ false,
11668 SDLoc(Op)).first;
11669 }
11670
LowerFP_ROUND(SDValue Op,SelectionDAG & DAG) const11671 SDValue ARMTargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
11672 assert(Op.getOperand(0).getValueType() == MVT::f64 &&
11673 Subtarget->isFPOnlySP() &&
11674 "Unexpected type for custom-lowering FP_ROUND");
11675
11676 RTLIB::Libcall LC;
11677 LC = RTLIB::getFPROUND(Op.getOperand(0).getValueType(), Op.getValueType());
11678
11679 SDValue SrcVal = Op.getOperand(0);
11680 return makeLibCall(DAG, LC, Op.getValueType(), SrcVal, /*isSigned*/ false,
11681 SDLoc(Op)).first;
11682 }
11683
11684 bool
isOffsetFoldingLegal(const GlobalAddressSDNode * GA) const11685 ARMTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
11686 // The ARM target isn't yet aware of offsets.
11687 return false;
11688 }
11689
isBitFieldInvertedMask(unsigned v)11690 bool ARM::isBitFieldInvertedMask(unsigned v) {
11691 if (v == 0xffffffff)
11692 return false;
11693
11694 // there can be 1's on either or both "outsides", all the "inside"
11695 // bits must be 0's
11696 return isShiftedMask_32(~v);
11697 }
11698
11699 /// isFPImmLegal - Returns true if the target can instruction select the
11700 /// specified FP immediate natively. If false, the legalizer will
11701 /// materialize the FP immediate as a load from a constant pool.
isFPImmLegal(const APFloat & Imm,EVT VT) const11702 bool ARMTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const {
11703 if (!Subtarget->hasVFP3())
11704 return false;
11705 if (VT == MVT::f32)
11706 return ARM_AM::getFP32Imm(Imm) != -1;
11707 if (VT == MVT::f64 && !Subtarget->isFPOnlySP())
11708 return ARM_AM::getFP64Imm(Imm) != -1;
11709 return false;
11710 }
11711
11712 /// getTgtMemIntrinsic - Represent NEON load and store intrinsics as
11713 /// MemIntrinsicNodes. The associated MachineMemOperands record the alignment
11714 /// specified in the intrinsic calls.
getTgtMemIntrinsic(IntrinsicInfo & Info,const CallInst & I,unsigned Intrinsic) const11715 bool ARMTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
11716 const CallInst &I,
11717 unsigned Intrinsic) const {
11718 switch (Intrinsic) {
11719 case Intrinsic::arm_neon_vld1:
11720 case Intrinsic::arm_neon_vld2:
11721 case Intrinsic::arm_neon_vld3:
11722 case Intrinsic::arm_neon_vld4:
11723 case Intrinsic::arm_neon_vld2lane:
11724 case Intrinsic::arm_neon_vld3lane:
11725 case Intrinsic::arm_neon_vld4lane: {
11726 Info.opc = ISD::INTRINSIC_W_CHAIN;
11727 // Conservatively set memVT to the entire set of vectors loaded.
11728 auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
11729 uint64_t NumElts = DL.getTypeSizeInBits(I.getType()) / 64;
11730 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts);
11731 Info.ptrVal = I.getArgOperand(0);
11732 Info.offset = 0;
11733 Value *AlignArg = I.getArgOperand(I.getNumArgOperands() - 1);
11734 Info.align = cast<ConstantInt>(AlignArg)->getZExtValue();
11735 Info.vol = false; // volatile loads with NEON intrinsics not supported
11736 Info.readMem = true;
11737 Info.writeMem = false;
11738 return true;
11739 }
11740 case Intrinsic::arm_neon_vst1:
11741 case Intrinsic::arm_neon_vst2:
11742 case Intrinsic::arm_neon_vst3:
11743 case Intrinsic::arm_neon_vst4:
11744 case Intrinsic::arm_neon_vst2lane:
11745 case Intrinsic::arm_neon_vst3lane:
11746 case Intrinsic::arm_neon_vst4lane: {
11747 Info.opc = ISD::INTRINSIC_VOID;
11748 // Conservatively set memVT to the entire set of vectors stored.
11749 auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
11750 unsigned NumElts = 0;
11751 for (unsigned ArgI = 1, ArgE = I.getNumArgOperands(); ArgI < ArgE; ++ArgI) {
11752 Type *ArgTy = I.getArgOperand(ArgI)->getType();
11753 if (!ArgTy->isVectorTy())
11754 break;
11755 NumElts += DL.getTypeSizeInBits(ArgTy) / 64;
11756 }
11757 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts);
11758 Info.ptrVal = I.getArgOperand(0);
11759 Info.offset = 0;
11760 Value *AlignArg = I.getArgOperand(I.getNumArgOperands() - 1);
11761 Info.align = cast<ConstantInt>(AlignArg)->getZExtValue();
11762 Info.vol = false; // volatile stores with NEON intrinsics not supported
11763 Info.readMem = false;
11764 Info.writeMem = true;
11765 return true;
11766 }
11767 case Intrinsic::arm_ldaex:
11768 case Intrinsic::arm_ldrex: {
11769 auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
11770 PointerType *PtrTy = cast<PointerType>(I.getArgOperand(0)->getType());
11771 Info.opc = ISD::INTRINSIC_W_CHAIN;
11772 Info.memVT = MVT::getVT(PtrTy->getElementType());
11773 Info.ptrVal = I.getArgOperand(0);
11774 Info.offset = 0;
11775 Info.align = DL.getABITypeAlignment(PtrTy->getElementType());
11776 Info.vol = true;
11777 Info.readMem = true;
11778 Info.writeMem = false;
11779 return true;
11780 }
11781 case Intrinsic::arm_stlex:
11782 case Intrinsic::arm_strex: {
11783 auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
11784 PointerType *PtrTy = cast<PointerType>(I.getArgOperand(1)->getType());
11785 Info.opc = ISD::INTRINSIC_W_CHAIN;
11786 Info.memVT = MVT::getVT(PtrTy->getElementType());
11787 Info.ptrVal = I.getArgOperand(1);
11788 Info.offset = 0;
11789 Info.align = DL.getABITypeAlignment(PtrTy->getElementType());
11790 Info.vol = true;
11791 Info.readMem = false;
11792 Info.writeMem = true;
11793 return true;
11794 }
11795 case Intrinsic::arm_stlexd:
11796 case Intrinsic::arm_strexd: {
11797 Info.opc = ISD::INTRINSIC_W_CHAIN;
11798 Info.memVT = MVT::i64;
11799 Info.ptrVal = I.getArgOperand(2);
11800 Info.offset = 0;
11801 Info.align = 8;
11802 Info.vol = true;
11803 Info.readMem = false;
11804 Info.writeMem = true;
11805 return true;
11806 }
11807 case Intrinsic::arm_ldaexd:
11808 case Intrinsic::arm_ldrexd: {
11809 Info.opc = ISD::INTRINSIC_W_CHAIN;
11810 Info.memVT = MVT::i64;
11811 Info.ptrVal = I.getArgOperand(0);
11812 Info.offset = 0;
11813 Info.align = 8;
11814 Info.vol = true;
11815 Info.readMem = true;
11816 Info.writeMem = false;
11817 return true;
11818 }
11819 default:
11820 break;
11821 }
11822
11823 return false;
11824 }
11825
11826 /// \brief Returns true if it is beneficial to convert a load of a constant
11827 /// to just the constant itself.
shouldConvertConstantLoadToIntImm(const APInt & Imm,Type * Ty) const11828 bool ARMTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
11829 Type *Ty) const {
11830 assert(Ty->isIntegerTy());
11831
11832 unsigned Bits = Ty->getPrimitiveSizeInBits();
11833 if (Bits == 0 || Bits > 32)
11834 return false;
11835 return true;
11836 }
11837
makeDMB(IRBuilder<> & Builder,ARM_MB::MemBOpt Domain) const11838 Instruction* ARMTargetLowering::makeDMB(IRBuilder<> &Builder,
11839 ARM_MB::MemBOpt Domain) const {
11840 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
11841
11842 // First, if the target has no DMB, see what fallback we can use.
11843 if (!Subtarget->hasDataBarrier()) {
11844 // Some ARMv6 cpus can support data barriers with an mcr instruction.
11845 // Thumb1 and pre-v6 ARM mode use a libcall instead and should never get
11846 // here.
11847 if (Subtarget->hasV6Ops() && !Subtarget->isThumb()) {
11848 Function *MCR = llvm::Intrinsic::getDeclaration(M, Intrinsic::arm_mcr);
11849 Value* args[6] = {Builder.getInt32(15), Builder.getInt32(0),
11850 Builder.getInt32(0), Builder.getInt32(7),
11851 Builder.getInt32(10), Builder.getInt32(5)};
11852 return Builder.CreateCall(MCR, args);
11853 } else {
11854 // Instead of using barriers, atomic accesses on these subtargets use
11855 // libcalls.
11856 llvm_unreachable("makeDMB on a target so old that it has no barriers");
11857 }
11858 } else {
11859 Function *DMB = llvm::Intrinsic::getDeclaration(M, Intrinsic::arm_dmb);
11860 // Only a full system barrier exists in the M-class architectures.
11861 Domain = Subtarget->isMClass() ? ARM_MB::SY : Domain;
11862 Constant *CDomain = Builder.getInt32(Domain);
11863 return Builder.CreateCall(DMB, CDomain);
11864 }
11865 }
11866
11867 // Based on http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
emitLeadingFence(IRBuilder<> & Builder,AtomicOrdering Ord,bool IsStore,bool IsLoad) const11868 Instruction* ARMTargetLowering::emitLeadingFence(IRBuilder<> &Builder,
11869 AtomicOrdering Ord, bool IsStore,
11870 bool IsLoad) const {
11871 if (!getInsertFencesForAtomic())
11872 return nullptr;
11873
11874 switch (Ord) {
11875 case NotAtomic:
11876 case Unordered:
11877 llvm_unreachable("Invalid fence: unordered/non-atomic");
11878 case Monotonic:
11879 case Acquire:
11880 return nullptr; // Nothing to do
11881 case SequentiallyConsistent:
11882 if (!IsStore)
11883 return nullptr; // Nothing to do
11884 /*FALLTHROUGH*/
11885 case Release:
11886 case AcquireRelease:
11887 if (Subtarget->isSwift())
11888 return makeDMB(Builder, ARM_MB::ISHST);
11889 // FIXME: add a comment with a link to documentation justifying this.
11890 else
11891 return makeDMB(Builder, ARM_MB::ISH);
11892 }
11893 llvm_unreachable("Unknown fence ordering in emitLeadingFence");
11894 }
11895
emitTrailingFence(IRBuilder<> & Builder,AtomicOrdering Ord,bool IsStore,bool IsLoad) const11896 Instruction* ARMTargetLowering::emitTrailingFence(IRBuilder<> &Builder,
11897 AtomicOrdering Ord, bool IsStore,
11898 bool IsLoad) const {
11899 if (!getInsertFencesForAtomic())
11900 return nullptr;
11901
11902 switch (Ord) {
11903 case NotAtomic:
11904 case Unordered:
11905 llvm_unreachable("Invalid fence: unordered/not-atomic");
11906 case Monotonic:
11907 case Release:
11908 return nullptr; // Nothing to do
11909 case Acquire:
11910 case AcquireRelease:
11911 case SequentiallyConsistent:
11912 return makeDMB(Builder, ARM_MB::ISH);
11913 }
11914 llvm_unreachable("Unknown fence ordering in emitTrailingFence");
11915 }
11916
11917 // Loads and stores less than 64-bits are already atomic; ones above that
11918 // are doomed anyway, so defer to the default libcall and blame the OS when
11919 // things go wrong. Cortex M doesn't have ldrexd/strexd though, so don't emit
11920 // anything for those.
shouldExpandAtomicStoreInIR(StoreInst * SI) const11921 bool ARMTargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
11922 unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
11923 return (Size == 64) && !Subtarget->isMClass();
11924 }
11925
11926 // Loads and stores less than 64-bits are already atomic; ones above that
11927 // are doomed anyway, so defer to the default libcall and blame the OS when
11928 // things go wrong. Cortex M doesn't have ldrexd/strexd though, so don't emit
11929 // anything for those.
11930 // FIXME: ldrd and strd are atomic if the CPU has LPAE (e.g. A15 has that
11931 // guarantee, see DDI0406C ARM architecture reference manual,
11932 // sections A8.8.72-74 LDRD)
11933 TargetLowering::AtomicExpansionKind
shouldExpandAtomicLoadInIR(LoadInst * LI) const11934 ARMTargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
11935 unsigned Size = LI->getType()->getPrimitiveSizeInBits();
11936 return ((Size == 64) && !Subtarget->isMClass()) ? AtomicExpansionKind::LLOnly
11937 : AtomicExpansionKind::None;
11938 }
11939
11940 // For the real atomic operations, we have ldrex/strex up to 32 bits,
11941 // and up to 64 bits on the non-M profiles
11942 TargetLowering::AtomicExpansionKind
shouldExpandAtomicRMWInIR(AtomicRMWInst * AI) const11943 ARMTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
11944 unsigned Size = AI->getType()->getPrimitiveSizeInBits();
11945 return (Size <= (Subtarget->isMClass() ? 32U : 64U))
11946 ? AtomicExpansionKind::LLSC
11947 : AtomicExpansionKind::None;
11948 }
11949
shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst * AI) const11950 bool ARMTargetLowering::shouldExpandAtomicCmpXchgInIR(
11951 AtomicCmpXchgInst *AI) const {
11952 return true;
11953 }
11954
11955 // This has so far only been implemented for MachO.
useLoadStackGuardNode() const11956 bool ARMTargetLowering::useLoadStackGuardNode() const {
11957 return Subtarget->isTargetMachO();
11958 }
11959
canCombineStoreAndExtract(Type * VectorTy,Value * Idx,unsigned & Cost) const11960 bool ARMTargetLowering::canCombineStoreAndExtract(Type *VectorTy, Value *Idx,
11961 unsigned &Cost) const {
11962 // If we do not have NEON, vector types are not natively supported.
11963 if (!Subtarget->hasNEON())
11964 return false;
11965
11966 // Floating point values and vector values map to the same register file.
11967 // Therefore, although we could do a store extract of a vector type, this is
11968 // better to leave at float as we have more freedom in the addressing mode for
11969 // those.
11970 if (VectorTy->isFPOrFPVectorTy())
11971 return false;
11972
11973 // If the index is unknown at compile time, this is very expensive to lower
11974 // and it is not possible to combine the store with the extract.
11975 if (!isa<ConstantInt>(Idx))
11976 return false;
11977
11978 assert(VectorTy->isVectorTy() && "VectorTy is not a vector type");
11979 unsigned BitWidth = cast<VectorType>(VectorTy)->getBitWidth();
11980 // We can do a store + vector extract on any vector that fits perfectly in a D
11981 // or Q register.
11982 if (BitWidth == 64 || BitWidth == 128) {
11983 Cost = 0;
11984 return true;
11985 }
11986 return false;
11987 }
11988
isCheapToSpeculateCttz() const11989 bool ARMTargetLowering::isCheapToSpeculateCttz() const {
11990 return Subtarget->hasV6T2Ops();
11991 }
11992
isCheapToSpeculateCtlz() const11993 bool ARMTargetLowering::isCheapToSpeculateCtlz() const {
11994 return Subtarget->hasV6T2Ops();
11995 }
11996
emitLoadLinked(IRBuilder<> & Builder,Value * Addr,AtomicOrdering Ord) const11997 Value *ARMTargetLowering::emitLoadLinked(IRBuilder<> &Builder, Value *Addr,
11998 AtomicOrdering Ord) const {
11999 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
12000 Type *ValTy = cast<PointerType>(Addr->getType())->getElementType();
12001 bool IsAcquire = isAtLeastAcquire(Ord);
12002
12003 // Since i64 isn't legal and intrinsics don't get type-lowered, the ldrexd
12004 // intrinsic must return {i32, i32} and we have to recombine them into a
12005 // single i64 here.
12006 if (ValTy->getPrimitiveSizeInBits() == 64) {
12007 Intrinsic::ID Int =
12008 IsAcquire ? Intrinsic::arm_ldaexd : Intrinsic::arm_ldrexd;
12009 Function *Ldrex = llvm::Intrinsic::getDeclaration(M, Int);
12010
12011 Addr = Builder.CreateBitCast(Addr, Type::getInt8PtrTy(M->getContext()));
12012 Value *LoHi = Builder.CreateCall(Ldrex, Addr, "lohi");
12013
12014 Value *Lo = Builder.CreateExtractValue(LoHi, 0, "lo");
12015 Value *Hi = Builder.CreateExtractValue(LoHi, 1, "hi");
12016 if (!Subtarget->isLittle())
12017 std::swap (Lo, Hi);
12018 Lo = Builder.CreateZExt(Lo, ValTy, "lo64");
12019 Hi = Builder.CreateZExt(Hi, ValTy, "hi64");
12020 return Builder.CreateOr(
12021 Lo, Builder.CreateShl(Hi, ConstantInt::get(ValTy, 32)), "val64");
12022 }
12023
12024 Type *Tys[] = { Addr->getType() };
12025 Intrinsic::ID Int = IsAcquire ? Intrinsic::arm_ldaex : Intrinsic::arm_ldrex;
12026 Function *Ldrex = llvm::Intrinsic::getDeclaration(M, Int, Tys);
12027
12028 return Builder.CreateTruncOrBitCast(
12029 Builder.CreateCall(Ldrex, Addr),
12030 cast<PointerType>(Addr->getType())->getElementType());
12031 }
12032
emitAtomicCmpXchgNoStoreLLBalance(IRBuilder<> & Builder) const12033 void ARMTargetLowering::emitAtomicCmpXchgNoStoreLLBalance(
12034 IRBuilder<> &Builder) const {
12035 if (!Subtarget->hasV7Ops())
12036 return;
12037 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
12038 Builder.CreateCall(llvm::Intrinsic::getDeclaration(M, Intrinsic::arm_clrex));
12039 }
12040
emitStoreConditional(IRBuilder<> & Builder,Value * Val,Value * Addr,AtomicOrdering Ord) const12041 Value *ARMTargetLowering::emitStoreConditional(IRBuilder<> &Builder, Value *Val,
12042 Value *Addr,
12043 AtomicOrdering Ord) const {
12044 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
12045 bool IsRelease = isAtLeastRelease(Ord);
12046
12047 // Since the intrinsics must have legal type, the i64 intrinsics take two
12048 // parameters: "i32, i32". We must marshal Val into the appropriate form
12049 // before the call.
12050 if (Val->getType()->getPrimitiveSizeInBits() == 64) {
12051 Intrinsic::ID Int =
12052 IsRelease ? Intrinsic::arm_stlexd : Intrinsic::arm_strexd;
12053 Function *Strex = Intrinsic::getDeclaration(M, Int);
12054 Type *Int32Ty = Type::getInt32Ty(M->getContext());
12055
12056 Value *Lo = Builder.CreateTrunc(Val, Int32Ty, "lo");
12057 Value *Hi = Builder.CreateTrunc(Builder.CreateLShr(Val, 32), Int32Ty, "hi");
12058 if (!Subtarget->isLittle())
12059 std::swap (Lo, Hi);
12060 Addr = Builder.CreateBitCast(Addr, Type::getInt8PtrTy(M->getContext()));
12061 return Builder.CreateCall(Strex, {Lo, Hi, Addr});
12062 }
12063
12064 Intrinsic::ID Int = IsRelease ? Intrinsic::arm_stlex : Intrinsic::arm_strex;
12065 Type *Tys[] = { Addr->getType() };
12066 Function *Strex = Intrinsic::getDeclaration(M, Int, Tys);
12067
12068 return Builder.CreateCall(
12069 Strex, {Builder.CreateZExtOrBitCast(
12070 Val, Strex->getFunctionType()->getParamType(0)),
12071 Addr});
12072 }
12073
12074 /// \brief Lower an interleaved load into a vldN intrinsic.
12075 ///
12076 /// E.g. Lower an interleaved load (Factor = 2):
12077 /// %wide.vec = load <8 x i32>, <8 x i32>* %ptr, align 4
12078 /// %v0 = shuffle %wide.vec, undef, <0, 2, 4, 6> ; Extract even elements
12079 /// %v1 = shuffle %wide.vec, undef, <1, 3, 5, 7> ; Extract odd elements
12080 ///
12081 /// Into:
12082 /// %vld2 = { <4 x i32>, <4 x i32> } call llvm.arm.neon.vld2(%ptr, 4)
12083 /// %vec0 = extractelement { <4 x i32>, <4 x i32> } %vld2, i32 0
12084 /// %vec1 = extractelement { <4 x i32>, <4 x i32> } %vld2, i32 1
lowerInterleavedLoad(LoadInst * LI,ArrayRef<ShuffleVectorInst * > Shuffles,ArrayRef<unsigned> Indices,unsigned Factor) const12085 bool ARMTargetLowering::lowerInterleavedLoad(
12086 LoadInst *LI, ArrayRef<ShuffleVectorInst *> Shuffles,
12087 ArrayRef<unsigned> Indices, unsigned Factor) const {
12088 assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
12089 "Invalid interleave factor");
12090 assert(!Shuffles.empty() && "Empty shufflevector input");
12091 assert(Shuffles.size() == Indices.size() &&
12092 "Unmatched number of shufflevectors and indices");
12093
12094 VectorType *VecTy = Shuffles[0]->getType();
12095 Type *EltTy = VecTy->getVectorElementType();
12096
12097 const DataLayout &DL = LI->getModule()->getDataLayout();
12098 unsigned VecSize = DL.getTypeSizeInBits(VecTy);
12099 bool EltIs64Bits = DL.getTypeSizeInBits(EltTy) == 64;
12100
12101 // Skip if we do not have NEON and skip illegal vector types and vector types
12102 // with i64/f64 elements (vldN doesn't support i64/f64 elements).
12103 if (!Subtarget->hasNEON() || (VecSize != 64 && VecSize != 128) || EltIs64Bits)
12104 return false;
12105
12106 // A pointer vector can not be the return type of the ldN intrinsics. Need to
12107 // load integer vectors first and then convert to pointer vectors.
12108 if (EltTy->isPointerTy())
12109 VecTy =
12110 VectorType::get(DL.getIntPtrType(EltTy), VecTy->getVectorNumElements());
12111
12112 static const Intrinsic::ID LoadInts[3] = {Intrinsic::arm_neon_vld2,
12113 Intrinsic::arm_neon_vld3,
12114 Intrinsic::arm_neon_vld4};
12115
12116 IRBuilder<> Builder(LI);
12117 SmallVector<Value *, 2> Ops;
12118
12119 Type *Int8Ptr = Builder.getInt8PtrTy(LI->getPointerAddressSpace());
12120 Ops.push_back(Builder.CreateBitCast(LI->getPointerOperand(), Int8Ptr));
12121 Ops.push_back(Builder.getInt32(LI->getAlignment()));
12122
12123 Type *Tys[] = { VecTy, Int8Ptr };
12124 Function *VldnFunc =
12125 Intrinsic::getDeclaration(LI->getModule(), LoadInts[Factor - 2], Tys);
12126 CallInst *VldN = Builder.CreateCall(VldnFunc, Ops, "vldN");
12127
12128 // Replace uses of each shufflevector with the corresponding vector loaded
12129 // by ldN.
12130 for (unsigned i = 0; i < Shuffles.size(); i++) {
12131 ShuffleVectorInst *SV = Shuffles[i];
12132 unsigned Index = Indices[i];
12133
12134 Value *SubVec = Builder.CreateExtractValue(VldN, Index);
12135
12136 // Convert the integer vector to pointer vector if the element is pointer.
12137 if (EltTy->isPointerTy())
12138 SubVec = Builder.CreateIntToPtr(SubVec, SV->getType());
12139
12140 SV->replaceAllUsesWith(SubVec);
12141 }
12142
12143 return true;
12144 }
12145
12146 /// \brief Get a mask consisting of sequential integers starting from \p Start.
12147 ///
12148 /// I.e. <Start, Start + 1, ..., Start + NumElts - 1>
getSequentialMask(IRBuilder<> & Builder,unsigned Start,unsigned NumElts)12149 static Constant *getSequentialMask(IRBuilder<> &Builder, unsigned Start,
12150 unsigned NumElts) {
12151 SmallVector<Constant *, 16> Mask;
12152 for (unsigned i = 0; i < NumElts; i++)
12153 Mask.push_back(Builder.getInt32(Start + i));
12154
12155 return ConstantVector::get(Mask);
12156 }
12157
12158 /// \brief Lower an interleaved store into a vstN intrinsic.
12159 ///
12160 /// E.g. Lower an interleaved store (Factor = 3):
12161 /// %i.vec = shuffle <8 x i32> %v0, <8 x i32> %v1,
12162 /// <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11>
12163 /// store <12 x i32> %i.vec, <12 x i32>* %ptr, align 4
12164 ///
12165 /// Into:
12166 /// %sub.v0 = shuffle <8 x i32> %v0, <8 x i32> v1, <0, 1, 2, 3>
12167 /// %sub.v1 = shuffle <8 x i32> %v0, <8 x i32> v1, <4, 5, 6, 7>
12168 /// %sub.v2 = shuffle <8 x i32> %v0, <8 x i32> v1, <8, 9, 10, 11>
12169 /// call void llvm.arm.neon.vst3(%ptr, %sub.v0, %sub.v1, %sub.v2, 4)
12170 ///
12171 /// Note that the new shufflevectors will be removed and we'll only generate one
12172 /// vst3 instruction in CodeGen.
lowerInterleavedStore(StoreInst * SI,ShuffleVectorInst * SVI,unsigned Factor) const12173 bool ARMTargetLowering::lowerInterleavedStore(StoreInst *SI,
12174 ShuffleVectorInst *SVI,
12175 unsigned Factor) const {
12176 assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
12177 "Invalid interleave factor");
12178
12179 VectorType *VecTy = SVI->getType();
12180 assert(VecTy->getVectorNumElements() % Factor == 0 &&
12181 "Invalid interleaved store");
12182
12183 unsigned NumSubElts = VecTy->getVectorNumElements() / Factor;
12184 Type *EltTy = VecTy->getVectorElementType();
12185 VectorType *SubVecTy = VectorType::get(EltTy, NumSubElts);
12186
12187 const DataLayout &DL = SI->getModule()->getDataLayout();
12188 unsigned SubVecSize = DL.getTypeSizeInBits(SubVecTy);
12189 bool EltIs64Bits = DL.getTypeSizeInBits(EltTy) == 64;
12190
12191 // Skip if we do not have NEON and skip illegal vector types and vector types
12192 // with i64/f64 elements (vstN doesn't support i64/f64 elements).
12193 if (!Subtarget->hasNEON() || (SubVecSize != 64 && SubVecSize != 128) ||
12194 EltIs64Bits)
12195 return false;
12196
12197 Value *Op0 = SVI->getOperand(0);
12198 Value *Op1 = SVI->getOperand(1);
12199 IRBuilder<> Builder(SI);
12200
12201 // StN intrinsics don't support pointer vectors as arguments. Convert pointer
12202 // vectors to integer vectors.
12203 if (EltTy->isPointerTy()) {
12204 Type *IntTy = DL.getIntPtrType(EltTy);
12205
12206 // Convert to the corresponding integer vector.
12207 Type *IntVecTy =
12208 VectorType::get(IntTy, Op0->getType()->getVectorNumElements());
12209 Op0 = Builder.CreatePtrToInt(Op0, IntVecTy);
12210 Op1 = Builder.CreatePtrToInt(Op1, IntVecTy);
12211
12212 SubVecTy = VectorType::get(IntTy, NumSubElts);
12213 }
12214
12215 static const Intrinsic::ID StoreInts[3] = {Intrinsic::arm_neon_vst2,
12216 Intrinsic::arm_neon_vst3,
12217 Intrinsic::arm_neon_vst4};
12218 SmallVector<Value *, 6> Ops;
12219
12220 Type *Int8Ptr = Builder.getInt8PtrTy(SI->getPointerAddressSpace());
12221 Ops.push_back(Builder.CreateBitCast(SI->getPointerOperand(), Int8Ptr));
12222
12223 Type *Tys[] = { Int8Ptr, SubVecTy };
12224 Function *VstNFunc = Intrinsic::getDeclaration(
12225 SI->getModule(), StoreInts[Factor - 2], Tys);
12226
12227 // Split the shufflevector operands into sub vectors for the new vstN call.
12228 for (unsigned i = 0; i < Factor; i++)
12229 Ops.push_back(Builder.CreateShuffleVector(
12230 Op0, Op1, getSequentialMask(Builder, NumSubElts * i, NumSubElts)));
12231
12232 Ops.push_back(Builder.getInt32(SI->getAlignment()));
12233 Builder.CreateCall(VstNFunc, Ops);
12234 return true;
12235 }
12236
12237 enum HABaseType {
12238 HA_UNKNOWN = 0,
12239 HA_FLOAT,
12240 HA_DOUBLE,
12241 HA_VECT64,
12242 HA_VECT128
12243 };
12244
isHomogeneousAggregate(Type * Ty,HABaseType & Base,uint64_t & Members)12245 static bool isHomogeneousAggregate(Type *Ty, HABaseType &Base,
12246 uint64_t &Members) {
12247 if (auto *ST = dyn_cast<StructType>(Ty)) {
12248 for (unsigned i = 0; i < ST->getNumElements(); ++i) {
12249 uint64_t SubMembers = 0;
12250 if (!isHomogeneousAggregate(ST->getElementType(i), Base, SubMembers))
12251 return false;
12252 Members += SubMembers;
12253 }
12254 } else if (auto *AT = dyn_cast<ArrayType>(Ty)) {
12255 uint64_t SubMembers = 0;
12256 if (!isHomogeneousAggregate(AT->getElementType(), Base, SubMembers))
12257 return false;
12258 Members += SubMembers * AT->getNumElements();
12259 } else if (Ty->isFloatTy()) {
12260 if (Base != HA_UNKNOWN && Base != HA_FLOAT)
12261 return false;
12262 Members = 1;
12263 Base = HA_FLOAT;
12264 } else if (Ty->isDoubleTy()) {
12265 if (Base != HA_UNKNOWN && Base != HA_DOUBLE)
12266 return false;
12267 Members = 1;
12268 Base = HA_DOUBLE;
12269 } else if (auto *VT = dyn_cast<VectorType>(Ty)) {
12270 Members = 1;
12271 switch (Base) {
12272 case HA_FLOAT:
12273 case HA_DOUBLE:
12274 return false;
12275 case HA_VECT64:
12276 return VT->getBitWidth() == 64;
12277 case HA_VECT128:
12278 return VT->getBitWidth() == 128;
12279 case HA_UNKNOWN:
12280 switch (VT->getBitWidth()) {
12281 case 64:
12282 Base = HA_VECT64;
12283 return true;
12284 case 128:
12285 Base = HA_VECT128;
12286 return true;
12287 default:
12288 return false;
12289 }
12290 }
12291 }
12292
12293 return (Members > 0 && Members <= 4);
12294 }
12295
12296 /// \brief Return true if a type is an AAPCS-VFP homogeneous aggregate or one of
12297 /// [N x i32] or [N x i64]. This allows front-ends to skip emitting padding when
12298 /// passing according to AAPCS rules.
functionArgumentNeedsConsecutiveRegisters(Type * Ty,CallingConv::ID CallConv,bool isVarArg) const12299 bool ARMTargetLowering::functionArgumentNeedsConsecutiveRegisters(
12300 Type *Ty, CallingConv::ID CallConv, bool isVarArg) const {
12301 if (getEffectiveCallingConv(CallConv, isVarArg) !=
12302 CallingConv::ARM_AAPCS_VFP)
12303 return false;
12304
12305 HABaseType Base = HA_UNKNOWN;
12306 uint64_t Members = 0;
12307 bool IsHA = isHomogeneousAggregate(Ty, Base, Members);
12308 DEBUG(dbgs() << "isHA: " << IsHA << " "; Ty->dump());
12309
12310 bool IsIntArray = Ty->isArrayTy() && Ty->getArrayElementType()->isIntegerTy();
12311 return IsHA || IsIntArray;
12312 }
12313
getExceptionPointerRegister(const Constant * PersonalityFn) const12314 unsigned ARMTargetLowering::getExceptionPointerRegister(
12315 const Constant *PersonalityFn) const {
12316 // Platforms which do not use SjLj EH may return values in these registers
12317 // via the personality function.
12318 return Subtarget->useSjLjEH() ? ARM::NoRegister : ARM::R0;
12319 }
12320
getExceptionSelectorRegister(const Constant * PersonalityFn) const12321 unsigned ARMTargetLowering::getExceptionSelectorRegister(
12322 const Constant *PersonalityFn) const {
12323 // Platforms which do not use SjLj EH may return values in these registers
12324 // via the personality function.
12325 return Subtarget->useSjLjEH() ? ARM::NoRegister : ARM::R1;
12326 }
12327