1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements integer type expansion and promotion for LegalizeTypes.
11 // Promotion is the act of changing a computation in an illegal type into a
12 // computation in a larger type. For example, implementing i8 arithmetic in an
13 // i32 register (often needed on powerpc).
14 // Expansion is the act of changing a computation in an illegal type into a
15 // computation in two identical registers of a smaller type. For example,
16 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit
17 // targets).
18 //
19 //===----------------------------------------------------------------------===//
20
21 #include "LegalizeTypes.h"
22 #include "llvm/IR/DerivedTypes.h"
23 #include "llvm/Support/ErrorHandling.h"
24 #include "llvm/Support/raw_ostream.h"
25 using namespace llvm;
26
27 #define DEBUG_TYPE "legalize-types"
28
29 //===----------------------------------------------------------------------===//
30 // Integer Result Promotion
31 //===----------------------------------------------------------------------===//
32
33 /// PromoteIntegerResult - This method is called when a result of a node is
34 /// found to be in need of promotion to a larger type. At this point, the node
35 /// may also have invalid operands or may have other results that need
36 /// expansion, we just know that (at least) one result needs promotion.
PromoteIntegerResult(SDNode * N,unsigned ResNo)37 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
38 DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG); dbgs() << "\n");
39 SDValue Res = SDValue();
40
41 // See if the target wants to custom expand this node.
42 if (CustomLowerNode(N, N->getValueType(ResNo), true))
43 return;
44
45 switch (N->getOpcode()) {
46 default:
47 #ifndef NDEBUG
48 dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
49 N->dump(&DAG); dbgs() << "\n";
50 #endif
51 llvm_unreachable("Do not know how to promote this operator!");
52 case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
53 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
54 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
55 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
56 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
57 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
58 case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
59 case ISD::CONVERT_RNDSAT:
60 Res = PromoteIntRes_CONVERT_RNDSAT(N); break;
61 case ISD::CTLZ_ZERO_UNDEF:
62 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
63 case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break;
64 case ISD::CTTZ_ZERO_UNDEF:
65 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
66 case ISD::EXTRACT_VECTOR_ELT:
67 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
68 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N));break;
69 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));break;
70 case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break;
71 case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break;
72 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
73 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
74 case ISD::SHL: Res = PromoteIntRes_SHL(N); break;
75 case ISD::SIGN_EXTEND_INREG:
76 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
77 case ISD::SRA: Res = PromoteIntRes_SRA(N); break;
78 case ISD::SRL: Res = PromoteIntRes_SRL(N); break;
79 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
80 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
81 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
82
83 case ISD::EXTRACT_SUBVECTOR:
84 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
85 case ISD::VECTOR_SHUFFLE:
86 Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
87 case ISD::INSERT_VECTOR_ELT:
88 Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
89 case ISD::BUILD_VECTOR:
90 Res = PromoteIntRes_BUILD_VECTOR(N); break;
91 case ISD::SCALAR_TO_VECTOR:
92 Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
93 case ISD::CONCAT_VECTORS:
94 Res = PromoteIntRes_CONCAT_VECTORS(N); break;
95
96 case ISD::SIGN_EXTEND:
97 case ISD::ZERO_EXTEND:
98 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
99
100 case ISD::FP_TO_SINT:
101 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
102
103 case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break;
104
105 case ISD::AND:
106 case ISD::OR:
107 case ISD::XOR:
108 case ISD::ADD:
109 case ISD::SUB:
110 case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break;
111
112 case ISD::SDIV:
113 case ISD::SREM: Res = PromoteIntRes_SDIV(N); break;
114
115 case ISD::UDIV:
116 case ISD::UREM: Res = PromoteIntRes_UDIV(N); break;
117
118 case ISD::SADDO:
119 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
120 case ISD::UADDO:
121 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
122 case ISD::SMULO:
123 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
124
125 case ISD::ATOMIC_LOAD:
126 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
127
128 case ISD::ATOMIC_LOAD_ADD:
129 case ISD::ATOMIC_LOAD_SUB:
130 case ISD::ATOMIC_LOAD_AND:
131 case ISD::ATOMIC_LOAD_OR:
132 case ISD::ATOMIC_LOAD_XOR:
133 case ISD::ATOMIC_LOAD_NAND:
134 case ISD::ATOMIC_LOAD_MIN:
135 case ISD::ATOMIC_LOAD_MAX:
136 case ISD::ATOMIC_LOAD_UMIN:
137 case ISD::ATOMIC_LOAD_UMAX:
138 case ISD::ATOMIC_SWAP:
139 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
140
141 case ISD::ATOMIC_CMP_SWAP:
142 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
143 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
144 break;
145 }
146
147 // If the result is null then the sub-method took care of registering it.
148 if (Res.getNode())
149 SetPromotedInteger(SDValue(N, ResNo), Res);
150 }
151
PromoteIntRes_MERGE_VALUES(SDNode * N,unsigned ResNo)152 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
153 unsigned ResNo) {
154 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
155 return GetPromotedInteger(Op);
156 }
157
PromoteIntRes_AssertSext(SDNode * N)158 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
159 // Sign-extend the new bits, and continue the assertion.
160 SDValue Op = SExtPromotedInteger(N->getOperand(0));
161 return DAG.getNode(ISD::AssertSext, SDLoc(N),
162 Op.getValueType(), Op, N->getOperand(1));
163 }
164
PromoteIntRes_AssertZext(SDNode * N)165 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
166 // Zero the new bits, and continue the assertion.
167 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
168 return DAG.getNode(ISD::AssertZext, SDLoc(N),
169 Op.getValueType(), Op, N->getOperand(1));
170 }
171
PromoteIntRes_Atomic0(AtomicSDNode * N)172 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
173 EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
174 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
175 N->getMemoryVT(), ResVT,
176 N->getChain(), N->getBasePtr(),
177 N->getMemOperand(), N->getOrdering(),
178 N->getSynchScope());
179 // Legalized the chain result - switch anything that used the old chain to
180 // use the new one.
181 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
182 return Res;
183 }
184
PromoteIntRes_Atomic1(AtomicSDNode * N)185 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
186 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
187 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
188 N->getMemoryVT(),
189 N->getChain(), N->getBasePtr(),
190 Op2, N->getMemOperand(), N->getOrdering(),
191 N->getSynchScope());
192 // Legalized the chain result - switch anything that used the old chain to
193 // use the new one.
194 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
195 return Res;
196 }
197
PromoteIntRes_AtomicCmpSwap(AtomicSDNode * N,unsigned ResNo)198 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
199 unsigned ResNo) {
200 if (ResNo == 1) {
201 assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
202 EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
203 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
204
205 // Only use the result of getSetCCResultType if it is legal,
206 // otherwise just use the promoted result type (NVT).
207 if (!TLI.isTypeLegal(SVT))
208 SVT = NVT;
209
210 SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
211 SDValue Res = DAG.getAtomicCmpSwap(
212 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs,
213 N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
214 N->getMemOperand(), N->getSuccessOrdering(), N->getFailureOrdering(),
215 N->getSynchScope());
216 ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
217 ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
218 return Res.getValue(1);
219 }
220
221 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
222 SDValue Op3 = GetPromotedInteger(N->getOperand(3));
223 SDVTList VTs =
224 DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
225 SDValue Res = DAG.getAtomicCmpSwap(
226 N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
227 N->getBasePtr(), Op2, Op3, N->getMemOperand(), N->getSuccessOrdering(),
228 N->getFailureOrdering(), N->getSynchScope());
229 // Update the use to N with the newly created Res.
230 for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
231 ReplaceValueWith(SDValue(N, i), Res.getValue(i));
232 return Res;
233 }
234
PromoteIntRes_BITCAST(SDNode * N)235 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
236 SDValue InOp = N->getOperand(0);
237 EVT InVT = InOp.getValueType();
238 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
239 EVT OutVT = N->getValueType(0);
240 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
241 SDLoc dl(N);
242
243 switch (getTypeAction(InVT)) {
244 case TargetLowering::TypeLegal:
245 break;
246 case TargetLowering::TypePromoteInteger:
247 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
248 // The input promotes to the same size. Convert the promoted value.
249 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
250 break;
251 case TargetLowering::TypeSoftenFloat:
252 // Promote the integer operand by hand.
253 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
254 case TargetLowering::TypePromoteFloat: {
255 // Convert the promoted float by hand.
256 if (NOutVT.bitsEq(NInVT)) {
257 SDValue PromotedOp = GetPromotedFloat(InOp);
258 SDValue Trunc = DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, PromotedOp);
259 return DAG.getNode(ISD::AssertZext, dl, NOutVT, Trunc,
260 DAG.getValueType(OutVT));
261 }
262 break;
263 }
264 case TargetLowering::TypeExpandInteger:
265 case TargetLowering::TypeExpandFloat:
266 break;
267 case TargetLowering::TypeScalarizeVector:
268 // Convert the element to an integer and promote it by hand.
269 if (!NOutVT.isVector())
270 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
271 BitConvertToInteger(GetScalarizedVector(InOp)));
272 break;
273 case TargetLowering::TypeSplitVector: {
274 // For example, i32 = BITCAST v2i16 on alpha. Convert the split
275 // pieces of the input into integers and reassemble in the final type.
276 SDValue Lo, Hi;
277 GetSplitVector(N->getOperand(0), Lo, Hi);
278 Lo = BitConvertToInteger(Lo);
279 Hi = BitConvertToInteger(Hi);
280
281 if (TLI.isBigEndian())
282 std::swap(Lo, Hi);
283
284 InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
285 EVT::getIntegerVT(*DAG.getContext(),
286 NOutVT.getSizeInBits()),
287 JoinIntegers(Lo, Hi));
288 return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
289 }
290 case TargetLowering::TypeWidenVector:
291 // The input is widened to the same size. Convert to the widened value.
292 // Make sure that the outgoing value is not a vector, because this would
293 // make us bitcast between two vectors which are legalized in different ways.
294 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector())
295 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
296 }
297
298 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
299 CreateStackStoreLoad(InOp, OutVT));
300 }
301
PromoteIntRes_BSWAP(SDNode * N)302 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
303 SDValue Op = GetPromotedInteger(N->getOperand(0));
304 EVT OVT = N->getValueType(0);
305 EVT NVT = Op.getValueType();
306 SDLoc dl(N);
307
308 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
309 return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
310 DAG.getConstant(DiffBits, TLI.getShiftAmountTy(NVT)));
311 }
312
PromoteIntRes_BUILD_PAIR(SDNode * N)313 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
314 // The pair element type may be legal, or may not promote to the same type as
315 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
316 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
317 TLI.getTypeToTransformTo(*DAG.getContext(),
318 N->getValueType(0)), JoinIntegers(N->getOperand(0),
319 N->getOperand(1)));
320 }
321
PromoteIntRes_Constant(SDNode * N)322 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
323 EVT VT = N->getValueType(0);
324 // FIXME there is no actual debug info here
325 SDLoc dl(N);
326 // Zero extend things like i1, sign extend everything else. It shouldn't
327 // matter in theory which one we pick, but this tends to give better code?
328 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
329 SDValue Result = DAG.getNode(Opc, dl,
330 TLI.getTypeToTransformTo(*DAG.getContext(), VT),
331 SDValue(N, 0));
332 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
333 return Result;
334 }
335
PromoteIntRes_CONVERT_RNDSAT(SDNode * N)336 SDValue DAGTypeLegalizer::PromoteIntRes_CONVERT_RNDSAT(SDNode *N) {
337 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode();
338 assert ((CvtCode == ISD::CVT_SS || CvtCode == ISD::CVT_SU ||
339 CvtCode == ISD::CVT_US || CvtCode == ISD::CVT_UU ||
340 CvtCode == ISD::CVT_SF || CvtCode == ISD::CVT_UF) &&
341 "can only promote integers");
342 EVT OutVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
343 return DAG.getConvertRndSat(OutVT, SDLoc(N), N->getOperand(0),
344 N->getOperand(1), N->getOperand(2),
345 N->getOperand(3), N->getOperand(4), CvtCode);
346 }
347
PromoteIntRes_CTLZ(SDNode * N)348 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
349 // Zero extend to the promoted type and do the count there.
350 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
351 SDLoc dl(N);
352 EVT OVT = N->getValueType(0);
353 EVT NVT = Op.getValueType();
354 Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
355 // Subtract off the extra leading bits in the bigger type.
356 return DAG.getNode(
357 ISD::SUB, dl, NVT, Op,
358 DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(),
359 NVT));
360 }
361
PromoteIntRes_CTPOP(SDNode * N)362 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) {
363 // Zero extend to the promoted type and do the count there.
364 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
365 return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op);
366 }
367
PromoteIntRes_CTTZ(SDNode * N)368 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
369 SDValue Op = GetPromotedInteger(N->getOperand(0));
370 EVT OVT = N->getValueType(0);
371 EVT NVT = Op.getValueType();
372 SDLoc dl(N);
373 if (N->getOpcode() == ISD::CTTZ) {
374 // The count is the same in the promoted type except if the original
375 // value was zero. This can be handled by setting the bit just off
376 // the top of the original type.
377 auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
378 OVT.getScalarSizeInBits());
379 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, NVT));
380 }
381 return DAG.getNode(N->getOpcode(), dl, NVT, Op);
382 }
383
PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode * N)384 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
385 SDLoc dl(N);
386 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
387 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, N->getOperand(0),
388 N->getOperand(1));
389 }
390
PromoteIntRes_FP_TO_XINT(SDNode * N)391 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
392 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
393 unsigned NewOpc = N->getOpcode();
394 SDLoc dl(N);
395
396 // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
397 // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT
398 // and SINT conversions are Custom, there is no way to tell which is
399 // preferable. We choose SINT because that's the right thing on PPC.)
400 if (N->getOpcode() == ISD::FP_TO_UINT &&
401 !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
402 TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT))
403 NewOpc = ISD::FP_TO_SINT;
404
405 SDValue Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
406
407 // Assert that the converted value fits in the original type. If it doesn't
408 // (eg: because the value being converted is too big), then the result of the
409 // original operation was undefined anyway, so the assert is still correct.
410 return DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ?
411 ISD::AssertZext : ISD::AssertSext, dl, NVT, Res,
412 DAG.getValueType(N->getValueType(0).getScalarType()));
413 }
414
PromoteIntRes_FP_TO_FP16(SDNode * N)415 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
416 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
417 SDLoc dl(N);
418
419 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
420
421 return DAG.getNode(ISD::AssertZext, dl,
422 NVT, Res, DAG.getValueType(N->getValueType(0)));
423 }
424
PromoteIntRes_INT_EXTEND(SDNode * N)425 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
426 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
427 SDLoc dl(N);
428
429 if (getTypeAction(N->getOperand(0).getValueType())
430 == TargetLowering::TypePromoteInteger) {
431 SDValue Res = GetPromotedInteger(N->getOperand(0));
432 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
433
434 // If the result and operand types are the same after promotion, simplify
435 // to an in-register extension.
436 if (NVT == Res.getValueType()) {
437 // The high bits are not guaranteed to be anything. Insert an extend.
438 if (N->getOpcode() == ISD::SIGN_EXTEND)
439 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
440 DAG.getValueType(N->getOperand(0).getValueType()));
441 if (N->getOpcode() == ISD::ZERO_EXTEND)
442 return DAG.getZeroExtendInReg(Res, dl,
443 N->getOperand(0).getValueType().getScalarType());
444 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
445 return Res;
446 }
447 }
448
449 // Otherwise, just extend the original operand all the way to the larger type.
450 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
451 }
452
PromoteIntRes_LOAD(LoadSDNode * N)453 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
454 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
455 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
456 ISD::LoadExtType ExtType =
457 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType();
458 SDLoc dl(N);
459 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
460 N->getMemoryVT(), N->getMemOperand());
461
462 // Legalized the chain result - switch anything that used the old chain to
463 // use the new one.
464 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
465 return Res;
466 }
467
PromoteIntRes_MLOAD(MaskedLoadSDNode * N)468 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
469 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
470 SDValue ExtSrc0 = GetPromotedInteger(N->getSrc0());
471
472 SDValue Mask = N->getMask();
473 EVT NewMaskVT = getSetCCResultType(NVT);
474 if (NewMaskVT != N->getMask().getValueType())
475 Mask = PromoteTargetBoolean(Mask, NewMaskVT);
476 SDLoc dl(N);
477
478 SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
479 Mask, ExtSrc0, N->getMemoryVT(),
480 N->getMemOperand(), ISD::SEXTLOAD);
481 // Legalized the chain result - switch anything that used the old chain to
482 // use the new one.
483 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
484 return Res;
485 }
486 /// Promote the overflow flag of an overflowing arithmetic node.
PromoteIntRes_Overflow(SDNode * N)487 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
488 // Simply change the return type of the boolean result.
489 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
490 EVT ValueVTs[] = { N->getValueType(0), NVT };
491 SDValue Ops[] = { N->getOperand(0), N->getOperand(1) };
492 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
493 DAG.getVTList(ValueVTs), Ops);
494
495 // Modified the sum result - switch anything that used the old sum to use
496 // the new one.
497 ReplaceValueWith(SDValue(N, 0), Res);
498
499 return SDValue(Res.getNode(), 1);
500 }
501
PromoteIntRes_SADDSUBO(SDNode * N,unsigned ResNo)502 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
503 if (ResNo == 1)
504 return PromoteIntRes_Overflow(N);
505
506 // The operation overflowed iff the result in the larger type is not the
507 // sign extension of its truncation to the original type.
508 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
509 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
510 EVT OVT = N->getOperand(0).getValueType();
511 EVT NVT = LHS.getValueType();
512 SDLoc dl(N);
513
514 // Do the arithmetic in the larger type.
515 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
516 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
517
518 // Calculate the overflow flag: sign extend the arithmetic result from
519 // the original type.
520 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
521 DAG.getValueType(OVT));
522 // Overflowed if and only if this is not equal to Res.
523 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
524
525 // Use the calculated overflow everywhere.
526 ReplaceValueWith(SDValue(N, 1), Ofl);
527
528 return Res;
529 }
530
PromoteIntRes_SDIV(SDNode * N)531 SDValue DAGTypeLegalizer::PromoteIntRes_SDIV(SDNode *N) {
532 // Sign extend the input.
533 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
534 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
535 return DAG.getNode(N->getOpcode(), SDLoc(N),
536 LHS.getValueType(), LHS, RHS);
537 }
538
PromoteIntRes_SELECT(SDNode * N)539 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
540 SDValue LHS = GetPromotedInteger(N->getOperand(1));
541 SDValue RHS = GetPromotedInteger(N->getOperand(2));
542 return DAG.getSelect(SDLoc(N),
543 LHS.getValueType(), N->getOperand(0), LHS, RHS);
544 }
545
PromoteIntRes_VSELECT(SDNode * N)546 SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
547 SDValue Mask = N->getOperand(0);
548 EVT OpTy = N->getOperand(1).getValueType();
549
550 // Promote all the way up to the canonical SetCC type.
551 Mask = PromoteTargetBoolean(Mask, OpTy);
552 SDValue LHS = GetPromotedInteger(N->getOperand(1));
553 SDValue RHS = GetPromotedInteger(N->getOperand(2));
554 return DAG.getNode(ISD::VSELECT, SDLoc(N),
555 LHS.getValueType(), Mask, LHS, RHS);
556 }
557
PromoteIntRes_SELECT_CC(SDNode * N)558 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
559 SDValue LHS = GetPromotedInteger(N->getOperand(2));
560 SDValue RHS = GetPromotedInteger(N->getOperand(3));
561 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
562 LHS.getValueType(), N->getOperand(0),
563 N->getOperand(1), LHS, RHS, N->getOperand(4));
564 }
565
PromoteIntRes_SETCC(SDNode * N)566 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
567 EVT SVT = getSetCCResultType(N->getOperand(0).getValueType());
568
569 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
570
571 // Only use the result of getSetCCResultType if it is legal,
572 // otherwise just use the promoted result type (NVT).
573 if (!TLI.isTypeLegal(SVT))
574 SVT = NVT;
575
576 SDLoc dl(N);
577 assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() &&
578 "Vector compare must return a vector result!");
579
580 SDValue LHS = N->getOperand(0);
581 SDValue RHS = N->getOperand(1);
582 if (LHS.getValueType() != RHS.getValueType()) {
583 if (getTypeAction(LHS.getValueType()) == TargetLowering::TypePromoteInteger &&
584 !LHS.getValueType().isVector())
585 LHS = GetPromotedInteger(LHS);
586 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger &&
587 !RHS.getValueType().isVector())
588 RHS = GetPromotedInteger(RHS);
589 }
590
591 // Get the SETCC result using the canonical SETCC type.
592 SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, LHS, RHS,
593 N->getOperand(2));
594
595 assert(NVT.bitsLE(SVT) && "Integer type overpromoted?");
596 // Convert to the expected type.
597 return DAG.getNode(ISD::TRUNCATE, dl, NVT, SetCC);
598 }
599
PromoteIntRes_SHL(SDNode * N)600 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
601 SDValue Res = GetPromotedInteger(N->getOperand(0));
602 SDValue Amt = N->getOperand(1);
603 Amt = Amt.getValueType().isVector() ? ZExtPromotedInteger(Amt) : Amt;
604 return DAG.getNode(ISD::SHL, SDLoc(N), Res.getValueType(), Res, Amt);
605 }
606
PromoteIntRes_SIGN_EXTEND_INREG(SDNode * N)607 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
608 SDValue Op = GetPromotedInteger(N->getOperand(0));
609 return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
610 Op.getValueType(), Op, N->getOperand(1));
611 }
612
PromoteIntRes_SimpleIntBinOp(SDNode * N)613 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
614 // The input may have strange things in the top bits of the registers, but
615 // these operations don't care. They may have weird bits going out, but
616 // that too is okay if they are integer operations.
617 SDValue LHS = GetPromotedInteger(N->getOperand(0));
618 SDValue RHS = GetPromotedInteger(N->getOperand(1));
619 return DAG.getNode(N->getOpcode(), SDLoc(N),
620 LHS.getValueType(), LHS, RHS);
621 }
622
PromoteIntRes_SRA(SDNode * N)623 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
624 // The input value must be properly sign extended.
625 SDValue Res = SExtPromotedInteger(N->getOperand(0));
626 SDValue Amt = N->getOperand(1);
627 Amt = Amt.getValueType().isVector() ? ZExtPromotedInteger(Amt) : Amt;
628 return DAG.getNode(ISD::SRA, SDLoc(N), Res.getValueType(), Res, Amt);
629 }
630
PromoteIntRes_SRL(SDNode * N)631 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
632 // The input value must be properly zero extended.
633 SDValue Res = ZExtPromotedInteger(N->getOperand(0));
634 SDValue Amt = N->getOperand(1);
635 Amt = Amt.getValueType().isVector() ? ZExtPromotedInteger(Amt) : Amt;
636 return DAG.getNode(ISD::SRL, SDLoc(N), Res.getValueType(), Res, Amt);
637 }
638
PromoteIntRes_TRUNCATE(SDNode * N)639 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
640 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
641 SDValue Res;
642 SDValue InOp = N->getOperand(0);
643 SDLoc dl(N);
644
645 switch (getTypeAction(InOp.getValueType())) {
646 default: llvm_unreachable("Unknown type action!");
647 case TargetLowering::TypeLegal:
648 case TargetLowering::TypeExpandInteger:
649 Res = InOp;
650 break;
651 case TargetLowering::TypePromoteInteger:
652 Res = GetPromotedInteger(InOp);
653 break;
654 case TargetLowering::TypeSplitVector:
655 EVT InVT = InOp.getValueType();
656 assert(InVT.isVector() && "Cannot split scalar types");
657 unsigned NumElts = InVT.getVectorNumElements();
658 assert(NumElts == NVT.getVectorNumElements() &&
659 "Dst and Src must have the same number of elements");
660 assert(isPowerOf2_32(NumElts) &&
661 "Promoted vector type must be a power of two");
662
663 SDValue EOp1, EOp2;
664 GetSplitVector(InOp, EOp1, EOp2);
665
666 EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
667 NumElts/2);
668 EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
669 EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
670
671 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
672 }
673
674 // Truncate to NVT instead of VT
675 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
676 }
677
PromoteIntRes_UADDSUBO(SDNode * N,unsigned ResNo)678 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
679 if (ResNo == 1)
680 return PromoteIntRes_Overflow(N);
681
682 // The operation overflowed iff the result in the larger type is not the
683 // zero extension of its truncation to the original type.
684 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
685 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
686 EVT OVT = N->getOperand(0).getValueType();
687 EVT NVT = LHS.getValueType();
688 SDLoc dl(N);
689
690 // Do the arithmetic in the larger type.
691 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
692 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
693
694 // Calculate the overflow flag: zero extend the arithmetic result from
695 // the original type.
696 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
697 // Overflowed if and only if this is not equal to Res.
698 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
699
700 // Use the calculated overflow everywhere.
701 ReplaceValueWith(SDValue(N, 1), Ofl);
702
703 return Res;
704 }
705
PromoteIntRes_XMULO(SDNode * N,unsigned ResNo)706 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
707 // Promote the overflow bit trivially.
708 if (ResNo == 1)
709 return PromoteIntRes_Overflow(N);
710
711 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
712 SDLoc DL(N);
713 EVT SmallVT = LHS.getValueType();
714
715 // To determine if the result overflowed in a larger type, we extend the
716 // input to the larger type, do the multiply (checking if it overflows),
717 // then also check the high bits of the result to see if overflow happened
718 // there.
719 if (N->getOpcode() == ISD::SMULO) {
720 LHS = SExtPromotedInteger(LHS);
721 RHS = SExtPromotedInteger(RHS);
722 } else {
723 LHS = ZExtPromotedInteger(LHS);
724 RHS = ZExtPromotedInteger(RHS);
725 }
726 SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
727 SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
728
729 // Overflow occurred if it occurred in the larger type, or if the high part
730 // of the result does not zero/sign-extend the low part. Check this second
731 // possibility first.
732 SDValue Overflow;
733 if (N->getOpcode() == ISD::UMULO) {
734 // Unsigned overflow occurred if the high part is non-zero.
735 SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
736 DAG.getIntPtrConstant(SmallVT.getSizeInBits()));
737 Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
738 DAG.getConstant(0, Hi.getValueType()), ISD::SETNE);
739 } else {
740 // Signed overflow occurred if the high part does not sign extend the low.
741 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
742 Mul, DAG.getValueType(SmallVT));
743 Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
744 }
745
746 // The only other way for overflow to occur is if the multiplication in the
747 // larger type itself overflowed.
748 Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
749 SDValue(Mul.getNode(), 1));
750
751 // Use the calculated overflow everywhere.
752 ReplaceValueWith(SDValue(N, 1), Overflow);
753 return Mul;
754 }
755
PromoteIntRes_UDIV(SDNode * N)756 SDValue DAGTypeLegalizer::PromoteIntRes_UDIV(SDNode *N) {
757 // Zero extend the input.
758 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
759 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
760 return DAG.getNode(N->getOpcode(), SDLoc(N),
761 LHS.getValueType(), LHS, RHS);
762 }
763
PromoteIntRes_UNDEF(SDNode * N)764 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
765 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
766 N->getValueType(0)));
767 }
768
PromoteIntRes_VAARG(SDNode * N)769 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
770 SDValue Chain = N->getOperand(0); // Get the chain.
771 SDValue Ptr = N->getOperand(1); // Get the pointer.
772 EVT VT = N->getValueType(0);
773 SDLoc dl(N);
774
775 MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
776 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
777 // The argument is passed as NumRegs registers of type RegVT.
778
779 SmallVector<SDValue, 8> Parts(NumRegs);
780 for (unsigned i = 0; i < NumRegs; ++i) {
781 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
782 N->getConstantOperandVal(3));
783 Chain = Parts[i].getValue(1);
784 }
785
786 // Handle endianness of the load.
787 if (TLI.isBigEndian())
788 std::reverse(Parts.begin(), Parts.end());
789
790 // Assemble the parts in the promoted type.
791 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
792 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
793 for (unsigned i = 1; i < NumRegs; ++i) {
794 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
795 // Shift it to the right position and "or" it in.
796 Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
797 DAG.getConstant(i * RegVT.getSizeInBits(),
798 TLI.getPointerTy()));
799 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
800 }
801
802 // Modified the chain result - switch anything that used the old chain to
803 // use the new one.
804 ReplaceValueWith(SDValue(N, 1), Chain);
805
806 return Res;
807 }
808
809 //===----------------------------------------------------------------------===//
810 // Integer Operand Promotion
811 //===----------------------------------------------------------------------===//
812
813 /// PromoteIntegerOperand - This method is called when the specified operand of
814 /// the specified node is found to need promotion. At this point, all of the
815 /// result types of the node are known to be legal, but other operands of the
816 /// node may need promotion or expansion as well as the specified one.
PromoteIntegerOperand(SDNode * N,unsigned OpNo)817 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
818 DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG); dbgs() << "\n");
819 SDValue Res = SDValue();
820
821 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
822 return false;
823
824 switch (N->getOpcode()) {
825 default:
826 #ifndef NDEBUG
827 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
828 N->dump(&DAG); dbgs() << "\n";
829 #endif
830 llvm_unreachable("Do not know how to promote this operator's operand!");
831
832 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
833 case ISD::ATOMIC_STORE:
834 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
835 break;
836 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
837 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
838 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
839 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
840 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
841 case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
842 case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
843 case ISD::CONVERT_RNDSAT:
844 Res = PromoteIntOp_CONVERT_RNDSAT(N); break;
845 case ISD::INSERT_VECTOR_ELT:
846 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
847 case ISD::SCALAR_TO_VECTOR:
848 Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
849 case ISD::VSELECT:
850 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
851 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
852 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
853 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
854 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
855 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
856 OpNo); break;
857 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
858 OpNo); break;
859 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
860 OpNo); break;
861 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
862 case ISD::FP16_TO_FP:
863 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
864 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
865
866 case ISD::SHL:
867 case ISD::SRA:
868 case ISD::SRL:
869 case ISD::ROTL:
870 case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
871 }
872
873 // If the result is null, the sub-method took care of registering results etc.
874 if (!Res.getNode()) return false;
875
876 // If the result is N, the sub-method updated N in place. Tell the legalizer
877 // core about this.
878 if (Res.getNode() == N)
879 return true;
880
881 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
882 "Invalid operand expansion");
883
884 ReplaceValueWith(SDValue(N, 0), Res);
885 return false;
886 }
887
888 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is
889 /// shared among BR_CC, SELECT_CC, and SETCC handlers.
PromoteSetCCOperands(SDValue & NewLHS,SDValue & NewRHS,ISD::CondCode CCCode)890 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
891 ISD::CondCode CCCode) {
892 // We have to insert explicit sign or zero extends. Note that we could
893 // insert sign extends for ALL conditions, but zero extend is cheaper on
894 // many machines (an AND instead of two shifts), so prefer it.
895 switch (CCCode) {
896 default: llvm_unreachable("Unknown integer comparison!");
897 case ISD::SETEQ:
898 case ISD::SETNE: {
899 SDValue OpL = GetPromotedInteger(NewLHS);
900 SDValue OpR = GetPromotedInteger(NewRHS);
901
902 // We would prefer to promote the comparison operand with sign extension,
903 // if we find the operand is actually to truncate an AssertSext. With this
904 // optimization, we can avoid inserting real truncate instruction, which
905 // is redudant eventually.
906 if (OpL->getOpcode() == ISD::AssertSext &&
907 cast<VTSDNode>(OpL->getOperand(1))->getVT() == NewLHS.getValueType() &&
908 OpR->getOpcode() == ISD::AssertSext &&
909 cast<VTSDNode>(OpR->getOperand(1))->getVT() == NewRHS.getValueType()) {
910 NewLHS = OpL;
911 NewRHS = OpR;
912 } else {
913 NewLHS = ZExtPromotedInteger(NewLHS);
914 NewRHS = ZExtPromotedInteger(NewRHS);
915 }
916 break;
917 }
918 case ISD::SETUGE:
919 case ISD::SETUGT:
920 case ISD::SETULE:
921 case ISD::SETULT:
922 // ALL of these operations will work if we either sign or zero extend
923 // the operands (including the unsigned comparisons!). Zero extend is
924 // usually a simpler/cheaper operation, so prefer it.
925 NewLHS = ZExtPromotedInteger(NewLHS);
926 NewRHS = ZExtPromotedInteger(NewRHS);
927 break;
928 case ISD::SETGE:
929 case ISD::SETGT:
930 case ISD::SETLT:
931 case ISD::SETLE:
932 NewLHS = SExtPromotedInteger(NewLHS);
933 NewRHS = SExtPromotedInteger(NewRHS);
934 break;
935 }
936 }
937
PromoteIntOp_ANY_EXTEND(SDNode * N)938 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
939 SDValue Op = GetPromotedInteger(N->getOperand(0));
940 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
941 }
942
PromoteIntOp_ATOMIC_STORE(AtomicSDNode * N)943 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
944 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
945 return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
946 N->getChain(), N->getBasePtr(), Op2, N->getMemOperand(),
947 N->getOrdering(), N->getSynchScope());
948 }
949
PromoteIntOp_BITCAST(SDNode * N)950 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
951 // This should only occur in unusual situations like bitcasting to an
952 // x86_fp80, so just turn it into a store+load
953 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
954 }
955
PromoteIntOp_BR_CC(SDNode * N,unsigned OpNo)956 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
957 assert(OpNo == 2 && "Don't know how to promote this operand!");
958
959 SDValue LHS = N->getOperand(2);
960 SDValue RHS = N->getOperand(3);
961 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
962
963 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
964 // legal types.
965 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
966 N->getOperand(1), LHS, RHS, N->getOperand(4)),
967 0);
968 }
969
PromoteIntOp_BRCOND(SDNode * N,unsigned OpNo)970 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
971 assert(OpNo == 1 && "only know how to promote condition");
972
973 // Promote all the way up to the canonical SetCC type.
974 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
975
976 // The chain (Op#0) and basic block destination (Op#2) are always legal types.
977 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
978 N->getOperand(2)), 0);
979 }
980
PromoteIntOp_BUILD_PAIR(SDNode * N)981 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
982 // Since the result type is legal, the operands must promote to it.
983 EVT OVT = N->getOperand(0).getValueType();
984 SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
985 SDValue Hi = GetPromotedInteger(N->getOperand(1));
986 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
987 SDLoc dl(N);
988
989 Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
990 DAG.getConstant(OVT.getSizeInBits(), TLI.getPointerTy()));
991 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
992 }
993
PromoteIntOp_BUILD_VECTOR(SDNode * N)994 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
995 // The vector type is legal but the element type is not. This implies
996 // that the vector is a power-of-two in length and that the element
997 // type does not have a strange size (eg: it is not i1).
998 EVT VecVT = N->getValueType(0);
999 unsigned NumElts = VecVT.getVectorNumElements();
1000 assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
1001 "Legal vector of one illegal element?");
1002
1003 // Promote the inserted value. The type does not need to match the
1004 // vector element type. Check that any extra bits introduced will be
1005 // truncated away.
1006 assert(N->getOperand(0).getValueType().getSizeInBits() >=
1007 N->getValueType(0).getVectorElementType().getSizeInBits() &&
1008 "Type of inserted value narrower than vector element type!");
1009
1010 SmallVector<SDValue, 16> NewOps;
1011 for (unsigned i = 0; i < NumElts; ++i)
1012 NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
1013
1014 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1015 }
1016
PromoteIntOp_CONVERT_RNDSAT(SDNode * N)1017 SDValue DAGTypeLegalizer::PromoteIntOp_CONVERT_RNDSAT(SDNode *N) {
1018 ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode();
1019 assert ((CvtCode == ISD::CVT_SS || CvtCode == ISD::CVT_SU ||
1020 CvtCode == ISD::CVT_US || CvtCode == ISD::CVT_UU ||
1021 CvtCode == ISD::CVT_FS || CvtCode == ISD::CVT_FU) &&
1022 "can only promote integer arguments");
1023 SDValue InOp = GetPromotedInteger(N->getOperand(0));
1024 return DAG.getConvertRndSat(N->getValueType(0), SDLoc(N), InOp,
1025 N->getOperand(1), N->getOperand(2),
1026 N->getOperand(3), N->getOperand(4), CvtCode);
1027 }
1028
PromoteIntOp_INSERT_VECTOR_ELT(SDNode * N,unsigned OpNo)1029 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
1030 unsigned OpNo) {
1031 if (OpNo == 1) {
1032 // Promote the inserted value. This is valid because the type does not
1033 // have to match the vector element type.
1034
1035 // Check that any extra bits introduced will be truncated away.
1036 assert(N->getOperand(1).getValueType().getSizeInBits() >=
1037 N->getValueType(0).getVectorElementType().getSizeInBits() &&
1038 "Type of inserted value narrower than vector element type!");
1039 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1040 GetPromotedInteger(N->getOperand(1)),
1041 N->getOperand(2)),
1042 0);
1043 }
1044
1045 assert(OpNo == 2 && "Different operand and result vector types?");
1046
1047 // Promote the index.
1048 SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
1049 TLI.getVectorIdxTy());
1050 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1051 N->getOperand(1), Idx), 0);
1052 }
1053
PromoteIntOp_SCALAR_TO_VECTOR(SDNode * N)1054 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
1055 // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
1056 // the operand in place.
1057 return SDValue(DAG.UpdateNodeOperands(N,
1058 GetPromotedInteger(N->getOperand(0))), 0);
1059 }
1060
PromoteIntOp_SELECT(SDNode * N,unsigned OpNo)1061 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
1062 assert(OpNo == 0 && "Only know how to promote the condition!");
1063 SDValue Cond = N->getOperand(0);
1064 EVT OpTy = N->getOperand(1).getValueType();
1065
1066 // Promote all the way up to the canonical SetCC type.
1067 EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
1068 Cond = PromoteTargetBoolean(Cond, OpVT);
1069
1070 return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
1071 N->getOperand(2)), 0);
1072 }
1073
PromoteIntOp_SELECT_CC(SDNode * N,unsigned OpNo)1074 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
1075 assert(OpNo == 0 && "Don't know how to promote this operand!");
1076
1077 SDValue LHS = N->getOperand(0);
1078 SDValue RHS = N->getOperand(1);
1079 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
1080
1081 // The CC (#4) and the possible return values (#2 and #3) have legal types.
1082 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
1083 N->getOperand(3), N->getOperand(4)), 0);
1084 }
1085
PromoteIntOp_SETCC(SDNode * N,unsigned OpNo)1086 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
1087 assert(OpNo == 0 && "Don't know how to promote this operand!");
1088
1089 SDValue LHS = N->getOperand(0);
1090 SDValue RHS = N->getOperand(1);
1091 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
1092
1093 // The CC (#2) is always legal.
1094 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
1095 }
1096
PromoteIntOp_Shift(SDNode * N)1097 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
1098 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1099 ZExtPromotedInteger(N->getOperand(1))), 0);
1100 }
1101
PromoteIntOp_SIGN_EXTEND(SDNode * N)1102 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
1103 SDValue Op = GetPromotedInteger(N->getOperand(0));
1104 SDLoc dl(N);
1105 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1106 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
1107 Op, DAG.getValueType(N->getOperand(0).getValueType()));
1108 }
1109
PromoteIntOp_SINT_TO_FP(SDNode * N)1110 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
1111 return SDValue(DAG.UpdateNodeOperands(N,
1112 SExtPromotedInteger(N->getOperand(0))), 0);
1113 }
1114
PromoteIntOp_STORE(StoreSDNode * N,unsigned OpNo)1115 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
1116 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1117 SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
1118 SDLoc dl(N);
1119
1120 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
1121
1122 // Truncate the value and store the result.
1123 return DAG.getTruncStore(Ch, dl, Val, Ptr,
1124 N->getMemoryVT(), N->getMemOperand());
1125 }
1126
PromoteIntOp_MSTORE(MaskedStoreSDNode * N,unsigned OpNo)1127 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N, unsigned OpNo){
1128
1129 SDValue DataOp = N->getValue();
1130 EVT DataVT = DataOp.getValueType();
1131 SDValue Mask = N->getMask();
1132 EVT MaskVT = Mask.getValueType();
1133 SDLoc dl(N);
1134
1135 bool TruncateStore = false;
1136 if (!TLI.isTypeLegal(DataVT)) {
1137 if (getTypeAction(DataVT) == TargetLowering::TypePromoteInteger) {
1138 DataOp = GetPromotedInteger(DataOp);
1139 if (!TLI.isTypeLegal(MaskVT))
1140 Mask = PromoteTargetBoolean(Mask, DataOp.getValueType());
1141 TruncateStore = true;
1142 }
1143 else {
1144 assert(getTypeAction(DataVT) == TargetLowering::TypeWidenVector &&
1145 "Unexpected data legalization in MSTORE");
1146 DataOp = GetWidenedVector(DataOp);
1147
1148 if (getTypeAction(MaskVT) == TargetLowering::TypeWidenVector)
1149 Mask = GetWidenedVector(Mask);
1150 else {
1151 EVT BoolVT = getSetCCResultType(DataOp.getValueType());
1152
1153 // We can't use ModifyToType() because we should fill the mask with
1154 // zeroes
1155 unsigned WidenNumElts = BoolVT.getVectorNumElements();
1156 unsigned MaskNumElts = MaskVT.getVectorNumElements();
1157
1158 unsigned NumConcat = WidenNumElts / MaskNumElts;
1159 SmallVector<SDValue, 16> Ops(NumConcat);
1160 SDValue ZeroVal = DAG.getConstant(0, MaskVT);
1161 Ops[0] = Mask;
1162 for (unsigned i = 1; i != NumConcat; ++i)
1163 Ops[i] = ZeroVal;
1164
1165 Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, BoolVT, Ops);
1166 }
1167 }
1168 }
1169 else
1170 Mask = PromoteTargetBoolean(N->getMask(), DataOp.getValueType());
1171 return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), Mask,
1172 N->getMemoryVT(), N->getMemOperand(),
1173 TruncateStore);
1174 }
1175
PromoteIntOp_MLOAD(MaskedLoadSDNode * N,unsigned OpNo)1176 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N, unsigned OpNo){
1177 assert(OpNo == 2 && "Only know how to promote the mask!");
1178 EVT DataVT = N->getValueType(0);
1179 SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1180 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1181 NewOps[OpNo] = Mask;
1182 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1183 }
1184
PromoteIntOp_TRUNCATE(SDNode * N)1185 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
1186 SDValue Op = GetPromotedInteger(N->getOperand(0));
1187 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
1188 }
1189
PromoteIntOp_UINT_TO_FP(SDNode * N)1190 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
1191 return SDValue(DAG.UpdateNodeOperands(N,
1192 ZExtPromotedInteger(N->getOperand(0))), 0);
1193 }
1194
PromoteIntOp_ZERO_EXTEND(SDNode * N)1195 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
1196 SDLoc dl(N);
1197 SDValue Op = GetPromotedInteger(N->getOperand(0));
1198 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1199 return DAG.getZeroExtendInReg(Op, dl,
1200 N->getOperand(0).getValueType().getScalarType());
1201 }
1202
1203
1204 //===----------------------------------------------------------------------===//
1205 // Integer Result Expansion
1206 //===----------------------------------------------------------------------===//
1207
1208 /// ExpandIntegerResult - This method is called when the specified result of the
1209 /// specified node is found to need expansion. At this point, the node may also
1210 /// have invalid operands or may have other results that need promotion, we just
1211 /// know that (at least) one result needs expansion.
ExpandIntegerResult(SDNode * N,unsigned ResNo)1212 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
1213 DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG); dbgs() << "\n");
1214 SDValue Lo, Hi;
1215 Lo = Hi = SDValue();
1216
1217 // See if the target wants to custom expand this node.
1218 if (CustomLowerNode(N, N->getValueType(ResNo), true))
1219 return;
1220
1221 switch (N->getOpcode()) {
1222 default:
1223 #ifndef NDEBUG
1224 dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
1225 N->dump(&DAG); dbgs() << "\n";
1226 #endif
1227 llvm_unreachable("Do not know how to expand the result of this operator!");
1228
1229 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1230 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
1231 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1232 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1233
1234 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1235 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1236 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1237 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1238 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1239
1240 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
1241 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
1242 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
1243 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
1244 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
1245 case ISD::CTLZ_ZERO_UNDEF:
1246 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
1247 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
1248 case ISD::CTTZ_ZERO_UNDEF:
1249 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
1250 case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
1251 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
1252 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
1253 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
1254 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
1255 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
1256 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
1257 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
1258 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
1259 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
1260 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
1261 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
1262 case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
1263
1264 case ISD::ATOMIC_LOAD_ADD:
1265 case ISD::ATOMIC_LOAD_SUB:
1266 case ISD::ATOMIC_LOAD_AND:
1267 case ISD::ATOMIC_LOAD_OR:
1268 case ISD::ATOMIC_LOAD_XOR:
1269 case ISD::ATOMIC_LOAD_NAND:
1270 case ISD::ATOMIC_LOAD_MIN:
1271 case ISD::ATOMIC_LOAD_MAX:
1272 case ISD::ATOMIC_LOAD_UMIN:
1273 case ISD::ATOMIC_LOAD_UMAX:
1274 case ISD::ATOMIC_SWAP:
1275 case ISD::ATOMIC_CMP_SWAP: {
1276 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
1277 SplitInteger(Tmp.first, Lo, Hi);
1278 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1279 break;
1280 }
1281 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
1282 AtomicSDNode *AN = cast<AtomicSDNode>(N);
1283 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
1284 SDValue Tmp = DAG.getAtomicCmpSwap(
1285 ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
1286 N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
1287 AN->getMemOperand(), AN->getSuccessOrdering(), AN->getFailureOrdering(),
1288 AN->getSynchScope());
1289
1290 // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
1291 // success simply by comparing the loaded value against the ingoing
1292 // comparison.
1293 SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
1294 N->getOperand(2), ISD::SETEQ);
1295
1296 SplitInteger(Tmp, Lo, Hi);
1297 ReplaceValueWith(SDValue(N, 1), Success);
1298 ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
1299 break;
1300 }
1301
1302 case ISD::AND:
1303 case ISD::OR:
1304 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
1305
1306 case ISD::ADD:
1307 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
1308
1309 case ISD::ADDC:
1310 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
1311
1312 case ISD::ADDE:
1313 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
1314
1315 case ISD::SHL:
1316 case ISD::SRA:
1317 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
1318
1319 case ISD::SADDO:
1320 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
1321 case ISD::UADDO:
1322 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
1323 case ISD::UMULO:
1324 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
1325 }
1326
1327 // If Lo/Hi is null, the sub-method took care of registering results etc.
1328 if (Lo.getNode())
1329 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
1330 }
1331
1332 /// Lower an atomic node to the appropriate builtin call.
ExpandAtomic(SDNode * Node)1333 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
1334 unsigned Opc = Node->getOpcode();
1335 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
1336 RTLIB::Libcall LC = RTLIB::getATOMIC(Opc, VT);
1337 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
1338
1339 return ExpandChainLibCall(LC, Node, false);
1340 }
1341
1342 /// ExpandShiftByConstant - N is a shift by a value that needs to be expanded,
1343 /// and the shift amount is a constant 'Amt'. Expand the operation.
ExpandShiftByConstant(SDNode * N,unsigned Amt,SDValue & Lo,SDValue & Hi)1344 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, unsigned Amt,
1345 SDValue &Lo, SDValue &Hi) {
1346 SDLoc DL(N);
1347 // Expand the incoming operand to be shifted, so that we have its parts
1348 SDValue InL, InH;
1349 GetExpandedInteger(N->getOperand(0), InL, InH);
1350
1351 // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
1352 // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
1353 if (!Amt) {
1354 Lo = InL;
1355 Hi = InH;
1356 return;
1357 }
1358
1359 EVT NVT = InL.getValueType();
1360 unsigned VTBits = N->getValueType(0).getSizeInBits();
1361 unsigned NVTBits = NVT.getSizeInBits();
1362 EVT ShTy = N->getOperand(1).getValueType();
1363
1364 if (N->getOpcode() == ISD::SHL) {
1365 if (Amt > VTBits) {
1366 Lo = Hi = DAG.getConstant(0, NVT);
1367 } else if (Amt > NVTBits) {
1368 Lo = DAG.getConstant(0, NVT);
1369 Hi = DAG.getNode(ISD::SHL, DL,
1370 NVT, InL, DAG.getConstant(Amt-NVTBits, ShTy));
1371 } else if (Amt == NVTBits) {
1372 Lo = DAG.getConstant(0, NVT);
1373 Hi = InL;
1374 } else if (Amt == 1 &&
1375 TLI.isOperationLegalOrCustom(ISD::ADDC,
1376 TLI.getTypeToExpandTo(*DAG.getContext(), NVT))) {
1377 // Emit this X << 1 as X+X.
1378 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
1379 SDValue LoOps[2] = { InL, InL };
1380 Lo = DAG.getNode(ISD::ADDC, DL, VTList, LoOps);
1381 SDValue HiOps[3] = { InH, InH, Lo.getValue(1) };
1382 Hi = DAG.getNode(ISD::ADDE, DL, VTList, HiOps);
1383 } else {
1384 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, ShTy));
1385 Hi = DAG.getNode(ISD::OR, DL, NVT,
1386 DAG.getNode(ISD::SHL, DL, NVT, InH,
1387 DAG.getConstant(Amt, ShTy)),
1388 DAG.getNode(ISD::SRL, DL, NVT, InL,
1389 DAG.getConstant(NVTBits-Amt, ShTy)));
1390 }
1391 return;
1392 }
1393
1394 if (N->getOpcode() == ISD::SRL) {
1395 if (Amt > VTBits) {
1396 Lo = DAG.getConstant(0, NVT);
1397 Hi = DAG.getConstant(0, NVT);
1398 } else if (Amt > NVTBits) {
1399 Lo = DAG.getNode(ISD::SRL, DL,
1400 NVT, InH, DAG.getConstant(Amt-NVTBits,ShTy));
1401 Hi = DAG.getConstant(0, NVT);
1402 } else if (Amt == NVTBits) {
1403 Lo = InH;
1404 Hi = DAG.getConstant(0, NVT);
1405 } else {
1406 Lo = DAG.getNode(ISD::OR, DL, NVT,
1407 DAG.getNode(ISD::SRL, DL, NVT, InL,
1408 DAG.getConstant(Amt, ShTy)),
1409 DAG.getNode(ISD::SHL, DL, NVT, InH,
1410 DAG.getConstant(NVTBits-Amt, ShTy)));
1411 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, ShTy));
1412 }
1413 return;
1414 }
1415
1416 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
1417 if (Amt > VTBits) {
1418 Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1419 DAG.getConstant(NVTBits-1, ShTy));
1420 } else if (Amt > NVTBits) {
1421 Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1422 DAG.getConstant(Amt-NVTBits, ShTy));
1423 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1424 DAG.getConstant(NVTBits-1, ShTy));
1425 } else if (Amt == NVTBits) {
1426 Lo = InH;
1427 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1428 DAG.getConstant(NVTBits-1, ShTy));
1429 } else {
1430 Lo = DAG.getNode(ISD::OR, DL, NVT,
1431 DAG.getNode(ISD::SRL, DL, NVT, InL,
1432 DAG.getConstant(Amt, ShTy)),
1433 DAG.getNode(ISD::SHL, DL, NVT, InH,
1434 DAG.getConstant(NVTBits-Amt, ShTy)));
1435 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, ShTy));
1436 }
1437 }
1438
1439 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
1440 /// this shift based on knowledge of the high bit of the shift amount. If we
1441 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
1442 /// shift amount.
1443 bool DAGTypeLegalizer::
ExpandShiftWithKnownAmountBit(SDNode * N,SDValue & Lo,SDValue & Hi)1444 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1445 SDValue Amt = N->getOperand(1);
1446 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1447 EVT ShTy = Amt.getValueType();
1448 unsigned ShBits = ShTy.getScalarType().getSizeInBits();
1449 unsigned NVTBits = NVT.getScalarType().getSizeInBits();
1450 assert(isPowerOf2_32(NVTBits) &&
1451 "Expanded integer type size not a power of two!");
1452 SDLoc dl(N);
1453
1454 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
1455 APInt KnownZero, KnownOne;
1456 DAG.computeKnownBits(N->getOperand(1), KnownZero, KnownOne);
1457
1458 // If we don't know anything about the high bits, exit.
1459 if (((KnownZero|KnownOne) & HighBitMask) == 0)
1460 return false;
1461
1462 // Get the incoming operand to be shifted.
1463 SDValue InL, InH;
1464 GetExpandedInteger(N->getOperand(0), InL, InH);
1465
1466 // If we know that any of the high bits of the shift amount are one, then we
1467 // can do this as a couple of simple shifts.
1468 if (KnownOne.intersects(HighBitMask)) {
1469 // Mask out the high bit, which we know is set.
1470 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
1471 DAG.getConstant(~HighBitMask, ShTy));
1472
1473 switch (N->getOpcode()) {
1474 default: llvm_unreachable("Unknown shift");
1475 case ISD::SHL:
1476 Lo = DAG.getConstant(0, NVT); // Low part is zero.
1477 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
1478 return true;
1479 case ISD::SRL:
1480 Hi = DAG.getConstant(0, NVT); // Hi part is zero.
1481 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
1482 return true;
1483 case ISD::SRA:
1484 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
1485 DAG.getConstant(NVTBits-1, ShTy));
1486 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
1487 return true;
1488 }
1489 }
1490
1491 // If we know that all of the high bits of the shift amount are zero, then we
1492 // can do this as a couple of simple shifts.
1493 if ((KnownZero & HighBitMask) == HighBitMask) {
1494 // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
1495 // shift if x is zero. We can use XOR here because x is known to be smaller
1496 // than 32.
1497 SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
1498 DAG.getConstant(NVTBits-1, ShTy));
1499
1500 unsigned Op1, Op2;
1501 switch (N->getOpcode()) {
1502 default: llvm_unreachable("Unknown shift");
1503 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
1504 case ISD::SRL:
1505 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
1506 }
1507
1508 // When shifting right the arithmetic for Lo and Hi is swapped.
1509 if (N->getOpcode() != ISD::SHL)
1510 std::swap(InL, InH);
1511
1512 // Use a little trick to get the bits that move from Lo to Hi. First
1513 // shift by one bit.
1514 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, ShTy));
1515 // Then compute the remaining shift with amount-1.
1516 SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
1517
1518 Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
1519 Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
1520
1521 if (N->getOpcode() != ISD::SHL)
1522 std::swap(Hi, Lo);
1523 return true;
1524 }
1525
1526 return false;
1527 }
1528
1529 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
1530 /// of any size.
1531 bool DAGTypeLegalizer::
ExpandShiftWithUnknownAmountBit(SDNode * N,SDValue & Lo,SDValue & Hi)1532 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1533 SDValue Amt = N->getOperand(1);
1534 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1535 EVT ShTy = Amt.getValueType();
1536 unsigned NVTBits = NVT.getSizeInBits();
1537 assert(isPowerOf2_32(NVTBits) &&
1538 "Expanded integer type size not a power of two!");
1539 SDLoc dl(N);
1540
1541 // Get the incoming operand to be shifted.
1542 SDValue InL, InH;
1543 GetExpandedInteger(N->getOperand(0), InL, InH);
1544
1545 SDValue NVBitsNode = DAG.getConstant(NVTBits, ShTy);
1546 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
1547 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
1548 SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
1549 Amt, NVBitsNode, ISD::SETULT);
1550
1551 SDValue LoS, HiS, LoL, HiL;
1552 switch (N->getOpcode()) {
1553 default: llvm_unreachable("Unknown shift");
1554 case ISD::SHL:
1555 // Short: ShAmt < NVTBits
1556 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
1557 HiS = DAG.getNode(ISD::OR, dl, NVT,
1558 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
1559 // FIXME: If Amt is zero, the following shift generates an undefined result
1560 // on some architectures.
1561 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
1562
1563 // Long: ShAmt >= NVTBits
1564 LoL = DAG.getConstant(0, NVT); // Lo part is zero.
1565 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
1566
1567 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
1568 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1569 return true;
1570 case ISD::SRL:
1571 // Short: ShAmt < NVTBits
1572 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
1573 LoS = DAG.getNode(ISD::OR, dl, NVT,
1574 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1575 // FIXME: If Amt is zero, the following shift generates an undefined result
1576 // on some architectures.
1577 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1578
1579 // Long: ShAmt >= NVTBits
1580 HiL = DAG.getConstant(0, NVT); // Hi part is zero.
1581 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1582
1583 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
1584 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1585 return true;
1586 case ISD::SRA:
1587 // Short: ShAmt < NVTBits
1588 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
1589 LoS = DAG.getNode(ISD::OR, dl, NVT,
1590 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
1591 // FIXME: If Amt is zero, the following shift generates an undefined result
1592 // on some architectures.
1593 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
1594
1595 // Long: ShAmt >= NVTBits
1596 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
1597 DAG.getConstant(NVTBits-1, ShTy));
1598 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
1599
1600 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
1601 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
1602 return true;
1603 }
1604 }
1605
ExpandIntRes_ADDSUB(SDNode * N,SDValue & Lo,SDValue & Hi)1606 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
1607 SDValue &Lo, SDValue &Hi) {
1608 SDLoc dl(N);
1609 // Expand the subcomponents.
1610 SDValue LHSL, LHSH, RHSL, RHSH;
1611 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1612 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1613
1614 EVT NVT = LHSL.getValueType();
1615 SDValue LoOps[2] = { LHSL, RHSL };
1616 SDValue HiOps[3] = { LHSH, RHSH };
1617
1618 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
1619 // them. TODO: Teach operation legalization how to expand unsupported
1620 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
1621 // a carry of type MVT::Glue, but there doesn't seem to be any way to
1622 // generate a value of this type in the expanded code sequence.
1623 bool hasCarry =
1624 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
1625 ISD::ADDC : ISD::SUBC,
1626 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
1627
1628 if (hasCarry) {
1629 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
1630 if (N->getOpcode() == ISD::ADD) {
1631 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
1632 HiOps[2] = Lo.getValue(1);
1633 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
1634 } else {
1635 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
1636 HiOps[2] = Lo.getValue(1);
1637 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
1638 }
1639 return;
1640 }
1641
1642 if (N->getOpcode() == ISD::ADD) {
1643 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
1644 Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
1645 SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
1646 ISD::SETULT);
1647 SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
1648 DAG.getConstant(1, NVT),
1649 DAG.getConstant(0, NVT));
1650 SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
1651 ISD::SETULT);
1652 SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
1653 DAG.getConstant(1, NVT), Carry1);
1654 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
1655 } else {
1656 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
1657 Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
1658 SDValue Cmp =
1659 DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
1660 LoOps[0], LoOps[1], ISD::SETULT);
1661 SDValue Borrow = DAG.getSelect(dl, NVT, Cmp,
1662 DAG.getConstant(1, NVT),
1663 DAG.getConstant(0, NVT));
1664 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
1665 }
1666 }
1667
ExpandIntRes_ADDSUBC(SDNode * N,SDValue & Lo,SDValue & Hi)1668 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
1669 SDValue &Lo, SDValue &Hi) {
1670 // Expand the subcomponents.
1671 SDValue LHSL, LHSH, RHSL, RHSH;
1672 SDLoc dl(N);
1673 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1674 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1675 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
1676 SDValue LoOps[2] = { LHSL, RHSL };
1677 SDValue HiOps[3] = { LHSH, RHSH };
1678
1679 if (N->getOpcode() == ISD::ADDC) {
1680 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
1681 HiOps[2] = Lo.getValue(1);
1682 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
1683 } else {
1684 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
1685 HiOps[2] = Lo.getValue(1);
1686 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
1687 }
1688
1689 // Legalized the flag result - switch anything that used the old flag to
1690 // use the new one.
1691 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
1692 }
1693
ExpandIntRes_ADDSUBE(SDNode * N,SDValue & Lo,SDValue & Hi)1694 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
1695 SDValue &Lo, SDValue &Hi) {
1696 // Expand the subcomponents.
1697 SDValue LHSL, LHSH, RHSL, RHSH;
1698 SDLoc dl(N);
1699 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
1700 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
1701 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
1702 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
1703 SDValue HiOps[3] = { LHSH, RHSH };
1704
1705 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
1706 HiOps[2] = Lo.getValue(1);
1707 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
1708
1709 // Legalized the flag result - switch anything that used the old flag to
1710 // use the new one.
1711 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
1712 }
1713
ExpandIntRes_MERGE_VALUES(SDNode * N,unsigned ResNo,SDValue & Lo,SDValue & Hi)1714 void DAGTypeLegalizer::ExpandIntRes_MERGE_VALUES(SDNode *N, unsigned ResNo,
1715 SDValue &Lo, SDValue &Hi) {
1716 SDValue Res = DisintegrateMERGE_VALUES(N, ResNo);
1717 SplitInteger(Res, Lo, Hi);
1718 }
1719
ExpandIntRes_ANY_EXTEND(SDNode * N,SDValue & Lo,SDValue & Hi)1720 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
1721 SDValue &Lo, SDValue &Hi) {
1722 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1723 SDLoc dl(N);
1724 SDValue Op = N->getOperand(0);
1725 if (Op.getValueType().bitsLE(NVT)) {
1726 // The low part is any extension of the input (which degenerates to a copy).
1727 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
1728 Hi = DAG.getUNDEF(NVT); // The high part is undefined.
1729 } else {
1730 // For example, extension of an i48 to an i64. The operand type necessarily
1731 // promotes to the result type, so will end up being expanded too.
1732 assert(getTypeAction(Op.getValueType()) ==
1733 TargetLowering::TypePromoteInteger &&
1734 "Only know how to promote this result!");
1735 SDValue Res = GetPromotedInteger(Op);
1736 assert(Res.getValueType() == N->getValueType(0) &&
1737 "Operand over promoted?");
1738 // Split the promoted operand. This will simplify when it is expanded.
1739 SplitInteger(Res, Lo, Hi);
1740 }
1741 }
1742
ExpandIntRes_AssertSext(SDNode * N,SDValue & Lo,SDValue & Hi)1743 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
1744 SDValue &Lo, SDValue &Hi) {
1745 SDLoc dl(N);
1746 GetExpandedInteger(N->getOperand(0), Lo, Hi);
1747 EVT NVT = Lo.getValueType();
1748 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
1749 unsigned NVTBits = NVT.getSizeInBits();
1750 unsigned EVTBits = EVT.getSizeInBits();
1751
1752 if (NVTBits < EVTBits) {
1753 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
1754 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
1755 EVTBits - NVTBits)));
1756 } else {
1757 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
1758 // The high part replicates the sign bit of Lo, make it explicit.
1759 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
1760 DAG.getConstant(NVTBits-1, TLI.getPointerTy()));
1761 }
1762 }
1763
ExpandIntRes_AssertZext(SDNode * N,SDValue & Lo,SDValue & Hi)1764 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
1765 SDValue &Lo, SDValue &Hi) {
1766 SDLoc dl(N);
1767 GetExpandedInteger(N->getOperand(0), Lo, Hi);
1768 EVT NVT = Lo.getValueType();
1769 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
1770 unsigned NVTBits = NVT.getSizeInBits();
1771 unsigned EVTBits = EVT.getSizeInBits();
1772
1773 if (NVTBits < EVTBits) {
1774 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
1775 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
1776 EVTBits - NVTBits)));
1777 } else {
1778 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
1779 // The high part must be zero, make it explicit.
1780 Hi = DAG.getConstant(0, NVT);
1781 }
1782 }
1783
ExpandIntRes_BSWAP(SDNode * N,SDValue & Lo,SDValue & Hi)1784 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
1785 SDValue &Lo, SDValue &Hi) {
1786 SDLoc dl(N);
1787 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
1788 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
1789 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
1790 }
1791
ExpandIntRes_Constant(SDNode * N,SDValue & Lo,SDValue & Hi)1792 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
1793 SDValue &Lo, SDValue &Hi) {
1794 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1795 unsigned NBitWidth = NVT.getSizeInBits();
1796 auto Constant = cast<ConstantSDNode>(N);
1797 const APInt &Cst = Constant->getAPIntValue();
1798 bool IsTarget = Constant->isTargetOpcode();
1799 bool IsOpaque = Constant->isOpaque();
1800 Lo = DAG.getConstant(Cst.trunc(NBitWidth), NVT, IsTarget, IsOpaque);
1801 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), NVT, IsTarget,
1802 IsOpaque);
1803 }
1804
ExpandIntRes_CTLZ(SDNode * N,SDValue & Lo,SDValue & Hi)1805 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
1806 SDValue &Lo, SDValue &Hi) {
1807 SDLoc dl(N);
1808 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
1809 GetExpandedInteger(N->getOperand(0), Lo, Hi);
1810 EVT NVT = Lo.getValueType();
1811
1812 SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
1813 DAG.getConstant(0, NVT), ISD::SETNE);
1814
1815 SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
1816 SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
1817
1818 Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
1819 DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
1820 DAG.getConstant(NVT.getSizeInBits(), NVT)));
1821 Hi = DAG.getConstant(0, NVT);
1822 }
1823
ExpandIntRes_CTPOP(SDNode * N,SDValue & Lo,SDValue & Hi)1824 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
1825 SDValue &Lo, SDValue &Hi) {
1826 SDLoc dl(N);
1827 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
1828 GetExpandedInteger(N->getOperand(0), Lo, Hi);
1829 EVT NVT = Lo.getValueType();
1830 Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
1831 DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
1832 Hi = DAG.getConstant(0, NVT);
1833 }
1834
ExpandIntRes_CTTZ(SDNode * N,SDValue & Lo,SDValue & Hi)1835 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
1836 SDValue &Lo, SDValue &Hi) {
1837 SDLoc dl(N);
1838 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
1839 GetExpandedInteger(N->getOperand(0), Lo, Hi);
1840 EVT NVT = Lo.getValueType();
1841
1842 SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
1843 DAG.getConstant(0, NVT), ISD::SETNE);
1844
1845 SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
1846 SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
1847
1848 Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
1849 DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
1850 DAG.getConstant(NVT.getSizeInBits(), NVT)));
1851 Hi = DAG.getConstant(0, NVT);
1852 }
1853
ExpandIntRes_FP_TO_SINT(SDNode * N,SDValue & Lo,SDValue & Hi)1854 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
1855 SDValue &Hi) {
1856 SDLoc dl(N);
1857 EVT VT = N->getValueType(0);
1858
1859 SDValue Op = N->getOperand(0);
1860 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
1861 Op = GetPromotedFloat(Op);
1862
1863 RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT);
1864 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
1865 SplitInteger(TLI.makeLibCall(DAG, LC, VT, &Op, 1, true/*irrelevant*/,
1866 dl).first,
1867 Lo, Hi);
1868 }
1869
ExpandIntRes_FP_TO_UINT(SDNode * N,SDValue & Lo,SDValue & Hi)1870 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
1871 SDValue &Hi) {
1872 SDLoc dl(N);
1873 EVT VT = N->getValueType(0);
1874
1875 SDValue Op = N->getOperand(0);
1876 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
1877 Op = GetPromotedFloat(Op);
1878
1879 RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT);
1880 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
1881 SplitInteger(TLI.makeLibCall(DAG, LC, VT, &Op, 1, false/*irrelevant*/,
1882 dl).first,
1883 Lo, Hi);
1884 }
1885
ExpandIntRes_LOAD(LoadSDNode * N,SDValue & Lo,SDValue & Hi)1886 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
1887 SDValue &Lo, SDValue &Hi) {
1888 if (ISD::isNormalLoad(N)) {
1889 ExpandRes_NormalLoad(N, Lo, Hi);
1890 return;
1891 }
1892
1893 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
1894
1895 EVT VT = N->getValueType(0);
1896 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1897 SDValue Ch = N->getChain();
1898 SDValue Ptr = N->getBasePtr();
1899 ISD::LoadExtType ExtType = N->getExtensionType();
1900 unsigned Alignment = N->getAlignment();
1901 bool isVolatile = N->isVolatile();
1902 bool isNonTemporal = N->isNonTemporal();
1903 bool isInvariant = N->isInvariant();
1904 AAMDNodes AAInfo = N->getAAInfo();
1905 SDLoc dl(N);
1906
1907 assert(NVT.isByteSized() && "Expanded type not byte sized!");
1908
1909 if (N->getMemoryVT().bitsLE(NVT)) {
1910 EVT MemVT = N->getMemoryVT();
1911
1912 Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
1913 MemVT, isVolatile, isNonTemporal, isInvariant,
1914 Alignment, AAInfo);
1915
1916 // Remember the chain.
1917 Ch = Lo.getValue(1);
1918
1919 if (ExtType == ISD::SEXTLOAD) {
1920 // The high part is obtained by SRA'ing all but one of the bits of the
1921 // lo part.
1922 unsigned LoSize = Lo.getValueType().getSizeInBits();
1923 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
1924 DAG.getConstant(LoSize-1, TLI.getPointerTy()));
1925 } else if (ExtType == ISD::ZEXTLOAD) {
1926 // The high part is just a zero.
1927 Hi = DAG.getConstant(0, NVT);
1928 } else {
1929 assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
1930 // The high part is undefined.
1931 Hi = DAG.getUNDEF(NVT);
1932 }
1933 } else if (TLI.isLittleEndian()) {
1934 // Little-endian - low bits are at low addresses.
1935 Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(),
1936 isVolatile, isNonTemporal, isInvariant, Alignment,
1937 AAInfo);
1938
1939 unsigned ExcessBits =
1940 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
1941 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
1942
1943 // Increment the pointer to the other half.
1944 unsigned IncrementSize = NVT.getSizeInBits()/8;
1945 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
1946 DAG.getConstant(IncrementSize, Ptr.getValueType()));
1947 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
1948 N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
1949 isVolatile, isNonTemporal, isInvariant,
1950 MinAlign(Alignment, IncrementSize), AAInfo);
1951
1952 // Build a factor node to remember that this load is independent of the
1953 // other one.
1954 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1955 Hi.getValue(1));
1956 } else {
1957 // Big-endian - high bits are at low addresses. Favor aligned loads at
1958 // the cost of some bit-fiddling.
1959 EVT MemVT = N->getMemoryVT();
1960 unsigned EBytes = MemVT.getStoreSize();
1961 unsigned IncrementSize = NVT.getSizeInBits()/8;
1962 unsigned ExcessBits = (EBytes - IncrementSize)*8;
1963
1964 // Load both the high bits and maybe some of the low bits.
1965 Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
1966 EVT::getIntegerVT(*DAG.getContext(),
1967 MemVT.getSizeInBits() - ExcessBits),
1968 isVolatile, isNonTemporal, isInvariant, Alignment,
1969 AAInfo);
1970
1971 // Increment the pointer to the other half.
1972 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
1973 DAG.getConstant(IncrementSize, Ptr.getValueType()));
1974 // Load the rest of the low bits.
1975 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
1976 N->getPointerInfo().getWithOffset(IncrementSize),
1977 EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
1978 isVolatile, isNonTemporal, isInvariant,
1979 MinAlign(Alignment, IncrementSize), AAInfo);
1980
1981 // Build a factor node to remember that this load is independent of the
1982 // other one.
1983 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1984 Hi.getValue(1));
1985
1986 if (ExcessBits < NVT.getSizeInBits()) {
1987 // Transfer low bits from the bottom of Hi to the top of Lo.
1988 Lo = DAG.getNode(ISD::OR, dl, NVT, Lo,
1989 DAG.getNode(ISD::SHL, dl, NVT, Hi,
1990 DAG.getConstant(ExcessBits,
1991 TLI.getPointerTy())));
1992 // Move high bits to the right position in Hi.
1993 Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl,
1994 NVT, Hi,
1995 DAG.getConstant(NVT.getSizeInBits() - ExcessBits,
1996 TLI.getPointerTy()));
1997 }
1998 }
1999
2000 // Legalized the chain result - switch anything that used the old chain to
2001 // use the new one.
2002 ReplaceValueWith(SDValue(N, 1), Ch);
2003 }
2004
ExpandIntRes_Logical(SDNode * N,SDValue & Lo,SDValue & Hi)2005 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
2006 SDValue &Lo, SDValue &Hi) {
2007 SDLoc dl(N);
2008 SDValue LL, LH, RL, RH;
2009 GetExpandedInteger(N->getOperand(0), LL, LH);
2010 GetExpandedInteger(N->getOperand(1), RL, RH);
2011 Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
2012 Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
2013 }
2014
ExpandIntRes_MUL(SDNode * N,SDValue & Lo,SDValue & Hi)2015 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
2016 SDValue &Lo, SDValue &Hi) {
2017 EVT VT = N->getValueType(0);
2018 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2019 SDLoc dl(N);
2020
2021 SDValue LL, LH, RL, RH;
2022 GetExpandedInteger(N->getOperand(0), LL, LH);
2023 GetExpandedInteger(N->getOperand(1), RL, RH);
2024
2025 if (TLI.expandMUL(N, Lo, Hi, NVT, DAG, LL, LH, RL, RH))
2026 return;
2027
2028 // If nothing else, we can make a libcall.
2029 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2030 if (VT == MVT::i16)
2031 LC = RTLIB::MUL_I16;
2032 else if (VT == MVT::i32)
2033 LC = RTLIB::MUL_I32;
2034 else if (VT == MVT::i64)
2035 LC = RTLIB::MUL_I64;
2036 else if (VT == MVT::i128)
2037 LC = RTLIB::MUL_I128;
2038 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported MUL!");
2039
2040 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2041 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, 2, true/*irrelevant*/,
2042 dl).first,
2043 Lo, Hi);
2044 }
2045
ExpandIntRes_SADDSUBO(SDNode * Node,SDValue & Lo,SDValue & Hi)2046 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
2047 SDValue &Lo, SDValue &Hi) {
2048 SDValue LHS = Node->getOperand(0);
2049 SDValue RHS = Node->getOperand(1);
2050 SDLoc dl(Node);
2051
2052 // Expand the result by simply replacing it with the equivalent
2053 // non-overflow-checking operation.
2054 SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
2055 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
2056 LHS, RHS);
2057 SplitInteger(Sum, Lo, Hi);
2058
2059 // Compute the overflow.
2060 //
2061 // LHSSign -> LHS >= 0
2062 // RHSSign -> RHS >= 0
2063 // SumSign -> Sum >= 0
2064 //
2065 // Add:
2066 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
2067 // Sub:
2068 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
2069 //
2070 EVT OType = Node->getValueType(1);
2071 SDValue Zero = DAG.getConstant(0, LHS.getValueType());
2072
2073 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
2074 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
2075 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
2076 Node->getOpcode() == ISD::SADDO ?
2077 ISD::SETEQ : ISD::SETNE);
2078
2079 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
2080 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
2081
2082 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
2083
2084 // Use the calculated overflow everywhere.
2085 ReplaceValueWith(SDValue(Node, 1), Cmp);
2086 }
2087
ExpandIntRes_SDIV(SDNode * N,SDValue & Lo,SDValue & Hi)2088 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
2089 SDValue &Lo, SDValue &Hi) {
2090 EVT VT = N->getValueType(0);
2091 SDLoc dl(N);
2092
2093 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2094 if (VT == MVT::i16)
2095 LC = RTLIB::SDIV_I16;
2096 else if (VT == MVT::i32)
2097 LC = RTLIB::SDIV_I32;
2098 else if (VT == MVT::i64)
2099 LC = RTLIB::SDIV_I64;
2100 else if (VT == MVT::i128)
2101 LC = RTLIB::SDIV_I128;
2102 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
2103
2104 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2105 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, 2, true, dl).first, Lo, Hi);
2106 }
2107
ExpandIntRes_Shift(SDNode * N,SDValue & Lo,SDValue & Hi)2108 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
2109 SDValue &Lo, SDValue &Hi) {
2110 EVT VT = N->getValueType(0);
2111 SDLoc dl(N);
2112
2113 // If we can emit an efficient shift operation, do so now. Check to see if
2114 // the RHS is a constant.
2115 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
2116 return ExpandShiftByConstant(N, CN->getZExtValue(), Lo, Hi);
2117
2118 // If we can determine that the high bit of the shift is zero or one, even if
2119 // the low bits are variable, emit this shift in an optimized form.
2120 if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
2121 return;
2122
2123 // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
2124 unsigned PartsOpc;
2125 if (N->getOpcode() == ISD::SHL) {
2126 PartsOpc = ISD::SHL_PARTS;
2127 } else if (N->getOpcode() == ISD::SRL) {
2128 PartsOpc = ISD::SRL_PARTS;
2129 } else {
2130 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2131 PartsOpc = ISD::SRA_PARTS;
2132 }
2133
2134 // Next check to see if the target supports this SHL_PARTS operation or if it
2135 // will custom expand it.
2136 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2137 TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
2138 if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
2139 Action == TargetLowering::Custom) {
2140 // Expand the subcomponents.
2141 SDValue LHSL, LHSH;
2142 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2143 EVT VT = LHSL.getValueType();
2144
2145 // If the shift amount operand is coming from a vector legalization it may
2146 // have an illegal type. Fix that first by casting the operand, otherwise
2147 // the new SHL_PARTS operation would need further legalization.
2148 SDValue ShiftOp = N->getOperand(1);
2149 EVT ShiftTy = TLI.getShiftAmountTy(VT);
2150 assert(ShiftTy.getScalarType().getSizeInBits() >=
2151 Log2_32_Ceil(VT.getScalarType().getSizeInBits()) &&
2152 "ShiftAmountTy is too small to cover the range of this type!");
2153 if (ShiftOp.getValueType() != ShiftTy)
2154 ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
2155
2156 SDValue Ops[] = { LHSL, LHSH, ShiftOp };
2157 Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
2158 Hi = Lo.getValue(1);
2159 return;
2160 }
2161
2162 // Otherwise, emit a libcall.
2163 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2164 bool isSigned;
2165 if (N->getOpcode() == ISD::SHL) {
2166 isSigned = false; /*sign irrelevant*/
2167 if (VT == MVT::i16)
2168 LC = RTLIB::SHL_I16;
2169 else if (VT == MVT::i32)
2170 LC = RTLIB::SHL_I32;
2171 else if (VT == MVT::i64)
2172 LC = RTLIB::SHL_I64;
2173 else if (VT == MVT::i128)
2174 LC = RTLIB::SHL_I128;
2175 } else if (N->getOpcode() == ISD::SRL) {
2176 isSigned = false;
2177 if (VT == MVT::i16)
2178 LC = RTLIB::SRL_I16;
2179 else if (VT == MVT::i32)
2180 LC = RTLIB::SRL_I32;
2181 else if (VT == MVT::i64)
2182 LC = RTLIB::SRL_I64;
2183 else if (VT == MVT::i128)
2184 LC = RTLIB::SRL_I128;
2185 } else {
2186 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2187 isSigned = true;
2188 if (VT == MVT::i16)
2189 LC = RTLIB::SRA_I16;
2190 else if (VT == MVT::i32)
2191 LC = RTLIB::SRA_I32;
2192 else if (VT == MVT::i64)
2193 LC = RTLIB::SRA_I64;
2194 else if (VT == MVT::i128)
2195 LC = RTLIB::SRA_I128;
2196 }
2197
2198 if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
2199 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2200 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, 2, isSigned, dl).first, Lo,
2201 Hi);
2202 return;
2203 }
2204
2205 if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
2206 llvm_unreachable("Unsupported shift!");
2207 }
2208
ExpandIntRes_SIGN_EXTEND(SDNode * N,SDValue & Lo,SDValue & Hi)2209 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
2210 SDValue &Lo, SDValue &Hi) {
2211 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2212 SDLoc dl(N);
2213 SDValue Op = N->getOperand(0);
2214 if (Op.getValueType().bitsLE(NVT)) {
2215 // The low part is sign extension of the input (degenerates to a copy).
2216 Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
2217 // The high part is obtained by SRA'ing all but one of the bits of low part.
2218 unsigned LoSize = NVT.getSizeInBits();
2219 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2220 DAG.getConstant(LoSize-1, TLI.getPointerTy()));
2221 } else {
2222 // For example, extension of an i48 to an i64. The operand type necessarily
2223 // promotes to the result type, so will end up being expanded too.
2224 assert(getTypeAction(Op.getValueType()) ==
2225 TargetLowering::TypePromoteInteger &&
2226 "Only know how to promote this result!");
2227 SDValue Res = GetPromotedInteger(Op);
2228 assert(Res.getValueType() == N->getValueType(0) &&
2229 "Operand over promoted?");
2230 // Split the promoted operand. This will simplify when it is expanded.
2231 SplitInteger(Res, Lo, Hi);
2232 unsigned ExcessBits =
2233 Op.getValueType().getSizeInBits() - NVT.getSizeInBits();
2234 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
2235 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
2236 ExcessBits)));
2237 }
2238 }
2239
2240 void DAGTypeLegalizer::
ExpandIntRes_SIGN_EXTEND_INREG(SDNode * N,SDValue & Lo,SDValue & Hi)2241 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
2242 SDLoc dl(N);
2243 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2244 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2245
2246 if (EVT.bitsLE(Lo.getValueType())) {
2247 // sext_inreg the low part if needed.
2248 Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
2249 N->getOperand(1));
2250
2251 // The high part gets the sign extension from the lo-part. This handles
2252 // things like sextinreg V:i64 from i8.
2253 Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
2254 DAG.getConstant(Hi.getValueType().getSizeInBits()-1,
2255 TLI.getPointerTy()));
2256 } else {
2257 // For example, extension of an i48 to an i64. Leave the low part alone,
2258 // sext_inreg the high part.
2259 unsigned ExcessBits =
2260 EVT.getSizeInBits() - Lo.getValueType().getSizeInBits();
2261 Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
2262 DAG.getValueType(EVT::getIntegerVT(*DAG.getContext(),
2263 ExcessBits)));
2264 }
2265 }
2266
ExpandIntRes_SREM(SDNode * N,SDValue & Lo,SDValue & Hi)2267 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
2268 SDValue &Lo, SDValue &Hi) {
2269 EVT VT = N->getValueType(0);
2270 SDLoc dl(N);
2271
2272 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2273 if (VT == MVT::i16)
2274 LC = RTLIB::SREM_I16;
2275 else if (VT == MVT::i32)
2276 LC = RTLIB::SREM_I32;
2277 else if (VT == MVT::i64)
2278 LC = RTLIB::SREM_I64;
2279 else if (VT == MVT::i128)
2280 LC = RTLIB::SREM_I128;
2281 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
2282
2283 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2284 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, 2, true, dl).first, Lo, Hi);
2285 }
2286
ExpandIntRes_TRUNCATE(SDNode * N,SDValue & Lo,SDValue & Hi)2287 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
2288 SDValue &Lo, SDValue &Hi) {
2289 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2290 SDLoc dl(N);
2291 Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
2292 Hi = DAG.getNode(ISD::SRL, dl,
2293 N->getOperand(0).getValueType(), N->getOperand(0),
2294 DAG.getConstant(NVT.getSizeInBits(), TLI.getPointerTy()));
2295 Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
2296 }
2297
ExpandIntRes_UADDSUBO(SDNode * N,SDValue & Lo,SDValue & Hi)2298 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
2299 SDValue &Lo, SDValue &Hi) {
2300 SDValue LHS = N->getOperand(0);
2301 SDValue RHS = N->getOperand(1);
2302 SDLoc dl(N);
2303
2304 // Expand the result by simply replacing it with the equivalent
2305 // non-overflow-checking operation.
2306 SDValue Sum = DAG.getNode(N->getOpcode() == ISD::UADDO ?
2307 ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
2308 LHS, RHS);
2309 SplitInteger(Sum, Lo, Hi);
2310
2311 // Calculate the overflow: addition overflows iff a + b < a, and subtraction
2312 // overflows iff a - b > a.
2313 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS,
2314 N->getOpcode () == ISD::UADDO ?
2315 ISD::SETULT : ISD::SETUGT);
2316
2317 // Use the calculated overflow everywhere.
2318 ReplaceValueWith(SDValue(N, 1), Ofl);
2319 }
2320
ExpandIntRes_XMULO(SDNode * N,SDValue & Lo,SDValue & Hi)2321 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
2322 SDValue &Lo, SDValue &Hi) {
2323 EVT VT = N->getValueType(0);
2324 SDLoc dl(N);
2325
2326 // A divide for UMULO should be faster than a function call.
2327 if (N->getOpcode() == ISD::UMULO) {
2328 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
2329
2330 SDValue MUL = DAG.getNode(ISD::MUL, dl, LHS.getValueType(), LHS, RHS);
2331 SplitInteger(MUL, Lo, Hi);
2332
2333 // A divide for UMULO will be faster than a function call. Select to
2334 // make sure we aren't using 0.
2335 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(VT),
2336 RHS, DAG.getConstant(0, VT), ISD::SETEQ);
2337 SDValue NotZero = DAG.getSelect(dl, VT, isZero,
2338 DAG.getConstant(1, VT), RHS);
2339 SDValue DIV = DAG.getNode(ISD::UDIV, dl, VT, MUL, NotZero);
2340 SDValue Overflow = DAG.getSetCC(dl, N->getValueType(1), DIV, LHS,
2341 ISD::SETNE);
2342 Overflow = DAG.getSelect(dl, N->getValueType(1), isZero,
2343 DAG.getConstant(0, N->getValueType(1)),
2344 Overflow);
2345 ReplaceValueWith(SDValue(N, 1), Overflow);
2346 return;
2347 }
2348
2349 Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
2350 EVT PtrVT = TLI.getPointerTy();
2351 Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
2352
2353 // Replace this with a libcall that will check overflow.
2354 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2355 if (VT == MVT::i32)
2356 LC = RTLIB::MULO_I32;
2357 else if (VT == MVT::i64)
2358 LC = RTLIB::MULO_I64;
2359 else if (VT == MVT::i128)
2360 LC = RTLIB::MULO_I128;
2361 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!");
2362
2363 SDValue Temp = DAG.CreateStackTemporary(PtrVT);
2364 // Temporary for the overflow value, default it to zero.
2365 SDValue Chain = DAG.getStore(DAG.getEntryNode(), dl,
2366 DAG.getConstant(0, PtrVT), Temp,
2367 MachinePointerInfo(), false, false, 0);
2368
2369 TargetLowering::ArgListTy Args;
2370 TargetLowering::ArgListEntry Entry;
2371 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
2372 EVT ArgVT = N->getOperand(i).getValueType();
2373 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2374 Entry.Node = N->getOperand(i);
2375 Entry.Ty = ArgTy;
2376 Entry.isSExt = true;
2377 Entry.isZExt = false;
2378 Args.push_back(Entry);
2379 }
2380
2381 // Also pass the address of the overflow check.
2382 Entry.Node = Temp;
2383 Entry.Ty = PtrTy->getPointerTo();
2384 Entry.isSExt = true;
2385 Entry.isZExt = false;
2386 Args.push_back(Entry);
2387
2388 SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
2389
2390 TargetLowering::CallLoweringInfo CLI(DAG);
2391 CLI.setDebugLoc(dl).setChain(Chain)
2392 .setCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args), 0)
2393 .setSExtResult();
2394
2395 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2396
2397 SplitInteger(CallInfo.first, Lo, Hi);
2398 SDValue Temp2 = DAG.getLoad(PtrVT, dl, CallInfo.second, Temp,
2399 MachinePointerInfo(), false, false, false, 0);
2400 SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
2401 DAG.getConstant(0, PtrVT),
2402 ISD::SETNE);
2403 // Use the overflow from the libcall everywhere.
2404 ReplaceValueWith(SDValue(N, 1), Ofl);
2405 }
2406
ExpandIntRes_UDIV(SDNode * N,SDValue & Lo,SDValue & Hi)2407 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
2408 SDValue &Lo, SDValue &Hi) {
2409 EVT VT = N->getValueType(0);
2410 SDLoc dl(N);
2411
2412 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2413 if (VT == MVT::i16)
2414 LC = RTLIB::UDIV_I16;
2415 else if (VT == MVT::i32)
2416 LC = RTLIB::UDIV_I32;
2417 else if (VT == MVT::i64)
2418 LC = RTLIB::UDIV_I64;
2419 else if (VT == MVT::i128)
2420 LC = RTLIB::UDIV_I128;
2421 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
2422
2423 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2424 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, 2, false, dl).first, Lo, Hi);
2425 }
2426
ExpandIntRes_UREM(SDNode * N,SDValue & Lo,SDValue & Hi)2427 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
2428 SDValue &Lo, SDValue &Hi) {
2429 EVT VT = N->getValueType(0);
2430 SDLoc dl(N);
2431
2432 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2433 if (VT == MVT::i16)
2434 LC = RTLIB::UREM_I16;
2435 else if (VT == MVT::i32)
2436 LC = RTLIB::UREM_I32;
2437 else if (VT == MVT::i64)
2438 LC = RTLIB::UREM_I64;
2439 else if (VT == MVT::i128)
2440 LC = RTLIB::UREM_I128;
2441 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
2442
2443 SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2444 SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, 2, false, dl).first, Lo, Hi);
2445 }
2446
ExpandIntRes_ZERO_EXTEND(SDNode * N,SDValue & Lo,SDValue & Hi)2447 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
2448 SDValue &Lo, SDValue &Hi) {
2449 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2450 SDLoc dl(N);
2451 SDValue Op = N->getOperand(0);
2452 if (Op.getValueType().bitsLE(NVT)) {
2453 // The low part is zero extension of the input (degenerates to a copy).
2454 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
2455 Hi = DAG.getConstant(0, NVT); // The high part is just a zero.
2456 } else {
2457 // For example, extension of an i48 to an i64. The operand type necessarily
2458 // promotes to the result type, so will end up being expanded too.
2459 assert(getTypeAction(Op.getValueType()) ==
2460 TargetLowering::TypePromoteInteger &&
2461 "Only know how to promote this result!");
2462 SDValue Res = GetPromotedInteger(Op);
2463 assert(Res.getValueType() == N->getValueType(0) &&
2464 "Operand over promoted?");
2465 // Split the promoted operand. This will simplify when it is expanded.
2466 SplitInteger(Res, Lo, Hi);
2467 unsigned ExcessBits =
2468 Op.getValueType().getSizeInBits() - NVT.getSizeInBits();
2469 Hi = DAG.getZeroExtendInReg(Hi, dl,
2470 EVT::getIntegerVT(*DAG.getContext(),
2471 ExcessBits));
2472 }
2473 }
2474
ExpandIntRes_ATOMIC_LOAD(SDNode * N,SDValue & Lo,SDValue & Hi)2475 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
2476 SDValue &Lo, SDValue &Hi) {
2477 SDLoc dl(N);
2478 EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
2479 SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
2480 SDValue Zero = DAG.getConstant(0, VT);
2481 SDValue Swap = DAG.getAtomicCmpSwap(
2482 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl,
2483 cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
2484 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand(),
2485 cast<AtomicSDNode>(N)->getOrdering(),
2486 cast<AtomicSDNode>(N)->getOrdering(),
2487 cast<AtomicSDNode>(N)->getSynchScope());
2488
2489 ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
2490 ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
2491 }
2492
2493 //===----------------------------------------------------------------------===//
2494 // Integer Operand Expansion
2495 //===----------------------------------------------------------------------===//
2496
2497 /// ExpandIntegerOperand - This method is called when the specified operand of
2498 /// the specified node is found to need expansion. At this point, all of the
2499 /// result types of the node are known to be legal, but other operands of the
2500 /// node may need promotion or expansion as well as the specified one.
ExpandIntegerOperand(SDNode * N,unsigned OpNo)2501 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
2502 DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG); dbgs() << "\n");
2503 SDValue Res = SDValue();
2504
2505 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2506 return false;
2507
2508 switch (N->getOpcode()) {
2509 default:
2510 #ifndef NDEBUG
2511 dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
2512 N->dump(&DAG); dbgs() << "\n";
2513 #endif
2514 llvm_unreachable("Do not know how to expand this operator's operand!");
2515
2516 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
2517 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break;
2518 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
2519 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
2520 case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
2521 case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
2522 case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
2523 case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
2524 case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break;
2525 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
2526 case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break;
2527 case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break;
2528
2529 case ISD::SHL:
2530 case ISD::SRA:
2531 case ISD::SRL:
2532 case ISD::ROTL:
2533 case ISD::ROTR: Res = ExpandIntOp_Shift(N); break;
2534 case ISD::RETURNADDR:
2535 case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break;
2536
2537 case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break;
2538 }
2539
2540 // If the result is null, the sub-method took care of registering results etc.
2541 if (!Res.getNode()) return false;
2542
2543 // If the result is N, the sub-method updated N in place. Tell the legalizer
2544 // core about this.
2545 if (Res.getNode() == N)
2546 return true;
2547
2548 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2549 "Invalid operand expansion");
2550
2551 ReplaceValueWith(SDValue(N, 0), Res);
2552 return false;
2553 }
2554
2555 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code
2556 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
IntegerExpandSetCCOperands(SDValue & NewLHS,SDValue & NewRHS,ISD::CondCode & CCCode,SDLoc dl)2557 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
2558 SDValue &NewRHS,
2559 ISD::CondCode &CCCode,
2560 SDLoc dl) {
2561 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
2562 GetExpandedInteger(NewLHS, LHSLo, LHSHi);
2563 GetExpandedInteger(NewRHS, RHSLo, RHSHi);
2564
2565 if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
2566 if (RHSLo == RHSHi) {
2567 if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) {
2568 if (RHSCST->isAllOnesValue()) {
2569 // Equality comparison to -1.
2570 NewLHS = DAG.getNode(ISD::AND, dl,
2571 LHSLo.getValueType(), LHSLo, LHSHi);
2572 NewRHS = RHSLo;
2573 return;
2574 }
2575 }
2576 }
2577
2578 NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
2579 NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
2580 NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
2581 NewRHS = DAG.getConstant(0, NewLHS.getValueType());
2582 return;
2583 }
2584
2585 // If this is a comparison of the sign bit, just look at the top part.
2586 // X > -1, x < 0
2587 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
2588 if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0
2589 (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1
2590 NewLHS = LHSHi;
2591 NewRHS = RHSHi;
2592 return;
2593 }
2594
2595 // FIXME: This generated code sucks.
2596 ISD::CondCode LowCC;
2597 switch (CCCode) {
2598 default: llvm_unreachable("Unknown integer setcc!");
2599 case ISD::SETLT:
2600 case ISD::SETULT: LowCC = ISD::SETULT; break;
2601 case ISD::SETGT:
2602 case ISD::SETUGT: LowCC = ISD::SETUGT; break;
2603 case ISD::SETLE:
2604 case ISD::SETULE: LowCC = ISD::SETULE; break;
2605 case ISD::SETGE:
2606 case ISD::SETUGE: LowCC = ISD::SETUGE; break;
2607 }
2608
2609 // Tmp1 = lo(op1) < lo(op2) // Always unsigned comparison
2610 // Tmp2 = hi(op1) < hi(op2) // Signedness depends on operands
2611 // dest = hi(op1) == hi(op2) ? Tmp1 : Tmp2;
2612
2613 // NOTE: on targets without efficient SELECT of bools, we can always use
2614 // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
2615 TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
2616 nullptr);
2617 SDValue Tmp1, Tmp2;
2618 if (TLI.isTypeLegal(LHSLo.getValueType()) &&
2619 TLI.isTypeLegal(RHSLo.getValueType()))
2620 Tmp1 = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()),
2621 LHSLo, RHSLo, LowCC, false, DagCombineInfo, dl);
2622 if (!Tmp1.getNode())
2623 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()),
2624 LHSLo, RHSLo, LowCC);
2625 if (TLI.isTypeLegal(LHSHi.getValueType()) &&
2626 TLI.isTypeLegal(RHSHi.getValueType()))
2627 Tmp2 = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()),
2628 LHSHi, RHSHi, CCCode, false, DagCombineInfo, dl);
2629 if (!Tmp2.getNode())
2630 Tmp2 = DAG.getNode(ISD::SETCC, dl,
2631 getSetCCResultType(LHSHi.getValueType()),
2632 LHSHi, RHSHi, DAG.getCondCode(CCCode));
2633
2634 ConstantSDNode *Tmp1C = dyn_cast<ConstantSDNode>(Tmp1.getNode());
2635 ConstantSDNode *Tmp2C = dyn_cast<ConstantSDNode>(Tmp2.getNode());
2636 if ((Tmp1C && Tmp1C->isNullValue()) ||
2637 (Tmp2C && Tmp2C->isNullValue() &&
2638 (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
2639 CCCode == ISD::SETUGE || CCCode == ISD::SETULE)) ||
2640 (Tmp2C && Tmp2C->getAPIntValue() == 1 &&
2641 (CCCode == ISD::SETLT || CCCode == ISD::SETGT ||
2642 CCCode == ISD::SETUGT || CCCode == ISD::SETULT))) {
2643 // low part is known false, returns high part.
2644 // For LE / GE, if high part is known false, ignore the low part.
2645 // For LT / GT, if high part is known true, ignore the low part.
2646 NewLHS = Tmp2;
2647 NewRHS = SDValue();
2648 return;
2649 }
2650
2651 NewLHS = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()),
2652 LHSHi, RHSHi, ISD::SETEQ, false,
2653 DagCombineInfo, dl);
2654 if (!NewLHS.getNode())
2655 NewLHS = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()),
2656 LHSHi, RHSHi, ISD::SETEQ);
2657 NewLHS = DAG.getSelect(dl, Tmp1.getValueType(),
2658 NewLHS, Tmp1, Tmp2);
2659 NewRHS = SDValue();
2660 }
2661
ExpandIntOp_BR_CC(SDNode * N)2662 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
2663 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
2664 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
2665 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
2666
2667 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2668 // against zero to select between true and false values.
2669 if (!NewRHS.getNode()) {
2670 NewRHS = DAG.getConstant(0, NewLHS.getValueType());
2671 CCCode = ISD::SETNE;
2672 }
2673
2674 // Update N to have the operands specified.
2675 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2676 DAG.getCondCode(CCCode), NewLHS, NewRHS,
2677 N->getOperand(4)), 0);
2678 }
2679
ExpandIntOp_SELECT_CC(SDNode * N)2680 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
2681 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
2682 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
2683 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
2684
2685 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2686 // against zero to select between true and false values.
2687 if (!NewRHS.getNode()) {
2688 NewRHS = DAG.getConstant(0, NewLHS.getValueType());
2689 CCCode = ISD::SETNE;
2690 }
2691
2692 // Update N to have the operands specified.
2693 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
2694 N->getOperand(2), N->getOperand(3),
2695 DAG.getCondCode(CCCode)), 0);
2696 }
2697
ExpandIntOp_SETCC(SDNode * N)2698 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
2699 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
2700 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
2701 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
2702
2703 // If ExpandSetCCOperands returned a scalar, use it.
2704 if (!NewRHS.getNode()) {
2705 assert(NewLHS.getValueType() == N->getValueType(0) &&
2706 "Unexpected setcc expansion!");
2707 return NewLHS;
2708 }
2709
2710 // Otherwise, update N to have the operands specified.
2711 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
2712 DAG.getCondCode(CCCode)), 0);
2713 }
2714
ExpandIntOp_Shift(SDNode * N)2715 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
2716 // The value being shifted is legal, but the shift amount is too big.
2717 // It follows that either the result of the shift is undefined, or the
2718 // upper half of the shift amount is zero. Just use the lower half.
2719 SDValue Lo, Hi;
2720 GetExpandedInteger(N->getOperand(1), Lo, Hi);
2721 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
2722 }
2723
ExpandIntOp_RETURNADDR(SDNode * N)2724 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
2725 // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This
2726 // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
2727 // constant to valid type.
2728 SDValue Lo, Hi;
2729 GetExpandedInteger(N->getOperand(0), Lo, Hi);
2730 return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
2731 }
2732
ExpandIntOp_SINT_TO_FP(SDNode * N)2733 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
2734 SDValue Op = N->getOperand(0);
2735 EVT DstVT = N->getValueType(0);
2736 RTLIB::Libcall LC = RTLIB::getSINTTOFP(Op.getValueType(), DstVT);
2737 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
2738 "Don't know how to expand this SINT_TO_FP!");
2739 return TLI.makeLibCall(DAG, LC, DstVT, &Op, 1, true, SDLoc(N)).first;
2740 }
2741
ExpandIntOp_STORE(StoreSDNode * N,unsigned OpNo)2742 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
2743 if (ISD::isNormalStore(N))
2744 return ExpandOp_NormalStore(N, OpNo);
2745
2746 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2747 assert(OpNo == 1 && "Can only expand the stored value so far");
2748
2749 EVT VT = N->getOperand(1).getValueType();
2750 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2751 SDValue Ch = N->getChain();
2752 SDValue Ptr = N->getBasePtr();
2753 unsigned Alignment = N->getAlignment();
2754 bool isVolatile = N->isVolatile();
2755 bool isNonTemporal = N->isNonTemporal();
2756 AAMDNodes AAInfo = N->getAAInfo();
2757 SDLoc dl(N);
2758 SDValue Lo, Hi;
2759
2760 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2761
2762 if (N->getMemoryVT().bitsLE(NVT)) {
2763 GetExpandedInteger(N->getValue(), Lo, Hi);
2764 return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
2765 N->getMemoryVT(), isVolatile, isNonTemporal,
2766 Alignment, AAInfo);
2767 }
2768
2769 if (TLI.isLittleEndian()) {
2770 // Little-endian - low bits are at low addresses.
2771 GetExpandedInteger(N->getValue(), Lo, Hi);
2772
2773 Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
2774 isVolatile, isNonTemporal, Alignment, AAInfo);
2775
2776 unsigned ExcessBits =
2777 N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
2778 EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
2779
2780 // Increment the pointer to the other half.
2781 unsigned IncrementSize = NVT.getSizeInBits()/8;
2782 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2783 DAG.getConstant(IncrementSize, Ptr.getValueType()));
2784 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr,
2785 N->getPointerInfo().getWithOffset(IncrementSize),
2786 NEVT, isVolatile, isNonTemporal,
2787 MinAlign(Alignment, IncrementSize), AAInfo);
2788 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
2789 }
2790
2791 // Big-endian - high bits are at low addresses. Favor aligned stores at
2792 // the cost of some bit-fiddling.
2793 GetExpandedInteger(N->getValue(), Lo, Hi);
2794
2795 EVT ExtVT = N->getMemoryVT();
2796 unsigned EBytes = ExtVT.getStoreSize();
2797 unsigned IncrementSize = NVT.getSizeInBits()/8;
2798 unsigned ExcessBits = (EBytes - IncrementSize)*8;
2799 EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
2800 ExtVT.getSizeInBits() - ExcessBits);
2801
2802 if (ExcessBits < NVT.getSizeInBits()) {
2803 // Transfer high bits from the top of Lo to the bottom of Hi.
2804 Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
2805 DAG.getConstant(NVT.getSizeInBits() - ExcessBits,
2806 TLI.getPointerTy()));
2807 Hi = DAG.getNode(ISD::OR, dl, NVT, Hi,
2808 DAG.getNode(ISD::SRL, dl, NVT, Lo,
2809 DAG.getConstant(ExcessBits,
2810 TLI.getPointerTy())));
2811 }
2812
2813 // Store both the high bits and maybe some of the low bits.
2814 Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(),
2815 HiVT, isVolatile, isNonTemporal, Alignment, AAInfo);
2816
2817 // Increment the pointer to the other half.
2818 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2819 DAG.getConstant(IncrementSize, Ptr.getValueType()));
2820 // Store the lowest ExcessBits bits in the second half.
2821 Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
2822 N->getPointerInfo().getWithOffset(IncrementSize),
2823 EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
2824 isVolatile, isNonTemporal,
2825 MinAlign(Alignment, IncrementSize), AAInfo);
2826 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
2827 }
2828
ExpandIntOp_TRUNCATE(SDNode * N)2829 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
2830 SDValue InL, InH;
2831 GetExpandedInteger(N->getOperand(0), InL, InH);
2832 // Just truncate the low part of the source.
2833 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
2834 }
2835
ExpandIntOp_UINT_TO_FP(SDNode * N)2836 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
2837 SDValue Op = N->getOperand(0);
2838 EVT SrcVT = Op.getValueType();
2839 EVT DstVT = N->getValueType(0);
2840 SDLoc dl(N);
2841
2842 // The following optimization is valid only if every value in SrcVT (when
2843 // treated as signed) is representable in DstVT. Check that the mantissa
2844 // size of DstVT is >= than the number of bits in SrcVT -1.
2845 const fltSemantics &sem = DAG.EVTToAPFloatSemantics(DstVT);
2846 if (APFloat::semanticsPrecision(sem) >= SrcVT.getSizeInBits()-1 &&
2847 TLI.getOperationAction(ISD::SINT_TO_FP, SrcVT) == TargetLowering::Custom){
2848 // Do a signed conversion then adjust the result.
2849 SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op);
2850 SignedConv = TLI.LowerOperation(SignedConv, DAG);
2851
2852 // The result of the signed conversion needs adjusting if the 'sign bit' of
2853 // the incoming integer was set. To handle this, we dynamically test to see
2854 // if it is set, and, if so, add a fudge factor.
2855
2856 const uint64_t F32TwoE32 = 0x4F800000ULL;
2857 const uint64_t F32TwoE64 = 0x5F800000ULL;
2858 const uint64_t F32TwoE128 = 0x7F800000ULL;
2859
2860 APInt FF(32, 0);
2861 if (SrcVT == MVT::i32)
2862 FF = APInt(32, F32TwoE32);
2863 else if (SrcVT == MVT::i64)
2864 FF = APInt(32, F32TwoE64);
2865 else if (SrcVT == MVT::i128)
2866 FF = APInt(32, F32TwoE128);
2867 else
2868 llvm_unreachable("Unsupported UINT_TO_FP!");
2869
2870 // Check whether the sign bit is set.
2871 SDValue Lo, Hi;
2872 GetExpandedInteger(Op, Lo, Hi);
2873 SDValue SignSet = DAG.getSetCC(dl,
2874 getSetCCResultType(Hi.getValueType()),
2875 Hi, DAG.getConstant(0, Hi.getValueType()),
2876 ISD::SETLT);
2877
2878 // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits.
2879 SDValue FudgePtr = DAG.getConstantPool(
2880 ConstantInt::get(*DAG.getContext(), FF.zext(64)),
2881 TLI.getPointerTy());
2882
2883 // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
2884 SDValue Zero = DAG.getIntPtrConstant(0);
2885 SDValue Four = DAG.getIntPtrConstant(4);
2886 if (TLI.isBigEndian()) std::swap(Zero, Four);
2887 SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet,
2888 Zero, Four);
2889 unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment();
2890 FudgePtr = DAG.getNode(ISD::ADD, dl, FudgePtr.getValueType(),
2891 FudgePtr, Offset);
2892 Alignment = std::min(Alignment, 4u);
2893
2894 // Load the value out, extending it from f32 to the destination float type.
2895 // FIXME: Avoid the extend by constructing the right constant pool?
2896 SDValue Fudge = DAG.getExtLoad(ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(),
2897 FudgePtr,
2898 MachinePointerInfo::getConstantPool(),
2899 MVT::f32,
2900 false, false, false, Alignment);
2901 return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge);
2902 }
2903
2904 // Otherwise, use a libcall.
2905 RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT);
2906 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
2907 "Don't know how to expand this UINT_TO_FP!");
2908 return TLI.makeLibCall(DAG, LC, DstVT, &Op, 1, true, dl).first;
2909 }
2910
ExpandIntOp_ATOMIC_STORE(SDNode * N)2911 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
2912 SDLoc dl(N);
2913 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
2914 cast<AtomicSDNode>(N)->getMemoryVT(),
2915 N->getOperand(0),
2916 N->getOperand(1), N->getOperand(2),
2917 cast<AtomicSDNode>(N)->getMemOperand(),
2918 cast<AtomicSDNode>(N)->getOrdering(),
2919 cast<AtomicSDNode>(N)->getSynchScope());
2920 return Swap.getValue(1);
2921 }
2922
2923
PromoteIntRes_EXTRACT_SUBVECTOR(SDNode * N)2924 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
2925 SDValue InOp0 = N->getOperand(0);
2926 EVT InVT = InOp0.getValueType();
2927
2928 EVT OutVT = N->getValueType(0);
2929 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
2930 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
2931 unsigned OutNumElems = OutVT.getVectorNumElements();
2932 EVT NOutVTElem = NOutVT.getVectorElementType();
2933
2934 SDLoc dl(N);
2935 SDValue BaseIdx = N->getOperand(1);
2936
2937 SmallVector<SDValue, 8> Ops;
2938 Ops.reserve(OutNumElems);
2939 for (unsigned i = 0; i != OutNumElems; ++i) {
2940
2941 // Extract the element from the original vector.
2942 SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
2943 BaseIdx, DAG.getConstant(i, BaseIdx.getValueType()));
2944 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
2945 InVT.getVectorElementType(), N->getOperand(0), Index);
2946
2947 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, Ext);
2948 // Insert the converted element to the new vector.
2949 Ops.push_back(Op);
2950 }
2951
2952 return DAG.getNode(ISD::BUILD_VECTOR, dl, NOutVT, Ops);
2953 }
2954
2955
PromoteIntRes_VECTOR_SHUFFLE(SDNode * N)2956 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
2957 ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
2958 EVT VT = N->getValueType(0);
2959 SDLoc dl(N);
2960
2961 ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
2962
2963 SDValue V0 = GetPromotedInteger(N->getOperand(0));
2964 SDValue V1 = GetPromotedInteger(N->getOperand(1));
2965 EVT OutVT = V0.getValueType();
2966
2967 return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
2968 }
2969
2970
PromoteIntRes_BUILD_VECTOR(SDNode * N)2971 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
2972 EVT OutVT = N->getValueType(0);
2973 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
2974 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
2975 unsigned NumElems = N->getNumOperands();
2976 EVT NOutVTElem = NOutVT.getVectorElementType();
2977
2978 SDLoc dl(N);
2979
2980 SmallVector<SDValue, 8> Ops;
2981 Ops.reserve(NumElems);
2982 for (unsigned i = 0; i != NumElems; ++i) {
2983 SDValue Op;
2984 // BUILD_VECTOR integer operand types are allowed to be larger than the
2985 // result's element type. This may still be true after the promotion. For
2986 // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
2987 // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
2988 if (N->getOperand(i).getValueType().bitsLT(NOutVTElem))
2989 Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
2990 else
2991 Op = N->getOperand(i);
2992 Ops.push_back(Op);
2993 }
2994
2995 return DAG.getNode(ISD::BUILD_VECTOR, dl, NOutVT, Ops);
2996 }
2997
PromoteIntRes_SCALAR_TO_VECTOR(SDNode * N)2998 SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
2999
3000 SDLoc dl(N);
3001
3002 assert(!N->getOperand(0).getValueType().isVector() &&
3003 "Input must be a scalar");
3004
3005 EVT OutVT = N->getValueType(0);
3006 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3007 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3008 EVT NOutVTElem = NOutVT.getVectorElementType();
3009
3010 SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
3011
3012 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
3013 }
3014
PromoteIntRes_CONCAT_VECTORS(SDNode * N)3015 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
3016 SDLoc dl(N);
3017
3018 EVT OutVT = N->getValueType(0);
3019 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3020 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3021
3022 EVT InElemTy = OutVT.getVectorElementType();
3023 EVT OutElemTy = NOutVT.getVectorElementType();
3024
3025 unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
3026 unsigned NumOutElem = NOutVT.getVectorNumElements();
3027 unsigned NumOperands = N->getNumOperands();
3028 assert(NumElem * NumOperands == NumOutElem &&
3029 "Unexpected number of elements");
3030
3031 // Take the elements from the first vector.
3032 SmallVector<SDValue, 8> Ops(NumOutElem);
3033 for (unsigned i = 0; i < NumOperands; ++i) {
3034 SDValue Op = N->getOperand(i);
3035 for (unsigned j = 0; j < NumElem; ++j) {
3036 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
3037 InElemTy, Op, DAG.getConstant(j,
3038 TLI.getVectorIdxTy()));
3039 Ops[i * NumElem + j] = DAG.getNode(ISD::ANY_EXTEND, dl, OutElemTy, Ext);
3040 }
3041 }
3042
3043 return DAG.getNode(ISD::BUILD_VECTOR, dl, NOutVT, Ops);
3044 }
3045
PromoteIntRes_INSERT_VECTOR_ELT(SDNode * N)3046 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
3047 EVT OutVT = N->getValueType(0);
3048 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
3049 assert(NOutVT.isVector() && "This type must be promoted to a vector type");
3050
3051 EVT NOutVTElem = NOutVT.getVectorElementType();
3052
3053 SDLoc dl(N);
3054 SDValue V0 = GetPromotedInteger(N->getOperand(0));
3055
3056 SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
3057 NOutVTElem, N->getOperand(1));
3058 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
3059 V0, ConvElem, N->getOperand(2));
3060 }
3061
PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode * N)3062 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
3063 SDLoc dl(N);
3064 SDValue V0 = GetPromotedInteger(N->getOperand(0));
3065 SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl, TLI.getVectorIdxTy());
3066 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
3067 V0->getValueType(0).getScalarType(), V0, V1);
3068
3069 // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
3070 // element types. If this is the case then we need to expand the outgoing
3071 // value and not truncate it.
3072 return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
3073 }
3074
PromoteIntOp_CONCAT_VECTORS(SDNode * N)3075 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
3076 SDLoc dl(N);
3077 unsigned NumElems = N->getNumOperands();
3078
3079 EVT RetSclrTy = N->getValueType(0).getVectorElementType();
3080
3081 SmallVector<SDValue, 8> NewOps;
3082 NewOps.reserve(NumElems);
3083
3084 // For each incoming vector
3085 for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
3086 SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
3087 EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
3088 unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
3089
3090 for (unsigned i=0; i<NumElem; ++i) {
3091 // Extract element from incoming vector
3092 SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SclrTy,
3093 Incoming, DAG.getConstant(i, TLI.getVectorIdxTy()));
3094 SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
3095 NewOps.push_back(Tr);
3096 }
3097 }
3098
3099 return DAG.getNode(ISD::BUILD_VECTOR, dl, N->getValueType(0), NewOps);
3100 }
3101