LegalizeVectorTypes.cpp revision aad3460086a1b29c55f7490c6d8743ea4e53f07d 
1//===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file performs vector type splitting and scalarization for LegalizeTypes.
11// Scalarization is the act of changing a computation in an illegal one-element
12// vector type to be a computation in its scalar element type.  For example,
13// implementing <1 x f32> arithmetic in a scalar f32 register.  This is needed
14// as a base case when scalarizing vector arithmetic like <4 x f32>, which
15// eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
16// types.
17// Splitting is the act of changing a computation in an invalid vector type to
18// be a computation in two vectors of half the size.  For example, implementing
19// <128 x f32> operations in terms of two <64 x f32> operations.
20//
21//===----------------------------------------------------------------------===//
22
23#include "LegalizeTypes.h"
24#include "llvm/CodeGen/PseudoSourceValue.h"
25#include "llvm/Target/TargetData.h"
26using namespace llvm;
27
28//===----------------------------------------------------------------------===//
29//  Result Vector Scalarization: <1 x ty> -> ty.
30//===----------------------------------------------------------------------===//
31
32void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
33  DEBUG(cerr << "Scalarize node result " << ResNo << ": "; N->dump(&DAG);
34        cerr << "\n");
35  SDValue R = SDValue();
36
37  switch (N->getOpcode()) {
38  default:
39#ifndef NDEBUG
40    cerr << "ScalarizeVectorResult #" << ResNo << ": ";
41    N->dump(&DAG); cerr << "\n";
42#endif
43    assert(0 && "Do not know how to scalarize the result of this operator!");
44    abort();
45
46  case ISD::BIT_CONVERT:       R = ScalarizeVecRes_BIT_CONVERT(N); break;
47  case ISD::BUILD_VECTOR:      R = N->getOperand(0); break;
48  case ISD::CONVERT_RNDSAT:    R = ScalarizeVecRes_CONVERT_RNDSAT(N); break;
49  case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
50  case ISD::FPOWI:             R = ScalarizeVecRes_FPOWI(N); break;
51  case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
52  case ISD::LOAD:           R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
53  case ISD::SCALAR_TO_VECTOR:  R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
54  case ISD::SELECT:            R = ScalarizeVecRes_SELECT(N); break;
55  case ISD::SELECT_CC:         R = ScalarizeVecRes_SELECT_CC(N); break;
56  case ISD::SETCC:             R = ScalarizeVecRes_SETCC(N); break;
57  case ISD::UNDEF:             R = ScalarizeVecRes_UNDEF(N); break;
58  case ISD::VECTOR_SHUFFLE:    R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
59  case ISD::VSETCC:            R = ScalarizeVecRes_VSETCC(N); break;
60
61  case ISD::CTLZ:
62  case ISD::CTPOP:
63  case ISD::CTTZ:
64  case ISD::FABS:
65  case ISD::FCOS:
66  case ISD::FNEG:
67  case ISD::FP_TO_SINT:
68  case ISD::FP_TO_UINT:
69  case ISD::FSIN:
70  case ISD::FSQRT:
71  case ISD::FTRUNC:
72  case ISD::FFLOOR:
73  case ISD::FCEIL:
74  case ISD::FRINT:
75  case ISD::FNEARBYINT:
76  case ISD::UINT_TO_FP:
77  case ISD::SINT_TO_FP:
78  case ISD::TRUNCATE:
79  case ISD::SIGN_EXTEND:
80  case ISD::ZERO_EXTEND:
81  case ISD::ANY_EXTEND:
82    R = ScalarizeVecRes_UnaryOp(N);
83    break;
84
85  case ISD::ADD:
86  case ISD::AND:
87  case ISD::FADD:
88  case ISD::FDIV:
89  case ISD::FMUL:
90  case ISD::FPOW:
91  case ISD::FREM:
92  case ISD::FSUB:
93  case ISD::MUL:
94  case ISD::OR:
95  case ISD::SDIV:
96  case ISD::SREM:
97  case ISD::SUB:
98  case ISD::UDIV:
99  case ISD::UREM:
100  case ISD::XOR:
101  case ISD::SHL:
102  case ISD::SRA:
103  case ISD::SRL:
104    R = ScalarizeVecRes_BinOp(N);
105    break;
106  }
107
108  // If R is null, the sub-method took care of registering the result.
109  if (R.getNode())
110    SetScalarizedVector(SDValue(N, ResNo), R);
111}
112
113SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
114  SDValue LHS = GetScalarizedVector(N->getOperand(0));
115  SDValue RHS = GetScalarizedVector(N->getOperand(1));
116  return DAG.getNode(N->getOpcode(), N->getDebugLoc(),
117                     LHS.getValueType(), LHS, RHS);
118}
119
120SDValue DAGTypeLegalizer::ScalarizeVecRes_BIT_CONVERT(SDNode *N) {
121  MVT NewVT = N->getValueType(0).getVectorElementType();
122  return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(),
123                     NewVT, N->getOperand(0));
124}
125
126SDValue DAGTypeLegalizer::ScalarizeVecRes_CONVERT_RNDSAT(SDNode *N) {
127  MVT NewVT = N->getValueType(0).getVectorElementType();
128  SDValue Op0 = GetScalarizedVector(N->getOperand(0));
129  return DAG.getConvertRndSat(NewVT, N->getDebugLoc(),
130                              Op0, DAG.getValueType(NewVT),
131                              DAG.getValueType(Op0.getValueType()),
132                              N->getOperand(3),
133                              N->getOperand(4),
134                              cast<CvtRndSatSDNode>(N)->getCvtCode());
135}
136
137SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
138  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(),
139                     N->getValueType(0).getVectorElementType(),
140                     N->getOperand(0), N->getOperand(1));
141}
142
143SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
144  SDValue Op = GetScalarizedVector(N->getOperand(0));
145  return DAG.getNode(ISD::FPOWI, N->getDebugLoc(),
146                     Op.getValueType(), Op, N->getOperand(1));
147}
148
149SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
150  // The value to insert may have a wider type than the vector element type,
151  // so be sure to truncate it to the element type if necessary.
152  SDValue Op = N->getOperand(1);
153  MVT EltVT = N->getValueType(0).getVectorElementType();
154  if (Op.getValueType() != EltVT)
155    // FIXME: Can this happen for floating point types?
156    Op = DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, Op);
157  return Op;
158}
159
160SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
161  assert(N->isUnindexed() && "Indexed vector load?");
162
163  SDValue Result = DAG.getLoad(ISD::UNINDEXED, N->getDebugLoc(),
164                               N->getExtensionType(),
165                               N->getValueType(0).getVectorElementType(),
166                               N->getChain(), N->getBasePtr(),
167                               DAG.getUNDEF(N->getBasePtr().getValueType()),
168                               N->getSrcValue(), N->getSrcValueOffset(),
169                               N->getMemoryVT().getVectorElementType(),
170                               N->isVolatile(), N->getAlignment());
171
172  // Legalized the chain result - switch anything that used the old chain to
173  // use the new one.
174  ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
175  return Result;
176}
177
178SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
179  // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
180  MVT DestVT = N->getValueType(0).getVectorElementType();
181  SDValue Op = GetScalarizedVector(N->getOperand(0));
182  return DAG.getNode(N->getOpcode(), N->getDebugLoc(), DestVT, Op);
183}
184
185SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
186  // If the operand is wider than the vector element type then it is implicitly
187  // truncated.  Make that explicit here.
188  MVT EltVT = N->getValueType(0).getVectorElementType();
189  SDValue InOp = N->getOperand(0);
190  if (InOp.getValueType() != EltVT)
191    return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), EltVT, InOp);
192  return InOp;
193}
194
195SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
196  SDValue LHS = GetScalarizedVector(N->getOperand(1));
197  return DAG.getNode(ISD::SELECT, N->getDebugLoc(),
198                     LHS.getValueType(), N->getOperand(0), LHS,
199                     GetScalarizedVector(N->getOperand(2)));
200}
201
202SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
203  SDValue LHS = GetScalarizedVector(N->getOperand(2));
204  return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(), LHS.getValueType(),
205                     N->getOperand(0), N->getOperand(1),
206                     LHS, GetScalarizedVector(N->getOperand(3)),
207                     N->getOperand(4));
208}
209
210SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
211  SDValue LHS = GetScalarizedVector(N->getOperand(0));
212  SDValue RHS = GetScalarizedVector(N->getOperand(1));
213  DebugLoc DL = N->getDebugLoc();
214
215  // Turn it into a scalar SETCC.
216  return DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, N->getOperand(2));
217}
218
219SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
220  return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
221}
222
223SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
224  // Figure out if the scalar is the LHS or RHS and return it.
225  SDValue Arg = N->getOperand(2).getOperand(0);
226  if (Arg.getOpcode() == ISD::UNDEF)
227    return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
228  unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue();
229  return GetScalarizedVector(N->getOperand(Op));
230}
231
232SDValue DAGTypeLegalizer::ScalarizeVecRes_VSETCC(SDNode *N) {
233  SDValue LHS = GetScalarizedVector(N->getOperand(0));
234  SDValue RHS = GetScalarizedVector(N->getOperand(1));
235  MVT NVT = N->getValueType(0).getVectorElementType();
236  MVT SVT = TLI.getSetCCResultType(LHS.getValueType());
237  DebugLoc DL = N->getDebugLoc();
238
239  // Turn it into a scalar SETCC.
240  SDValue Res = DAG.getNode(ISD::SETCC, DL, SVT, LHS, RHS, N->getOperand(2));
241
242  // VSETCC always returns a sign-extended value, while SETCC may not.  The
243  // SETCC result type may not match the vector element type.  Correct these.
244  if (NVT.bitsLE(SVT)) {
245    // The SETCC result type is bigger than the vector element type.
246    // Ensure the SETCC result is sign-extended.
247    if (TLI.getBooleanContents() !=
248        TargetLowering::ZeroOrNegativeOneBooleanContent)
249      Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, SVT, Res,
250                        DAG.getValueType(MVT::i1));
251    // Truncate to the final type.
252    return DAG.getNode(ISD::TRUNCATE, DL, NVT, Res);
253  }
254
255  // The SETCC result type is smaller than the vector element type.
256  // If the SetCC result is not sign-extended, chop it down to MVT::i1.
257  if (TLI.getBooleanContents() !=
258        TargetLowering::ZeroOrNegativeOneBooleanContent)
259    Res = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Res);
260  // Sign extend to the final type.
261  return DAG.getNode(ISD::SIGN_EXTEND, DL, NVT, Res);
262}
263
264
265//===----------------------------------------------------------------------===//
266//  Operand Vector Scalarization <1 x ty> -> ty.
267//===----------------------------------------------------------------------===//
268
269bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
270  DEBUG(cerr << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG);
271        cerr << "\n");
272  SDValue Res = SDValue();
273
274  if (Res.getNode() == 0) {
275    switch (N->getOpcode()) {
276    default:
277#ifndef NDEBUG
278      cerr << "ScalarizeVectorOperand Op #" << OpNo << ": ";
279      N->dump(&DAG); cerr << "\n";
280#endif
281      assert(0 && "Do not know how to scalarize this operator's operand!");
282    case ISD::BIT_CONVERT:
283      Res = ScalarizeVecOp_BIT_CONVERT(N);
284      break;
285    case ISD::CONCAT_VECTORS:
286      Res = ScalarizeVecOp_CONCAT_VECTORS(N);
287      break;
288    case ISD::EXTRACT_VECTOR_ELT:
289      Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
290      break;
291    case ISD::STORE:
292      Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
293      break;
294    }
295  }
296
297  // If the result is null, the sub-method took care of registering results etc.
298  if (!Res.getNode()) return false;
299
300  // If the result is N, the sub-method updated N in place.  Tell the legalizer
301  // core about this.
302  if (Res.getNode() == N)
303    return true;
304
305  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
306         "Invalid operand expansion");
307
308  ReplaceValueWith(SDValue(N, 0), Res);
309  return false;
310}
311
312/// ScalarizeVecOp_BIT_CONVERT - If the value to convert is a vector that needs
313/// to be scalarized, it must be <1 x ty>.  Convert the element instead.
314SDValue DAGTypeLegalizer::ScalarizeVecOp_BIT_CONVERT(SDNode *N) {
315  SDValue Elt = GetScalarizedVector(N->getOperand(0));
316  return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(),
317                     N->getValueType(0), Elt);
318}
319
320/// ScalarizeVecOp_CONCAT_VECTORS - The vectors to concatenate have length one -
321/// use a BUILD_VECTOR instead.
322SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
323  SmallVector<SDValue, 8> Ops(N->getNumOperands());
324  for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
325    Ops[i] = GetScalarizedVector(N->getOperand(i));
326  return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(), N->getValueType(0),
327                     &Ops[0], Ops.size());
328}
329
330/// ScalarizeVecOp_EXTRACT_VECTOR_ELT - If the input is a vector that needs to
331/// be scalarized, it must be <1 x ty>, so just return the element, ignoring the
332/// index.
333SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
334  SDValue Res = GetScalarizedVector(N->getOperand(0));
335  if (Res.getValueType() != N->getValueType(0))
336    Res = DAG.getNode(ISD::ANY_EXTEND, N->getDebugLoc(), N->getValueType(0),
337                      Res);
338  return Res;
339}
340
341/// ScalarizeVecOp_STORE - If the value to store is a vector that needs to be
342/// scalarized, it must be <1 x ty>.  Just store the element.
343SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
344  assert(N->isUnindexed() && "Indexed store of one-element vector?");
345  assert(OpNo == 1 && "Do not know how to scalarize this operand!");
346  DebugLoc dl = N->getDebugLoc();
347
348  if (N->isTruncatingStore())
349    return DAG.getTruncStore(N->getChain(), dl,
350                             GetScalarizedVector(N->getOperand(1)),
351                             N->getBasePtr(),
352                             N->getSrcValue(), N->getSrcValueOffset(),
353                             N->getMemoryVT().getVectorElementType(),
354                             N->isVolatile(), N->getAlignment());
355
356  return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
357                      N->getBasePtr(), N->getSrcValue(), N->getSrcValueOffset(),
358                      N->isVolatile(), N->getAlignment());
359}
360
361
362//===----------------------------------------------------------------------===//
363//  Result Vector Splitting
364//===----------------------------------------------------------------------===//
365
366/// SplitVectorResult - This method is called when the specified result of the
367/// specified node is found to need vector splitting.  At this point, the node
368/// may also have invalid operands or may have other results that need
369/// legalization, we just know that (at least) one result needs vector
370/// splitting.
371void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
372  DEBUG(cerr << "Split node result: "; N->dump(&DAG); cerr << "\n");
373  SDValue Lo, Hi;
374
375  switch (N->getOpcode()) {
376  default:
377#ifndef NDEBUG
378    cerr << "SplitVectorResult #" << ResNo << ": ";
379    N->dump(&DAG); cerr << "\n";
380#endif
381    assert(0 && "Do not know how to split the result of this operator!");
382    abort();
383
384  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, Lo, Hi); break;
385  case ISD::SELECT:       SplitRes_SELECT(N, Lo, Hi); break;
386  case ISD::SELECT_CC:    SplitRes_SELECT_CC(N, Lo, Hi); break;
387  case ISD::UNDEF:        SplitRes_UNDEF(N, Lo, Hi); break;
388
389  case ISD::BIT_CONVERT:       SplitVecRes_BIT_CONVERT(N, Lo, Hi); break;
390  case ISD::BUILD_VECTOR:      SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
391  case ISD::CONCAT_VECTORS:    SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
392  case ISD::CONVERT_RNDSAT:    SplitVecRes_CONVERT_RNDSAT(N, Lo, Hi); break;
393  case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
394  case ISD::FPOWI:             SplitVecRes_FPOWI(N, Lo, Hi); break;
395  case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
396  case ISD::SCALAR_TO_VECTOR:  SplitVecRes_SCALAR_TO_VECTOR(N, Lo, Hi); break;
397  case ISD::LOAD:
398    SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
399    break;
400  case ISD::SETCC:
401  case ISD::VSETCC:
402    SplitVecRes_SETCC(N, Lo, Hi);
403    break;
404  case ISD::VECTOR_SHUFFLE:
405    SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
406    break;
407
408  case ISD::CTTZ:
409  case ISD::CTLZ:
410  case ISD::CTPOP:
411  case ISD::FNEG:
412  case ISD::FABS:
413  case ISD::FSQRT:
414  case ISD::FSIN:
415  case ISD::FCOS:
416  case ISD::FTRUNC:
417  case ISD::FFLOOR:
418  case ISD::FCEIL:
419  case ISD::FRINT:
420  case ISD::FNEARBYINT:
421  case ISD::FP_TO_SINT:
422  case ISD::FP_TO_UINT:
423  case ISD::SINT_TO_FP:
424  case ISD::UINT_TO_FP:
425  case ISD::TRUNCATE:
426  case ISD::SIGN_EXTEND:
427  case ISD::ZERO_EXTEND:
428  case ISD::ANY_EXTEND:
429    SplitVecRes_UnaryOp(N, Lo, Hi);
430    break;
431
432  case ISD::ADD:
433  case ISD::SUB:
434  case ISD::MUL:
435  case ISD::FADD:
436  case ISD::FSUB:
437  case ISD::FMUL:
438  case ISD::SDIV:
439  case ISD::UDIV:
440  case ISD::FDIV:
441  case ISD::FPOW:
442  case ISD::AND:
443  case ISD::OR:
444  case ISD::XOR:
445  case ISD::SHL:
446  case ISD::SRA:
447  case ISD::SRL:
448  case ISD::UREM:
449  case ISD::SREM:
450  case ISD::FREM:
451    SplitVecRes_BinOp(N, Lo, Hi);
452    break;
453  }
454
455  // If Lo/Hi is null, the sub-method took care of registering results etc.
456  if (Lo.getNode())
457    SetSplitVector(SDValue(N, ResNo), Lo, Hi);
458}
459
460void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo,
461                                         SDValue &Hi) {
462  SDValue LHSLo, LHSHi;
463  GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
464  SDValue RHSLo, RHSHi;
465  GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
466  DebugLoc dl = N->getDebugLoc();
467
468  Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo, RHSLo);
469  Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi, RHSHi);
470}
471
472void DAGTypeLegalizer::SplitVecRes_BIT_CONVERT(SDNode *N, SDValue &Lo,
473                                               SDValue &Hi) {
474  // We know the result is a vector.  The input may be either a vector or a
475  // scalar value.
476  MVT LoVT, HiVT;
477  GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
478  DebugLoc dl = N->getDebugLoc();
479
480  SDValue InOp = N->getOperand(0);
481  MVT InVT = InOp.getValueType();
482
483  // Handle some special cases efficiently.
484  switch (getTypeAction(InVT)) {
485  default:
486    assert(false && "Unknown type action!");
487  case Legal:
488  case PromoteInteger:
489  case SoftenFloat:
490  case ScalarizeVector:
491    break;
492  case ExpandInteger:
493  case ExpandFloat:
494    // A scalar to vector conversion, where the scalar needs expansion.
495    // If the vector is being split in two then we can just convert the
496    // expanded pieces.
497    if (LoVT == HiVT) {
498      GetExpandedOp(InOp, Lo, Hi);
499      if (TLI.isBigEndian())
500        std::swap(Lo, Hi);
501      Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo);
502      Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi);
503      return;
504    }
505    break;
506  case SplitVector:
507    // If the input is a vector that needs to be split, convert each split
508    // piece of the input now.
509    GetSplitVector(InOp, Lo, Hi);
510    Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo);
511    Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi);
512    return;
513  }
514
515  // In the general case, convert the input to an integer and split it by hand.
516  MVT LoIntVT = MVT::getIntegerVT(LoVT.getSizeInBits());
517  MVT HiIntVT = MVT::getIntegerVT(HiVT.getSizeInBits());
518  if (TLI.isBigEndian())
519    std::swap(LoIntVT, HiIntVT);
520
521  SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
522
523  if (TLI.isBigEndian())
524    std::swap(Lo, Hi);
525  Lo = DAG.getNode(ISD::BIT_CONVERT, dl, LoVT, Lo);
526  Hi = DAG.getNode(ISD::BIT_CONVERT, dl, HiVT, Hi);
527}
528
529void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
530                                                SDValue &Hi) {
531  MVT LoVT, HiVT;
532  DebugLoc dl = N->getDebugLoc();
533  GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
534  unsigned LoNumElts = LoVT.getVectorNumElements();
535  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
536  Lo = DAG.getNode(ISD::BUILD_VECTOR, dl, LoVT, &LoOps[0], LoOps.size());
537
538  SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
539  Hi = DAG.getNode(ISD::BUILD_VECTOR, dl, HiVT, &HiOps[0], HiOps.size());
540}
541
542void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
543                                                  SDValue &Hi) {
544  assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
545  DebugLoc dl = N->getDebugLoc();
546  unsigned NumSubvectors = N->getNumOperands() / 2;
547  if (NumSubvectors == 1) {
548    Lo = N->getOperand(0);
549    Hi = N->getOperand(1);
550    return;
551  }
552
553  MVT LoVT, HiVT;
554  GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
555
556  SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
557  Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, &LoOps[0], LoOps.size());
558
559  SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
560  Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, &HiOps[0], HiOps.size());
561}
562
563void DAGTypeLegalizer::SplitVecRes_CONVERT_RNDSAT(SDNode *N, SDValue &Lo,
564                                                  SDValue &Hi) {
565  MVT LoVT, HiVT;
566  DebugLoc dl = N->getDebugLoc();
567  GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
568
569  SDValue DTyOpLo =  DAG.getValueType(LoVT);
570  SDValue DTyOpHi =  DAG.getValueType(HiVT);
571
572  SDValue RndOp = N->getOperand(3);
573  SDValue SatOp = N->getOperand(4);
574  ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode();
575
576  // Split the input.
577  SDValue VLo, VHi;
578  MVT InVT = N->getOperand(0).getValueType();
579  switch (getTypeAction(InVT)) {
580  default: assert(0 && "Unexpected type action!");
581  case Legal: {
582    MVT InNVT = MVT::getVectorVT(InVT.getVectorElementType(),
583                                 LoVT.getVectorNumElements());
584    VLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0),
585                      DAG.getIntPtrConstant(0));
586    VHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0),
587                      DAG.getIntPtrConstant(InNVT.getVectorNumElements()));
588    break;
589  }
590  case SplitVector:
591    GetSplitVector(N->getOperand(0), VLo, VHi);
592    break;
593  case WidenVector: {
594    // If the result needs to be split and the input needs to be widened,
595    // the two types must have different lengths. Use the widened result
596    // and extract from it to do the split.
597    SDValue InOp = GetWidenedVector(N->getOperand(0));
598    MVT InNVT = MVT::getVectorVT(InVT.getVectorElementType(),
599                                 LoVT.getVectorNumElements());
600    VLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp,
601                     DAG.getIntPtrConstant(0));
602    VHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp,
603                     DAG.getIntPtrConstant(InNVT.getVectorNumElements()));
604    break;
605  }
606  }
607
608  SDValue STyOpLo =  DAG.getValueType(VLo.getValueType());
609  SDValue STyOpHi =  DAG.getValueType(VHi.getValueType());
610
611  Lo = DAG.getConvertRndSat(LoVT, dl, VLo, DTyOpLo, STyOpLo, RndOp, SatOp,
612                            CvtCode);
613  Hi = DAG.getConvertRndSat(HiVT, dl, VHi, DTyOpHi, STyOpHi, RndOp, SatOp,
614                            CvtCode);
615}
616
617void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
618                                                     SDValue &Hi) {
619  SDValue Vec = N->getOperand(0);
620  SDValue Idx = N->getOperand(1);
621  MVT IdxVT = Idx.getValueType();
622  DebugLoc dl = N->getDebugLoc();
623
624  MVT LoVT, HiVT;
625  GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
626
627  Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
628  Idx = DAG.getNode(ISD::ADD, dl, IdxVT, Idx,
629                    DAG.getConstant(LoVT.getVectorNumElements(), IdxVT));
630  Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec, Idx);
631}
632
633void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
634                                         SDValue &Hi) {
635  DebugLoc dl = N->getDebugLoc();
636  GetSplitVector(N->getOperand(0), Lo, Hi);
637  Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
638  Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
639}
640
641void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
642                                                     SDValue &Hi) {
643  SDValue Vec = N->getOperand(0);
644  SDValue Elt = N->getOperand(1);
645  SDValue Idx = N->getOperand(2);
646  DebugLoc dl = N->getDebugLoc();
647  GetSplitVector(Vec, Lo, Hi);
648
649  if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
650    unsigned IdxVal = CIdx->getZExtValue();
651    unsigned LoNumElts = Lo.getValueType().getVectorNumElements();
652    if (IdxVal < LoNumElts)
653      Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
654                       Lo.getValueType(), Lo, Elt, Idx);
655    else
656      Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
657                       DAG.getIntPtrConstant(IdxVal - LoNumElts));
658    return;
659  }
660
661  // Spill the vector to the stack.
662  MVT VecVT = Vec.getValueType();
663  MVT EltVT = VecVT.getVectorElementType();
664  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
665  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, NULL, 0);
666
667  // Store the new element.  This may be larger than the vector element type,
668  // so use a truncating store.
669  SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx);
670  unsigned Alignment =
671    TLI.getTargetData()->getPrefTypeAlignment(VecVT.getTypeForMVT(
672                                                            *DAG.getContext()));
673  Store = DAG.getTruncStore(Store, dl, Elt, EltPtr, NULL, 0, EltVT);
674
675  // Load the Lo part from the stack slot.
676  Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, NULL, 0);
677
678  // Increment the pointer to the other part.
679  unsigned IncrementSize = Lo.getValueType().getSizeInBits() / 8;
680  StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
681                         DAG.getIntPtrConstant(IncrementSize));
682
683  // Load the Hi part from the stack slot.
684  Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, NULL, 0, false,
685                   MinAlign(Alignment, IncrementSize));
686}
687
688void DAGTypeLegalizer::SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo,
689                                                    SDValue &Hi) {
690  MVT LoVT, HiVT;
691  DebugLoc dl = N->getDebugLoc();
692  GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
693  Lo = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoVT, N->getOperand(0));
694  Hi = DAG.getUNDEF(HiVT);
695}
696
697void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
698                                        SDValue &Hi) {
699  assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
700  MVT LoVT, HiVT;
701  DebugLoc dl = LD->getDebugLoc();
702  GetSplitDestVTs(LD->getValueType(0), LoVT, HiVT);
703
704  ISD::LoadExtType ExtType = LD->getExtensionType();
705  SDValue Ch = LD->getChain();
706  SDValue Ptr = LD->getBasePtr();
707  SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
708  const Value *SV = LD->getSrcValue();
709  int SVOffset = LD->getSrcValueOffset();
710  MVT MemoryVT = LD->getMemoryVT();
711  unsigned Alignment = LD->getAlignment();
712  bool isVolatile = LD->isVolatile();
713
714  MVT LoMemVT, HiMemVT;
715  GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT);
716
717  Lo = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, LoVT, Ch, Ptr, Offset,
718                   SV, SVOffset, LoMemVT, isVolatile, Alignment);
719
720  unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
721  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
722                    DAG.getIntPtrConstant(IncrementSize));
723  SVOffset += IncrementSize;
724  Alignment = MinAlign(Alignment, IncrementSize);
725  Hi = DAG.getLoad(ISD::UNINDEXED, dl, ExtType, HiVT, Ch, Ptr, Offset,
726                   SV, SVOffset, HiMemVT, isVolatile, Alignment);
727
728  // Build a factor node to remember that this load is independent of the
729  // other one.
730  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
731                   Hi.getValue(1));
732
733  // Legalized the chain result - switch anything that used the old chain to
734  // use the new one.
735  ReplaceValueWith(SDValue(LD, 1), Ch);
736}
737
738void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
739  MVT LoVT, HiVT;
740  DebugLoc DL = N->getDebugLoc();
741  GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
742
743  // Split the input.
744  MVT InVT = N->getOperand(0).getValueType();
745  SDValue LL, LH, RL, RH;
746  MVT InNVT = MVT::getVectorVT(InVT.getVectorElementType(),
747                               LoVT.getVectorNumElements());
748  LL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0),
749                   DAG.getIntPtrConstant(0));
750  LH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(0),
751                   DAG.getIntPtrConstant(InNVT.getVectorNumElements()));
752
753  RL = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1),
754                   DAG.getIntPtrConstant(0));
755  RH = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, N->getOperand(1),
756                   DAG.getIntPtrConstant(InNVT.getVectorNumElements()));
757
758  Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
759  Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
760}
761
762void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
763                                           SDValue &Hi) {
764  // Get the dest types - they may not match the input types, e.g. int_to_fp.
765  MVT LoVT, HiVT;
766  DebugLoc dl = N->getDebugLoc();
767  GetSplitDestVTs(N->getValueType(0), LoVT, HiVT);
768
769  // Split the input.
770  MVT InVT = N->getOperand(0).getValueType();
771  switch (getTypeAction(InVT)) {
772  default: assert(0 && "Unexpected type action!");
773  case Legal: {
774    MVT InNVT = MVT::getVectorVT(InVT.getVectorElementType(),
775                                 LoVT.getVectorNumElements());
776    Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0),
777                     DAG.getIntPtrConstant(0));
778    Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, N->getOperand(0),
779                     DAG.getIntPtrConstant(InNVT.getVectorNumElements()));
780    break;
781  }
782  case SplitVector:
783    GetSplitVector(N->getOperand(0), Lo, Hi);
784    break;
785  case WidenVector: {
786    // If the result needs to be split and the input needs to be widened,
787    // the two types must have different lengths. Use the widened result
788    // and extract from it to do the split.
789    SDValue InOp = GetWidenedVector(N->getOperand(0));
790    MVT InNVT = MVT::getVectorVT(InVT.getVectorElementType(),
791                                 LoVT.getVectorNumElements());
792    Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp,
793                     DAG.getIntPtrConstant(0));
794    Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InNVT, InOp,
795                     DAG.getIntPtrConstant(InNVT.getVectorNumElements()));
796    break;
797  }
798  }
799
800  Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
801  Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
802}
803
804void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
805                                                  SDValue &Lo, SDValue &Hi) {
806  // The low and high parts of the original input give four input vectors.
807  SDValue Inputs[4];
808  DebugLoc dl = N->getDebugLoc();
809  GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
810  GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
811  MVT NewVT = Inputs[0].getValueType();
812  unsigned NewElts = NewVT.getVectorNumElements();
813
814  // If Lo or Hi uses elements from at most two of the four input vectors, then
815  // express it as a vector shuffle of those two inputs.  Otherwise extract the
816  // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
817  SmallVector<int, 16> Ops;
818  for (unsigned High = 0; High < 2; ++High) {
819    SDValue &Output = High ? Hi : Lo;
820
821    // Build a shuffle mask for the output, discovering on the fly which
822    // input vectors to use as shuffle operands (recorded in InputUsed).
823    // If building a suitable shuffle vector proves too hard, then bail
824    // out with useBuildVector set.
825    unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered.
826    unsigned FirstMaskIdx = High * NewElts;
827    bool useBuildVector = false;
828    for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
829      // The mask element.  This indexes into the input.
830      int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
831
832      // The input vector this mask element indexes into.
833      unsigned Input = (unsigned)Idx / NewElts;
834
835      if (Input >= array_lengthof(Inputs)) {
836        // The mask element does not index into any input vector.
837        Ops.push_back(-1);
838        continue;
839      }
840
841      // Turn the index into an offset from the start of the input vector.
842      Idx -= Input * NewElts;
843
844      // Find or create a shuffle vector operand to hold this input.
845      unsigned OpNo;
846      for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) {
847        if (InputUsed[OpNo] == Input) {
848          // This input vector is already an operand.
849          break;
850        } else if (InputUsed[OpNo] == -1U) {
851          // Create a new operand for this input vector.
852          InputUsed[OpNo] = Input;
853          break;
854        }
855      }
856
857      if (OpNo >= array_lengthof(InputUsed)) {
858        // More than two input vectors used!  Give up on trying to create a
859        // shuffle vector.  Insert all elements into a BUILD_VECTOR instead.
860        useBuildVector = true;
861        break;
862      }
863
864      // Add the mask index for the new shuffle vector.
865      Ops.push_back(Idx + OpNo * NewElts);
866    }
867
868    if (useBuildVector) {
869      MVT EltVT = NewVT.getVectorElementType();
870      SmallVector<SDValue, 16> SVOps;
871
872      // Extract the input elements by hand.
873      for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
874        // The mask element.  This indexes into the input.
875        int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
876
877        // The input vector this mask element indexes into.
878        unsigned Input = (unsigned)Idx / NewElts;
879
880        if (Input >= array_lengthof(Inputs)) {
881          // The mask element is "undef" or indexes off the end of the input.
882          SVOps.push_back(DAG.getUNDEF(EltVT));
883          continue;
884        }
885
886        // Turn the index into an offset from the start of the input vector.
887        Idx -= Input * NewElts;
888
889        // Extract the vector element by hand.
890        SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
891                                    Inputs[Input], DAG.getIntPtrConstant(Idx)));
892      }
893
894      // Construct the Lo/Hi output using a BUILD_VECTOR.
895      Output = DAG.getNode(ISD::BUILD_VECTOR,dl,NewVT, &SVOps[0], SVOps.size());
896    } else if (InputUsed[0] == -1U) {
897      // No input vectors were used!  The result is undefined.
898      Output = DAG.getUNDEF(NewVT);
899    } else {
900      SDValue Op0 = Inputs[InputUsed[0]];
901      // If only one input was used, use an undefined vector for the other.
902      SDValue Op1 = InputUsed[1] == -1U ?
903        DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]];
904      // At least one input vector was used.  Create a new shuffle vector.
905      Output =  DAG.getVectorShuffle(NewVT, dl, Op0, Op1, &Ops[0]);
906    }
907
908    Ops.clear();
909  }
910}
911
912
913//===----------------------------------------------------------------------===//
914//  Operand Vector Splitting
915//===----------------------------------------------------------------------===//
916
917/// SplitVectorOperand - This method is called when the specified operand of the
918/// specified node is found to need vector splitting.  At this point, all of the
919/// result types of the node are known to be legal, but other operands of the
920/// node may need legalization as well as the specified one.
921bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
922  DEBUG(cerr << "Split node operand: "; N->dump(&DAG); cerr << "\n");
923  SDValue Res = SDValue();
924
925  if (Res.getNode() == 0) {
926    switch (N->getOpcode()) {
927    default:
928#ifndef NDEBUG
929      cerr << "SplitVectorOperand Op #" << OpNo << ": ";
930      N->dump(&DAG); cerr << "\n";
931#endif
932      assert(0 && "Do not know how to split this operator's operand!");
933      abort();
934
935    case ISD::BIT_CONVERT:       Res = SplitVecOp_BIT_CONVERT(N); break;
936    case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
937    case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
938    case ISD::STORE:
939      Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
940      break;
941
942    case ISD::CTTZ:
943    case ISD::CTLZ:
944    case ISD::CTPOP:
945    case ISD::FP_TO_SINT:
946    case ISD::FP_TO_UINT:
947    case ISD::SINT_TO_FP:
948    case ISD::UINT_TO_FP:
949    case ISD::TRUNCATE:
950    case ISD::SIGN_EXTEND:
951    case ISD::ZERO_EXTEND:
952    case ISD::ANY_EXTEND:
953      Res = SplitVecOp_UnaryOp(N);
954      break;
955    }
956  }
957
958  // If the result is null, the sub-method took care of registering results etc.
959  if (!Res.getNode()) return false;
960
961  // If the result is N, the sub-method updated N in place.  Tell the legalizer
962  // core about this.
963  if (Res.getNode() == N)
964    return true;
965
966  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
967         "Invalid operand expansion");
968
969  ReplaceValueWith(SDValue(N, 0), Res);
970  return false;
971}
972
973SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
974  // The result has a legal vector type, but the input needs splitting.
975  MVT ResVT = N->getValueType(0);
976  SDValue Lo, Hi;
977  DebugLoc dl = N->getDebugLoc();
978  GetSplitVector(N->getOperand(0), Lo, Hi);
979  MVT InVT = Lo.getValueType();
980
981  MVT OutVT = MVT::getVectorVT(ResVT.getVectorElementType(),
982                               InVT.getVectorNumElements());
983
984  Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
985  Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
986
987  return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
988}
989
990SDValue DAGTypeLegalizer::SplitVecOp_BIT_CONVERT(SDNode *N) {
991  // For example, i64 = BIT_CONVERT v4i16 on alpha.  Typically the vector will
992  // end up being split all the way down to individual components.  Convert the
993  // split pieces into integers and reassemble.
994  SDValue Lo, Hi;
995  GetSplitVector(N->getOperand(0), Lo, Hi);
996  Lo = BitConvertToInteger(Lo);
997  Hi = BitConvertToInteger(Hi);
998
999  if (TLI.isBigEndian())
1000    std::swap(Lo, Hi);
1001
1002  return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), N->getValueType(0),
1003                     JoinIntegers(Lo, Hi));
1004}
1005
1006SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
1007  // We know that the extracted result type is legal.  For now, assume the index
1008  // is a constant.
1009  MVT SubVT = N->getValueType(0);
1010  SDValue Idx = N->getOperand(1);
1011  DebugLoc dl = N->getDebugLoc();
1012  SDValue Lo, Hi;
1013  GetSplitVector(N->getOperand(0), Lo, Hi);
1014
1015  uint64_t LoElts = Lo.getValueType().getVectorNumElements();
1016  uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1017
1018  if (IdxVal < LoElts) {
1019    assert(IdxVal + SubVT.getVectorNumElements() <= LoElts &&
1020           "Extracted subvector crosses vector split!");
1021    return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
1022  } else {
1023    return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
1024                       DAG.getConstant(IdxVal - LoElts, Idx.getValueType()));
1025  }
1026}
1027
1028SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
1029  SDValue Vec = N->getOperand(0);
1030  SDValue Idx = N->getOperand(1);
1031  MVT VecVT = Vec.getValueType();
1032
1033  if (isa<ConstantSDNode>(Idx)) {
1034    uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1035    assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!");
1036
1037    SDValue Lo, Hi;
1038    GetSplitVector(Vec, Lo, Hi);
1039
1040    uint64_t LoElts = Lo.getValueType().getVectorNumElements();
1041
1042    if (IdxVal < LoElts)
1043      return DAG.UpdateNodeOperands(SDValue(N, 0), Lo, Idx);
1044    return DAG.UpdateNodeOperands(SDValue(N, 0), Hi,
1045                                  DAG.getConstant(IdxVal - LoElts,
1046                                                  Idx.getValueType()));
1047  }
1048
1049  // Store the vector to the stack.
1050  MVT EltVT = VecVT.getVectorElementType();
1051  DebugLoc dl = N->getDebugLoc();
1052  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1053  int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1054  const Value *SV = PseudoSourceValue::getFixedStack(SPFI);
1055  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, SV, 0);
1056
1057  // Load back the required element.
1058  StackPtr = GetVectorElementPointer(StackPtr, EltVT, Idx);
1059  return DAG.getExtLoad(ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
1060                        SV, 0, EltVT);
1061}
1062
1063SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
1064  assert(N->isUnindexed() && "Indexed store of vector?");
1065  assert(OpNo == 1 && "Can only split the stored value");
1066  DebugLoc dl = N->getDebugLoc();
1067
1068  bool isTruncating = N->isTruncatingStore();
1069  SDValue Ch  = N->getChain();
1070  SDValue Ptr = N->getBasePtr();
1071  int SVOffset = N->getSrcValueOffset();
1072  MVT MemoryVT = N->getMemoryVT();
1073  unsigned Alignment = N->getAlignment();
1074  bool isVol = N->isVolatile();
1075  SDValue Lo, Hi;
1076  GetSplitVector(N->getOperand(1), Lo, Hi);
1077
1078  MVT LoMemVT, HiMemVT;
1079  GetSplitDestVTs(MemoryVT, LoMemVT, HiMemVT);
1080
1081  unsigned IncrementSize = LoMemVT.getSizeInBits()/8;
1082
1083  if (isTruncating)
1084    Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getSrcValue(), SVOffset,
1085                           LoMemVT, isVol, Alignment);
1086  else
1087    Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getSrcValue(), SVOffset,
1088                      isVol, Alignment);
1089
1090  // Increment the pointer to the other half.
1091  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
1092                    DAG.getIntPtrConstant(IncrementSize));
1093
1094  if (isTruncating)
1095    Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr,
1096                           N->getSrcValue(), SVOffset+IncrementSize,
1097                           HiMemVT,
1098                           isVol, MinAlign(Alignment, IncrementSize));
1099  else
1100    Hi = DAG.getStore(Ch, dl, Hi, Ptr, N->getSrcValue(), SVOffset+IncrementSize,
1101                      isVol, MinAlign(Alignment, IncrementSize));
1102
1103  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
1104}
1105
1106
1107//===----------------------------------------------------------------------===//
1108//  Result Vector Widening
1109//===----------------------------------------------------------------------===//
1110
1111void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
1112  DEBUG(cerr << "Widen node result " << ResNo << ": "; N->dump(&DAG);
1113        cerr << "\n");
1114  SDValue Res = SDValue();
1115
1116  switch (N->getOpcode()) {
1117  default:
1118#ifndef NDEBUG
1119    cerr << "WidenVectorResult #" << ResNo << ": ";
1120    N->dump(&DAG); cerr << "\n";
1121#endif
1122    assert(0 && "Do not know how to widen the result of this operator!");
1123    abort();
1124
1125  case ISD::BIT_CONVERT:       Res = WidenVecRes_BIT_CONVERT(N); break;
1126  case ISD::BUILD_VECTOR:      Res = WidenVecRes_BUILD_VECTOR(N); break;
1127  case ISD::CONCAT_VECTORS:    Res = WidenVecRes_CONCAT_VECTORS(N); break;
1128  case ISD::CONVERT_RNDSAT:    Res = WidenVecRes_CONVERT_RNDSAT(N); break;
1129  case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
1130  case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
1131  case ISD::LOAD:              Res = WidenVecRes_LOAD(N); break;
1132  case ISD::SCALAR_TO_VECTOR:  Res = WidenVecRes_SCALAR_TO_VECTOR(N); break;
1133  case ISD::SELECT:            Res = WidenVecRes_SELECT(N); break;
1134  case ISD::SELECT_CC:         Res = WidenVecRes_SELECT_CC(N); break;
1135  case ISD::UNDEF:             Res = WidenVecRes_UNDEF(N); break;
1136  case ISD::VECTOR_SHUFFLE:
1137    Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
1138    break;
1139  case ISD::VSETCC:
1140    Res = WidenVecRes_VSETCC(N);
1141    break;
1142
1143  case ISD::ADD:
1144  case ISD::AND:
1145  case ISD::BSWAP:
1146  case ISD::FADD:
1147  case ISD::FCOPYSIGN:
1148  case ISD::FDIV:
1149  case ISD::FMUL:
1150  case ISD::FPOW:
1151  case ISD::FPOWI:
1152  case ISD::FREM:
1153  case ISD::FSUB:
1154  case ISD::MUL:
1155  case ISD::MULHS:
1156  case ISD::MULHU:
1157  case ISD::OR:
1158  case ISD::SDIV:
1159  case ISD::SREM:
1160  case ISD::UDIV:
1161  case ISD::UREM:
1162  case ISD::SUB:
1163  case ISD::XOR:
1164    Res = WidenVecRes_Binary(N);
1165    break;
1166
1167  case ISD::SHL:
1168  case ISD::SRA:
1169  case ISD::SRL:
1170    Res = WidenVecRes_Shift(N);
1171    break;
1172
1173  case ISD::FP_ROUND:
1174  case ISD::FP_TO_SINT:
1175  case ISD::FP_TO_UINT:
1176  case ISD::SINT_TO_FP:
1177  case ISD::UINT_TO_FP:
1178  case ISD::TRUNCATE:
1179  case ISD::SIGN_EXTEND:
1180  case ISD::ZERO_EXTEND:
1181  case ISD::ANY_EXTEND:
1182    Res = WidenVecRes_Convert(N);
1183    break;
1184
1185  case ISD::CTLZ:
1186  case ISD::CTPOP:
1187  case ISD::CTTZ:
1188  case ISD::FABS:
1189  case ISD::FCOS:
1190  case ISD::FNEG:
1191  case ISD::FSIN:
1192  case ISD::FSQRT:
1193    Res = WidenVecRes_Unary(N);
1194    break;
1195  }
1196
1197  // If Res is null, the sub-method took care of registering the result.
1198  if (Res.getNode())
1199    SetWidenedVector(SDValue(N, ResNo), Res);
1200}
1201
1202SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
1203  // Binary op widening.
1204  MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0));
1205  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
1206  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
1207  return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp1, InOp2);
1208}
1209
1210SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
1211  SDValue InOp = N->getOperand(0);
1212  DebugLoc dl = N->getDebugLoc();
1213
1214  MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0));
1215  unsigned WidenNumElts = WidenVT.getVectorNumElements();
1216
1217  MVT InVT = InOp.getValueType();
1218  MVT InEltVT = InVT.getVectorElementType();
1219  MVT InWidenVT = MVT::getVectorVT(InEltVT, WidenNumElts);
1220
1221  unsigned Opcode = N->getOpcode();
1222  unsigned InVTNumElts = InVT.getVectorNumElements();
1223
1224  if (getTypeAction(InVT) == WidenVector) {
1225    InOp = GetWidenedVector(N->getOperand(0));
1226    InVT = InOp.getValueType();
1227    InVTNumElts = InVT.getVectorNumElements();
1228    if (InVTNumElts == WidenNumElts)
1229      return DAG.getNode(Opcode, dl, WidenVT, InOp);
1230  }
1231
1232  if (TLI.isTypeLegal(InWidenVT)) {
1233    // Because the result and the input are different vector types, widening
1234    // the result could create a legal type but widening the input might make
1235    // it an illegal type that might lead to repeatedly splitting the input
1236    // and then widening it. To avoid this, we widen the input only if
1237    // it results in a legal type.
1238    if (WidenNumElts % InVTNumElts == 0) {
1239      // Widen the input and call convert on the widened input vector.
1240      unsigned NumConcat = WidenNumElts/InVTNumElts;
1241      SmallVector<SDValue, 16> Ops(NumConcat);
1242      Ops[0] = InOp;
1243      SDValue UndefVal = DAG.getUNDEF(InVT);
1244      for (unsigned i = 1; i != NumConcat; ++i)
1245        Ops[i] = UndefVal;
1246      return DAG.getNode(Opcode, dl, WidenVT,
1247                         DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT,
1248                         &Ops[0], NumConcat));
1249    }
1250
1251    if (InVTNumElts % WidenNumElts == 0) {
1252      // Extract the input and convert the shorten input vector.
1253      return DAG.getNode(Opcode, dl, WidenVT,
1254                         DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT,
1255                                     InOp, DAG.getIntPtrConstant(0)));
1256    }
1257  }
1258
1259  // Otherwise unroll into some nasty scalar code and rebuild the vector.
1260  SmallVector<SDValue, 16> Ops(WidenNumElts);
1261  MVT EltVT = WidenVT.getVectorElementType();
1262  unsigned MinElts = std::min(InVTNumElts, WidenNumElts);
1263  unsigned i;
1264  for (i=0; i < MinElts; ++i)
1265    Ops[i] = DAG.getNode(Opcode, dl, EltVT,
1266                         DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
1267                                     DAG.getIntPtrConstant(i)));
1268
1269  SDValue UndefVal = DAG.getUNDEF(EltVT);
1270  for (; i < WidenNumElts; ++i)
1271    Ops[i] = UndefVal;
1272
1273  return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts);
1274}
1275
1276SDValue DAGTypeLegalizer::WidenVecRes_Shift(SDNode *N) {
1277  MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0));
1278  SDValue InOp = GetWidenedVector(N->getOperand(0));
1279  SDValue ShOp = N->getOperand(1);
1280
1281  MVT ShVT = ShOp.getValueType();
1282  if (getTypeAction(ShVT) == WidenVector) {
1283    ShOp = GetWidenedVector(ShOp);
1284    ShVT = ShOp.getValueType();
1285  }
1286  MVT ShWidenVT = MVT::getVectorVT(ShVT.getVectorElementType(),
1287                                   WidenVT.getVectorNumElements());
1288  if (ShVT != ShWidenVT)
1289    ShOp = ModifyToType(ShOp, ShWidenVT);
1290
1291  return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp, ShOp);
1292}
1293
1294SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
1295  // Unary op widening.
1296  MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0));
1297  SDValue InOp = GetWidenedVector(N->getOperand(0));
1298  return DAG.getNode(N->getOpcode(), N->getDebugLoc(), WidenVT, InOp);
1299}
1300
1301SDValue DAGTypeLegalizer::WidenVecRes_BIT_CONVERT(SDNode *N) {
1302  SDValue InOp = N->getOperand(0);
1303  MVT InVT = InOp.getValueType();
1304  MVT VT = N->getValueType(0);
1305  MVT WidenVT = TLI.getTypeToTransformTo(VT);
1306  DebugLoc dl = N->getDebugLoc();
1307
1308  switch (getTypeAction(InVT)) {
1309  default:
1310    assert(false && "Unknown type action!");
1311    break;
1312  case Legal:
1313    break;
1314  case PromoteInteger:
1315    // If the InOp is promoted to the same size, convert it.  Otherwise,
1316    // fall out of the switch and widen the promoted input.
1317    InOp = GetPromotedInteger(InOp);
1318    InVT = InOp.getValueType();
1319    if (WidenVT.bitsEq(InVT))
1320      return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, InOp);
1321    break;
1322  case SoftenFloat:
1323  case ExpandInteger:
1324  case ExpandFloat:
1325  case ScalarizeVector:
1326  case SplitVector:
1327    break;
1328  case WidenVector:
1329    // If the InOp is widened to the same size, convert it.  Otherwise, fall
1330    // out of the switch and widen the widened input.
1331    InOp = GetWidenedVector(InOp);
1332    InVT = InOp.getValueType();
1333    if (WidenVT.bitsEq(InVT))
1334      // The input widens to the same size. Convert to the widen value.
1335      return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, InOp);
1336    break;
1337  }
1338
1339  unsigned WidenSize = WidenVT.getSizeInBits();
1340  unsigned InSize = InVT.getSizeInBits();
1341  if (WidenSize % InSize == 0) {
1342    // Determine new input vector type.  The new input vector type will use
1343    // the same element type (if its a vector) or use the input type as a
1344    // vector.  It is the same size as the type to widen to.
1345    MVT NewInVT;
1346    unsigned NewNumElts = WidenSize / InSize;
1347    if (InVT.isVector()) {
1348      MVT InEltVT = InVT.getVectorElementType();
1349      NewInVT= MVT::getVectorVT(InEltVT, WidenSize / InEltVT.getSizeInBits());
1350    } else {
1351      NewInVT = MVT::getVectorVT(InVT, NewNumElts);
1352    }
1353
1354    if (TLI.isTypeLegal(NewInVT)) {
1355      // Because the result and the input are different vector types, widening
1356      // the result could create a legal type but widening the input might make
1357      // it an illegal type that might lead to repeatedly splitting the input
1358      // and then widening it. To avoid this, we widen the input only if
1359      // it results in a legal type.
1360      SmallVector<SDValue, 16> Ops(NewNumElts);
1361      SDValue UndefVal = DAG.getUNDEF(InVT);
1362      Ops[0] = InOp;
1363      for (unsigned i = 1; i < NewNumElts; ++i)
1364        Ops[i] = UndefVal;
1365
1366      SDValue NewVec;
1367      if (InVT.isVector())
1368        NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl,
1369                             NewInVT, &Ops[0], NewNumElts);
1370      else
1371        NewVec = DAG.getNode(ISD::BUILD_VECTOR, dl,
1372                             NewInVT, &Ops[0], NewNumElts);
1373      return DAG.getNode(ISD::BIT_CONVERT, dl, WidenVT, NewVec);
1374    }
1375  }
1376
1377  return CreateStackStoreLoad(InOp, WidenVT);
1378}
1379
1380SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
1381  DebugLoc dl = N->getDebugLoc();
1382  // Build a vector with undefined for the new nodes.
1383  MVT VT = N->getValueType(0);
1384  MVT EltVT = VT.getVectorElementType();
1385  unsigned NumElts = VT.getVectorNumElements();
1386
1387  MVT WidenVT = TLI.getTypeToTransformTo(VT);
1388  unsigned WidenNumElts = WidenVT.getVectorNumElements();
1389
1390  SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
1391  NewOps.reserve(WidenNumElts);
1392  for (unsigned i = NumElts; i < WidenNumElts; ++i)
1393    NewOps.push_back(DAG.getUNDEF(EltVT));
1394
1395  return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &NewOps[0], NewOps.size());
1396}
1397
1398SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
1399  MVT InVT = N->getOperand(0).getValueType();
1400  MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0));
1401  DebugLoc dl = N->getDebugLoc();
1402  unsigned WidenNumElts = WidenVT.getVectorNumElements();
1403  unsigned NumOperands = N->getNumOperands();
1404
1405  bool InputWidened = false; // Indicates we need to widen the input.
1406  if (getTypeAction(InVT) != WidenVector) {
1407    if (WidenVT.getVectorNumElements() % InVT.getVectorNumElements() == 0) {
1408      // Add undef vectors to widen to correct length.
1409      unsigned NumConcat = WidenVT.getVectorNumElements() /
1410                           InVT.getVectorNumElements();
1411      SDValue UndefVal = DAG.getUNDEF(InVT);
1412      SmallVector<SDValue, 16> Ops(NumConcat);
1413      for (unsigned i=0; i < NumOperands; ++i)
1414        Ops[i] = N->getOperand(i);
1415      for (unsigned i = NumOperands; i != NumConcat; ++i)
1416        Ops[i] = UndefVal;
1417      return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, &Ops[0], NumConcat);
1418    }
1419  } else {
1420    InputWidened = true;
1421    if (WidenVT == TLI.getTypeToTransformTo(InVT)) {
1422      // The inputs and the result are widen to the same value.
1423      unsigned i;
1424      for (i=1; i < NumOperands; ++i)
1425        if (N->getOperand(i).getOpcode() != ISD::UNDEF)
1426          break;
1427
1428      if (i > NumOperands)
1429        // Everything but the first operand is an UNDEF so just return the
1430        // widened first operand.
1431        return GetWidenedVector(N->getOperand(0));
1432
1433      if (NumOperands == 2) {
1434        // Replace concat of two operands with a shuffle.
1435        SmallVector<int, 16> MaskOps(WidenNumElts);
1436        for (unsigned i=0; i < WidenNumElts/2; ++i) {
1437          MaskOps[i] = i;
1438          MaskOps[i+WidenNumElts/2] = i+WidenNumElts;
1439        }
1440        return DAG.getVectorShuffle(WidenVT, dl,
1441                                    GetWidenedVector(N->getOperand(0)),
1442                                    GetWidenedVector(N->getOperand(1)),
1443                                    &MaskOps[0]);
1444      }
1445    }
1446  }
1447
1448  // Fall back to use extracts and build vector.
1449  MVT EltVT = WidenVT.getVectorElementType();
1450  unsigned NumInElts = InVT.getVectorNumElements();
1451  SmallVector<SDValue, 16> Ops(WidenNumElts);
1452  unsigned Idx = 0;
1453  for (unsigned i=0; i < NumOperands; ++i) {
1454    SDValue InOp = N->getOperand(i);
1455    if (InputWidened)
1456      InOp = GetWidenedVector(InOp);
1457    for (unsigned j=0; j < NumInElts; ++j)
1458        Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
1459                                 DAG.getIntPtrConstant(j));
1460  }
1461  SDValue UndefVal = DAG.getUNDEF(EltVT);
1462  for (; Idx < WidenNumElts; ++Idx)
1463    Ops[Idx] = UndefVal;
1464  return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts);
1465}
1466
1467SDValue DAGTypeLegalizer::WidenVecRes_CONVERT_RNDSAT(SDNode *N) {
1468  DebugLoc dl = N->getDebugLoc();
1469  SDValue InOp  = N->getOperand(0);
1470  SDValue RndOp = N->getOperand(3);
1471  SDValue SatOp = N->getOperand(4);
1472
1473  MVT      WidenVT = TLI.getTypeToTransformTo(N->getValueType(0));
1474  unsigned WidenNumElts = WidenVT.getVectorNumElements();
1475
1476  MVT InVT = InOp.getValueType();
1477  MVT InEltVT = InVT.getVectorElementType();
1478  MVT InWidenVT = MVT::getVectorVT(InEltVT, WidenNumElts);
1479
1480  SDValue DTyOp = DAG.getValueType(WidenVT);
1481  SDValue STyOp = DAG.getValueType(InWidenVT);
1482  ISD::CvtCode CvtCode = cast<CvtRndSatSDNode>(N)->getCvtCode();
1483
1484  unsigned InVTNumElts = InVT.getVectorNumElements();
1485  if (getTypeAction(InVT) == WidenVector) {
1486    InOp = GetWidenedVector(InOp);
1487    InVT = InOp.getValueType();
1488    InVTNumElts = InVT.getVectorNumElements();
1489    if (InVTNumElts == WidenNumElts)
1490      return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp,
1491                                  SatOp, CvtCode);
1492  }
1493
1494  if (TLI.isTypeLegal(InWidenVT)) {
1495    // Because the result and the input are different vector types, widening
1496    // the result could create a legal type but widening the input might make
1497    // it an illegal type that might lead to repeatedly splitting the input
1498    // and then widening it. To avoid this, we widen the input only if
1499    // it results in a legal type.
1500    if (WidenNumElts % InVTNumElts == 0) {
1501      // Widen the input and call convert on the widened input vector.
1502      unsigned NumConcat = WidenNumElts/InVTNumElts;
1503      SmallVector<SDValue, 16> Ops(NumConcat);
1504      Ops[0] = InOp;
1505      SDValue UndefVal = DAG.getUNDEF(InVT);
1506      for (unsigned i = 1; i != NumConcat; ++i) {
1507        Ops[i] = UndefVal;
1508      }
1509      InOp = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWidenVT, &Ops[0],NumConcat);
1510      return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp,
1511                                  SatOp, CvtCode);
1512    }
1513
1514    if (InVTNumElts % WidenNumElts == 0) {
1515      // Extract the input and convert the shorten input vector.
1516      InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, InWidenVT, InOp,
1517                         DAG.getIntPtrConstant(0));
1518      return DAG.getConvertRndSat(WidenVT, dl, InOp, DTyOp, STyOp, RndOp,
1519                                SatOp, CvtCode);
1520    }
1521  }
1522
1523  // Otherwise unroll into some nasty scalar code and rebuild the vector.
1524  SmallVector<SDValue, 16> Ops(WidenNumElts);
1525  MVT EltVT = WidenVT.getVectorElementType();
1526  DTyOp = DAG.getValueType(EltVT);
1527  STyOp = DAG.getValueType(InEltVT);
1528
1529  unsigned MinElts = std::min(InVTNumElts, WidenNumElts);
1530  unsigned i;
1531  for (i=0; i < MinElts; ++i) {
1532    SDValue ExtVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
1533                                 DAG.getIntPtrConstant(i));
1534    Ops[i] = DAG.getConvertRndSat(WidenVT, dl, ExtVal, DTyOp, STyOp, RndOp,
1535                                        SatOp, CvtCode);
1536  }
1537
1538  SDValue UndefVal = DAG.getUNDEF(EltVT);
1539  for (; i < WidenNumElts; ++i)
1540    Ops[i] = UndefVal;
1541
1542  return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts);
1543}
1544
1545SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
1546  MVT      VT = N->getValueType(0);
1547  MVT      WidenVT = TLI.getTypeToTransformTo(VT);
1548  unsigned WidenNumElts = WidenVT.getVectorNumElements();
1549  SDValue  InOp = N->getOperand(0);
1550  SDValue  Idx  = N->getOperand(1);
1551  DebugLoc dl = N->getDebugLoc();
1552
1553  if (getTypeAction(InOp.getValueType()) == WidenVector)
1554    InOp = GetWidenedVector(InOp);
1555
1556  MVT InVT = InOp.getValueType();
1557
1558  ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx);
1559  if (CIdx) {
1560    unsigned IdxVal = CIdx->getZExtValue();
1561    // Check if we can just return the input vector after widening.
1562    if (IdxVal == 0 && InVT == WidenVT)
1563      return InOp;
1564
1565    // Check if we can extract from the vector.
1566    unsigned InNumElts = InVT.getVectorNumElements();
1567    if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
1568        return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
1569  }
1570
1571  // We could try widening the input to the right length but for now, extract
1572  // the original elements, fill the rest with undefs and build a vector.
1573  SmallVector<SDValue, 16> Ops(WidenNumElts);
1574  MVT EltVT = VT.getVectorElementType();
1575  MVT IdxVT = Idx.getValueType();
1576  unsigned NumElts = VT.getVectorNumElements();
1577  unsigned i;
1578  if (CIdx) {
1579    unsigned IdxVal = CIdx->getZExtValue();
1580    for (i=0; i < NumElts; ++i)
1581      Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
1582                           DAG.getConstant(IdxVal+i, IdxVT));
1583  } else {
1584    Ops[0] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, Idx);
1585    for (i=1; i < NumElts; ++i) {
1586      SDValue NewIdx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx,
1587                                   DAG.getConstant(i, IdxVT));
1588      Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp, NewIdx);
1589    }
1590  }
1591
1592  SDValue UndefVal = DAG.getUNDEF(EltVT);
1593  for (; i < WidenNumElts; ++i)
1594    Ops[i] = UndefVal;
1595  return DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], WidenNumElts);
1596}
1597
1598SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
1599  SDValue InOp = GetWidenedVector(N->getOperand(0));
1600  return DAG.getNode(ISD::INSERT_VECTOR_ELT, N->getDebugLoc(),
1601                     InOp.getValueType(), InOp,
1602                     N->getOperand(1), N->getOperand(2));
1603}
1604
1605SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
1606  LoadSDNode *LD = cast<LoadSDNode>(N);
1607  MVT WidenVT = TLI.getTypeToTransformTo(LD->getValueType(0));
1608  MVT LdVT    = LD->getMemoryVT();
1609  DebugLoc dl = N->getDebugLoc();
1610  assert(LdVT.isVector() && WidenVT.isVector());
1611
1612  // Load information
1613  SDValue   Chain = LD->getChain();
1614  SDValue   BasePtr = LD->getBasePtr();
1615  int       SVOffset = LD->getSrcValueOffset();
1616  unsigned  Align    = LD->getAlignment();
1617  bool      isVolatile = LD->isVolatile();
1618  const Value *SV = LD->getSrcValue();
1619  ISD::LoadExtType ExtType = LD->getExtensionType();
1620
1621  SDValue Result;
1622  SmallVector<SDValue, 16> LdChain;  // Chain for the series of load
1623  if (ExtType != ISD::NON_EXTLOAD) {
1624    // For extension loads, we can not play the tricks of chopping legal
1625    // vector types and bit cast it to the right type.  Instead, we unroll
1626    // the load and build a vector.
1627    MVT EltVT = WidenVT.getVectorElementType();
1628    MVT LdEltVT = LdVT.getVectorElementType();
1629    unsigned NumElts = LdVT.getVectorNumElements();
1630
1631    // Load each element and widen
1632    unsigned WidenNumElts = WidenVT.getVectorNumElements();
1633    SmallVector<SDValue, 16> Ops(WidenNumElts);
1634    unsigned Increment = LdEltVT.getSizeInBits() / 8;
1635    Ops[0] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, SV, SVOffset,
1636                            LdEltVT, isVolatile, Align);
1637    LdChain.push_back(Ops[0].getValue(1));
1638    unsigned i = 0, Offset = Increment;
1639    for (i=1; i < NumElts; ++i, Offset += Increment) {
1640      SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(),
1641                                       BasePtr, DAG.getIntPtrConstant(Offset));
1642      Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr, SV,
1643                              SVOffset + Offset, LdEltVT, isVolatile, Align);
1644      LdChain.push_back(Ops[i].getValue(1));
1645    }
1646
1647    // Fill the rest with undefs
1648    SDValue UndefVal = DAG.getUNDEF(EltVT);
1649    for (; i != WidenNumElts; ++i)
1650      Ops[i] = UndefVal;
1651
1652    Result =  DAG.getNode(ISD::BUILD_VECTOR, dl, WidenVT, &Ops[0], Ops.size());
1653  } else {
1654    assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
1655    unsigned int LdWidth = LdVT.getSizeInBits();
1656    Result = GenWidenVectorLoads(LdChain, Chain, BasePtr, SV, SVOffset,
1657                                 Align, isVolatile, LdWidth, WidenVT, dl);
1658  }
1659
1660 // If we generate a single load, we can use that for the chain.  Otherwise,
1661 // build a factor node to remember the multiple loads are independent and
1662 // chain to that.
1663 SDValue NewChain;
1664 if (LdChain.size() == 1)
1665   NewChain = LdChain[0];
1666 else
1667   NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, &LdChain[0],
1668                          LdChain.size());
1669
1670  // Modified the chain - switch anything that used the old chain to use
1671  // the new one.
1672  ReplaceValueWith(SDValue(N, 1), Chain);
1673
1674  return Result;
1675}
1676
1677SDValue DAGTypeLegalizer::WidenVecRes_SCALAR_TO_VECTOR(SDNode *N) {
1678  MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0));
1679  return DAG.getNode(ISD::SCALAR_TO_VECTOR, N->getDebugLoc(),
1680                     WidenVT, N->getOperand(0));
1681}
1682
1683SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) {
1684  MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0));
1685  unsigned WidenNumElts = WidenVT.getVectorNumElements();
1686
1687  SDValue Cond1 = N->getOperand(0);
1688  MVT CondVT = Cond1.getValueType();
1689  if (CondVT.isVector()) {
1690    MVT CondEltVT = CondVT.getVectorElementType();
1691    MVT CondWidenVT =  MVT::getVectorVT(CondEltVT, WidenNumElts);
1692    if (getTypeAction(CondVT) == WidenVector)
1693      Cond1 = GetWidenedVector(Cond1);
1694
1695    if (Cond1.getValueType() != CondWidenVT)
1696       Cond1 = ModifyToType(Cond1, CondWidenVT);
1697  }
1698
1699  SDValue InOp1 = GetWidenedVector(N->getOperand(1));
1700  SDValue InOp2 = GetWidenedVector(N->getOperand(2));
1701  assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
1702  return DAG.getNode(ISD::SELECT, N->getDebugLoc(),
1703                     WidenVT, Cond1, InOp1, InOp2);
1704}
1705
1706SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
1707  SDValue InOp1 = GetWidenedVector(N->getOperand(2));
1708  SDValue InOp2 = GetWidenedVector(N->getOperand(3));
1709  return DAG.getNode(ISD::SELECT_CC, N->getDebugLoc(),
1710                     InOp1.getValueType(), N->getOperand(0),
1711                     N->getOperand(1), InOp1, InOp2, N->getOperand(4));
1712}
1713
1714SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
1715 MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0));
1716 return DAG.getUNDEF(WidenVT);
1717}
1718
1719SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
1720  MVT VT = N->getValueType(0);
1721  DebugLoc dl = N->getDebugLoc();
1722
1723  MVT WidenVT = TLI.getTypeToTransformTo(VT);
1724  unsigned NumElts = VT.getVectorNumElements();
1725  unsigned WidenNumElts = WidenVT.getVectorNumElements();
1726
1727  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
1728  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
1729
1730  // Adjust mask based on new input vector length.
1731  SmallVector<int, 16> NewMask;
1732  for (unsigned i = 0; i != NumElts; ++i) {
1733    int Idx = N->getMaskElt(i);
1734    if (Idx < (int)NumElts)
1735      NewMask.push_back(Idx);
1736    else
1737      NewMask.push_back(Idx - NumElts + WidenNumElts);
1738  }
1739  for (unsigned i = NumElts; i != WidenNumElts; ++i)
1740    NewMask.push_back(-1);
1741  return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, &NewMask[0]);
1742}
1743
1744SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) {
1745  MVT WidenVT = TLI.getTypeToTransformTo(N->getValueType(0));
1746  unsigned WidenNumElts = WidenVT.getVectorNumElements();
1747
1748  SDValue InOp1 = N->getOperand(0);
1749  MVT InVT = InOp1.getValueType();
1750  assert(InVT.isVector() && "can not widen non vector type");
1751  MVT WidenInVT = MVT::getVectorVT(InVT.getVectorElementType(), WidenNumElts);
1752  InOp1 = GetWidenedVector(InOp1);
1753  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
1754
1755  // Assume that the input and output will be widen appropriately.  If not,
1756  // we will have to unroll it at some point.
1757  assert(InOp1.getValueType() == WidenInVT &&
1758         InOp2.getValueType() == WidenInVT &&
1759         "Input not widened to expected type!");
1760  return DAG.getNode(ISD::VSETCC, N->getDebugLoc(),
1761                     WidenVT, InOp1, InOp2, N->getOperand(2));
1762}
1763
1764
1765//===----------------------------------------------------------------------===//
1766// Widen Vector Operand
1767//===----------------------------------------------------------------------===//
1768bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned ResNo) {
1769  DEBUG(cerr << "Widen node operand " << ResNo << ": "; N->dump(&DAG);
1770        cerr << "\n");
1771  SDValue Res = SDValue();
1772
1773  switch (N->getOpcode()) {
1774  default:
1775#ifndef NDEBUG
1776    cerr << "WidenVectorOperand op #" << ResNo << ": ";
1777    N->dump(&DAG); cerr << "\n";
1778#endif
1779    assert(0 && "Do not know how to widen this operator's operand!");
1780    abort();
1781
1782  case ISD::BIT_CONVERT:        Res = WidenVecOp_BIT_CONVERT(N); break;
1783  case ISD::CONCAT_VECTORS:     Res = WidenVecOp_CONCAT_VECTORS(N); break;
1784  case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
1785  case ISD::STORE:              Res = WidenVecOp_STORE(N); break;
1786
1787  case ISD::FP_ROUND:
1788  case ISD::FP_TO_SINT:
1789  case ISD::FP_TO_UINT:
1790  case ISD::SINT_TO_FP:
1791  case ISD::UINT_TO_FP:
1792  case ISD::TRUNCATE:
1793  case ISD::SIGN_EXTEND:
1794  case ISD::ZERO_EXTEND:
1795  case ISD::ANY_EXTEND:
1796    Res = WidenVecOp_Convert(N);
1797    break;
1798  }
1799
1800  // If Res is null, the sub-method took care of registering the result.
1801  if (!Res.getNode()) return false;
1802
1803  // If the result is N, the sub-method updated N in place.  Tell the legalizer
1804  // core about this.
1805  if (Res.getNode() == N)
1806    return true;
1807
1808
1809  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1810         "Invalid operand expansion");
1811
1812  ReplaceValueWith(SDValue(N, 0), Res);
1813  return false;
1814}
1815
1816SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
1817  // Since the result is legal and the input is illegal, it is unlikely
1818  // that we can fix the input to a legal type so unroll the convert
1819  // into some scalar code and create a nasty build vector.
1820  MVT VT = N->getValueType(0);
1821  MVT EltVT = VT.getVectorElementType();
1822  DebugLoc dl = N->getDebugLoc();
1823  unsigned NumElts = VT.getVectorNumElements();
1824  SDValue InOp = N->getOperand(0);
1825  if (getTypeAction(InOp.getValueType()) == WidenVector)
1826    InOp = GetWidenedVector(InOp);
1827  MVT InVT = InOp.getValueType();
1828  MVT InEltVT = InVT.getVectorElementType();
1829
1830  unsigned Opcode = N->getOpcode();
1831  SmallVector<SDValue, 16> Ops(NumElts);
1832  for (unsigned i=0; i < NumElts; ++i)
1833    Ops[i] = DAG.getNode(Opcode, dl, EltVT,
1834                         DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
1835                                     DAG.getIntPtrConstant(i)));
1836
1837  return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts);
1838}
1839
1840SDValue DAGTypeLegalizer::WidenVecOp_BIT_CONVERT(SDNode *N) {
1841  MVT VT = N->getValueType(0);
1842  SDValue InOp = GetWidenedVector(N->getOperand(0));
1843  MVT InWidenVT = InOp.getValueType();
1844  DebugLoc dl = N->getDebugLoc();
1845
1846  // Check if we can convert between two legal vector types and extract.
1847  unsigned InWidenSize = InWidenVT.getSizeInBits();
1848  unsigned Size = VT.getSizeInBits();
1849  if (InWidenSize % Size == 0 && !VT.isVector()) {
1850    unsigned NewNumElts = InWidenSize / Size;
1851    MVT NewVT = MVT::getVectorVT(VT, NewNumElts);
1852    if (TLI.isTypeLegal(NewVT)) {
1853      SDValue BitOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVT, InOp);
1854      return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp,
1855                         DAG.getIntPtrConstant(0));
1856    }
1857  }
1858
1859  return CreateStackStoreLoad(InOp, VT);
1860}
1861
1862SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
1863  // If the input vector is not legal, it is likely that we will not find a
1864  // legal vector of the same size. Replace the concatenate vector with a
1865  // nasty build vector.
1866  MVT VT = N->getValueType(0);
1867  MVT EltVT = VT.getVectorElementType();
1868  DebugLoc dl = N->getDebugLoc();
1869  unsigned NumElts = VT.getVectorNumElements();
1870  SmallVector<SDValue, 16> Ops(NumElts);
1871
1872  MVT InVT = N->getOperand(0).getValueType();
1873  unsigned NumInElts = InVT.getVectorNumElements();
1874
1875  unsigned Idx = 0;
1876  unsigned NumOperands = N->getNumOperands();
1877  for (unsigned i=0; i < NumOperands; ++i) {
1878    SDValue InOp = N->getOperand(i);
1879    if (getTypeAction(InOp.getValueType()) == WidenVector)
1880      InOp = GetWidenedVector(InOp);
1881    for (unsigned j=0; j < NumInElts; ++j)
1882      Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
1883                               DAG.getIntPtrConstant(j));
1884  }
1885  return DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &Ops[0], NumElts);
1886}
1887
1888SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
1889  SDValue InOp = GetWidenedVector(N->getOperand(0));
1890  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, N->getDebugLoc(),
1891                     N->getValueType(0), InOp, N->getOperand(1));
1892}
1893
1894SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
1895  // We have to widen the value but we want only to store the original
1896  // vector type.
1897  StoreSDNode *ST = cast<StoreSDNode>(N);
1898  SDValue  Chain = ST->getChain();
1899  SDValue  BasePtr = ST->getBasePtr();
1900  const    Value *SV = ST->getSrcValue();
1901  int      SVOffset = ST->getSrcValueOffset();
1902  unsigned Align = ST->getAlignment();
1903  bool     isVolatile = ST->isVolatile();
1904  SDValue  ValOp = GetWidenedVector(ST->getValue());
1905  DebugLoc dl = N->getDebugLoc();
1906
1907  MVT StVT = ST->getMemoryVT();
1908  MVT ValVT = ValOp.getValueType();
1909  // It must be true that we the widen vector type is bigger than where
1910  // we need to store.
1911  assert(StVT.isVector() && ValOp.getValueType().isVector());
1912  assert(StVT.bitsLT(ValOp.getValueType()));
1913
1914  SmallVector<SDValue, 16> StChain;
1915  if (ST->isTruncatingStore()) {
1916    // For truncating stores, we can not play the tricks of chopping legal
1917    // vector types and bit cast it to the right type.  Instead, we unroll
1918    // the store.
1919    MVT StEltVT  = StVT.getVectorElementType();
1920    MVT ValEltVT = ValVT.getVectorElementType();
1921    unsigned Increment = ValEltVT.getSizeInBits() / 8;
1922    unsigned NumElts = StVT.getVectorNumElements();
1923    SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
1924                              DAG.getIntPtrConstant(0));
1925    StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, BasePtr, SV,
1926                                        SVOffset, StEltVT,
1927                                        isVolatile, Align));
1928    unsigned Offset = Increment;
1929    for (unsigned i=1; i < NumElts; ++i, Offset += Increment) {
1930      SDValue NewBasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(),
1931                                       BasePtr, DAG.getIntPtrConstant(Offset));
1932      SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ValEltVT, ValOp,
1933                              DAG.getIntPtrConstant(0));
1934      StChain.push_back(DAG.getTruncStore(Chain, dl, EOp, NewBasePtr, SV,
1935                                          SVOffset + Offset, StEltVT,
1936                                          isVolatile, MinAlign(Align, Offset)));
1937    }
1938  }
1939  else {
1940    assert(StVT.getVectorElementType() == ValVT.getVectorElementType());
1941    // Store value
1942    GenWidenVectorStores(StChain, Chain, BasePtr, SV, SVOffset,
1943                         Align, isVolatile, ValOp, StVT.getSizeInBits(), dl);
1944  }
1945  if (StChain.size() == 1)
1946    return StChain[0];
1947  else
1948    return DAG.getNode(ISD::TokenFactor, dl,
1949                       MVT::Other,&StChain[0],StChain.size());
1950}
1951
1952//===----------------------------------------------------------------------===//
1953// Vector Widening Utilities
1954//===----------------------------------------------------------------------===//
1955
1956
1957// Utility function to find a vector type and its associated element
1958// type from a preferred width and whose vector type must be the same size
1959// as the VecVT.
1960//  TLI:   Target lowering used to determine legal types.
1961//  Width: Preferred width to store.
1962//  VecVT: Vector value type whose size we must match.
1963// Returns NewVecVT and NewEltVT - the vector type and its associated
1964// element type.
1965static void FindAssocWidenVecType(const TargetLowering &TLI, unsigned Width,
1966                                  MVT VecVT,
1967                                  MVT& NewEltVT, MVT& NewVecVT) {
1968  unsigned EltWidth = Width + 1;
1969  if (TLI.isTypeLegal(VecVT)) {
1970    // We start with the preferred with, making it a power of 2 and find a
1971    // legal vector type of that width.  If not, we reduce it by another of 2.
1972    // For incoming type is legal, this process will end as a vector of the
1973    // smallest loadable type should always be legal.
1974    do {
1975      assert(EltWidth > 0);
1976      EltWidth = 1 << Log2_32(EltWidth - 1);
1977      NewEltVT = MVT::getIntegerVT(EltWidth);
1978      unsigned NumElts = VecVT.getSizeInBits() / EltWidth;
1979      NewVecVT = MVT::getVectorVT(NewEltVT, NumElts);
1980    } while (!TLI.isTypeLegal(NewVecVT) ||
1981             VecVT.getSizeInBits() != NewVecVT.getSizeInBits());
1982  } else {
1983    // The incoming vector type is illegal and is the result of widening
1984    // a vector to a power of 2. In this case, we will use the preferred
1985    // with as long as it is a multiple of the incoming vector length.
1986    // The legalization process will eventually make this into a legal type
1987    // and remove the illegal bit converts (which would turn to stack converts
1988    // if they are allow to exist).
1989     do {
1990      assert(EltWidth > 0);
1991      EltWidth = 1 << Log2_32(EltWidth - 1);
1992      NewEltVT = MVT::getIntegerVT(EltWidth);
1993      unsigned NumElts = VecVT.getSizeInBits() / EltWidth;
1994      NewVecVT = MVT::getVectorVT(NewEltVT, NumElts);
1995    } while (!TLI.isTypeLegal(NewEltVT) ||
1996             VecVT.getSizeInBits() != NewVecVT.getSizeInBits());
1997  }
1998}
1999
2000SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVector<SDValue, 16>& LdChain,
2001                                              SDValue      Chain,
2002                                              SDValue      BasePtr,
2003                                              const Value *SV,
2004                                              int          SVOffset,
2005                                              unsigned     Alignment,
2006                                              bool         isVolatile,
2007                                              unsigned     LdWidth,
2008                                              MVT          ResType,
2009                                              DebugLoc     dl) {
2010  // The strategy assumes that we can efficiently load powers of two widths.
2011  // The routines chops the vector into the largest power of 2 load and
2012  // can be inserted into a legal vector and then cast the result into the
2013  // vector type we want.  This avoids unnecessary stack converts.
2014
2015  // TODO: If the Ldwidth is legal, alignment is the same as the LdWidth, and
2016  //       the load is nonvolatile, we an use a wider load for the value.
2017
2018  // Find the vector type that can load from.
2019  MVT NewEltVT, NewVecVT;
2020  unsigned NewEltVTWidth;
2021  FindAssocWidenVecType(TLI, LdWidth, ResType, NewEltVT, NewVecVT);
2022  NewEltVTWidth = NewEltVT.getSizeInBits();
2023
2024  SDValue LdOp = DAG.getLoad(NewEltVT, dl, Chain, BasePtr, SV, SVOffset,
2025                             isVolatile, Alignment);
2026  SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
2027  LdChain.push_back(LdOp.getValue(1));
2028
2029  // Check if we can load the element with one instruction
2030  if (LdWidth == NewEltVTWidth) {
2031    return DAG.getNode(ISD::BIT_CONVERT, dl, ResType, VecOp);
2032  }
2033
2034  unsigned Idx = 1;
2035  LdWidth -= NewEltVTWidth;
2036  unsigned Offset = 0;
2037
2038  while (LdWidth > 0) {
2039    unsigned Increment = NewEltVTWidth / 8;
2040    Offset += Increment;
2041    BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
2042                          DAG.getIntPtrConstant(Increment));
2043
2044    if (LdWidth < NewEltVTWidth) {
2045      // Our current type we are using is too large, use a smaller size by
2046      // using a smaller power of 2
2047      unsigned oNewEltVTWidth = NewEltVTWidth;
2048      FindAssocWidenVecType(TLI, LdWidth, ResType, NewEltVT, NewVecVT);
2049      NewEltVTWidth = NewEltVT.getSizeInBits();
2050      // Readjust position and vector position based on new load type
2051      Idx = Idx * (oNewEltVTWidth/NewEltVTWidth);
2052      VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, VecOp);
2053    }
2054
2055    SDValue LdOp = DAG.getLoad(NewEltVT, dl, Chain, BasePtr, SV,
2056                                 SVOffset+Offset, isVolatile,
2057                                 MinAlign(Alignment, Offset));
2058    LdChain.push_back(LdOp.getValue(1));
2059    VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOp,
2060                        DAG.getIntPtrConstant(Idx++));
2061
2062    LdWidth -= NewEltVTWidth;
2063  }
2064
2065  return DAG.getNode(ISD::BIT_CONVERT, dl, ResType, VecOp);
2066}
2067
2068void DAGTypeLegalizer::GenWidenVectorStores(SmallVector<SDValue, 16>& StChain,
2069                                            SDValue   Chain,
2070                                            SDValue   BasePtr,
2071                                            const Value *SV,
2072                                            int         SVOffset,
2073                                            unsigned    Alignment,
2074                                            bool        isVolatile,
2075                                            SDValue     ValOp,
2076                                            unsigned    StWidth,
2077                                            DebugLoc    dl) {
2078  // Breaks the stores into a series of power of 2 width stores.  For any
2079  // width, we convert the vector to the vector of element size that we
2080  // want to store.  This avoids requiring a stack convert.
2081
2082  // Find a width of the element type we can store with
2083  MVT WidenVT = ValOp.getValueType();
2084  MVT NewEltVT, NewVecVT;
2085
2086  FindAssocWidenVecType(TLI, StWidth, WidenVT, NewEltVT, NewVecVT);
2087  unsigned NewEltVTWidth = NewEltVT.getSizeInBits();
2088
2089  SDValue VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, ValOp);
2090  SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewEltVT, VecOp,
2091                            DAG.getIntPtrConstant(0));
2092  SDValue StOp = DAG.getStore(Chain, dl, EOp, BasePtr, SV, SVOffset,
2093                               isVolatile, Alignment);
2094  StChain.push_back(StOp);
2095
2096  // Check if we are done
2097  if (StWidth == NewEltVTWidth) {
2098    return;
2099  }
2100
2101  unsigned Idx = 1;
2102  StWidth -= NewEltVTWidth;
2103  unsigned Offset = 0;
2104
2105  while (StWidth > 0) {
2106    unsigned Increment = NewEltVTWidth / 8;
2107    Offset += Increment;
2108    BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr,
2109                          DAG.getIntPtrConstant(Increment));
2110
2111    if (StWidth < NewEltVTWidth) {
2112      // Our current type we are using is too large, use a smaller size by
2113      // using a smaller power of 2
2114      unsigned oNewEltVTWidth = NewEltVTWidth;
2115      FindAssocWidenVecType(TLI, StWidth, WidenVT, NewEltVT, NewVecVT);
2116      NewEltVTWidth = NewEltVT.getSizeInBits();
2117      // Readjust position and vector position based on new load type
2118      Idx = Idx * (oNewEltVTWidth/NewEltVTWidth);
2119      VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, NewVecVT, VecOp);
2120    }
2121
2122    EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewEltVT, VecOp,
2123                      DAG.getIntPtrConstant(Idx++));
2124    StChain.push_back(DAG.getStore(Chain, dl, EOp, BasePtr, SV,
2125                                   SVOffset + Offset, isVolatile,
2126                                   MinAlign(Alignment, Offset)));
2127    StWidth -= NewEltVTWidth;
2128  }
2129}
2130
2131/// Modifies a vector input (widen or narrows) to a vector of NVT.  The
2132/// input vector must have the same element type as NVT.
2133SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, MVT NVT) {
2134  // Note that InOp might have been widened so it might already have
2135  // the right width or it might need be narrowed.
2136  MVT InVT = InOp.getValueType();
2137  assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
2138         "input and widen element type must match");
2139  DebugLoc dl = InOp.getDebugLoc();
2140
2141  // Check if InOp already has the right width.
2142  if (InVT == NVT)
2143    return InOp;
2144
2145  unsigned InNumElts = InVT.getVectorNumElements();
2146  unsigned WidenNumElts = NVT.getVectorNumElements();
2147  if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) {
2148    unsigned NumConcat = WidenNumElts / InNumElts;
2149    SmallVector<SDValue, 16> Ops(NumConcat);
2150    SDValue UndefVal = DAG.getUNDEF(InVT);
2151    Ops[0] = InOp;
2152    for (unsigned i = 1; i != NumConcat; ++i)
2153      Ops[i] = UndefVal;
2154
2155    return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, &Ops[0], NumConcat);
2156  }
2157
2158  if (WidenNumElts < InNumElts && InNumElts % WidenNumElts)
2159    return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp,
2160                       DAG.getIntPtrConstant(0));
2161
2162  // Fall back to extract and build.
2163  SmallVector<SDValue, 16> Ops(WidenNumElts);
2164  MVT EltVT = NVT.getVectorElementType();
2165  unsigned MinNumElts = std::min(WidenNumElts, InNumElts);
2166  unsigned Idx;
2167  for (Idx = 0; Idx < MinNumElts; ++Idx)
2168    Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
2169                           DAG.getIntPtrConstant(Idx));
2170
2171  SDValue UndefVal = DAG.getUNDEF(EltVT);
2172  for ( ; Idx < WidenNumElts; ++Idx)
2173    Ops[Idx] = UndefVal;
2174  return DAG.getNode(ISD::BUILD_VECTOR, dl, NVT, &Ops[0], WidenNumElts);
2175}
2176