LegalizeDAG.cpp revision 2920d2b7b601d9dfcec783cb91316f35b6811be1
1//===-- LegalizeDAG.cpp - Implement SelectionDAG::Legalize ----------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the SelectionDAG::Legalize method.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/CodeGen/SelectionDAG.h"
15#include "llvm/CodeGen/MachineFunction.h"
16#include "llvm/CodeGen/MachineFrameInfo.h"
17#include "llvm/CodeGen/MachineJumpTableInfo.h"
18#include "llvm/CodeGen/MachineModuleInfo.h"
19#include "llvm/CodeGen/PseudoSourceValue.h"
20#include "llvm/Target/TargetFrameInfo.h"
21#include "llvm/Target/TargetLowering.h"
22#include "llvm/Target/TargetData.h"
23#include "llvm/Target/TargetMachine.h"
24#include "llvm/Target/TargetOptions.h"
25#include "llvm/Target/TargetSubtarget.h"
26#include "llvm/CallingConv.h"
27#include "llvm/Constants.h"
28#include "llvm/DerivedTypes.h"
29#include "llvm/Support/CommandLine.h"
30#include "llvm/Support/Compiler.h"
31#include "llvm/Support/MathExtras.h"
32#include "llvm/ADT/DenseMap.h"
33#include "llvm/ADT/SmallVector.h"
34#include "llvm/ADT/SmallPtrSet.h"
35#include <map>
36using namespace llvm;
37
38#ifndef NDEBUG
39static cl::opt<bool>
40ViewLegalizeDAGs("view-legalize-dags", cl::Hidden,
41                 cl::desc("Pop up a window to show dags before legalize"));
42#else
43static const bool ViewLegalizeDAGs = 0;
44#endif
45
46//===----------------------------------------------------------------------===//
47/// SelectionDAGLegalize - This takes an arbitrary SelectionDAG as input and
48/// hacks on it until the target machine can handle it.  This involves
49/// eliminating value sizes the machine cannot handle (promoting small sizes to
50/// large sizes or splitting up large values into small values) as well as
51/// eliminating operations the machine cannot handle.
52///
53/// This code also does a small amount of optimization and recognition of idioms
54/// as part of its processing.  For example, if a target does not support a
55/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
56/// will attempt merge setcc and brc instructions into brcc's.
57///
58namespace {
59class VISIBILITY_HIDDEN SelectionDAGLegalize {
60  TargetLowering &TLI;
61  SelectionDAG &DAG;
62
63  // Libcall insertion helpers.
64
65  /// LastCALLSEQ_END - This keeps track of the CALLSEQ_END node that has been
66  /// legalized.  We use this to ensure that calls are properly serialized
67  /// against each other, including inserted libcalls.
68  SDOperand LastCALLSEQ_END;
69
70  /// IsLegalizingCall - This member is used *only* for purposes of providing
71  /// helpful assertions that a libcall isn't created while another call is
72  /// being legalized (which could lead to non-serialized call sequences).
73  bool IsLegalizingCall;
74
75  enum LegalizeAction {
76    Legal,      // The target natively supports this operation.
77    Promote,    // This operation should be executed in a larger type.
78    Expand      // Try to expand this to other ops, otherwise use a libcall.
79  };
80
81  /// ValueTypeActions - This is a bitvector that contains two bits for each
82  /// value type, where the two bits correspond to the LegalizeAction enum.
83  /// This can be queried with "getTypeAction(VT)".
84  TargetLowering::ValueTypeActionImpl ValueTypeActions;
85
86  /// LegalizedNodes - For nodes that are of legal width, and that have more
87  /// than one use, this map indicates what regularized operand to use.  This
88  /// allows us to avoid legalizing the same thing more than once.
89  DenseMap<SDOperand, SDOperand> LegalizedNodes;
90
91  /// PromotedNodes - For nodes that are below legal width, and that have more
92  /// than one use, this map indicates what promoted value to use.  This allows
93  /// us to avoid promoting the same thing more than once.
94  DenseMap<SDOperand, SDOperand> PromotedNodes;
95
96  /// ExpandedNodes - For nodes that need to be expanded this map indicates
97  /// which which operands are the expanded version of the input.  This allows
98  /// us to avoid expanding the same node more than once.
99  DenseMap<SDOperand, std::pair<SDOperand, SDOperand> > ExpandedNodes;
100
101  /// SplitNodes - For vector nodes that need to be split, this map indicates
102  /// which which operands are the split version of the input.  This allows us
103  /// to avoid splitting the same node more than once.
104  std::map<SDOperand, std::pair<SDOperand, SDOperand> > SplitNodes;
105
106  /// ScalarizedNodes - For nodes that need to be converted from vector types to
107  /// scalar types, this contains the mapping of ones we have already
108  /// processed to the result.
109  std::map<SDOperand, SDOperand> ScalarizedNodes;
110
111  void AddLegalizedOperand(SDOperand From, SDOperand To) {
112    LegalizedNodes.insert(std::make_pair(From, To));
113    // If someone requests legalization of the new node, return itself.
114    if (From != To)
115      LegalizedNodes.insert(std::make_pair(To, To));
116  }
117  void AddPromotedOperand(SDOperand From, SDOperand To) {
118    bool isNew = PromotedNodes.insert(std::make_pair(From, To)).second;
119    assert(isNew && "Got into the map somehow?");
120    // If someone requests legalization of the new node, return itself.
121    LegalizedNodes.insert(std::make_pair(To, To));
122  }
123
124public:
125  explicit SelectionDAGLegalize(SelectionDAG &DAG);
126
127  /// getTypeAction - Return how we should legalize values of this type, either
128  /// it is already legal or we need to expand it into multiple registers of
129  /// smaller integer type, or we need to promote it to a larger type.
130  LegalizeAction getTypeAction(MVT VT) const {
131    return (LegalizeAction)ValueTypeActions.getTypeAction(VT);
132  }
133
134  /// isTypeLegal - Return true if this type is legal on this target.
135  ///
136  bool isTypeLegal(MVT VT) const {
137    return getTypeAction(VT) == Legal;
138  }
139
140  void LegalizeDAG();
141
142private:
143  /// HandleOp - Legalize, Promote, or Expand the specified operand as
144  /// appropriate for its type.
145  void HandleOp(SDOperand Op);
146
147  /// LegalizeOp - We know that the specified value has a legal type.
148  /// Recursively ensure that the operands have legal types, then return the
149  /// result.
150  SDOperand LegalizeOp(SDOperand O);
151
152  /// UnrollVectorOp - We know that the given vector has a legal type, however
153  /// the operation it performs is not legal and is an operation that we have
154  /// no way of lowering.  "Unroll" the vector, splitting out the scalars and
155  /// operating on each element individually.
156  SDOperand UnrollVectorOp(SDOperand O);
157
158  /// PerformInsertVectorEltInMemory - Some target cannot handle a variable
159  /// insertion index for the INSERT_VECTOR_ELT instruction.  In this case, it
160  /// is necessary to spill the vector being inserted into to memory, perform
161  /// the insert there, and then read the result back.
162  SDOperand PerformInsertVectorEltInMemory(SDOperand Vec, SDOperand Val,
163                                           SDOperand Idx);
164
165  /// PromoteOp - Given an operation that produces a value in an invalid type,
166  /// promote it to compute the value into a larger type.  The produced value
167  /// will have the correct bits for the low portion of the register, but no
168  /// guarantee is made about the top bits: it may be zero, sign-extended, or
169  /// garbage.
170  SDOperand PromoteOp(SDOperand O);
171
172  /// ExpandOp - Expand the specified SDOperand into its two component pieces
173  /// Lo&Hi.  Note that the Op MUST be an expanded type.  As a result of this,
174  /// the LegalizeNodes map is filled in for any results that are not expanded,
175  /// the ExpandedNodes map is filled in for any results that are expanded, and
176  /// the Lo/Hi values are returned.   This applies to integer types and Vector
177  /// types.
178  void ExpandOp(SDOperand O, SDOperand &Lo, SDOperand &Hi);
179
180  /// SplitVectorOp - Given an operand of vector type, break it down into
181  /// two smaller values.
182  void SplitVectorOp(SDOperand O, SDOperand &Lo, SDOperand &Hi);
183
184  /// ScalarizeVectorOp - Given an operand of single-element vector type
185  /// (e.g. v1f32), convert it into the equivalent operation that returns a
186  /// scalar (e.g. f32) value.
187  SDOperand ScalarizeVectorOp(SDOperand O);
188
189  /// isShuffleLegal - Return true if a vector shuffle is legal with the
190  /// specified mask and type.  Targets can specify exactly which masks they
191  /// support and the code generator is tasked with not creating illegal masks.
192  ///
193  /// Note that this will also return true for shuffles that are promoted to a
194  /// different type.
195  ///
196  /// If this is a legal shuffle, this method returns the (possibly promoted)
197  /// build_vector Mask.  If it's not a legal shuffle, it returns null.
198  SDNode *isShuffleLegal(MVT VT, SDOperand Mask) const;
199
200  bool LegalizeAllNodesNotLeadingTo(SDNode *N, SDNode *Dest,
201                                    SmallPtrSet<SDNode*, 32> &NodesLeadingTo);
202
203  void LegalizeSetCCOperands(SDOperand &LHS, SDOperand &RHS, SDOperand &CC);
204
205  SDOperand ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned,
206                          SDOperand &Hi);
207  SDOperand ExpandIntToFP(bool isSigned, MVT DestTy, SDOperand Source);
208
209  SDOperand EmitStackConvert(SDOperand SrcOp, MVT SlotVT, MVT DestVT);
210  SDOperand ExpandBUILD_VECTOR(SDNode *Node);
211  SDOperand ExpandSCALAR_TO_VECTOR(SDNode *Node);
212  SDOperand ExpandLegalINT_TO_FP(bool isSigned, SDOperand LegalOp, MVT DestVT);
213  SDOperand PromoteLegalINT_TO_FP(SDOperand LegalOp, MVT DestVT, bool isSigned);
214  SDOperand PromoteLegalFP_TO_INT(SDOperand LegalOp, MVT DestVT, bool isSigned);
215
216  SDOperand ExpandBSWAP(SDOperand Op);
217  SDOperand ExpandBitCount(unsigned Opc, SDOperand Op);
218  bool ExpandShift(unsigned Opc, SDOperand Op, SDOperand Amt,
219                   SDOperand &Lo, SDOperand &Hi);
220  void ExpandShiftParts(unsigned NodeOp, SDOperand Op, SDOperand Amt,
221                        SDOperand &Lo, SDOperand &Hi);
222
223  SDOperand ExpandEXTRACT_SUBVECTOR(SDOperand Op);
224  SDOperand ExpandEXTRACT_VECTOR_ELT(SDOperand Op);
225};
226}
227
228/// isVectorShuffleLegal - Return true if a vector shuffle is legal with the
229/// specified mask and type.  Targets can specify exactly which masks they
230/// support and the code generator is tasked with not creating illegal masks.
231///
232/// Note that this will also return true for shuffles that are promoted to a
233/// different type.
234SDNode *SelectionDAGLegalize::isShuffleLegal(MVT VT, SDOperand Mask) const {
235  switch (TLI.getOperationAction(ISD::VECTOR_SHUFFLE, VT)) {
236  default: return 0;
237  case TargetLowering::Legal:
238  case TargetLowering::Custom:
239    break;
240  case TargetLowering::Promote: {
241    // If this is promoted to a different type, convert the shuffle mask and
242    // ask if it is legal in the promoted type!
243    MVT NVT = TLI.getTypeToPromoteTo(ISD::VECTOR_SHUFFLE, VT);
244
245    // If we changed # elements, change the shuffle mask.
246    unsigned NumEltsGrowth =
247      NVT.getVectorNumElements() / VT.getVectorNumElements();
248    assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
249    if (NumEltsGrowth > 1) {
250      // Renumber the elements.
251      SmallVector<SDOperand, 8> Ops;
252      for (unsigned i = 0, e = Mask.getNumOperands(); i != e; ++i) {
253        SDOperand InOp = Mask.getOperand(i);
254        for (unsigned j = 0; j != NumEltsGrowth; ++j) {
255          if (InOp.getOpcode() == ISD::UNDEF)
256            Ops.push_back(DAG.getNode(ISD::UNDEF, MVT::i32));
257          else {
258            unsigned InEltNo = cast<ConstantSDNode>(InOp)->getValue();
259            Ops.push_back(DAG.getConstant(InEltNo*NumEltsGrowth+j, MVT::i32));
260          }
261        }
262      }
263      Mask = DAG.getNode(ISD::BUILD_VECTOR, NVT, &Ops[0], Ops.size());
264    }
265    VT = NVT;
266    break;
267  }
268  }
269  return TLI.isShuffleMaskLegal(Mask, VT) ? Mask.Val : 0;
270}
271
272SelectionDAGLegalize::SelectionDAGLegalize(SelectionDAG &dag)
273  : TLI(dag.getTargetLoweringInfo()), DAG(dag),
274    ValueTypeActions(TLI.getValueTypeActions()) {
275  assert(MVT::LAST_VALUETYPE <= 32 &&
276         "Too many value types for ValueTypeActions to hold!");
277}
278
279/// ComputeTopDownOrdering - Compute a top-down ordering of the dag, where Order
280/// contains all of a nodes operands before it contains the node.
281static void ComputeTopDownOrdering(SelectionDAG &DAG,
282                                   SmallVector<SDNode*, 64> &Order) {
283
284  DenseMap<SDNode*, unsigned> Visited;
285  std::vector<SDNode*> Worklist;
286  Worklist.reserve(128);
287
288  // Compute ordering from all of the leaves in the graphs, those (like the
289  // entry node) that have no operands.
290  for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
291       E = DAG.allnodes_end(); I != E; ++I) {
292    if (I->getNumOperands() == 0) {
293      Visited[I] = 0 - 1U;
294      Worklist.push_back(I);
295    }
296  }
297
298  while (!Worklist.empty()) {
299    SDNode *N = Worklist.back();
300    Worklist.pop_back();
301
302    if (++Visited[N] != N->getNumOperands())
303      continue;  // Haven't visited all operands yet
304
305    Order.push_back(N);
306
307    // Now that we have N in, add anything that uses it if all of their operands
308    // are now done.
309    for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end();
310         UI != E; ++UI)
311      Worklist.push_back(UI->getUser());
312  }
313
314  assert(Order.size() == Visited.size() &&
315         Order.size() == DAG.allnodes_size() &&
316         "Error: DAG is cyclic!");
317}
318
319
320void SelectionDAGLegalize::LegalizeDAG() {
321  LastCALLSEQ_END = DAG.getEntryNode();
322  IsLegalizingCall = false;
323
324  // The legalize process is inherently a bottom-up recursive process (users
325  // legalize their uses before themselves).  Given infinite stack space, we
326  // could just start legalizing on the root and traverse the whole graph.  In
327  // practice however, this causes us to run out of stack space on large basic
328  // blocks.  To avoid this problem, compute an ordering of the nodes where each
329  // node is only legalized after all of its operands are legalized.
330  SmallVector<SDNode*, 64> Order;
331  ComputeTopDownOrdering(DAG, Order);
332
333  for (unsigned i = 0, e = Order.size(); i != e; ++i)
334    HandleOp(SDOperand(Order[i], 0));
335
336  // Finally, it's possible the root changed.  Get the new root.
337  SDOperand OldRoot = DAG.getRoot();
338  assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?");
339  DAG.setRoot(LegalizedNodes[OldRoot]);
340
341  ExpandedNodes.clear();
342  LegalizedNodes.clear();
343  PromotedNodes.clear();
344  SplitNodes.clear();
345  ScalarizedNodes.clear();
346
347  // Remove dead nodes now.
348  DAG.RemoveDeadNodes();
349}
350
351
352/// FindCallEndFromCallStart - Given a chained node that is part of a call
353/// sequence, find the CALLSEQ_END node that terminates the call sequence.
354static SDNode *FindCallEndFromCallStart(SDNode *Node) {
355  if (Node->getOpcode() == ISD::CALLSEQ_END)
356    return Node;
357  if (Node->use_empty())
358    return 0;   // No CallSeqEnd
359
360  // The chain is usually at the end.
361  SDOperand TheChain(Node, Node->getNumValues()-1);
362  if (TheChain.getValueType() != MVT::Other) {
363    // Sometimes it's at the beginning.
364    TheChain = SDOperand(Node, 0);
365    if (TheChain.getValueType() != MVT::Other) {
366      // Otherwise, hunt for it.
367      for (unsigned i = 1, e = Node->getNumValues(); i != e; ++i)
368        if (Node->getValueType(i) == MVT::Other) {
369          TheChain = SDOperand(Node, i);
370          break;
371        }
372
373      // Otherwise, we walked into a node without a chain.
374      if (TheChain.getValueType() != MVT::Other)
375        return 0;
376    }
377  }
378
379  for (SDNode::use_iterator UI = Node->use_begin(),
380       E = Node->use_end(); UI != E; ++UI) {
381
382    // Make sure to only follow users of our token chain.
383    SDNode *User = UI->getUser();
384    for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i)
385      if (User->getOperand(i) == TheChain)
386        if (SDNode *Result = FindCallEndFromCallStart(User))
387          return Result;
388  }
389  return 0;
390}
391
392/// FindCallStartFromCallEnd - Given a chained node that is part of a call
393/// sequence, find the CALLSEQ_START node that initiates the call sequence.
394static SDNode *FindCallStartFromCallEnd(SDNode *Node) {
395  assert(Node && "Didn't find callseq_start for a call??");
396  if (Node->getOpcode() == ISD::CALLSEQ_START) return Node;
397
398  assert(Node->getOperand(0).getValueType() == MVT::Other &&
399         "Node doesn't have a token chain argument!");
400  return FindCallStartFromCallEnd(Node->getOperand(0).Val);
401}
402
403/// LegalizeAllNodesNotLeadingTo - Recursively walk the uses of N, looking to
404/// see if any uses can reach Dest.  If no dest operands can get to dest,
405/// legalize them, legalize ourself, and return false, otherwise, return true.
406///
407/// Keep track of the nodes we fine that actually do lead to Dest in
408/// NodesLeadingTo.  This avoids retraversing them exponential number of times.
409///
410bool SelectionDAGLegalize::LegalizeAllNodesNotLeadingTo(SDNode *N, SDNode *Dest,
411                                     SmallPtrSet<SDNode*, 32> &NodesLeadingTo) {
412  if (N == Dest) return true;  // N certainly leads to Dest :)
413
414  // If we've already processed this node and it does lead to Dest, there is no
415  // need to reprocess it.
416  if (NodesLeadingTo.count(N)) return true;
417
418  // If the first result of this node has been already legalized, then it cannot
419  // reach N.
420  switch (getTypeAction(N->getValueType(0))) {
421  case Legal:
422    if (LegalizedNodes.count(SDOperand(N, 0))) return false;
423    break;
424  case Promote:
425    if (PromotedNodes.count(SDOperand(N, 0))) return false;
426    break;
427  case Expand:
428    if (ExpandedNodes.count(SDOperand(N, 0))) return false;
429    break;
430  }
431
432  // Okay, this node has not already been legalized.  Check and legalize all
433  // operands.  If none lead to Dest, then we can legalize this node.
434  bool OperandsLeadToDest = false;
435  for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
436    OperandsLeadToDest |=     // If an operand leads to Dest, so do we.
437      LegalizeAllNodesNotLeadingTo(N->getOperand(i).Val, Dest, NodesLeadingTo);
438
439  if (OperandsLeadToDest) {
440    NodesLeadingTo.insert(N);
441    return true;
442  }
443
444  // Okay, this node looks safe, legalize it and return false.
445  HandleOp(SDOperand(N, 0));
446  return false;
447}
448
449/// HandleOp - Legalize, Promote, or Expand the specified operand as
450/// appropriate for its type.
451void SelectionDAGLegalize::HandleOp(SDOperand Op) {
452  MVT VT = Op.getValueType();
453  switch (getTypeAction(VT)) {
454  default: assert(0 && "Bad type action!");
455  case Legal:   (void)LegalizeOp(Op); break;
456  case Promote: (void)PromoteOp(Op); break;
457  case Expand:
458    if (!VT.isVector()) {
459      // If this is an illegal scalar, expand it into its two component
460      // pieces.
461      SDOperand X, Y;
462      if (Op.getOpcode() == ISD::TargetConstant)
463        break;  // Allow illegal target nodes.
464      ExpandOp(Op, X, Y);
465    } else if (VT.getVectorNumElements() == 1) {
466      // If this is an illegal single element vector, convert it to a
467      // scalar operation.
468      (void)ScalarizeVectorOp(Op);
469    } else {
470      // Otherwise, this is an illegal multiple element vector.
471      // Split it in half and legalize both parts.
472      SDOperand X, Y;
473      SplitVectorOp(Op, X, Y);
474    }
475    break;
476  }
477}
478
479/// ExpandConstantFP - Expands the ConstantFP node to an integer constant or
480/// a load from the constant pool.
481static SDOperand ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP,
482                                  SelectionDAG &DAG, TargetLowering &TLI) {
483  bool Extend = false;
484
485  // If a FP immediate is precise when represented as a float and if the
486  // target can do an extending load from float to double, we put it into
487  // the constant pool as a float, even if it's is statically typed as a
488  // double.  This shrinks FP constants and canonicalizes them for targets where
489  // an FP extending load is the same cost as a normal load (such as on the x87
490  // fp stack or PPC FP unit).
491  MVT VT = CFP->getValueType(0);
492  ConstantFP *LLVMC = ConstantFP::get(CFP->getValueAPF());
493  if (!UseCP) {
494    if (VT!=MVT::f64 && VT!=MVT::f32)
495      assert(0 && "Invalid type expansion");
496    return DAG.getConstant(LLVMC->getValueAPF().convertToAPInt(),
497                           (VT == MVT::f64) ? MVT::i64 : MVT::i32);
498  }
499
500  MVT OrigVT = VT;
501  MVT SVT = VT;
502  while (SVT != MVT::f32) {
503    SVT = (MVT::SimpleValueType)(SVT.getSimpleVT() - 1);
504    if (CFP->isValueValidForType(SVT, CFP->getValueAPF()) &&
505        // Only do this if the target has a native EXTLOAD instruction from
506        // smaller type.
507        TLI.isLoadXLegal(ISD::EXTLOAD, SVT) &&
508        TLI.ShouldShrinkFPConstant(OrigVT)) {
509      const Type *SType = SVT.getTypeForMVT();
510      LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType));
511      VT = SVT;
512      Extend = true;
513    }
514  }
515
516  SDOperand CPIdx = DAG.getConstantPool(LLVMC, TLI.getPointerTy());
517  if (Extend)
518    return DAG.getExtLoad(ISD::EXTLOAD, OrigVT, DAG.getEntryNode(),
519                          CPIdx, PseudoSourceValue::getConstantPool(),
520                          0, VT);
521  return DAG.getLoad(OrigVT, DAG.getEntryNode(), CPIdx,
522                     PseudoSourceValue::getConstantPool(), 0);
523}
524
525
526/// ExpandFCOPYSIGNToBitwiseOps - Expands fcopysign to a series of bitwise
527/// operations.
528static
529SDOperand ExpandFCOPYSIGNToBitwiseOps(SDNode *Node, MVT NVT,
530                                      SelectionDAG &DAG, TargetLowering &TLI) {
531  MVT VT = Node->getValueType(0);
532  MVT SrcVT = Node->getOperand(1).getValueType();
533  assert((SrcVT == MVT::f32 || SrcVT == MVT::f64) &&
534         "fcopysign expansion only supported for f32 and f64");
535  MVT SrcNVT = (SrcVT == MVT::f64) ? MVT::i64 : MVT::i32;
536
537  // First get the sign bit of second operand.
538  SDOperand Mask1 = (SrcVT == MVT::f64)
539    ? DAG.getConstantFP(BitsToDouble(1ULL << 63), SrcVT)
540    : DAG.getConstantFP(BitsToFloat(1U << 31), SrcVT);
541  Mask1 = DAG.getNode(ISD::BIT_CONVERT, SrcNVT, Mask1);
542  SDOperand SignBit= DAG.getNode(ISD::BIT_CONVERT, SrcNVT, Node->getOperand(1));
543  SignBit = DAG.getNode(ISD::AND, SrcNVT, SignBit, Mask1);
544  // Shift right or sign-extend it if the two operands have different types.
545  int SizeDiff = SrcNVT.getSizeInBits() - NVT.getSizeInBits();
546  if (SizeDiff > 0) {
547    SignBit = DAG.getNode(ISD::SRL, SrcNVT, SignBit,
548                          DAG.getConstant(SizeDiff, TLI.getShiftAmountTy()));
549    SignBit = DAG.getNode(ISD::TRUNCATE, NVT, SignBit);
550  } else if (SizeDiff < 0) {
551    SignBit = DAG.getNode(ISD::ZERO_EXTEND, NVT, SignBit);
552    SignBit = DAG.getNode(ISD::SHL, NVT, SignBit,
553                          DAG.getConstant(-SizeDiff, TLI.getShiftAmountTy()));
554  }
555
556  // Clear the sign bit of first operand.
557  SDOperand Mask2 = (VT == MVT::f64)
558    ? DAG.getConstantFP(BitsToDouble(~(1ULL << 63)), VT)
559    : DAG.getConstantFP(BitsToFloat(~(1U << 31)), VT);
560  Mask2 = DAG.getNode(ISD::BIT_CONVERT, NVT, Mask2);
561  SDOperand Result = DAG.getNode(ISD::BIT_CONVERT, NVT, Node->getOperand(0));
562  Result = DAG.getNode(ISD::AND, NVT, Result, Mask2);
563
564  // Or the value with the sign bit.
565  Result = DAG.getNode(ISD::OR, NVT, Result, SignBit);
566  return Result;
567}
568
569/// ExpandUnalignedStore - Expands an unaligned store to 2 half-size stores.
570static
571SDOperand ExpandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG,
572                               TargetLowering &TLI) {
573  SDOperand Chain = ST->getChain();
574  SDOperand Ptr = ST->getBasePtr();
575  SDOperand Val = ST->getValue();
576  MVT VT = Val.getValueType();
577  int Alignment = ST->getAlignment();
578  int SVOffset = ST->getSrcValueOffset();
579  if (ST->getMemoryVT().isFloatingPoint() ||
580      ST->getMemoryVT().isVector()) {
581    // Expand to a bitconvert of the value to the integer type of the
582    // same size, then a (misaligned) int store.
583    MVT intVT;
584    if (VT.is128BitVector() || VT == MVT::ppcf128 || VT == MVT::f128)
585      intVT = MVT::i128;
586    else if (VT.is64BitVector() || VT==MVT::f64)
587      intVT = MVT::i64;
588    else if (VT==MVT::f32)
589      intVT = MVT::i32;
590    else
591      assert(0 && "Unaligned store of unsupported type");
592
593    SDOperand Result = DAG.getNode(ISD::BIT_CONVERT, intVT, Val);
594    return DAG.getStore(Chain, Result, Ptr, ST->getSrcValue(),
595                        SVOffset, ST->isVolatile(), Alignment);
596  }
597  assert(ST->getMemoryVT().isInteger() &&
598         !ST->getMemoryVT().isVector() &&
599         "Unaligned store of unknown type.");
600  // Get the half-size VT
601  MVT NewStoredVT =
602    (MVT::SimpleValueType)(ST->getMemoryVT().getSimpleVT() - 1);
603  int NumBits = NewStoredVT.getSizeInBits();
604  int IncrementSize = NumBits / 8;
605
606  // Divide the stored value in two parts.
607  SDOperand ShiftAmount = DAG.getConstant(NumBits, TLI.getShiftAmountTy());
608  SDOperand Lo = Val;
609  SDOperand Hi = DAG.getNode(ISD::SRL, VT, Val, ShiftAmount);
610
611  // Store the two parts
612  SDOperand Store1, Store2;
613  Store1 = DAG.getTruncStore(Chain, TLI.isLittleEndian()?Lo:Hi, Ptr,
614                             ST->getSrcValue(), SVOffset, NewStoredVT,
615                             ST->isVolatile(), Alignment);
616  Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
617                    DAG.getConstant(IncrementSize, TLI.getPointerTy()));
618  Alignment = MinAlign(Alignment, IncrementSize);
619  Store2 = DAG.getTruncStore(Chain, TLI.isLittleEndian()?Hi:Lo, Ptr,
620                             ST->getSrcValue(), SVOffset + IncrementSize,
621                             NewStoredVT, ST->isVolatile(), Alignment);
622
623  return DAG.getNode(ISD::TokenFactor, MVT::Other, Store1, Store2);
624}
625
626/// ExpandUnalignedLoad - Expands an unaligned load to 2 half-size loads.
627static
628SDOperand ExpandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG,
629                              TargetLowering &TLI) {
630  int SVOffset = LD->getSrcValueOffset();
631  SDOperand Chain = LD->getChain();
632  SDOperand Ptr = LD->getBasePtr();
633  MVT VT = LD->getValueType(0);
634  MVT LoadedVT = LD->getMemoryVT();
635  if (VT.isFloatingPoint() || VT.isVector()) {
636    // Expand to a (misaligned) integer load of the same size,
637    // then bitconvert to floating point or vector.
638    MVT intVT;
639    if (LoadedVT.is128BitVector() ||
640         LoadedVT == MVT::ppcf128 || LoadedVT == MVT::f128)
641      intVT = MVT::i128;
642    else if (LoadedVT.is64BitVector() || LoadedVT == MVT::f64)
643      intVT = MVT::i64;
644    else if (LoadedVT == MVT::f32)
645      intVT = MVT::i32;
646    else
647      assert(0 && "Unaligned load of unsupported type");
648
649    SDOperand newLoad = DAG.getLoad(intVT, Chain, Ptr, LD->getSrcValue(),
650                                    SVOffset, LD->isVolatile(),
651                                    LD->getAlignment());
652    SDOperand Result = DAG.getNode(ISD::BIT_CONVERT, LoadedVT, newLoad);
653    if (VT.isFloatingPoint() && LoadedVT != VT)
654      Result = DAG.getNode(ISD::FP_EXTEND, VT, Result);
655
656    SDOperand Ops[] = { Result, Chain };
657    return DAG.getMergeValues(Ops, 2);
658  }
659  assert(LoadedVT.isInteger() && !LoadedVT.isVector() &&
660         "Unaligned load of unsupported type.");
661
662  // Compute the new VT that is half the size of the old one.  This is an
663  // integer MVT.
664  unsigned NumBits = LoadedVT.getSizeInBits();
665  MVT NewLoadedVT;
666  NewLoadedVT = MVT::getIntegerVT(NumBits/2);
667  NumBits >>= 1;
668
669  unsigned Alignment = LD->getAlignment();
670  unsigned IncrementSize = NumBits / 8;
671  ISD::LoadExtType HiExtType = LD->getExtensionType();
672
673  // If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD.
674  if (HiExtType == ISD::NON_EXTLOAD)
675    HiExtType = ISD::ZEXTLOAD;
676
677  // Load the value in two parts
678  SDOperand Lo, Hi;
679  if (TLI.isLittleEndian()) {
680    Lo = DAG.getExtLoad(ISD::ZEXTLOAD, VT, Chain, Ptr, LD->getSrcValue(),
681                        SVOffset, NewLoadedVT, LD->isVolatile(), Alignment);
682    Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
683                      DAG.getConstant(IncrementSize, TLI.getPointerTy()));
684    Hi = DAG.getExtLoad(HiExtType, VT, Chain, Ptr, LD->getSrcValue(),
685                        SVOffset + IncrementSize, NewLoadedVT, LD->isVolatile(),
686                        MinAlign(Alignment, IncrementSize));
687  } else {
688    Hi = DAG.getExtLoad(HiExtType, VT, Chain, Ptr, LD->getSrcValue(), SVOffset,
689                        NewLoadedVT,LD->isVolatile(), Alignment);
690    Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
691                      DAG.getConstant(IncrementSize, TLI.getPointerTy()));
692    Lo = DAG.getExtLoad(ISD::ZEXTLOAD, VT, Chain, Ptr, LD->getSrcValue(),
693                        SVOffset + IncrementSize, NewLoadedVT, LD->isVolatile(),
694                        MinAlign(Alignment, IncrementSize));
695  }
696
697  // aggregate the two parts
698  SDOperand ShiftAmount = DAG.getConstant(NumBits, TLI.getShiftAmountTy());
699  SDOperand Result = DAG.getNode(ISD::SHL, VT, Hi, ShiftAmount);
700  Result = DAG.getNode(ISD::OR, VT, Result, Lo);
701
702  SDOperand TF = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1),
703                             Hi.getValue(1));
704
705  SDOperand Ops[] = { Result, TF };
706  return DAG.getMergeValues(Ops, 2);
707}
708
709/// UnrollVectorOp - We know that the given vector has a legal type, however
710/// the operation it performs is not legal and is an operation that we have
711/// no way of lowering.  "Unroll" the vector, splitting out the scalars and
712/// operating on each element individually.
713SDOperand SelectionDAGLegalize::UnrollVectorOp(SDOperand Op) {
714  MVT VT = Op.getValueType();
715  assert(isTypeLegal(VT) &&
716         "Caller should expand or promote operands that are not legal!");
717  assert(Op.Val->getNumValues() == 1 &&
718         "Can't unroll a vector with multiple results!");
719  unsigned NE = VT.getVectorNumElements();
720  MVT EltVT = VT.getVectorElementType();
721
722  SmallVector<SDOperand, 8> Scalars;
723  SmallVector<SDOperand, 4> Operands(Op.getNumOperands());
724  for (unsigned i = 0; i != NE; ++i) {
725    for (unsigned j = 0; j != Op.getNumOperands(); ++j) {
726      SDOperand Operand = Op.getOperand(j);
727      MVT OperandVT = Operand.getValueType();
728      if (OperandVT.isVector()) {
729        // A vector operand; extract a single element.
730        MVT OperandEltVT = OperandVT.getVectorElementType();
731        Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT,
732                                  OperandEltVT,
733                                  Operand,
734                                  DAG.getConstant(i, MVT::i32));
735      } else {
736        // A scalar operand; just use it as is.
737        Operands[j] = Operand;
738      }
739    }
740    Scalars.push_back(DAG.getNode(Op.getOpcode(), EltVT,
741                                  &Operands[0], Operands.size()));
742  }
743
744  return DAG.getNode(ISD::BUILD_VECTOR, VT, &Scalars[0], Scalars.size());
745}
746
747/// GetFPLibCall - Return the right libcall for the given floating point type.
748static RTLIB::Libcall GetFPLibCall(MVT VT,
749                                   RTLIB::Libcall Call_F32,
750                                   RTLIB::Libcall Call_F64,
751                                   RTLIB::Libcall Call_F80,
752                                   RTLIB::Libcall Call_PPCF128) {
753  return
754    VT == MVT::f32 ? Call_F32 :
755    VT == MVT::f64 ? Call_F64 :
756    VT == MVT::f80 ? Call_F80 :
757    VT == MVT::ppcf128 ? Call_PPCF128 :
758    RTLIB::UNKNOWN_LIBCALL;
759}
760
761/// PerformInsertVectorEltInMemory - Some target cannot handle a variable
762/// insertion index for the INSERT_VECTOR_ELT instruction.  In this case, it
763/// is necessary to spill the vector being inserted into to memory, perform
764/// the insert there, and then read the result back.
765SDOperand SelectionDAGLegalize::
766PerformInsertVectorEltInMemory(SDOperand Vec, SDOperand Val, SDOperand Idx) {
767  SDOperand Tmp1 = Vec;
768  SDOperand Tmp2 = Val;
769  SDOperand Tmp3 = Idx;
770
771  // If the target doesn't support this, we have to spill the input vector
772  // to a temporary stack slot, update the element, then reload it.  This is
773  // badness.  We could also load the value into a vector register (either
774  // with a "move to register" or "extload into register" instruction, then
775  // permute it into place, if the idx is a constant and if the idx is
776  // supported by the target.
777  MVT VT    = Tmp1.getValueType();
778  MVT EltVT = VT.getVectorElementType();
779  MVT IdxVT = Tmp3.getValueType();
780  MVT PtrVT = TLI.getPointerTy();
781  SDOperand StackPtr = DAG.CreateStackTemporary(VT);
782
783  int SPFI = cast<FrameIndexSDNode>(StackPtr.Val)->getIndex();
784
785  // Store the vector.
786  SDOperand Ch = DAG.getStore(DAG.getEntryNode(), Tmp1, StackPtr,
787                              PseudoSourceValue::getFixedStack(SPFI), 0);
788
789  // Truncate or zero extend offset to target pointer type.
790  unsigned CastOpc = IdxVT.bitsGT(PtrVT) ? ISD::TRUNCATE : ISD::ZERO_EXTEND;
791  Tmp3 = DAG.getNode(CastOpc, PtrVT, Tmp3);
792  // Add the offset to the index.
793  unsigned EltSize = EltVT.getSizeInBits()/8;
794  Tmp3 = DAG.getNode(ISD::MUL, IdxVT, Tmp3,DAG.getConstant(EltSize, IdxVT));
795  SDOperand StackPtr2 = DAG.getNode(ISD::ADD, IdxVT, Tmp3, StackPtr);
796  // Store the scalar value.
797  Ch = DAG.getTruncStore(Ch, Tmp2, StackPtr2,
798                         PseudoSourceValue::getFixedStack(SPFI), 0, EltVT);
799  // Load the updated vector.
800  return DAG.getLoad(VT, Ch, StackPtr,
801                     PseudoSourceValue::getFixedStack(SPFI), 0);
802}
803
804/// LegalizeOp - We know that the specified value has a legal type, and
805/// that its operands are legal.  Now ensure that the operation itself
806/// is legal, recursively ensuring that the operands' operations remain
807/// legal.
808SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) {
809  if (Op.getOpcode() == ISD::TargetConstant) // Allow illegal target nodes.
810    return Op;
811
812  assert(isTypeLegal(Op.getValueType()) &&
813         "Caller should expand or promote operands that are not legal!");
814  SDNode *Node = Op.Val;
815
816  // If this operation defines any values that cannot be represented in a
817  // register on this target, make sure to expand or promote them.
818  if (Node->getNumValues() > 1) {
819    for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
820      if (getTypeAction(Node->getValueType(i)) != Legal) {
821        HandleOp(Op.getValue(i));
822        assert(LegalizedNodes.count(Op) &&
823               "Handling didn't add legal operands!");
824        return LegalizedNodes[Op];
825      }
826  }
827
828  // Note that LegalizeOp may be reentered even from single-use nodes, which
829  // means that we always must cache transformed nodes.
830  DenseMap<SDOperand, SDOperand>::iterator I = LegalizedNodes.find(Op);
831  if (I != LegalizedNodes.end()) return I->second;
832
833  SDOperand Tmp1, Tmp2, Tmp3, Tmp4;
834  SDOperand Result = Op;
835  bool isCustom = false;
836
837  switch (Node->getOpcode()) {
838  case ISD::FrameIndex:
839  case ISD::EntryToken:
840  case ISD::Register:
841  case ISD::BasicBlock:
842  case ISD::TargetFrameIndex:
843  case ISD::TargetJumpTable:
844  case ISD::TargetConstant:
845  case ISD::TargetConstantFP:
846  case ISD::TargetConstantPool:
847  case ISD::TargetGlobalAddress:
848  case ISD::TargetGlobalTLSAddress:
849  case ISD::TargetExternalSymbol:
850  case ISD::VALUETYPE:
851  case ISD::SRCVALUE:
852  case ISD::MEMOPERAND:
853  case ISD::CONDCODE:
854  case ISD::ARG_FLAGS:
855    // Primitives must all be legal.
856    assert(TLI.isOperationLegal(Node->getOpcode(), Node->getValueType(0)) &&
857           "This must be legal!");
858    break;
859  default:
860    if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
861      // If this is a target node, legalize it by legalizing the operands then
862      // passing it through.
863      SmallVector<SDOperand, 8> Ops;
864      for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i)
865        Ops.push_back(LegalizeOp(Node->getOperand(i)));
866
867      Result = DAG.UpdateNodeOperands(Result.getValue(0), &Ops[0], Ops.size());
868
869      for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
870        AddLegalizedOperand(Op.getValue(i), Result.getValue(i));
871      return Result.getValue(Op.ResNo);
872    }
873    // Otherwise this is an unhandled builtin node.  splat.
874#ifndef NDEBUG
875    cerr << "NODE: "; Node->dump(&DAG); cerr << "\n";
876#endif
877    assert(0 && "Do not know how to legalize this operator!");
878    abort();
879  case ISD::GLOBAL_OFFSET_TABLE:
880  case ISD::GlobalAddress:
881  case ISD::GlobalTLSAddress:
882  case ISD::ExternalSymbol:
883  case ISD::ConstantPool:
884  case ISD::JumpTable: // Nothing to do.
885    switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
886    default: assert(0 && "This action is not supported yet!");
887    case TargetLowering::Custom:
888      Tmp1 = TLI.LowerOperation(Op, DAG);
889      if (Tmp1.Val) Result = Tmp1;
890      // FALLTHROUGH if the target doesn't want to lower this op after all.
891    case TargetLowering::Legal:
892      break;
893    }
894    break;
895  case ISD::FRAMEADDR:
896  case ISD::RETURNADDR:
897    // The only option for these nodes is to custom lower them.  If the target
898    // does not custom lower them, then return zero.
899    Tmp1 = TLI.LowerOperation(Op, DAG);
900    if (Tmp1.Val)
901      Result = Tmp1;
902    else
903      Result = DAG.getConstant(0, TLI.getPointerTy());
904    break;
905  case ISD::FRAME_TO_ARGS_OFFSET: {
906    MVT VT = Node->getValueType(0);
907    switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
908    default: assert(0 && "This action is not supported yet!");
909    case TargetLowering::Custom:
910      Result = TLI.LowerOperation(Op, DAG);
911      if (Result.Val) break;
912      // Fall Thru
913    case TargetLowering::Legal:
914      Result = DAG.getConstant(0, VT);
915      break;
916    }
917    }
918    break;
919  case ISD::EXCEPTIONADDR: {
920    Tmp1 = LegalizeOp(Node->getOperand(0));
921    MVT VT = Node->getValueType(0);
922    switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
923    default: assert(0 && "This action is not supported yet!");
924    case TargetLowering::Expand: {
925        unsigned Reg = TLI.getExceptionAddressRegister();
926        Result = DAG.getCopyFromReg(Tmp1, Reg, VT);
927      }
928      break;
929    case TargetLowering::Custom:
930      Result = TLI.LowerOperation(Op, DAG);
931      if (Result.Val) break;
932      // Fall Thru
933    case TargetLowering::Legal: {
934      SDOperand Ops[] = { DAG.getConstant(0, VT), Tmp1 };
935      Result = DAG.getMergeValues(Ops, 2);
936      break;
937    }
938    }
939    }
940    if (Result.Val->getNumValues() == 1) break;
941
942    assert(Result.Val->getNumValues() == 2 &&
943           "Cannot return more than two values!");
944
945    // Since we produced two values, make sure to remember that we
946    // legalized both of them.
947    Tmp1 = LegalizeOp(Result);
948    Tmp2 = LegalizeOp(Result.getValue(1));
949    AddLegalizedOperand(Op.getValue(0), Tmp1);
950    AddLegalizedOperand(Op.getValue(1), Tmp2);
951    return Op.ResNo ? Tmp2 : Tmp1;
952  case ISD::EHSELECTION: {
953    Tmp1 = LegalizeOp(Node->getOperand(0));
954    Tmp2 = LegalizeOp(Node->getOperand(1));
955    MVT VT = Node->getValueType(0);
956    switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
957    default: assert(0 && "This action is not supported yet!");
958    case TargetLowering::Expand: {
959        unsigned Reg = TLI.getExceptionSelectorRegister();
960        Result = DAG.getCopyFromReg(Tmp2, Reg, VT);
961      }
962      break;
963    case TargetLowering::Custom:
964      Result = TLI.LowerOperation(Op, DAG);
965      if (Result.Val) break;
966      // Fall Thru
967    case TargetLowering::Legal: {
968      SDOperand Ops[] = { DAG.getConstant(0, VT), Tmp2 };
969      Result = DAG.getMergeValues(Ops, 2);
970      break;
971    }
972    }
973    }
974    if (Result.Val->getNumValues() == 1) break;
975
976    assert(Result.Val->getNumValues() == 2 &&
977           "Cannot return more than two values!");
978
979    // Since we produced two values, make sure to remember that we
980    // legalized both of them.
981    Tmp1 = LegalizeOp(Result);
982    Tmp2 = LegalizeOp(Result.getValue(1));
983    AddLegalizedOperand(Op.getValue(0), Tmp1);
984    AddLegalizedOperand(Op.getValue(1), Tmp2);
985    return Op.ResNo ? Tmp2 : Tmp1;
986  case ISD::EH_RETURN: {
987    MVT VT = Node->getValueType(0);
988    // The only "good" option for this node is to custom lower it.
989    switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
990    default: assert(0 && "This action is not supported at all!");
991    case TargetLowering::Custom:
992      Result = TLI.LowerOperation(Op, DAG);
993      if (Result.Val) break;
994      // Fall Thru
995    case TargetLowering::Legal:
996      // Target does not know, how to lower this, lower to noop
997      Result = LegalizeOp(Node->getOperand(0));
998      break;
999    }
1000    }
1001    break;
1002  case ISD::AssertSext:
1003  case ISD::AssertZext:
1004    Tmp1 = LegalizeOp(Node->getOperand(0));
1005    Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1));
1006    break;
1007  case ISD::MERGE_VALUES:
1008    // Legalize eliminates MERGE_VALUES nodes.
1009    Result = Node->getOperand(Op.ResNo);
1010    break;
1011  case ISD::CopyFromReg:
1012    Tmp1 = LegalizeOp(Node->getOperand(0));
1013    Result = Op.getValue(0);
1014    if (Node->getNumValues() == 2) {
1015      Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1));
1016    } else {
1017      assert(Node->getNumValues() == 3 && "Invalid copyfromreg!");
1018      if (Node->getNumOperands() == 3) {
1019        Tmp2 = LegalizeOp(Node->getOperand(2));
1020        Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1),Tmp2);
1021      } else {
1022        Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1));
1023      }
1024      AddLegalizedOperand(Op.getValue(2), Result.getValue(2));
1025    }
1026    // Since CopyFromReg produces two values, make sure to remember that we
1027    // legalized both of them.
1028    AddLegalizedOperand(Op.getValue(0), Result);
1029    AddLegalizedOperand(Op.getValue(1), Result.getValue(1));
1030    return Result.getValue(Op.ResNo);
1031  case ISD::UNDEF: {
1032    MVT VT = Op.getValueType();
1033    switch (TLI.getOperationAction(ISD::UNDEF, VT)) {
1034    default: assert(0 && "This action is not supported yet!");
1035    case TargetLowering::Expand:
1036      if (VT.isInteger())
1037        Result = DAG.getConstant(0, VT);
1038      else if (VT.isFloatingPoint())
1039        Result = DAG.getConstantFP(APFloat(APInt(VT.getSizeInBits(), 0)),
1040                                   VT);
1041      else
1042        assert(0 && "Unknown value type!");
1043      break;
1044    case TargetLowering::Legal:
1045      break;
1046    }
1047    break;
1048  }
1049
1050  case ISD::INTRINSIC_W_CHAIN:
1051  case ISD::INTRINSIC_WO_CHAIN:
1052  case ISD::INTRINSIC_VOID: {
1053    SmallVector<SDOperand, 8> Ops;
1054    for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i)
1055      Ops.push_back(LegalizeOp(Node->getOperand(i)));
1056    Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size());
1057
1058    // Allow the target to custom lower its intrinsics if it wants to.
1059    if (TLI.getOperationAction(Node->getOpcode(), MVT::Other) ==
1060        TargetLowering::Custom) {
1061      Tmp3 = TLI.LowerOperation(Result, DAG);
1062      if (Tmp3.Val) Result = Tmp3;
1063    }
1064
1065    if (Result.Val->getNumValues() == 1) break;
1066
1067    // Must have return value and chain result.
1068    assert(Result.Val->getNumValues() == 2 &&
1069           "Cannot return more than two values!");
1070
1071    // Since loads produce two values, make sure to remember that we
1072    // legalized both of them.
1073    AddLegalizedOperand(SDOperand(Node, 0), Result.getValue(0));
1074    AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1));
1075    return Result.getValue(Op.ResNo);
1076  }
1077
1078  case ISD::DBG_STOPPOINT:
1079    assert(Node->getNumOperands() == 1 && "Invalid DBG_STOPPOINT node!");
1080    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the input chain.
1081
1082    switch (TLI.getOperationAction(ISD::DBG_STOPPOINT, MVT::Other)) {
1083    case TargetLowering::Promote:
1084    default: assert(0 && "This action is not supported yet!");
1085    case TargetLowering::Expand: {
1086      MachineModuleInfo *MMI = DAG.getMachineModuleInfo();
1087      bool useDEBUG_LOC = TLI.isOperationLegal(ISD::DEBUG_LOC, MVT::Other);
1088      bool useLABEL = TLI.isOperationLegal(ISD::DBG_LABEL, MVT::Other);
1089
1090      const DbgStopPointSDNode *DSP = cast<DbgStopPointSDNode>(Node);
1091      if (MMI && (useDEBUG_LOC || useLABEL)) {
1092        const CompileUnitDesc *CompileUnit = DSP->getCompileUnit();
1093        unsigned SrcFile = MMI->RecordSource(CompileUnit);
1094
1095        unsigned Line = DSP->getLine();
1096        unsigned Col = DSP->getColumn();
1097
1098        if (useDEBUG_LOC) {
1099          SDOperand Ops[] = { Tmp1, DAG.getConstant(Line, MVT::i32),
1100                              DAG.getConstant(Col, MVT::i32),
1101                              DAG.getConstant(SrcFile, MVT::i32) };
1102          Result = DAG.getNode(ISD::DEBUG_LOC, MVT::Other, Ops, 4);
1103        } else {
1104          unsigned ID = MMI->RecordSourceLine(Line, Col, SrcFile);
1105          Result = DAG.getLabel(ISD::DBG_LABEL, Tmp1, ID);
1106        }
1107      } else {
1108        Result = Tmp1;  // chain
1109      }
1110      break;
1111    }
1112    case TargetLowering::Legal: {
1113      LegalizeAction Action = getTypeAction(Node->getOperand(1).getValueType());
1114      if (Action == Legal && Tmp1 == Node->getOperand(0))
1115        break;
1116
1117      SmallVector<SDOperand, 8> Ops;
1118      Ops.push_back(Tmp1);
1119      if (Action == Legal) {
1120        Ops.push_back(Node->getOperand(1));  // line # must be legal.
1121        Ops.push_back(Node->getOperand(2));  // col # must be legal.
1122      } else {
1123        // Otherwise promote them.
1124        Ops.push_back(PromoteOp(Node->getOperand(1)));
1125        Ops.push_back(PromoteOp(Node->getOperand(2)));
1126      }
1127      Ops.push_back(Node->getOperand(3));  // filename must be legal.
1128      Ops.push_back(Node->getOperand(4));  // working dir # must be legal.
1129      Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size());
1130      break;
1131    }
1132    }
1133    break;
1134
1135  case ISD::DECLARE:
1136    assert(Node->getNumOperands() == 3 && "Invalid DECLARE node!");
1137    switch (TLI.getOperationAction(ISD::DECLARE, MVT::Other)) {
1138    default: assert(0 && "This action is not supported yet!");
1139    case TargetLowering::Legal:
1140      Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
1141      Tmp2 = LegalizeOp(Node->getOperand(1));  // Legalize the address.
1142      Tmp3 = LegalizeOp(Node->getOperand(2));  // Legalize the variable.
1143      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
1144      break;
1145    case TargetLowering::Expand:
1146      Result = LegalizeOp(Node->getOperand(0));
1147      break;
1148    }
1149    break;
1150
1151  case ISD::DEBUG_LOC:
1152    assert(Node->getNumOperands() == 4 && "Invalid DEBUG_LOC node!");
1153    switch (TLI.getOperationAction(ISD::DEBUG_LOC, MVT::Other)) {
1154    default: assert(0 && "This action is not supported yet!");
1155    case TargetLowering::Legal: {
1156      LegalizeAction Action = getTypeAction(Node->getOperand(1).getValueType());
1157      Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
1158      if (Action == Legal && Tmp1 == Node->getOperand(0))
1159        break;
1160      if (Action == Legal) {
1161        Tmp2 = Node->getOperand(1);
1162        Tmp3 = Node->getOperand(2);
1163        Tmp4 = Node->getOperand(3);
1164      } else {
1165        Tmp2 = LegalizeOp(Node->getOperand(1));  // Legalize the line #.
1166        Tmp3 = LegalizeOp(Node->getOperand(2));  // Legalize the col #.
1167        Tmp4 = LegalizeOp(Node->getOperand(3));  // Legalize the source file id.
1168      }
1169      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3, Tmp4);
1170      break;
1171    }
1172    }
1173    break;
1174
1175  case ISD::DBG_LABEL:
1176  case ISD::EH_LABEL:
1177    assert(Node->getNumOperands() == 1 && "Invalid LABEL node!");
1178    switch (TLI.getOperationAction(Node->getOpcode(), MVT::Other)) {
1179    default: assert(0 && "This action is not supported yet!");
1180    case TargetLowering::Legal:
1181      Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
1182      Result = DAG.UpdateNodeOperands(Result, Tmp1);
1183      break;
1184    case TargetLowering::Expand:
1185      Result = LegalizeOp(Node->getOperand(0));
1186      break;
1187    }
1188    break;
1189
1190  case ISD::PREFETCH:
1191    assert(Node->getNumOperands() == 4 && "Invalid Prefetch node!");
1192    switch (TLI.getOperationAction(ISD::PREFETCH, MVT::Other)) {
1193    default: assert(0 && "This action is not supported yet!");
1194    case TargetLowering::Legal:
1195      Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
1196      Tmp2 = LegalizeOp(Node->getOperand(1));  // Legalize the address.
1197      Tmp3 = LegalizeOp(Node->getOperand(2));  // Legalize the rw specifier.
1198      Tmp4 = LegalizeOp(Node->getOperand(3));  // Legalize locality specifier.
1199      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3, Tmp4);
1200      break;
1201    case TargetLowering::Expand:
1202      // It's a noop.
1203      Result = LegalizeOp(Node->getOperand(0));
1204      break;
1205    }
1206    break;
1207
1208  case ISD::MEMBARRIER: {
1209    assert(Node->getNumOperands() == 6 && "Invalid MemBarrier node!");
1210    switch (TLI.getOperationAction(ISD::MEMBARRIER, MVT::Other)) {
1211    default: assert(0 && "This action is not supported yet!");
1212    case TargetLowering::Legal: {
1213      SDOperand Ops[6];
1214      Ops[0] = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
1215      for (int x = 1; x < 6; ++x) {
1216        Ops[x] = Node->getOperand(x);
1217        if (!isTypeLegal(Ops[x].getValueType()))
1218          Ops[x] = PromoteOp(Ops[x]);
1219      }
1220      Result = DAG.UpdateNodeOperands(Result, &Ops[0], 6);
1221      break;
1222    }
1223    case TargetLowering::Expand:
1224      //There is no libgcc call for this op
1225      Result = Node->getOperand(0);  // Noop
1226    break;
1227    }
1228    break;
1229  }
1230
1231  case ISD::ATOMIC_CMP_SWAP: {
1232    unsigned int num_operands = 4;
1233    assert(Node->getNumOperands() == num_operands && "Invalid Atomic node!");
1234    SDOperand Ops[4];
1235    for (unsigned int x = 0; x < num_operands; ++x)
1236      Ops[x] = LegalizeOp(Node->getOperand(x));
1237    Result = DAG.UpdateNodeOperands(Result, &Ops[0], num_operands);
1238
1239    switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
1240      default: assert(0 && "This action is not supported yet!");
1241      case TargetLowering::Custom:
1242        Result = TLI.LowerOperation(Result, DAG);
1243        break;
1244      case TargetLowering::Legal:
1245        break;
1246    }
1247    AddLegalizedOperand(SDOperand(Node, 0), Result.getValue(0));
1248    AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1));
1249    return Result.getValue(Op.ResNo);
1250  }
1251  case ISD::ATOMIC_LOAD_ADD:
1252  case ISD::ATOMIC_LOAD_SUB:
1253  case ISD::ATOMIC_LOAD_AND:
1254  case ISD::ATOMIC_LOAD_OR:
1255  case ISD::ATOMIC_LOAD_XOR:
1256  case ISD::ATOMIC_LOAD_NAND:
1257  case ISD::ATOMIC_LOAD_MIN:
1258  case ISD::ATOMIC_LOAD_MAX:
1259  case ISD::ATOMIC_LOAD_UMIN:
1260  case ISD::ATOMIC_LOAD_UMAX:
1261  case ISD::ATOMIC_SWAP: {
1262    unsigned int num_operands = 3;
1263    assert(Node->getNumOperands() == num_operands && "Invalid Atomic node!");
1264    SDOperand Ops[3];
1265    for (unsigned int x = 0; x < num_operands; ++x)
1266      Ops[x] = LegalizeOp(Node->getOperand(x));
1267    Result = DAG.UpdateNodeOperands(Result, &Ops[0], num_operands);
1268
1269    switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
1270    default: assert(0 && "This action is not supported yet!");
1271    case TargetLowering::Custom:
1272      Result = TLI.LowerOperation(Result, DAG);
1273      break;
1274    case TargetLowering::Expand:
1275      Result = SDOperand(TLI.ReplaceNodeResults(Op.Val, DAG),0);
1276      break;
1277    case TargetLowering::Legal:
1278      break;
1279    }
1280    AddLegalizedOperand(SDOperand(Node, 0), Result.getValue(0));
1281    AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1));
1282    return Result.getValue(Op.ResNo);
1283  }
1284  case ISD::Constant: {
1285    ConstantSDNode *CN = cast<ConstantSDNode>(Node);
1286    unsigned opAction =
1287      TLI.getOperationAction(ISD::Constant, CN->getValueType(0));
1288
1289    // We know we don't need to expand constants here, constants only have one
1290    // value and we check that it is fine above.
1291
1292    if (opAction == TargetLowering::Custom) {
1293      Tmp1 = TLI.LowerOperation(Result, DAG);
1294      if (Tmp1.Val)
1295        Result = Tmp1;
1296    }
1297    break;
1298  }
1299  case ISD::ConstantFP: {
1300    // Spill FP immediates to the constant pool if the target cannot directly
1301    // codegen them.  Targets often have some immediate values that can be
1302    // efficiently generated into an FP register without a load.  We explicitly
1303    // leave these constants as ConstantFP nodes for the target to deal with.
1304    ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
1305
1306    switch (TLI.getOperationAction(ISD::ConstantFP, CFP->getValueType(0))) {
1307    default: assert(0 && "This action is not supported yet!");
1308    case TargetLowering::Legal:
1309      break;
1310    case TargetLowering::Custom:
1311      Tmp3 = TLI.LowerOperation(Result, DAG);
1312      if (Tmp3.Val) {
1313        Result = Tmp3;
1314        break;
1315      }
1316      // FALLTHROUGH
1317    case TargetLowering::Expand: {
1318      // Check to see if this FP immediate is already legal.
1319      bool isLegal = false;
1320      for (TargetLowering::legal_fpimm_iterator I = TLI.legal_fpimm_begin(),
1321             E = TLI.legal_fpimm_end(); I != E; ++I) {
1322        if (CFP->isExactlyValue(*I)) {
1323          isLegal = true;
1324          break;
1325        }
1326      }
1327      // If this is a legal constant, turn it into a TargetConstantFP node.
1328      if (isLegal)
1329        break;
1330      Result = ExpandConstantFP(CFP, true, DAG, TLI);
1331    }
1332    }
1333    break;
1334  }
1335  case ISD::TokenFactor:
1336    if (Node->getNumOperands() == 2) {
1337      Tmp1 = LegalizeOp(Node->getOperand(0));
1338      Tmp2 = LegalizeOp(Node->getOperand(1));
1339      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
1340    } else if (Node->getNumOperands() == 3) {
1341      Tmp1 = LegalizeOp(Node->getOperand(0));
1342      Tmp2 = LegalizeOp(Node->getOperand(1));
1343      Tmp3 = LegalizeOp(Node->getOperand(2));
1344      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
1345    } else {
1346      SmallVector<SDOperand, 8> Ops;
1347      // Legalize the operands.
1348      for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i)
1349        Ops.push_back(LegalizeOp(Node->getOperand(i)));
1350      Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size());
1351    }
1352    break;
1353
1354  case ISD::FORMAL_ARGUMENTS:
1355  case ISD::CALL:
1356    // The only option for this is to custom lower it.
1357    Tmp3 = TLI.LowerOperation(Result.getValue(0), DAG);
1358    assert(Tmp3.Val && "Target didn't custom lower this node!");
1359    // A call within a calling sequence must be legalized to something
1360    // other than the normal CALLSEQ_END.  Violating this gets Legalize
1361    // into an infinite loop.
1362    assert ((!IsLegalizingCall ||
1363             Node->getOpcode() != ISD::CALL ||
1364             Tmp3.Val->getOpcode() != ISD::CALLSEQ_END) &&
1365            "Nested CALLSEQ_START..CALLSEQ_END not supported.");
1366
1367    // The number of incoming and outgoing values should match; unless the final
1368    // outgoing value is a flag.
1369    assert((Tmp3.Val->getNumValues() == Result.Val->getNumValues() ||
1370            (Tmp3.Val->getNumValues() == Result.Val->getNumValues() + 1 &&
1371             Tmp3.Val->getValueType(Tmp3.Val->getNumValues() - 1) ==
1372               MVT::Flag)) &&
1373           "Lowering call/formal_arguments produced unexpected # results!");
1374
1375    // Since CALL/FORMAL_ARGUMENTS nodes produce multiple values, make sure to
1376    // remember that we legalized all of them, so it doesn't get relegalized.
1377    for (unsigned i = 0, e = Tmp3.Val->getNumValues(); i != e; ++i) {
1378      if (Tmp3.Val->getValueType(i) == MVT::Flag)
1379        continue;
1380      Tmp1 = LegalizeOp(Tmp3.getValue(i));
1381      if (Op.ResNo == i)
1382        Tmp2 = Tmp1;
1383      AddLegalizedOperand(SDOperand(Node, i), Tmp1);
1384    }
1385    return Tmp2;
1386   case ISD::EXTRACT_SUBREG: {
1387      Tmp1 = LegalizeOp(Node->getOperand(0));
1388      ConstantSDNode *idx = dyn_cast<ConstantSDNode>(Node->getOperand(1));
1389      assert(idx && "Operand must be a constant");
1390      Tmp2 = DAG.getTargetConstant(idx->getValue(), idx->getValueType(0));
1391      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
1392    }
1393    break;
1394  case ISD::INSERT_SUBREG: {
1395      Tmp1 = LegalizeOp(Node->getOperand(0));
1396      Tmp2 = LegalizeOp(Node->getOperand(1));
1397      ConstantSDNode *idx = dyn_cast<ConstantSDNode>(Node->getOperand(2));
1398      assert(idx && "Operand must be a constant");
1399      Tmp3 = DAG.getTargetConstant(idx->getValue(), idx->getValueType(0));
1400      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
1401    }
1402    break;
1403  case ISD::BUILD_VECTOR:
1404    switch (TLI.getOperationAction(ISD::BUILD_VECTOR, Node->getValueType(0))) {
1405    default: assert(0 && "This action is not supported yet!");
1406    case TargetLowering::Custom:
1407      Tmp3 = TLI.LowerOperation(Result, DAG);
1408      if (Tmp3.Val) {
1409        Result = Tmp3;
1410        break;
1411      }
1412      // FALLTHROUGH
1413    case TargetLowering::Expand:
1414      Result = ExpandBUILD_VECTOR(Result.Val);
1415      break;
1416    }
1417    break;
1418  case ISD::INSERT_VECTOR_ELT:
1419    Tmp1 = LegalizeOp(Node->getOperand(0));  // InVec
1420    Tmp3 = LegalizeOp(Node->getOperand(2));  // InEltNo
1421
1422    // The type of the value to insert may not be legal, even though the vector
1423    // type is legal.  Legalize/Promote accordingly.  We do not handle Expand
1424    // here.
1425    switch (getTypeAction(Node->getOperand(1).getValueType())) {
1426    default: assert(0 && "Cannot expand insert element operand");
1427    case Legal:   Tmp2 = LegalizeOp(Node->getOperand(1)); break;
1428    case Promote: Tmp2 = PromoteOp(Node->getOperand(1));  break;
1429    }
1430    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
1431
1432    switch (TLI.getOperationAction(ISD::INSERT_VECTOR_ELT,
1433                                   Node->getValueType(0))) {
1434    default: assert(0 && "This action is not supported yet!");
1435    case TargetLowering::Legal:
1436      break;
1437    case TargetLowering::Custom:
1438      Tmp4 = TLI.LowerOperation(Result, DAG);
1439      if (Tmp4.Val) {
1440        Result = Tmp4;
1441        break;
1442      }
1443      // FALLTHROUGH
1444    case TargetLowering::Expand: {
1445      // If the insert index is a constant, codegen this as a scalar_to_vector,
1446      // then a shuffle that inserts it into the right position in the vector.
1447      if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Tmp3)) {
1448        // SCALAR_TO_VECTOR requires that the type of the value being inserted
1449        // match the element type of the vector being created.
1450        if (Tmp2.getValueType() ==
1451            Op.getValueType().getVectorElementType()) {
1452          SDOperand ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR,
1453                                        Tmp1.getValueType(), Tmp2);
1454
1455          unsigned NumElts = Tmp1.getValueType().getVectorNumElements();
1456          MVT ShufMaskVT =
1457            MVT::getIntVectorWithNumElements(NumElts);
1458          MVT ShufMaskEltVT = ShufMaskVT.getVectorElementType();
1459
1460          // We generate a shuffle of InVec and ScVec, so the shuffle mask
1461          // should be 0,1,2,3,4,5... with the appropriate element replaced with
1462          // elt 0 of the RHS.
1463          SmallVector<SDOperand, 8> ShufOps;
1464          for (unsigned i = 0; i != NumElts; ++i) {
1465            if (i != InsertPos->getValue())
1466              ShufOps.push_back(DAG.getConstant(i, ShufMaskEltVT));
1467            else
1468              ShufOps.push_back(DAG.getConstant(NumElts, ShufMaskEltVT));
1469          }
1470          SDOperand ShufMask = DAG.getNode(ISD::BUILD_VECTOR, ShufMaskVT,
1471                                           &ShufOps[0], ShufOps.size());
1472
1473          Result = DAG.getNode(ISD::VECTOR_SHUFFLE, Tmp1.getValueType(),
1474                               Tmp1, ScVec, ShufMask);
1475          Result = LegalizeOp(Result);
1476          break;
1477        }
1478      }
1479      Result = PerformInsertVectorEltInMemory(Tmp1, Tmp2, Tmp3);
1480      break;
1481    }
1482    }
1483    break;
1484  case ISD::SCALAR_TO_VECTOR:
1485    if (!TLI.isTypeLegal(Node->getOperand(0).getValueType())) {
1486      Result = LegalizeOp(ExpandSCALAR_TO_VECTOR(Node));
1487      break;
1488    }
1489
1490    Tmp1 = LegalizeOp(Node->getOperand(0));  // InVal
1491    Result = DAG.UpdateNodeOperands(Result, Tmp1);
1492    switch (TLI.getOperationAction(ISD::SCALAR_TO_VECTOR,
1493                                   Node->getValueType(0))) {
1494    default: assert(0 && "This action is not supported yet!");
1495    case TargetLowering::Legal:
1496      break;
1497    case TargetLowering::Custom:
1498      Tmp3 = TLI.LowerOperation(Result, DAG);
1499      if (Tmp3.Val) {
1500        Result = Tmp3;
1501        break;
1502      }
1503      // FALLTHROUGH
1504    case TargetLowering::Expand:
1505      Result = LegalizeOp(ExpandSCALAR_TO_VECTOR(Node));
1506      break;
1507    }
1508    break;
1509  case ISD::VECTOR_SHUFFLE:
1510    Tmp1 = LegalizeOp(Node->getOperand(0));   // Legalize the input vectors,
1511    Tmp2 = LegalizeOp(Node->getOperand(1));   // but not the shuffle mask.
1512    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2));
1513
1514    // Allow targets to custom lower the SHUFFLEs they support.
1515    switch (TLI.getOperationAction(ISD::VECTOR_SHUFFLE,Result.getValueType())) {
1516    default: assert(0 && "Unknown operation action!");
1517    case TargetLowering::Legal:
1518      assert(isShuffleLegal(Result.getValueType(), Node->getOperand(2)) &&
1519             "vector shuffle should not be created if not legal!");
1520      break;
1521    case TargetLowering::Custom:
1522      Tmp3 = TLI.LowerOperation(Result, DAG);
1523      if (Tmp3.Val) {
1524        Result = Tmp3;
1525        break;
1526      }
1527      // FALLTHROUGH
1528    case TargetLowering::Expand: {
1529      MVT VT = Node->getValueType(0);
1530      MVT EltVT = VT.getVectorElementType();
1531      MVT PtrVT = TLI.getPointerTy();
1532      SDOperand Mask = Node->getOperand(2);
1533      unsigned NumElems = Mask.getNumOperands();
1534      SmallVector<SDOperand,8> Ops;
1535      for (unsigned i = 0; i != NumElems; ++i) {
1536        SDOperand Arg = Mask.getOperand(i);
1537        if (Arg.getOpcode() == ISD::UNDEF) {
1538          Ops.push_back(DAG.getNode(ISD::UNDEF, EltVT));
1539        } else {
1540          assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!");
1541          unsigned Idx = cast<ConstantSDNode>(Arg)->getValue();
1542          if (Idx < NumElems)
1543            Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EltVT, Tmp1,
1544                                      DAG.getConstant(Idx, PtrVT)));
1545          else
1546            Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EltVT, Tmp2,
1547                                      DAG.getConstant(Idx - NumElems, PtrVT)));
1548        }
1549      }
1550      Result = DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size());
1551      break;
1552    }
1553    case TargetLowering::Promote: {
1554      // Change base type to a different vector type.
1555      MVT OVT = Node->getValueType(0);
1556      MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
1557
1558      // Cast the two input vectors.
1559      Tmp1 = DAG.getNode(ISD::BIT_CONVERT, NVT, Tmp1);
1560      Tmp2 = DAG.getNode(ISD::BIT_CONVERT, NVT, Tmp2);
1561
1562      // Convert the shuffle mask to the right # elements.
1563      Tmp3 = SDOperand(isShuffleLegal(OVT, Node->getOperand(2)), 0);
1564      assert(Tmp3.Val && "Shuffle not legal?");
1565      Result = DAG.getNode(ISD::VECTOR_SHUFFLE, NVT, Tmp1, Tmp2, Tmp3);
1566      Result = DAG.getNode(ISD::BIT_CONVERT, OVT, Result);
1567      break;
1568    }
1569    }
1570    break;
1571
1572  case ISD::EXTRACT_VECTOR_ELT:
1573    Tmp1 = Node->getOperand(0);
1574    Tmp2 = LegalizeOp(Node->getOperand(1));
1575    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
1576    Result = ExpandEXTRACT_VECTOR_ELT(Result);
1577    break;
1578
1579  case ISD::EXTRACT_SUBVECTOR:
1580    Tmp1 = Node->getOperand(0);
1581    Tmp2 = LegalizeOp(Node->getOperand(1));
1582    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
1583    Result = ExpandEXTRACT_SUBVECTOR(Result);
1584    break;
1585
1586  case ISD::CALLSEQ_START: {
1587    SDNode *CallEnd = FindCallEndFromCallStart(Node);
1588
1589    // Recursively Legalize all of the inputs of the call end that do not lead
1590    // to this call start.  This ensures that any libcalls that need be inserted
1591    // are inserted *before* the CALLSEQ_START.
1592    {SmallPtrSet<SDNode*, 32> NodesLeadingTo;
1593    for (unsigned i = 0, e = CallEnd->getNumOperands(); i != e; ++i)
1594      LegalizeAllNodesNotLeadingTo(CallEnd->getOperand(i).Val, Node,
1595                                   NodesLeadingTo);
1596    }
1597
1598    // Now that we legalized all of the inputs (which may have inserted
1599    // libcalls) create the new CALLSEQ_START node.
1600    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
1601
1602    // Merge in the last call, to ensure that this call start after the last
1603    // call ended.
1604    if (LastCALLSEQ_END.getOpcode() != ISD::EntryToken) {
1605      Tmp1 = DAG.getNode(ISD::TokenFactor, MVT::Other, Tmp1, LastCALLSEQ_END);
1606      Tmp1 = LegalizeOp(Tmp1);
1607    }
1608
1609    // Do not try to legalize the target-specific arguments (#1+).
1610    if (Tmp1 != Node->getOperand(0)) {
1611      SmallVector<SDOperand, 8> Ops(Node->op_begin(), Node->op_end());
1612      Ops[0] = Tmp1;
1613      Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size());
1614    }
1615
1616    // Remember that the CALLSEQ_START is legalized.
1617    AddLegalizedOperand(Op.getValue(0), Result);
1618    if (Node->getNumValues() == 2)    // If this has a flag result, remember it.
1619      AddLegalizedOperand(Op.getValue(1), Result.getValue(1));
1620
1621    // Now that the callseq_start and all of the non-call nodes above this call
1622    // sequence have been legalized, legalize the call itself.  During this
1623    // process, no libcalls can/will be inserted, guaranteeing that no calls
1624    // can overlap.
1625    assert(!IsLegalizingCall && "Inconsistent sequentialization of calls!");
1626    // Note that we are selecting this call!
1627    LastCALLSEQ_END = SDOperand(CallEnd, 0);
1628    IsLegalizingCall = true;
1629
1630    // Legalize the call, starting from the CALLSEQ_END.
1631    LegalizeOp(LastCALLSEQ_END);
1632    assert(!IsLegalizingCall && "CALLSEQ_END should have cleared this!");
1633    return Result;
1634  }
1635  case ISD::CALLSEQ_END:
1636    // If the CALLSEQ_START node hasn't been legalized first, legalize it.  This
1637    // will cause this node to be legalized as well as handling libcalls right.
1638    if (LastCALLSEQ_END.Val != Node) {
1639      LegalizeOp(SDOperand(FindCallStartFromCallEnd(Node), 0));
1640      DenseMap<SDOperand, SDOperand>::iterator I = LegalizedNodes.find(Op);
1641      assert(I != LegalizedNodes.end() &&
1642             "Legalizing the call start should have legalized this node!");
1643      return I->second;
1644    }
1645
1646    // Otherwise, the call start has been legalized and everything is going
1647    // according to plan.  Just legalize ourselves normally here.
1648    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
1649    // Do not try to legalize the target-specific arguments (#1+), except for
1650    // an optional flag input.
1651    if (Node->getOperand(Node->getNumOperands()-1).getValueType() != MVT::Flag){
1652      if (Tmp1 != Node->getOperand(0)) {
1653        SmallVector<SDOperand, 8> Ops(Node->op_begin(), Node->op_end());
1654        Ops[0] = Tmp1;
1655        Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size());
1656      }
1657    } else {
1658      Tmp2 = LegalizeOp(Node->getOperand(Node->getNumOperands()-1));
1659      if (Tmp1 != Node->getOperand(0) ||
1660          Tmp2 != Node->getOperand(Node->getNumOperands()-1)) {
1661        SmallVector<SDOperand, 8> Ops(Node->op_begin(), Node->op_end());
1662        Ops[0] = Tmp1;
1663        Ops.back() = Tmp2;
1664        Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size());
1665      }
1666    }
1667    assert(IsLegalizingCall && "Call sequence imbalance between start/end?");
1668    // This finishes up call legalization.
1669    IsLegalizingCall = false;
1670
1671    // If the CALLSEQ_END node has a flag, remember that we legalized it.
1672    AddLegalizedOperand(SDOperand(Node, 0), Result.getValue(0));
1673    if (Node->getNumValues() == 2)
1674      AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1));
1675    return Result.getValue(Op.ResNo);
1676  case ISD::DYNAMIC_STACKALLOC: {
1677    MVT VT = Node->getValueType(0);
1678    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
1679    Tmp2 = LegalizeOp(Node->getOperand(1));  // Legalize the size.
1680    Tmp3 = LegalizeOp(Node->getOperand(2));  // Legalize the alignment.
1681    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
1682
1683    Tmp1 = Result.getValue(0);
1684    Tmp2 = Result.getValue(1);
1685    switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
1686    default: assert(0 && "This action is not supported yet!");
1687    case TargetLowering::Expand: {
1688      unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore();
1689      assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1690             " not tell us which reg is the stack pointer!");
1691      SDOperand Chain = Tmp1.getOperand(0);
1692
1693      // Chain the dynamic stack allocation so that it doesn't modify the stack
1694      // pointer when other instructions are using the stack.
1695      Chain = DAG.getCALLSEQ_START(Chain,
1696                                   DAG.getConstant(0, TLI.getPointerTy()));
1697
1698      SDOperand Size  = Tmp2.getOperand(1);
1699      SDOperand SP = DAG.getCopyFromReg(Chain, SPReg, VT);
1700      Chain = SP.getValue(1);
1701      unsigned Align = cast<ConstantSDNode>(Tmp3)->getValue();
1702      unsigned StackAlign =
1703        TLI.getTargetMachine().getFrameInfo()->getStackAlignment();
1704      if (Align > StackAlign)
1705        SP = DAG.getNode(ISD::AND, VT, SP,
1706                         DAG.getConstant(-(uint64_t)Align, VT));
1707      Tmp1 = DAG.getNode(ISD::SUB, VT, SP, Size);       // Value
1708      Chain = DAG.getCopyToReg(Chain, SPReg, Tmp1);     // Output chain
1709
1710      Tmp2 =
1711        DAG.getCALLSEQ_END(Chain,
1712                           DAG.getConstant(0, TLI.getPointerTy()),
1713                           DAG.getConstant(0, TLI.getPointerTy()),
1714                           SDOperand());
1715
1716      Tmp1 = LegalizeOp(Tmp1);
1717      Tmp2 = LegalizeOp(Tmp2);
1718      break;
1719    }
1720    case TargetLowering::Custom:
1721      Tmp3 = TLI.LowerOperation(Tmp1, DAG);
1722      if (Tmp3.Val) {
1723        Tmp1 = LegalizeOp(Tmp3);
1724        Tmp2 = LegalizeOp(Tmp3.getValue(1));
1725      }
1726      break;
1727    case TargetLowering::Legal:
1728      break;
1729    }
1730    // Since this op produce two values, make sure to remember that we
1731    // legalized both of them.
1732    AddLegalizedOperand(SDOperand(Node, 0), Tmp1);
1733    AddLegalizedOperand(SDOperand(Node, 1), Tmp2);
1734    return Op.ResNo ? Tmp2 : Tmp1;
1735  }
1736  case ISD::INLINEASM: {
1737    SmallVector<SDOperand, 8> Ops(Node->op_begin(), Node->op_end());
1738    bool Changed = false;
1739    // Legalize all of the operands of the inline asm, in case they are nodes
1740    // that need to be expanded or something.  Note we skip the asm string and
1741    // all of the TargetConstant flags.
1742    SDOperand Op = LegalizeOp(Ops[0]);
1743    Changed = Op != Ops[0];
1744    Ops[0] = Op;
1745
1746    bool HasInFlag = Ops.back().getValueType() == MVT::Flag;
1747    for (unsigned i = 2, e = Ops.size()-HasInFlag; i < e; ) {
1748      unsigned NumVals = cast<ConstantSDNode>(Ops[i])->getValue() >> 3;
1749      for (++i; NumVals; ++i, --NumVals) {
1750        SDOperand Op = LegalizeOp(Ops[i]);
1751        if (Op != Ops[i]) {
1752          Changed = true;
1753          Ops[i] = Op;
1754        }
1755      }
1756    }
1757
1758    if (HasInFlag) {
1759      Op = LegalizeOp(Ops.back());
1760      Changed |= Op != Ops.back();
1761      Ops.back() = Op;
1762    }
1763
1764    if (Changed)
1765      Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size());
1766
1767    // INLINE asm returns a chain and flag, make sure to add both to the map.
1768    AddLegalizedOperand(SDOperand(Node, 0), Result.getValue(0));
1769    AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1));
1770    return Result.getValue(Op.ResNo);
1771  }
1772  case ISD::BR:
1773    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
1774    // Ensure that libcalls are emitted before a branch.
1775    Tmp1 = DAG.getNode(ISD::TokenFactor, MVT::Other, Tmp1, LastCALLSEQ_END);
1776    Tmp1 = LegalizeOp(Tmp1);
1777    LastCALLSEQ_END = DAG.getEntryNode();
1778
1779    Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1));
1780    break;
1781  case ISD::BRIND:
1782    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
1783    // Ensure that libcalls are emitted before a branch.
1784    Tmp1 = DAG.getNode(ISD::TokenFactor, MVT::Other, Tmp1, LastCALLSEQ_END);
1785    Tmp1 = LegalizeOp(Tmp1);
1786    LastCALLSEQ_END = DAG.getEntryNode();
1787
1788    switch (getTypeAction(Node->getOperand(1).getValueType())) {
1789    default: assert(0 && "Indirect target must be legal type (pointer)!");
1790    case Legal:
1791      Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the condition.
1792      break;
1793    }
1794    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
1795    break;
1796  case ISD::BR_JT:
1797    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
1798    // Ensure that libcalls are emitted before a branch.
1799    Tmp1 = DAG.getNode(ISD::TokenFactor, MVT::Other, Tmp1, LastCALLSEQ_END);
1800    Tmp1 = LegalizeOp(Tmp1);
1801    LastCALLSEQ_END = DAG.getEntryNode();
1802
1803    Tmp2 = LegalizeOp(Node->getOperand(1));  // Legalize the jumptable node.
1804    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2));
1805
1806    switch (TLI.getOperationAction(ISD::BR_JT, MVT::Other)) {
1807    default: assert(0 && "This action is not supported yet!");
1808    case TargetLowering::Legal: break;
1809    case TargetLowering::Custom:
1810      Tmp1 = TLI.LowerOperation(Result, DAG);
1811      if (Tmp1.Val) Result = Tmp1;
1812      break;
1813    case TargetLowering::Expand: {
1814      SDOperand Chain = Result.getOperand(0);
1815      SDOperand Table = Result.getOperand(1);
1816      SDOperand Index = Result.getOperand(2);
1817
1818      MVT PTy = TLI.getPointerTy();
1819      MachineFunction &MF = DAG.getMachineFunction();
1820      unsigned EntrySize = MF.getJumpTableInfo()->getEntrySize();
1821      Index= DAG.getNode(ISD::MUL, PTy, Index, DAG.getConstant(EntrySize, PTy));
1822      SDOperand Addr = DAG.getNode(ISD::ADD, PTy, Index, Table);
1823
1824      SDOperand LD;
1825      switch (EntrySize) {
1826      default: assert(0 && "Size of jump table not supported yet."); break;
1827      case 4: LD = DAG.getLoad(MVT::i32, Chain, Addr,
1828                               PseudoSourceValue::getJumpTable(), 0); break;
1829      case 8: LD = DAG.getLoad(MVT::i64, Chain, Addr,
1830                               PseudoSourceValue::getJumpTable(), 0); break;
1831      }
1832
1833      Addr = LD;
1834      if (TLI.getTargetMachine().getRelocationModel() == Reloc::PIC_) {
1835        // For PIC, the sequence is:
1836        // BRIND(load(Jumptable + index) + RelocBase)
1837        // RelocBase can be JumpTable, GOT or some sort of global base.
1838        if (PTy != MVT::i32)
1839          Addr = DAG.getNode(ISD::SIGN_EXTEND, PTy, Addr);
1840        Addr = DAG.getNode(ISD::ADD, PTy, Addr,
1841                           TLI.getPICJumpTableRelocBase(Table, DAG));
1842      }
1843      Result = DAG.getNode(ISD::BRIND, MVT::Other, LD.getValue(1), Addr);
1844    }
1845    }
1846    break;
1847  case ISD::BRCOND:
1848    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
1849    // Ensure that libcalls are emitted before a return.
1850    Tmp1 = DAG.getNode(ISD::TokenFactor, MVT::Other, Tmp1, LastCALLSEQ_END);
1851    Tmp1 = LegalizeOp(Tmp1);
1852    LastCALLSEQ_END = DAG.getEntryNode();
1853
1854    switch (getTypeAction(Node->getOperand(1).getValueType())) {
1855    case Expand: assert(0 && "It's impossible to expand bools");
1856    case Legal:
1857      Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the condition.
1858      break;
1859    case Promote: {
1860      Tmp2 = PromoteOp(Node->getOperand(1));  // Promote the condition.
1861
1862      // The top bits of the promoted condition are not necessarily zero, ensure
1863      // that the value is properly zero extended.
1864      unsigned BitWidth = Tmp2.getValueSizeInBits();
1865      if (!DAG.MaskedValueIsZero(Tmp2,
1866                                 APInt::getHighBitsSet(BitWidth, BitWidth-1)))
1867        Tmp2 = DAG.getZeroExtendInReg(Tmp2, MVT::i1);
1868      break;
1869    }
1870    }
1871
1872    // Basic block destination (Op#2) is always legal.
1873    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2));
1874
1875    switch (TLI.getOperationAction(ISD::BRCOND, MVT::Other)) {
1876    default: assert(0 && "This action is not supported yet!");
1877    case TargetLowering::Legal: break;
1878    case TargetLowering::Custom:
1879      Tmp1 = TLI.LowerOperation(Result, DAG);
1880      if (Tmp1.Val) Result = Tmp1;
1881      break;
1882    case TargetLowering::Expand:
1883      // Expand brcond's setcc into its constituent parts and create a BR_CC
1884      // Node.
1885      if (Tmp2.getOpcode() == ISD::SETCC) {
1886        Result = DAG.getNode(ISD::BR_CC, MVT::Other, Tmp1, Tmp2.getOperand(2),
1887                             Tmp2.getOperand(0), Tmp2.getOperand(1),
1888                             Node->getOperand(2));
1889      } else {
1890        Result = DAG.getNode(ISD::BR_CC, MVT::Other, Tmp1,
1891                             DAG.getCondCode(ISD::SETNE), Tmp2,
1892                             DAG.getConstant(0, Tmp2.getValueType()),
1893                             Node->getOperand(2));
1894      }
1895      break;
1896    }
1897    break;
1898  case ISD::BR_CC:
1899    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
1900    // Ensure that libcalls are emitted before a branch.
1901    Tmp1 = DAG.getNode(ISD::TokenFactor, MVT::Other, Tmp1, LastCALLSEQ_END);
1902    Tmp1 = LegalizeOp(Tmp1);
1903    Tmp2 = Node->getOperand(2);              // LHS
1904    Tmp3 = Node->getOperand(3);              // RHS
1905    Tmp4 = Node->getOperand(1);              // CC
1906
1907    LegalizeSetCCOperands(Tmp2, Tmp3, Tmp4);
1908    LastCALLSEQ_END = DAG.getEntryNode();
1909
1910    // If we didn't get both a LHS and RHS back from LegalizeSetCCOperands,
1911    // the LHS is a legal SETCC itself.  In this case, we need to compare
1912    // the result against zero to select between true and false values.
1913    if (Tmp3.Val == 0) {
1914      Tmp3 = DAG.getConstant(0, Tmp2.getValueType());
1915      Tmp4 = DAG.getCondCode(ISD::SETNE);
1916    }
1917
1918    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp4, Tmp2, Tmp3,
1919                                    Node->getOperand(4));
1920
1921    switch (TLI.getOperationAction(ISD::BR_CC, Tmp3.getValueType())) {
1922    default: assert(0 && "Unexpected action for BR_CC!");
1923    case TargetLowering::Legal: break;
1924    case TargetLowering::Custom:
1925      Tmp4 = TLI.LowerOperation(Result, DAG);
1926      if (Tmp4.Val) Result = Tmp4;
1927      break;
1928    }
1929    break;
1930  case ISD::LOAD: {
1931    LoadSDNode *LD = cast<LoadSDNode>(Node);
1932    Tmp1 = LegalizeOp(LD->getChain());   // Legalize the chain.
1933    Tmp2 = LegalizeOp(LD->getBasePtr()); // Legalize the base pointer.
1934
1935    ISD::LoadExtType ExtType = LD->getExtensionType();
1936    if (ExtType == ISD::NON_EXTLOAD) {
1937      MVT VT = Node->getValueType(0);
1938      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, LD->getOffset());
1939      Tmp3 = Result.getValue(0);
1940      Tmp4 = Result.getValue(1);
1941
1942      switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
1943      default: assert(0 && "This action is not supported yet!");
1944      case TargetLowering::Legal:
1945        // If this is an unaligned load and the target doesn't support it,
1946        // expand it.
1947        if (!TLI.allowsUnalignedMemoryAccesses()) {
1948          unsigned ABIAlignment = TLI.getTargetData()->
1949            getABITypeAlignment(LD->getMemoryVT().getTypeForMVT());
1950          if (LD->getAlignment() < ABIAlignment){
1951            Result = ExpandUnalignedLoad(cast<LoadSDNode>(Result.Val), DAG,
1952                                         TLI);
1953            Tmp3 = Result.getOperand(0);
1954            Tmp4 = Result.getOperand(1);
1955            Tmp3 = LegalizeOp(Tmp3);
1956            Tmp4 = LegalizeOp(Tmp4);
1957          }
1958        }
1959        break;
1960      case TargetLowering::Custom:
1961        Tmp1 = TLI.LowerOperation(Tmp3, DAG);
1962        if (Tmp1.Val) {
1963          Tmp3 = LegalizeOp(Tmp1);
1964          Tmp4 = LegalizeOp(Tmp1.getValue(1));
1965        }
1966        break;
1967      case TargetLowering::Promote: {
1968        // Only promote a load of vector type to another.
1969        assert(VT.isVector() && "Cannot promote this load!");
1970        // Change base type to a different vector type.
1971        MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
1972
1973        Tmp1 = DAG.getLoad(NVT, Tmp1, Tmp2, LD->getSrcValue(),
1974                           LD->getSrcValueOffset(),
1975                           LD->isVolatile(), LD->getAlignment());
1976        Tmp3 = LegalizeOp(DAG.getNode(ISD::BIT_CONVERT, VT, Tmp1));
1977        Tmp4 = LegalizeOp(Tmp1.getValue(1));
1978        break;
1979      }
1980      }
1981      // Since loads produce two values, make sure to remember that we
1982      // legalized both of them.
1983      AddLegalizedOperand(SDOperand(Node, 0), Tmp3);
1984      AddLegalizedOperand(SDOperand(Node, 1), Tmp4);
1985      return Op.ResNo ? Tmp4 : Tmp3;
1986    } else {
1987      MVT SrcVT = LD->getMemoryVT();
1988      unsigned SrcWidth = SrcVT.getSizeInBits();
1989      int SVOffset = LD->getSrcValueOffset();
1990      unsigned Alignment = LD->getAlignment();
1991      bool isVolatile = LD->isVolatile();
1992
1993      if (SrcWidth != SrcVT.getStoreSizeInBits() &&
1994          // Some targets pretend to have an i1 loading operation, and actually
1995          // load an i8.  This trick is correct for ZEXTLOAD because the top 7
1996          // bits are guaranteed to be zero; it helps the optimizers understand
1997          // that these bits are zero.  It is also useful for EXTLOAD, since it
1998          // tells the optimizers that those bits are undefined.  It would be
1999          // nice to have an effective generic way of getting these benefits...
2000          // Until such a way is found, don't insist on promoting i1 here.
2001          (SrcVT != MVT::i1 ||
2002           TLI.getLoadXAction(ExtType, MVT::i1) == TargetLowering::Promote)) {
2003        // Promote to a byte-sized load if not loading an integral number of
2004        // bytes.  For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
2005        unsigned NewWidth = SrcVT.getStoreSizeInBits();
2006        MVT NVT = MVT::getIntegerVT(NewWidth);
2007        SDOperand Ch;
2008
2009        // The extra bits are guaranteed to be zero, since we stored them that
2010        // way.  A zext load from NVT thus automatically gives zext from SrcVT.
2011
2012        ISD::LoadExtType NewExtType =
2013          ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
2014
2015        Result = DAG.getExtLoad(NewExtType, Node->getValueType(0),
2016                                Tmp1, Tmp2, LD->getSrcValue(), SVOffset,
2017                                NVT, isVolatile, Alignment);
2018
2019        Ch = Result.getValue(1); // The chain.
2020
2021        if (ExtType == ISD::SEXTLOAD)
2022          // Having the top bits zero doesn't help when sign extending.
2023          Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, Result.getValueType(),
2024                               Result, DAG.getValueType(SrcVT));
2025        else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
2026          // All the top bits are guaranteed to be zero - inform the optimizers.
2027          Result = DAG.getNode(ISD::AssertZext, Result.getValueType(), Result,
2028                               DAG.getValueType(SrcVT));
2029
2030        Tmp1 = LegalizeOp(Result);
2031        Tmp2 = LegalizeOp(Ch);
2032      } else if (SrcWidth & (SrcWidth - 1)) {
2033        // If not loading a power-of-2 number of bits, expand as two loads.
2034        assert(SrcVT.isExtended() && !SrcVT.isVector() &&
2035               "Unsupported extload!");
2036        unsigned RoundWidth = 1 << Log2_32(SrcWidth);
2037        assert(RoundWidth < SrcWidth);
2038        unsigned ExtraWidth = SrcWidth - RoundWidth;
2039        assert(ExtraWidth < RoundWidth);
2040        assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
2041               "Load size not an integral number of bytes!");
2042        MVT RoundVT = MVT::getIntegerVT(RoundWidth);
2043        MVT ExtraVT = MVT::getIntegerVT(ExtraWidth);
2044        SDOperand Lo, Hi, Ch;
2045        unsigned IncrementSize;
2046
2047        if (TLI.isLittleEndian()) {
2048          // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
2049          // Load the bottom RoundWidth bits.
2050          Lo = DAG.getExtLoad(ISD::ZEXTLOAD, Node->getValueType(0), Tmp1, Tmp2,
2051                              LD->getSrcValue(), SVOffset, RoundVT, isVolatile,
2052                              Alignment);
2053
2054          // Load the remaining ExtraWidth bits.
2055          IncrementSize = RoundWidth / 8;
2056          Tmp2 = DAG.getNode(ISD::ADD, Tmp2.getValueType(), Tmp2,
2057                             DAG.getIntPtrConstant(IncrementSize));
2058          Hi = DAG.getExtLoad(ExtType, Node->getValueType(0), Tmp1, Tmp2,
2059                              LD->getSrcValue(), SVOffset + IncrementSize,
2060                              ExtraVT, isVolatile,
2061                              MinAlign(Alignment, IncrementSize));
2062
2063          // Build a factor node to remember that this load is independent of the
2064          // other one.
2065          Ch = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1),
2066                           Hi.getValue(1));
2067
2068          // Move the top bits to the right place.
2069          Hi = DAG.getNode(ISD::SHL, Hi.getValueType(), Hi,
2070                           DAG.getConstant(RoundWidth, TLI.getShiftAmountTy()));
2071
2072          // Join the hi and lo parts.
2073          Result = DAG.getNode(ISD::OR, Node->getValueType(0), Lo, Hi);
2074        } else {
2075          // Big endian - avoid unaligned loads.
2076          // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
2077          // Load the top RoundWidth bits.
2078          Hi = DAG.getExtLoad(ExtType, Node->getValueType(0), Tmp1, Tmp2,
2079                              LD->getSrcValue(), SVOffset, RoundVT, isVolatile,
2080                              Alignment);
2081
2082          // Load the remaining ExtraWidth bits.
2083          IncrementSize = RoundWidth / 8;
2084          Tmp2 = DAG.getNode(ISD::ADD, Tmp2.getValueType(), Tmp2,
2085                             DAG.getIntPtrConstant(IncrementSize));
2086          Lo = DAG.getExtLoad(ISD::ZEXTLOAD, Node->getValueType(0), Tmp1, Tmp2,
2087                              LD->getSrcValue(), SVOffset + IncrementSize,
2088                              ExtraVT, isVolatile,
2089                              MinAlign(Alignment, IncrementSize));
2090
2091          // Build a factor node to remember that this load is independent of the
2092          // other one.
2093          Ch = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1),
2094                           Hi.getValue(1));
2095
2096          // Move the top bits to the right place.
2097          Hi = DAG.getNode(ISD::SHL, Hi.getValueType(), Hi,
2098                           DAG.getConstant(ExtraWidth, TLI.getShiftAmountTy()));
2099
2100          // Join the hi and lo parts.
2101          Result = DAG.getNode(ISD::OR, Node->getValueType(0), Lo, Hi);
2102        }
2103
2104        Tmp1 = LegalizeOp(Result);
2105        Tmp2 = LegalizeOp(Ch);
2106      } else {
2107        switch (TLI.getLoadXAction(ExtType, SrcVT)) {
2108        default: assert(0 && "This action is not supported yet!");
2109        case TargetLowering::Custom:
2110          isCustom = true;
2111          // FALLTHROUGH
2112        case TargetLowering::Legal:
2113          Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, LD->getOffset());
2114          Tmp1 = Result.getValue(0);
2115          Tmp2 = Result.getValue(1);
2116
2117          if (isCustom) {
2118            Tmp3 = TLI.LowerOperation(Result, DAG);
2119            if (Tmp3.Val) {
2120              Tmp1 = LegalizeOp(Tmp3);
2121              Tmp2 = LegalizeOp(Tmp3.getValue(1));
2122            }
2123          } else {
2124            // If this is an unaligned load and the target doesn't support it,
2125            // expand it.
2126            if (!TLI.allowsUnalignedMemoryAccesses()) {
2127              unsigned ABIAlignment = TLI.getTargetData()->
2128                getABITypeAlignment(LD->getMemoryVT().getTypeForMVT());
2129              if (LD->getAlignment() < ABIAlignment){
2130                Result = ExpandUnalignedLoad(cast<LoadSDNode>(Result.Val), DAG,
2131                                             TLI);
2132                Tmp1 = Result.getOperand(0);
2133                Tmp2 = Result.getOperand(1);
2134                Tmp1 = LegalizeOp(Tmp1);
2135                Tmp2 = LegalizeOp(Tmp2);
2136              }
2137            }
2138          }
2139          break;
2140        case TargetLowering::Expand:
2141          // f64 = EXTLOAD f32 should expand to LOAD, FP_EXTEND
2142          if (SrcVT == MVT::f32 && Node->getValueType(0) == MVT::f64) {
2143            SDOperand Load = DAG.getLoad(SrcVT, Tmp1, Tmp2, LD->getSrcValue(),
2144                                         LD->getSrcValueOffset(),
2145                                         LD->isVolatile(), LD->getAlignment());
2146            Result = DAG.getNode(ISD::FP_EXTEND, Node->getValueType(0), Load);
2147            Tmp1 = LegalizeOp(Result);  // Relegalize new nodes.
2148            Tmp2 = LegalizeOp(Load.getValue(1));
2149            break;
2150          }
2151          assert(ExtType != ISD::EXTLOAD &&"EXTLOAD should always be supported!");
2152          // Turn the unsupported load into an EXTLOAD followed by an explicit
2153          // zero/sign extend inreg.
2154          Result = DAG.getExtLoad(ISD::EXTLOAD, Node->getValueType(0),
2155                                  Tmp1, Tmp2, LD->getSrcValue(),
2156                                  LD->getSrcValueOffset(), SrcVT,
2157                                  LD->isVolatile(), LD->getAlignment());
2158          SDOperand ValRes;
2159          if (ExtType == ISD::SEXTLOAD)
2160            ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, Result.getValueType(),
2161                                 Result, DAG.getValueType(SrcVT));
2162          else
2163            ValRes = DAG.getZeroExtendInReg(Result, SrcVT);
2164          Tmp1 = LegalizeOp(ValRes);  // Relegalize new nodes.
2165          Tmp2 = LegalizeOp(Result.getValue(1));  // Relegalize new nodes.
2166          break;
2167        }
2168      }
2169
2170      // Since loads produce two values, make sure to remember that we legalized
2171      // both of them.
2172      AddLegalizedOperand(SDOperand(Node, 0), Tmp1);
2173      AddLegalizedOperand(SDOperand(Node, 1), Tmp2);
2174      return Op.ResNo ? Tmp2 : Tmp1;
2175    }
2176  }
2177  case ISD::EXTRACT_ELEMENT: {
2178    MVT OpTy = Node->getOperand(0).getValueType();
2179    switch (getTypeAction(OpTy)) {
2180    default: assert(0 && "EXTRACT_ELEMENT action for type unimplemented!");
2181    case Legal:
2182      if (cast<ConstantSDNode>(Node->getOperand(1))->getValue()) {
2183        // 1 -> Hi
2184        Result = DAG.getNode(ISD::SRL, OpTy, Node->getOperand(0),
2185                             DAG.getConstant(OpTy.getSizeInBits()/2,
2186                                             TLI.getShiftAmountTy()));
2187        Result = DAG.getNode(ISD::TRUNCATE, Node->getValueType(0), Result);
2188      } else {
2189        // 0 -> Lo
2190        Result = DAG.getNode(ISD::TRUNCATE, Node->getValueType(0),
2191                             Node->getOperand(0));
2192      }
2193      break;
2194    case Expand:
2195      // Get both the low and high parts.
2196      ExpandOp(Node->getOperand(0), Tmp1, Tmp2);
2197      if (cast<ConstantSDNode>(Node->getOperand(1))->getValue())
2198        Result = Tmp2;  // 1 -> Hi
2199      else
2200        Result = Tmp1;  // 0 -> Lo
2201      break;
2202    }
2203    break;
2204  }
2205
2206  case ISD::CopyToReg:
2207    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
2208
2209    assert(isTypeLegal(Node->getOperand(2).getValueType()) &&
2210           "Register type must be legal!");
2211    // Legalize the incoming value (must be a legal type).
2212    Tmp2 = LegalizeOp(Node->getOperand(2));
2213    if (Node->getNumValues() == 1) {
2214      Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1), Tmp2);
2215    } else {
2216      assert(Node->getNumValues() == 2 && "Unknown CopyToReg");
2217      if (Node->getNumOperands() == 4) {
2218        Tmp3 = LegalizeOp(Node->getOperand(3));
2219        Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1), Tmp2,
2220                                        Tmp3);
2221      } else {
2222        Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1),Tmp2);
2223      }
2224
2225      // Since this produces two values, make sure to remember that we legalized
2226      // both of them.
2227      AddLegalizedOperand(SDOperand(Node, 0), Result.getValue(0));
2228      AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1));
2229      return Result;
2230    }
2231    break;
2232
2233  case ISD::RET:
2234    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
2235
2236    // Ensure that libcalls are emitted before a return.
2237    Tmp1 = DAG.getNode(ISD::TokenFactor, MVT::Other, Tmp1, LastCALLSEQ_END);
2238    Tmp1 = LegalizeOp(Tmp1);
2239    LastCALLSEQ_END = DAG.getEntryNode();
2240
2241    switch (Node->getNumOperands()) {
2242    case 3:  // ret val
2243      Tmp2 = Node->getOperand(1);
2244      Tmp3 = Node->getOperand(2);  // Signness
2245      switch (getTypeAction(Tmp2.getValueType())) {
2246      case Legal:
2247        Result = DAG.UpdateNodeOperands(Result, Tmp1, LegalizeOp(Tmp2), Tmp3);
2248        break;
2249      case Expand:
2250        if (!Tmp2.getValueType().isVector()) {
2251          SDOperand Lo, Hi;
2252          ExpandOp(Tmp2, Lo, Hi);
2253
2254          // Big endian systems want the hi reg first.
2255          if (TLI.isBigEndian())
2256            std::swap(Lo, Hi);
2257
2258          if (Hi.Val)
2259            Result = DAG.getNode(ISD::RET, MVT::Other, Tmp1, Lo, Tmp3, Hi,Tmp3);
2260          else
2261            Result = DAG.getNode(ISD::RET, MVT::Other, Tmp1, Lo, Tmp3);
2262          Result = LegalizeOp(Result);
2263        } else {
2264          SDNode *InVal = Tmp2.Val;
2265          int InIx = Tmp2.ResNo;
2266          unsigned NumElems = InVal->getValueType(InIx).getVectorNumElements();
2267          MVT EVT = InVal->getValueType(InIx).getVectorElementType();
2268
2269          // Figure out if there is a simple type corresponding to this Vector
2270          // type.  If so, convert to the vector type.
2271          MVT TVT = MVT::getVectorVT(EVT, NumElems);
2272          if (TLI.isTypeLegal(TVT)) {
2273            // Turn this into a return of the vector type.
2274            Tmp2 = LegalizeOp(Tmp2);
2275            Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
2276          } else if (NumElems == 1) {
2277            // Turn this into a return of the scalar type.
2278            Tmp2 = ScalarizeVectorOp(Tmp2);
2279            Tmp2 = LegalizeOp(Tmp2);
2280            Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
2281
2282            // FIXME: Returns of gcc generic vectors smaller than a legal type
2283            // should be returned in integer registers!
2284
2285            // The scalarized value type may not be legal, e.g. it might require
2286            // promotion or expansion.  Relegalize the return.
2287            Result = LegalizeOp(Result);
2288          } else {
2289            // FIXME: Returns of gcc generic vectors larger than a legal vector
2290            // type should be returned by reference!
2291            SDOperand Lo, Hi;
2292            SplitVectorOp(Tmp2, Lo, Hi);
2293            Result = DAG.getNode(ISD::RET, MVT::Other, Tmp1, Lo, Tmp3, Hi,Tmp3);
2294            Result = LegalizeOp(Result);
2295          }
2296        }
2297        break;
2298      case Promote:
2299        Tmp2 = PromoteOp(Node->getOperand(1));
2300        Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
2301        Result = LegalizeOp(Result);
2302        break;
2303      }
2304      break;
2305    case 1:  // ret void
2306      Result = DAG.UpdateNodeOperands(Result, Tmp1);
2307      break;
2308    default: { // ret <values>
2309      SmallVector<SDOperand, 8> NewValues;
2310      NewValues.push_back(Tmp1);
2311      for (unsigned i = 1, e = Node->getNumOperands(); i < e; i += 2)
2312        switch (getTypeAction(Node->getOperand(i).getValueType())) {
2313        case Legal:
2314          NewValues.push_back(LegalizeOp(Node->getOperand(i)));
2315          NewValues.push_back(Node->getOperand(i+1));
2316          break;
2317        case Expand: {
2318          SDOperand Lo, Hi;
2319          assert(!Node->getOperand(i).getValueType().isExtended() &&
2320                 "FIXME: TODO: implement returning non-legal vector types!");
2321          ExpandOp(Node->getOperand(i), Lo, Hi);
2322          NewValues.push_back(Lo);
2323          NewValues.push_back(Node->getOperand(i+1));
2324          if (Hi.Val) {
2325            NewValues.push_back(Hi);
2326            NewValues.push_back(Node->getOperand(i+1));
2327          }
2328          break;
2329        }
2330        case Promote:
2331          assert(0 && "Can't promote multiple return value yet!");
2332        }
2333
2334      if (NewValues.size() == Node->getNumOperands())
2335        Result = DAG.UpdateNodeOperands(Result, &NewValues[0],NewValues.size());
2336      else
2337        Result = DAG.getNode(ISD::RET, MVT::Other,
2338                             &NewValues[0], NewValues.size());
2339      break;
2340    }
2341    }
2342
2343    if (Result.getOpcode() == ISD::RET) {
2344      switch (TLI.getOperationAction(Result.getOpcode(), MVT::Other)) {
2345      default: assert(0 && "This action is not supported yet!");
2346      case TargetLowering::Legal: break;
2347      case TargetLowering::Custom:
2348        Tmp1 = TLI.LowerOperation(Result, DAG);
2349        if (Tmp1.Val) Result = Tmp1;
2350        break;
2351      }
2352    }
2353    break;
2354  case ISD::STORE: {
2355    StoreSDNode *ST = cast<StoreSDNode>(Node);
2356    Tmp1 = LegalizeOp(ST->getChain());    // Legalize the chain.
2357    Tmp2 = LegalizeOp(ST->getBasePtr());  // Legalize the pointer.
2358    int SVOffset = ST->getSrcValueOffset();
2359    unsigned Alignment = ST->getAlignment();
2360    bool isVolatile = ST->isVolatile();
2361
2362    if (!ST->isTruncatingStore()) {
2363      // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
2364      // FIXME: We shouldn't do this for TargetConstantFP's.
2365      // FIXME: move this to the DAG Combiner!  Note that we can't regress due
2366      // to phase ordering between legalized code and the dag combiner.  This
2367      // probably means that we need to integrate dag combiner and legalizer
2368      // together.
2369      // We generally can't do this one for long doubles.
2370      if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) {
2371        if (CFP->getValueType(0) == MVT::f32 &&
2372            getTypeAction(MVT::i32) == Legal) {
2373          Tmp3 = DAG.getConstant(CFP->getValueAPF().
2374                                          convertToAPInt().zextOrTrunc(32),
2375                                  MVT::i32);
2376          Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
2377                                SVOffset, isVolatile, Alignment);
2378          break;
2379        } else if (CFP->getValueType(0) == MVT::f64) {
2380          // If this target supports 64-bit registers, do a single 64-bit store.
2381          if (getTypeAction(MVT::i64) == Legal) {
2382            Tmp3 = DAG.getConstant(CFP->getValueAPF().convertToAPInt().
2383                                     zextOrTrunc(64), MVT::i64);
2384            Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
2385                                  SVOffset, isVolatile, Alignment);
2386            break;
2387          } else if (getTypeAction(MVT::i32) == Legal && !ST->isVolatile()) {
2388            // Otherwise, if the target supports 32-bit registers, use 2 32-bit
2389            // stores.  If the target supports neither 32- nor 64-bits, this
2390            // xform is certainly not worth it.
2391            const APInt &IntVal =CFP->getValueAPF().convertToAPInt();
2392            SDOperand Lo = DAG.getConstant(APInt(IntVal).trunc(32), MVT::i32);
2393            SDOperand Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), MVT::i32);
2394            if (TLI.isBigEndian()) std::swap(Lo, Hi);
2395
2396            Lo = DAG.getStore(Tmp1, Lo, Tmp2, ST->getSrcValue(),
2397                              SVOffset, isVolatile, Alignment);
2398            Tmp2 = DAG.getNode(ISD::ADD, Tmp2.getValueType(), Tmp2,
2399                               DAG.getIntPtrConstant(4));
2400            Hi = DAG.getStore(Tmp1, Hi, Tmp2, ST->getSrcValue(), SVOffset+4,
2401                              isVolatile, MinAlign(Alignment, 4U));
2402
2403            Result = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi);
2404            break;
2405          }
2406        }
2407      }
2408
2409      switch (getTypeAction(ST->getMemoryVT())) {
2410      case Legal: {
2411        Tmp3 = LegalizeOp(ST->getValue());
2412        Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp3, Tmp2,
2413                                        ST->getOffset());
2414
2415        MVT VT = Tmp3.getValueType();
2416        switch (TLI.getOperationAction(ISD::STORE, VT)) {
2417        default: assert(0 && "This action is not supported yet!");
2418        case TargetLowering::Legal:
2419          // If this is an unaligned store and the target doesn't support it,
2420          // expand it.
2421          if (!TLI.allowsUnalignedMemoryAccesses()) {
2422            unsigned ABIAlignment = TLI.getTargetData()->
2423              getABITypeAlignment(ST->getMemoryVT().getTypeForMVT());
2424            if (ST->getAlignment() < ABIAlignment)
2425              Result = ExpandUnalignedStore(cast<StoreSDNode>(Result.Val), DAG,
2426                                            TLI);
2427          }
2428          break;
2429        case TargetLowering::Custom:
2430          Tmp1 = TLI.LowerOperation(Result, DAG);
2431          if (Tmp1.Val) Result = Tmp1;
2432          break;
2433        case TargetLowering::Promote:
2434          assert(VT.isVector() && "Unknown legal promote case!");
2435          Tmp3 = DAG.getNode(ISD::BIT_CONVERT,
2436                             TLI.getTypeToPromoteTo(ISD::STORE, VT), Tmp3);
2437          Result = DAG.getStore(Tmp1, Tmp3, Tmp2,
2438                                ST->getSrcValue(), SVOffset, isVolatile,
2439                                Alignment);
2440          break;
2441        }
2442        break;
2443      }
2444      case Promote:
2445        // Truncate the value and store the result.
2446        Tmp3 = PromoteOp(ST->getValue());
2447        Result = DAG.getTruncStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
2448                                   SVOffset, ST->getMemoryVT(),
2449                                   isVolatile, Alignment);
2450        break;
2451
2452      case Expand:
2453        unsigned IncrementSize = 0;
2454        SDOperand Lo, Hi;
2455
2456        // If this is a vector type, then we have to calculate the increment as
2457        // the product of the element size in bytes, and the number of elements
2458        // in the high half of the vector.
2459        if (ST->getValue().getValueType().isVector()) {
2460          SDNode *InVal = ST->getValue().Val;
2461          int InIx = ST->getValue().ResNo;
2462          MVT InVT = InVal->getValueType(InIx);
2463          unsigned NumElems = InVT.getVectorNumElements();
2464          MVT EVT = InVT.getVectorElementType();
2465
2466          // Figure out if there is a simple type corresponding to this Vector
2467          // type.  If so, convert to the vector type.
2468          MVT TVT = MVT::getVectorVT(EVT, NumElems);
2469          if (TLI.isTypeLegal(TVT)) {
2470            // Turn this into a normal store of the vector type.
2471            Tmp3 = LegalizeOp(ST->getValue());
2472            Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
2473                                  SVOffset, isVolatile, Alignment);
2474            Result = LegalizeOp(Result);
2475            break;
2476          } else if (NumElems == 1) {
2477            // Turn this into a normal store of the scalar type.
2478            Tmp3 = ScalarizeVectorOp(ST->getValue());
2479            Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
2480                                  SVOffset, isVolatile, Alignment);
2481            // The scalarized value type may not be legal, e.g. it might require
2482            // promotion or expansion.  Relegalize the scalar store.
2483            Result = LegalizeOp(Result);
2484            break;
2485          } else {
2486            SplitVectorOp(ST->getValue(), Lo, Hi);
2487            IncrementSize = Lo.Val->getValueType(0).getVectorNumElements() *
2488                            EVT.getSizeInBits()/8;
2489          }
2490        } else {
2491          ExpandOp(ST->getValue(), Lo, Hi);
2492          IncrementSize = Hi.Val ? Hi.getValueType().getSizeInBits()/8 : 0;
2493
2494          if (TLI.isBigEndian())
2495            std::swap(Lo, Hi);
2496        }
2497
2498        Lo = DAG.getStore(Tmp1, Lo, Tmp2, ST->getSrcValue(),
2499                          SVOffset, isVolatile, Alignment);
2500
2501        if (Hi.Val == NULL) {
2502          // Must be int <-> float one-to-one expansion.
2503          Result = Lo;
2504          break;
2505        }
2506
2507        Tmp2 = DAG.getNode(ISD::ADD, Tmp2.getValueType(), Tmp2,
2508                           DAG.getIntPtrConstant(IncrementSize));
2509        assert(isTypeLegal(Tmp2.getValueType()) &&
2510               "Pointers must be legal!");
2511        SVOffset += IncrementSize;
2512        Alignment = MinAlign(Alignment, IncrementSize);
2513        Hi = DAG.getStore(Tmp1, Hi, Tmp2, ST->getSrcValue(),
2514                          SVOffset, isVolatile, Alignment);
2515        Result = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi);
2516        break;
2517      }
2518    } else {
2519      switch (getTypeAction(ST->getValue().getValueType())) {
2520      case Legal:
2521        Tmp3 = LegalizeOp(ST->getValue());
2522        break;
2523      case Promote:
2524        // We can promote the value, the truncstore will still take care of it.
2525        Tmp3 = PromoteOp(ST->getValue());
2526        break;
2527      case Expand:
2528        // Just store the low part.  This may become a non-trunc store, so make
2529        // sure to use getTruncStore, not UpdateNodeOperands below.
2530        ExpandOp(ST->getValue(), Tmp3, Tmp4);
2531        return DAG.getTruncStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
2532                                 SVOffset, MVT::i8, isVolatile, Alignment);
2533      }
2534
2535      MVT StVT = ST->getMemoryVT();
2536      unsigned StWidth = StVT.getSizeInBits();
2537
2538      if (StWidth != StVT.getStoreSizeInBits()) {
2539        // Promote to a byte-sized store with upper bits zero if not
2540        // storing an integral number of bytes.  For example, promote
2541        // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
2542        MVT NVT = MVT::getIntegerVT(StVT.getStoreSizeInBits());
2543        Tmp3 = DAG.getZeroExtendInReg(Tmp3, StVT);
2544        Result = DAG.getTruncStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
2545                                   SVOffset, NVT, isVolatile, Alignment);
2546      } else if (StWidth & (StWidth - 1)) {
2547        // If not storing a power-of-2 number of bits, expand as two stores.
2548        assert(StVT.isExtended() && !StVT.isVector() &&
2549               "Unsupported truncstore!");
2550        unsigned RoundWidth = 1 << Log2_32(StWidth);
2551        assert(RoundWidth < StWidth);
2552        unsigned ExtraWidth = StWidth - RoundWidth;
2553        assert(ExtraWidth < RoundWidth);
2554        assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
2555               "Store size not an integral number of bytes!");
2556        MVT RoundVT = MVT::getIntegerVT(RoundWidth);
2557        MVT ExtraVT = MVT::getIntegerVT(ExtraWidth);
2558        SDOperand Lo, Hi;
2559        unsigned IncrementSize;
2560
2561        if (TLI.isLittleEndian()) {
2562          // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
2563          // Store the bottom RoundWidth bits.
2564          Lo = DAG.getTruncStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
2565                                 SVOffset, RoundVT,
2566                                 isVolatile, Alignment);
2567
2568          // Store the remaining ExtraWidth bits.
2569          IncrementSize = RoundWidth / 8;
2570          Tmp2 = DAG.getNode(ISD::ADD, Tmp2.getValueType(), Tmp2,
2571                             DAG.getIntPtrConstant(IncrementSize));
2572          Hi = DAG.getNode(ISD::SRL, Tmp3.getValueType(), Tmp3,
2573                           DAG.getConstant(RoundWidth, TLI.getShiftAmountTy()));
2574          Hi = DAG.getTruncStore(Tmp1, Hi, Tmp2, ST->getSrcValue(),
2575                                 SVOffset + IncrementSize, ExtraVT, isVolatile,
2576                                 MinAlign(Alignment, IncrementSize));
2577        } else {
2578          // Big endian - avoid unaligned stores.
2579          // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
2580          // Store the top RoundWidth bits.
2581          Hi = DAG.getNode(ISD::SRL, Tmp3.getValueType(), Tmp3,
2582                           DAG.getConstant(ExtraWidth, TLI.getShiftAmountTy()));
2583          Hi = DAG.getTruncStore(Tmp1, Hi, Tmp2, ST->getSrcValue(), SVOffset,
2584                                 RoundVT, isVolatile, Alignment);
2585
2586          // Store the remaining ExtraWidth bits.
2587          IncrementSize = RoundWidth / 8;
2588          Tmp2 = DAG.getNode(ISD::ADD, Tmp2.getValueType(), Tmp2,
2589                             DAG.getIntPtrConstant(IncrementSize));
2590          Lo = DAG.getTruncStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
2591                                 SVOffset + IncrementSize, ExtraVT, isVolatile,
2592                                 MinAlign(Alignment, IncrementSize));
2593        }
2594
2595        // The order of the stores doesn't matter.
2596        Result = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi);
2597      } else {
2598        if (Tmp1 != ST->getChain() || Tmp3 != ST->getValue() ||
2599            Tmp2 != ST->getBasePtr())
2600          Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp3, Tmp2,
2601                                          ST->getOffset());
2602
2603        switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
2604        default: assert(0 && "This action is not supported yet!");
2605        case TargetLowering::Legal:
2606          // If this is an unaligned store and the target doesn't support it,
2607          // expand it.
2608          if (!TLI.allowsUnalignedMemoryAccesses()) {
2609            unsigned ABIAlignment = TLI.getTargetData()->
2610              getABITypeAlignment(ST->getMemoryVT().getTypeForMVT());
2611            if (ST->getAlignment() < ABIAlignment)
2612              Result = ExpandUnalignedStore(cast<StoreSDNode>(Result.Val), DAG,
2613                                            TLI);
2614          }
2615          break;
2616        case TargetLowering::Custom:
2617          Result = TLI.LowerOperation(Result, DAG);
2618          break;
2619        case Expand:
2620          // TRUNCSTORE:i16 i32 -> STORE i16
2621          assert(isTypeLegal(StVT) && "Do not know how to expand this store!");
2622          Tmp3 = DAG.getNode(ISD::TRUNCATE, StVT, Tmp3);
2623          Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(), SVOffset,
2624                                isVolatile, Alignment);
2625          break;
2626        }
2627      }
2628    }
2629    break;
2630  }
2631  case ISD::PCMARKER:
2632    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
2633    Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1));
2634    break;
2635  case ISD::STACKSAVE:
2636    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
2637    Result = DAG.UpdateNodeOperands(Result, Tmp1);
2638    Tmp1 = Result.getValue(0);
2639    Tmp2 = Result.getValue(1);
2640
2641    switch (TLI.getOperationAction(ISD::STACKSAVE, MVT::Other)) {
2642    default: assert(0 && "This action is not supported yet!");
2643    case TargetLowering::Legal: break;
2644    case TargetLowering::Custom:
2645      Tmp3 = TLI.LowerOperation(Result, DAG);
2646      if (Tmp3.Val) {
2647        Tmp1 = LegalizeOp(Tmp3);
2648        Tmp2 = LegalizeOp(Tmp3.getValue(1));
2649      }
2650      break;
2651    case TargetLowering::Expand:
2652      // Expand to CopyFromReg if the target set
2653      // StackPointerRegisterToSaveRestore.
2654      if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
2655        Tmp1 = DAG.getCopyFromReg(Result.getOperand(0), SP,
2656                                  Node->getValueType(0));
2657        Tmp2 = Tmp1.getValue(1);
2658      } else {
2659        Tmp1 = DAG.getNode(ISD::UNDEF, Node->getValueType(0));
2660        Tmp2 = Node->getOperand(0);
2661      }
2662      break;
2663    }
2664
2665    // Since stacksave produce two values, make sure to remember that we
2666    // legalized both of them.
2667    AddLegalizedOperand(SDOperand(Node, 0), Tmp1);
2668    AddLegalizedOperand(SDOperand(Node, 1), Tmp2);
2669    return Op.ResNo ? Tmp2 : Tmp1;
2670
2671  case ISD::STACKRESTORE:
2672    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
2673    Tmp2 = LegalizeOp(Node->getOperand(1));  // Legalize the pointer.
2674    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
2675
2676    switch (TLI.getOperationAction(ISD::STACKRESTORE, MVT::Other)) {
2677    default: assert(0 && "This action is not supported yet!");
2678    case TargetLowering::Legal: break;
2679    case TargetLowering::Custom:
2680      Tmp1 = TLI.LowerOperation(Result, DAG);
2681      if (Tmp1.Val) Result = Tmp1;
2682      break;
2683    case TargetLowering::Expand:
2684      // Expand to CopyToReg if the target set
2685      // StackPointerRegisterToSaveRestore.
2686      if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
2687        Result = DAG.getCopyToReg(Tmp1, SP, Tmp2);
2688      } else {
2689        Result = Tmp1;
2690      }
2691      break;
2692    }
2693    break;
2694
2695  case ISD::READCYCLECOUNTER:
2696    Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain
2697    Result = DAG.UpdateNodeOperands(Result, Tmp1);
2698    switch (TLI.getOperationAction(ISD::READCYCLECOUNTER,
2699                                   Node->getValueType(0))) {
2700    default: assert(0 && "This action is not supported yet!");
2701    case TargetLowering::Legal:
2702      Tmp1 = Result.getValue(0);
2703      Tmp2 = Result.getValue(1);
2704      break;
2705    case TargetLowering::Custom:
2706      Result = TLI.LowerOperation(Result, DAG);
2707      Tmp1 = LegalizeOp(Result.getValue(0));
2708      Tmp2 = LegalizeOp(Result.getValue(1));
2709      break;
2710    }
2711
2712    // Since rdcc produce two values, make sure to remember that we legalized
2713    // both of them.
2714    AddLegalizedOperand(SDOperand(Node, 0), Tmp1);
2715    AddLegalizedOperand(SDOperand(Node, 1), Tmp2);
2716    return Result;
2717
2718  case ISD::SELECT:
2719    switch (getTypeAction(Node->getOperand(0).getValueType())) {
2720    case Expand: assert(0 && "It's impossible to expand bools");
2721    case Legal:
2722      Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the condition.
2723      break;
2724    case Promote: {
2725      Tmp1 = PromoteOp(Node->getOperand(0));  // Promote the condition.
2726      // Make sure the condition is either zero or one.
2727      unsigned BitWidth = Tmp1.getValueSizeInBits();
2728      if (!DAG.MaskedValueIsZero(Tmp1,
2729                                 APInt::getHighBitsSet(BitWidth, BitWidth-1)))
2730        Tmp1 = DAG.getZeroExtendInReg(Tmp1, MVT::i1);
2731      break;
2732    }
2733    }
2734    Tmp2 = LegalizeOp(Node->getOperand(1));   // TrueVal
2735    Tmp3 = LegalizeOp(Node->getOperand(2));   // FalseVal
2736
2737    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
2738
2739    switch (TLI.getOperationAction(ISD::SELECT, Tmp2.getValueType())) {
2740    default: assert(0 && "This action is not supported yet!");
2741    case TargetLowering::Legal: break;
2742    case TargetLowering::Custom: {
2743      Tmp1 = TLI.LowerOperation(Result, DAG);
2744      if (Tmp1.Val) Result = Tmp1;
2745      break;
2746    }
2747    case TargetLowering::Expand:
2748      if (Tmp1.getOpcode() == ISD::SETCC) {
2749        Result = DAG.getSelectCC(Tmp1.getOperand(0), Tmp1.getOperand(1),
2750                              Tmp2, Tmp3,
2751                              cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
2752      } else {
2753        Result = DAG.getSelectCC(Tmp1,
2754                                 DAG.getConstant(0, Tmp1.getValueType()),
2755                                 Tmp2, Tmp3, ISD::SETNE);
2756      }
2757      break;
2758    case TargetLowering::Promote: {
2759      MVT NVT =
2760        TLI.getTypeToPromoteTo(ISD::SELECT, Tmp2.getValueType());
2761      unsigned ExtOp, TruncOp;
2762      if (Tmp2.getValueType().isVector()) {
2763        ExtOp   = ISD::BIT_CONVERT;
2764        TruncOp = ISD::BIT_CONVERT;
2765      } else if (Tmp2.getValueType().isInteger()) {
2766        ExtOp   = ISD::ANY_EXTEND;
2767        TruncOp = ISD::TRUNCATE;
2768      } else {
2769        ExtOp   = ISD::FP_EXTEND;
2770        TruncOp = ISD::FP_ROUND;
2771      }
2772      // Promote each of the values to the new type.
2773      Tmp2 = DAG.getNode(ExtOp, NVT, Tmp2);
2774      Tmp3 = DAG.getNode(ExtOp, NVT, Tmp3);
2775      // Perform the larger operation, then round down.
2776      Result = DAG.getNode(ISD::SELECT, NVT, Tmp1, Tmp2,Tmp3);
2777      if (TruncOp != ISD::FP_ROUND)
2778        Result = DAG.getNode(TruncOp, Node->getValueType(0), Result);
2779      else
2780        Result = DAG.getNode(TruncOp, Node->getValueType(0), Result,
2781                             DAG.getIntPtrConstant(0));
2782      break;
2783    }
2784    }
2785    break;
2786  case ISD::SELECT_CC: {
2787    Tmp1 = Node->getOperand(0);               // LHS
2788    Tmp2 = Node->getOperand(1);               // RHS
2789    Tmp3 = LegalizeOp(Node->getOperand(2));   // True
2790    Tmp4 = LegalizeOp(Node->getOperand(3));   // False
2791    SDOperand CC = Node->getOperand(4);
2792
2793    LegalizeSetCCOperands(Tmp1, Tmp2, CC);
2794
2795    // If we didn't get both a LHS and RHS back from LegalizeSetCCOperands,
2796    // the LHS is a legal SETCC itself.  In this case, we need to compare
2797    // the result against zero to select between true and false values.
2798    if (Tmp2.Val == 0) {
2799      Tmp2 = DAG.getConstant(0, Tmp1.getValueType());
2800      CC = DAG.getCondCode(ISD::SETNE);
2801    }
2802    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3, Tmp4, CC);
2803
2804    // Everything is legal, see if we should expand this op or something.
2805    switch (TLI.getOperationAction(ISD::SELECT_CC, Tmp3.getValueType())) {
2806    default: assert(0 && "This action is not supported yet!");
2807    case TargetLowering::Legal: break;
2808    case TargetLowering::Custom:
2809      Tmp1 = TLI.LowerOperation(Result, DAG);
2810      if (Tmp1.Val) Result = Tmp1;
2811      break;
2812    }
2813    break;
2814  }
2815  case ISD::SETCC:
2816    Tmp1 = Node->getOperand(0);
2817    Tmp2 = Node->getOperand(1);
2818    Tmp3 = Node->getOperand(2);
2819    LegalizeSetCCOperands(Tmp1, Tmp2, Tmp3);
2820
2821    // If we had to Expand the SetCC operands into a SELECT node, then it may
2822    // not always be possible to return a true LHS & RHS.  In this case, just
2823    // return the value we legalized, returned in the LHS
2824    if (Tmp2.Val == 0) {
2825      Result = Tmp1;
2826      break;
2827    }
2828
2829    switch (TLI.getOperationAction(ISD::SETCC, Tmp1.getValueType())) {
2830    default: assert(0 && "Cannot handle this action for SETCC yet!");
2831    case TargetLowering::Custom:
2832      isCustom = true;
2833      // FALLTHROUGH.
2834    case TargetLowering::Legal:
2835      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
2836      if (isCustom) {
2837        Tmp4 = TLI.LowerOperation(Result, DAG);
2838        if (Tmp4.Val) Result = Tmp4;
2839      }
2840      break;
2841    case TargetLowering::Promote: {
2842      // First step, figure out the appropriate operation to use.
2843      // Allow SETCC to not be supported for all legal data types
2844      // Mostly this targets FP
2845      MVT NewInTy = Node->getOperand(0).getValueType();
2846      MVT OldVT = NewInTy; OldVT = OldVT;
2847
2848      // Scan for the appropriate larger type to use.
2849      while (1) {
2850        NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT()+1);
2851
2852        assert(NewInTy.isInteger() == OldVT.isInteger() &&
2853               "Fell off of the edge of the integer world");
2854        assert(NewInTy.isFloatingPoint() == OldVT.isFloatingPoint() &&
2855               "Fell off of the edge of the floating point world");
2856
2857        // If the target supports SETCC of this type, use it.
2858        if (TLI.isOperationLegal(ISD::SETCC, NewInTy))
2859          break;
2860      }
2861      if (NewInTy.isInteger())
2862        assert(0 && "Cannot promote Legal Integer SETCC yet");
2863      else {
2864        Tmp1 = DAG.getNode(ISD::FP_EXTEND, NewInTy, Tmp1);
2865        Tmp2 = DAG.getNode(ISD::FP_EXTEND, NewInTy, Tmp2);
2866      }
2867      Tmp1 = LegalizeOp(Tmp1);
2868      Tmp2 = LegalizeOp(Tmp2);
2869      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
2870      Result = LegalizeOp(Result);
2871      break;
2872    }
2873    case TargetLowering::Expand:
2874      // Expand a setcc node into a select_cc of the same condition, lhs, and
2875      // rhs that selects between const 1 (true) and const 0 (false).
2876      MVT VT = Node->getValueType(0);
2877      Result = DAG.getNode(ISD::SELECT_CC, VT, Tmp1, Tmp2,
2878                           DAG.getConstant(1, VT), DAG.getConstant(0, VT),
2879                           Tmp3);
2880      break;
2881    }
2882    break;
2883  case ISD::VSETCC: {
2884    Tmp1 = LegalizeOp(Node->getOperand(0));   // LHS
2885    Tmp2 = LegalizeOp(Node->getOperand(1));   // RHS
2886    SDOperand CC = Node->getOperand(2);
2887
2888    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, CC);
2889
2890    // Everything is legal, see if we should expand this op or something.
2891    switch (TLI.getOperationAction(ISD::VSETCC, Tmp1.getValueType())) {
2892    default: assert(0 && "This action is not supported yet!");
2893    case TargetLowering::Legal: break;
2894    case TargetLowering::Custom:
2895      Tmp1 = TLI.LowerOperation(Result, DAG);
2896      if (Tmp1.Val) Result = Tmp1;
2897      break;
2898    }
2899    break;
2900  }
2901
2902  case ISD::SHL_PARTS:
2903  case ISD::SRA_PARTS:
2904  case ISD::SRL_PARTS: {
2905    SmallVector<SDOperand, 8> Ops;
2906    bool Changed = false;
2907    for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
2908      Ops.push_back(LegalizeOp(Node->getOperand(i)));
2909      Changed |= Ops.back() != Node->getOperand(i);
2910    }
2911    if (Changed)
2912      Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size());
2913
2914    switch (TLI.getOperationAction(Node->getOpcode(),
2915                                   Node->getValueType(0))) {
2916    default: assert(0 && "This action is not supported yet!");
2917    case TargetLowering::Legal: break;
2918    case TargetLowering::Custom:
2919      Tmp1 = TLI.LowerOperation(Result, DAG);
2920      if (Tmp1.Val) {
2921        SDOperand Tmp2, RetVal(0, 0);
2922        for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
2923          Tmp2 = LegalizeOp(Tmp1.getValue(i));
2924          AddLegalizedOperand(SDOperand(Node, i), Tmp2);
2925          if (i == Op.ResNo)
2926            RetVal = Tmp2;
2927        }
2928        assert(RetVal.Val && "Illegal result number");
2929        return RetVal;
2930      }
2931      break;
2932    }
2933
2934    // Since these produce multiple values, make sure to remember that we
2935    // legalized all of them.
2936    for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
2937      AddLegalizedOperand(SDOperand(Node, i), Result.getValue(i));
2938    return Result.getValue(Op.ResNo);
2939  }
2940
2941    // Binary operators
2942  case ISD::ADD:
2943  case ISD::SUB:
2944  case ISD::MUL:
2945  case ISD::MULHS:
2946  case ISD::MULHU:
2947  case ISD::UDIV:
2948  case ISD::SDIV:
2949  case ISD::AND:
2950  case ISD::OR:
2951  case ISD::XOR:
2952  case ISD::SHL:
2953  case ISD::SRL:
2954  case ISD::SRA:
2955  case ISD::FADD:
2956  case ISD::FSUB:
2957  case ISD::FMUL:
2958  case ISD::FDIV:
2959  case ISD::FPOW:
2960    Tmp1 = LegalizeOp(Node->getOperand(0));   // LHS
2961    switch (getTypeAction(Node->getOperand(1).getValueType())) {
2962    case Expand: assert(0 && "Not possible");
2963    case Legal:
2964      Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the RHS.
2965      break;
2966    case Promote:
2967      Tmp2 = PromoteOp(Node->getOperand(1));  // Promote the RHS.
2968      break;
2969    }
2970
2971    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
2972
2973    switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
2974    default: assert(0 && "BinOp legalize operation not supported");
2975    case TargetLowering::Legal: break;
2976    case TargetLowering::Custom:
2977      Tmp1 = TLI.LowerOperation(Result, DAG);
2978      if (Tmp1.Val) Result = Tmp1;
2979      break;
2980    case TargetLowering::Expand: {
2981      MVT VT = Op.getValueType();
2982
2983      // See if multiply or divide can be lowered using two-result operations.
2984      SDVTList VTs = DAG.getVTList(VT, VT);
2985      if (Node->getOpcode() == ISD::MUL) {
2986        // We just need the low half of the multiply; try both the signed
2987        // and unsigned forms. If the target supports both SMUL_LOHI and
2988        // UMUL_LOHI, form a preference by checking which forms of plain
2989        // MULH it supports.
2990        bool HasSMUL_LOHI = TLI.isOperationLegal(ISD::SMUL_LOHI, VT);
2991        bool HasUMUL_LOHI = TLI.isOperationLegal(ISD::UMUL_LOHI, VT);
2992        bool HasMULHS = TLI.isOperationLegal(ISD::MULHS, VT);
2993        bool HasMULHU = TLI.isOperationLegal(ISD::MULHU, VT);
2994        unsigned OpToUse = 0;
2995        if (HasSMUL_LOHI && !HasMULHS) {
2996          OpToUse = ISD::SMUL_LOHI;
2997        } else if (HasUMUL_LOHI && !HasMULHU) {
2998          OpToUse = ISD::UMUL_LOHI;
2999        } else if (HasSMUL_LOHI) {
3000          OpToUse = ISD::SMUL_LOHI;
3001        } else if (HasUMUL_LOHI) {
3002          OpToUse = ISD::UMUL_LOHI;
3003        }
3004        if (OpToUse) {
3005          Result = SDOperand(DAG.getNode(OpToUse, VTs, Tmp1, Tmp2).Val, 0);
3006          break;
3007        }
3008      }
3009      if (Node->getOpcode() == ISD::MULHS &&
3010          TLI.isOperationLegal(ISD::SMUL_LOHI, VT)) {
3011        Result = SDOperand(DAG.getNode(ISD::SMUL_LOHI, VTs, Tmp1, Tmp2).Val, 1);
3012        break;
3013      }
3014      if (Node->getOpcode() == ISD::MULHU &&
3015          TLI.isOperationLegal(ISD::UMUL_LOHI, VT)) {
3016        Result = SDOperand(DAG.getNode(ISD::UMUL_LOHI, VTs, Tmp1, Tmp2).Val, 1);
3017        break;
3018      }
3019      if (Node->getOpcode() == ISD::SDIV &&
3020          TLI.isOperationLegal(ISD::SDIVREM, VT)) {
3021        Result = SDOperand(DAG.getNode(ISD::SDIVREM, VTs, Tmp1, Tmp2).Val, 0);
3022        break;
3023      }
3024      if (Node->getOpcode() == ISD::UDIV &&
3025          TLI.isOperationLegal(ISD::UDIVREM, VT)) {
3026        Result = SDOperand(DAG.getNode(ISD::UDIVREM, VTs, Tmp1, Tmp2).Val, 0);
3027        break;
3028      }
3029
3030      // Check to see if we have a libcall for this operator.
3031      RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3032      bool isSigned = false;
3033      switch (Node->getOpcode()) {
3034      case ISD::UDIV:
3035      case ISD::SDIV:
3036        if (VT == MVT::i32) {
3037          LC = Node->getOpcode() == ISD::UDIV
3038            ? RTLIB::UDIV_I32 : RTLIB::SDIV_I32;
3039          isSigned = Node->getOpcode() == ISD::SDIV;
3040        }
3041        break;
3042      case ISD::FPOW:
3043        LC = GetFPLibCall(VT, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80,
3044                          RTLIB::POW_PPCF128);
3045        break;
3046      default: break;
3047      }
3048      if (LC != RTLIB::UNKNOWN_LIBCALL) {
3049        SDOperand Dummy;
3050        Result = ExpandLibCall(LC, Node, isSigned, Dummy);
3051        break;
3052      }
3053
3054      assert(Node->getValueType(0).isVector() &&
3055             "Cannot expand this binary operator!");
3056      // Expand the operation into a bunch of nasty scalar code.
3057      Result = LegalizeOp(UnrollVectorOp(Op));
3058      break;
3059    }
3060    case TargetLowering::Promote: {
3061      switch (Node->getOpcode()) {
3062      default:  assert(0 && "Do not know how to promote this BinOp!");
3063      case ISD::AND:
3064      case ISD::OR:
3065      case ISD::XOR: {
3066        MVT OVT = Node->getValueType(0);
3067        MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
3068        assert(OVT.isVector() && "Cannot promote this BinOp!");
3069        // Bit convert each of the values to the new type.
3070        Tmp1 = DAG.getNode(ISD::BIT_CONVERT, NVT, Tmp1);
3071        Tmp2 = DAG.getNode(ISD::BIT_CONVERT, NVT, Tmp2);
3072        Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1, Tmp2);
3073        // Bit convert the result back the original type.
3074        Result = DAG.getNode(ISD::BIT_CONVERT, OVT, Result);
3075        break;
3076      }
3077      }
3078    }
3079    }
3080    break;
3081
3082  case ISD::SMUL_LOHI:
3083  case ISD::UMUL_LOHI:
3084  case ISD::SDIVREM:
3085  case ISD::UDIVREM:
3086    // These nodes will only be produced by target-specific lowering, so
3087    // they shouldn't be here if they aren't legal.
3088    assert(TLI.isOperationLegal(Node->getOpcode(), Node->getValueType(0)) &&
3089           "This must be legal!");
3090
3091    Tmp1 = LegalizeOp(Node->getOperand(0));   // LHS
3092    Tmp2 = LegalizeOp(Node->getOperand(1));   // RHS
3093    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
3094    break;
3095
3096  case ISD::FCOPYSIGN:  // FCOPYSIGN does not require LHS/RHS to match type!
3097    Tmp1 = LegalizeOp(Node->getOperand(0));   // LHS
3098    switch (getTypeAction(Node->getOperand(1).getValueType())) {
3099      case Expand: assert(0 && "Not possible");
3100      case Legal:
3101        Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the RHS.
3102        break;
3103      case Promote:
3104        Tmp2 = PromoteOp(Node->getOperand(1));  // Promote the RHS.
3105        break;
3106    }
3107
3108    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
3109
3110    switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
3111    default: assert(0 && "Operation not supported");
3112    case TargetLowering::Custom:
3113      Tmp1 = TLI.LowerOperation(Result, DAG);
3114      if (Tmp1.Val) Result = Tmp1;
3115      break;
3116    case TargetLowering::Legal: break;
3117    case TargetLowering::Expand: {
3118      // If this target supports fabs/fneg natively and select is cheap,
3119      // do this efficiently.
3120      if (!TLI.isSelectExpensive() &&
3121          TLI.getOperationAction(ISD::FABS, Tmp1.getValueType()) ==
3122          TargetLowering::Legal &&
3123          TLI.getOperationAction(ISD::FNEG, Tmp1.getValueType()) ==
3124          TargetLowering::Legal) {
3125        // Get the sign bit of the RHS.
3126        MVT IVT =
3127          Tmp2.getValueType() == MVT::f32 ? MVT::i32 : MVT::i64;
3128        SDOperand SignBit = DAG.getNode(ISD::BIT_CONVERT, IVT, Tmp2);
3129        SignBit = DAG.getSetCC(TLI.getSetCCResultType(SignBit),
3130                               SignBit, DAG.getConstant(0, IVT), ISD::SETLT);
3131        // Get the absolute value of the result.
3132        SDOperand AbsVal = DAG.getNode(ISD::FABS, Tmp1.getValueType(), Tmp1);
3133        // Select between the nabs and abs value based on the sign bit of
3134        // the input.
3135        Result = DAG.getNode(ISD::SELECT, AbsVal.getValueType(), SignBit,
3136                             DAG.getNode(ISD::FNEG, AbsVal.getValueType(),
3137                                         AbsVal),
3138                             AbsVal);
3139        Result = LegalizeOp(Result);
3140        break;
3141      }
3142
3143      // Otherwise, do bitwise ops!
3144      MVT NVT =
3145        Node->getValueType(0) == MVT::f32 ? MVT::i32 : MVT::i64;
3146      Result = ExpandFCOPYSIGNToBitwiseOps(Node, NVT, DAG, TLI);
3147      Result = DAG.getNode(ISD::BIT_CONVERT, Node->getValueType(0), Result);
3148      Result = LegalizeOp(Result);
3149      break;
3150    }
3151    }
3152    break;
3153
3154  case ISD::ADDC:
3155  case ISD::SUBC:
3156    Tmp1 = LegalizeOp(Node->getOperand(0));
3157    Tmp2 = LegalizeOp(Node->getOperand(1));
3158    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
3159    // Since this produces two values, make sure to remember that we legalized
3160    // both of them.
3161    AddLegalizedOperand(SDOperand(Node, 0), Result.getValue(0));
3162    AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1));
3163    return Result;
3164
3165  case ISD::ADDE:
3166  case ISD::SUBE:
3167    Tmp1 = LegalizeOp(Node->getOperand(0));
3168    Tmp2 = LegalizeOp(Node->getOperand(1));
3169    Tmp3 = LegalizeOp(Node->getOperand(2));
3170    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
3171    // Since this produces two values, make sure to remember that we legalized
3172    // both of them.
3173    AddLegalizedOperand(SDOperand(Node, 0), Result.getValue(0));
3174    AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1));
3175    return Result;
3176
3177  case ISD::BUILD_PAIR: {
3178    MVT PairTy = Node->getValueType(0);
3179    // TODO: handle the case where the Lo and Hi operands are not of legal type
3180    Tmp1 = LegalizeOp(Node->getOperand(0));   // Lo
3181    Tmp2 = LegalizeOp(Node->getOperand(1));   // Hi
3182    switch (TLI.getOperationAction(ISD::BUILD_PAIR, PairTy)) {
3183    case TargetLowering::Promote:
3184    case TargetLowering::Custom:
3185      assert(0 && "Cannot promote/custom this yet!");
3186    case TargetLowering::Legal:
3187      if (Tmp1 != Node->getOperand(0) || Tmp2 != Node->getOperand(1))
3188        Result = DAG.getNode(ISD::BUILD_PAIR, PairTy, Tmp1, Tmp2);
3189      break;
3190    case TargetLowering::Expand:
3191      Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, PairTy, Tmp1);
3192      Tmp2 = DAG.getNode(ISD::ANY_EXTEND, PairTy, Tmp2);
3193      Tmp2 = DAG.getNode(ISD::SHL, PairTy, Tmp2,
3194                         DAG.getConstant(PairTy.getSizeInBits()/2,
3195                                         TLI.getShiftAmountTy()));
3196      Result = DAG.getNode(ISD::OR, PairTy, Tmp1, Tmp2);
3197      break;
3198    }
3199    break;
3200  }
3201
3202  case ISD::UREM:
3203  case ISD::SREM:
3204  case ISD::FREM:
3205    Tmp1 = LegalizeOp(Node->getOperand(0));   // LHS
3206    Tmp2 = LegalizeOp(Node->getOperand(1));   // RHS
3207
3208    switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
3209    case TargetLowering::Promote: assert(0 && "Cannot promote this yet!");
3210    case TargetLowering::Custom:
3211      isCustom = true;
3212      // FALLTHROUGH
3213    case TargetLowering::Legal:
3214      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
3215      if (isCustom) {
3216        Tmp1 = TLI.LowerOperation(Result, DAG);
3217        if (Tmp1.Val) Result = Tmp1;
3218      }
3219      break;
3220    case TargetLowering::Expand: {
3221      unsigned DivOpc= (Node->getOpcode() == ISD::UREM) ? ISD::UDIV : ISD::SDIV;
3222      bool isSigned = DivOpc == ISD::SDIV;
3223      MVT VT = Node->getValueType(0);
3224
3225      // See if remainder can be lowered using two-result operations.
3226      SDVTList VTs = DAG.getVTList(VT, VT);
3227      if (Node->getOpcode() == ISD::SREM &&
3228          TLI.isOperationLegal(ISD::SDIVREM, VT)) {
3229        Result = SDOperand(DAG.getNode(ISD::SDIVREM, VTs, Tmp1, Tmp2).Val, 1);
3230        break;
3231      }
3232      if (Node->getOpcode() == ISD::UREM &&
3233          TLI.isOperationLegal(ISD::UDIVREM, VT)) {
3234        Result = SDOperand(DAG.getNode(ISD::UDIVREM, VTs, Tmp1, Tmp2).Val, 1);
3235        break;
3236      }
3237
3238      if (VT.isInteger()) {
3239        if (TLI.getOperationAction(DivOpc, VT) ==
3240            TargetLowering::Legal) {
3241          // X % Y -> X-X/Y*Y
3242          Result = DAG.getNode(DivOpc, VT, Tmp1, Tmp2);
3243          Result = DAG.getNode(ISD::MUL, VT, Result, Tmp2);
3244          Result = DAG.getNode(ISD::SUB, VT, Tmp1, Result);
3245        } else if (VT.isVector()) {
3246          Result = LegalizeOp(UnrollVectorOp(Op));
3247        } else {
3248          assert(VT == MVT::i32 &&
3249                 "Cannot expand this binary operator!");
3250          RTLIB::Libcall LC = Node->getOpcode() == ISD::UREM
3251            ? RTLIB::UREM_I32 : RTLIB::SREM_I32;
3252          SDOperand Dummy;
3253          Result = ExpandLibCall(LC, Node, isSigned, Dummy);
3254        }
3255      } else {
3256        assert(VT.isFloatingPoint() &&
3257               "remainder op must have integer or floating-point type");
3258        if (VT.isVector()) {
3259          Result = LegalizeOp(UnrollVectorOp(Op));
3260        } else {
3261          // Floating point mod -> fmod libcall.
3262          RTLIB::Libcall LC = GetFPLibCall(VT, RTLIB::REM_F32, RTLIB::REM_F64,
3263                                           RTLIB::REM_F80, RTLIB::REM_PPCF128);
3264          SDOperand Dummy;
3265          Result = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Dummy);
3266        }
3267      }
3268      break;
3269    }
3270    }
3271    break;
3272  case ISD::VAARG: {
3273    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
3274    Tmp2 = LegalizeOp(Node->getOperand(1));  // Legalize the pointer.
3275
3276    MVT VT = Node->getValueType(0);
3277    switch (TLI.getOperationAction(Node->getOpcode(), MVT::Other)) {
3278    default: assert(0 && "This action is not supported yet!");
3279    case TargetLowering::Custom:
3280      isCustom = true;
3281      // FALLTHROUGH
3282    case TargetLowering::Legal:
3283      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2));
3284      Result = Result.getValue(0);
3285      Tmp1 = Result.getValue(1);
3286
3287      if (isCustom) {
3288        Tmp2 = TLI.LowerOperation(Result, DAG);
3289        if (Tmp2.Val) {
3290          Result = LegalizeOp(Tmp2);
3291          Tmp1 = LegalizeOp(Tmp2.getValue(1));
3292        }
3293      }
3294      break;
3295    case TargetLowering::Expand: {
3296      const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
3297      SDOperand VAList = DAG.getLoad(TLI.getPointerTy(), Tmp1, Tmp2, V, 0);
3298      // Increment the pointer, VAList, to the next vaarg
3299      Tmp3 = DAG.getNode(ISD::ADD, TLI.getPointerTy(), VAList,
3300                         DAG.getConstant(VT.getSizeInBits()/8,
3301                                         TLI.getPointerTy()));
3302      // Store the incremented VAList to the legalized pointer
3303      Tmp3 = DAG.getStore(VAList.getValue(1), Tmp3, Tmp2, V, 0);
3304      // Load the actual argument out of the pointer VAList
3305      Result = DAG.getLoad(VT, Tmp3, VAList, NULL, 0);
3306      Tmp1 = LegalizeOp(Result.getValue(1));
3307      Result = LegalizeOp(Result);
3308      break;
3309    }
3310    }
3311    // Since VAARG produces two values, make sure to remember that we
3312    // legalized both of them.
3313    AddLegalizedOperand(SDOperand(Node, 0), Result);
3314    AddLegalizedOperand(SDOperand(Node, 1), Tmp1);
3315    return Op.ResNo ? Tmp1 : Result;
3316  }
3317
3318  case ISD::VACOPY:
3319    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
3320    Tmp2 = LegalizeOp(Node->getOperand(1));  // Legalize the dest pointer.
3321    Tmp3 = LegalizeOp(Node->getOperand(2));  // Legalize the source pointer.
3322
3323    switch (TLI.getOperationAction(ISD::VACOPY, MVT::Other)) {
3324    default: assert(0 && "This action is not supported yet!");
3325    case TargetLowering::Custom:
3326      isCustom = true;
3327      // FALLTHROUGH
3328    case TargetLowering::Legal:
3329      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3,
3330                                      Node->getOperand(3), Node->getOperand(4));
3331      if (isCustom) {
3332        Tmp1 = TLI.LowerOperation(Result, DAG);
3333        if (Tmp1.Val) Result = Tmp1;
3334      }
3335      break;
3336    case TargetLowering::Expand:
3337      // This defaults to loading a pointer from the input and storing it to the
3338      // output, returning the chain.
3339      const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue();
3340      const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue();
3341      Tmp4 = DAG.getLoad(TLI.getPointerTy(), Tmp1, Tmp3, VS, 0);
3342      Result = DAG.getStore(Tmp4.getValue(1), Tmp4, Tmp2, VD, 0);
3343      break;
3344    }
3345    break;
3346
3347  case ISD::VAEND:
3348    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
3349    Tmp2 = LegalizeOp(Node->getOperand(1));  // Legalize the pointer.
3350
3351    switch (TLI.getOperationAction(ISD::VAEND, MVT::Other)) {
3352    default: assert(0 && "This action is not supported yet!");
3353    case TargetLowering::Custom:
3354      isCustom = true;
3355      // FALLTHROUGH
3356    case TargetLowering::Legal:
3357      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2));
3358      if (isCustom) {
3359        Tmp1 = TLI.LowerOperation(Tmp1, DAG);
3360        if (Tmp1.Val) Result = Tmp1;
3361      }
3362      break;
3363    case TargetLowering::Expand:
3364      Result = Tmp1; // Default to a no-op, return the chain
3365      break;
3366    }
3367    break;
3368
3369  case ISD::VASTART:
3370    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
3371    Tmp2 = LegalizeOp(Node->getOperand(1));  // Legalize the pointer.
3372
3373    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2));
3374
3375    switch (TLI.getOperationAction(ISD::VASTART, MVT::Other)) {
3376    default: assert(0 && "This action is not supported yet!");
3377    case TargetLowering::Legal: break;
3378    case TargetLowering::Custom:
3379      Tmp1 = TLI.LowerOperation(Result, DAG);
3380      if (Tmp1.Val) Result = Tmp1;
3381      break;
3382    }
3383    break;
3384
3385  case ISD::ROTL:
3386  case ISD::ROTR:
3387    Tmp1 = LegalizeOp(Node->getOperand(0));   // LHS
3388    Tmp2 = LegalizeOp(Node->getOperand(1));   // RHS
3389    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
3390    switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
3391    default:
3392      assert(0 && "ROTL/ROTR legalize operation not supported");
3393      break;
3394    case TargetLowering::Legal:
3395      break;
3396    case TargetLowering::Custom:
3397      Tmp1 = TLI.LowerOperation(Result, DAG);
3398      if (Tmp1.Val) Result = Tmp1;
3399      break;
3400    case TargetLowering::Promote:
3401      assert(0 && "Do not know how to promote ROTL/ROTR");
3402      break;
3403    case TargetLowering::Expand:
3404      assert(0 && "Do not know how to expand ROTL/ROTR");
3405      break;
3406    }
3407    break;
3408
3409  case ISD::BSWAP:
3410    Tmp1 = LegalizeOp(Node->getOperand(0));   // Op
3411    switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
3412    case TargetLowering::Custom:
3413      assert(0 && "Cannot custom legalize this yet!");
3414    case TargetLowering::Legal:
3415      Result = DAG.UpdateNodeOperands(Result, Tmp1);
3416      break;
3417    case TargetLowering::Promote: {
3418      MVT OVT = Tmp1.getValueType();
3419      MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
3420      unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
3421
3422      Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, NVT, Tmp1);
3423      Tmp1 = DAG.getNode(ISD::BSWAP, NVT, Tmp1);
3424      Result = DAG.getNode(ISD::SRL, NVT, Tmp1,
3425                           DAG.getConstant(DiffBits, TLI.getShiftAmountTy()));
3426      break;
3427    }
3428    case TargetLowering::Expand:
3429      Result = ExpandBSWAP(Tmp1);
3430      break;
3431    }
3432    break;
3433
3434  case ISD::CTPOP:
3435  case ISD::CTTZ:
3436  case ISD::CTLZ:
3437    Tmp1 = LegalizeOp(Node->getOperand(0));   // Op
3438    switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
3439    case TargetLowering::Custom:
3440    case TargetLowering::Legal:
3441      Result = DAG.UpdateNodeOperands(Result, Tmp1);
3442      if (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)) ==
3443          TargetLowering::Custom) {
3444        Tmp1 = TLI.LowerOperation(Result, DAG);
3445        if (Tmp1.Val) {
3446          Result = Tmp1;
3447        }
3448      }
3449      break;
3450    case TargetLowering::Promote: {
3451      MVT OVT = Tmp1.getValueType();
3452      MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
3453
3454      // Zero extend the argument.
3455      Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, NVT, Tmp1);
3456      // Perform the larger operation, then subtract if needed.
3457      Tmp1 = DAG.getNode(Node->getOpcode(), Node->getValueType(0), Tmp1);
3458      switch (Node->getOpcode()) {
3459      case ISD::CTPOP:
3460        Result = Tmp1;
3461        break;
3462      case ISD::CTTZ:
3463        //if Tmp1 == sizeinbits(NVT) then Tmp1 = sizeinbits(Old VT)
3464        Tmp2 = DAG.getSetCC(TLI.getSetCCResultType(Tmp1), Tmp1,
3465                            DAG.getConstant(NVT.getSizeInBits(), NVT),
3466                            ISD::SETEQ);
3467        Result = DAG.getNode(ISD::SELECT, NVT, Tmp2,
3468                             DAG.getConstant(OVT.getSizeInBits(), NVT), Tmp1);
3469        break;
3470      case ISD::CTLZ:
3471        // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
3472        Result = DAG.getNode(ISD::SUB, NVT, Tmp1,
3473                             DAG.getConstant(NVT.getSizeInBits() -
3474                                             OVT.getSizeInBits(), NVT));
3475        break;
3476      }
3477      break;
3478    }
3479    case TargetLowering::Expand:
3480      Result = ExpandBitCount(Node->getOpcode(), Tmp1);
3481      break;
3482    }
3483    break;
3484
3485    // Unary operators
3486  case ISD::FABS:
3487  case ISD::FNEG:
3488  case ISD::FSQRT:
3489  case ISD::FSIN:
3490  case ISD::FCOS:
3491    Tmp1 = LegalizeOp(Node->getOperand(0));
3492    switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
3493    case TargetLowering::Promote:
3494    case TargetLowering::Custom:
3495     isCustom = true;
3496     // FALLTHROUGH
3497    case TargetLowering::Legal:
3498      Result = DAG.UpdateNodeOperands(Result, Tmp1);
3499      if (isCustom) {
3500        Tmp1 = TLI.LowerOperation(Result, DAG);
3501        if (Tmp1.Val) Result = Tmp1;
3502      }
3503      break;
3504    case TargetLowering::Expand:
3505      switch (Node->getOpcode()) {
3506      default: assert(0 && "Unreachable!");
3507      case ISD::FNEG:
3508        // Expand Y = FNEG(X) ->  Y = SUB -0.0, X
3509        Tmp2 = DAG.getConstantFP(-0.0, Node->getValueType(0));
3510        Result = DAG.getNode(ISD::FSUB, Node->getValueType(0), Tmp2, Tmp1);
3511        break;
3512      case ISD::FABS: {
3513        // Expand Y = FABS(X) -> Y = (X >u 0.0) ? X : fneg(X).
3514        MVT VT = Node->getValueType(0);
3515        Tmp2 = DAG.getConstantFP(0.0, VT);
3516        Tmp2 = DAG.getSetCC(TLI.getSetCCResultType(Tmp1), Tmp1, Tmp2,
3517                            ISD::SETUGT);
3518        Tmp3 = DAG.getNode(ISD::FNEG, VT, Tmp1);
3519        Result = DAG.getNode(ISD::SELECT, VT, Tmp2, Tmp1, Tmp3);
3520        break;
3521      }
3522      case ISD::FSQRT:
3523      case ISD::FSIN:
3524      case ISD::FCOS: {
3525        MVT VT = Node->getValueType(0);
3526
3527        // Expand unsupported unary vector operators by unrolling them.
3528        if (VT.isVector()) {
3529          Result = LegalizeOp(UnrollVectorOp(Op));
3530          break;
3531        }
3532
3533        RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3534        switch(Node->getOpcode()) {
3535        case ISD::FSQRT:
3536          LC = GetFPLibCall(VT, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
3537                            RTLIB::SQRT_F80, RTLIB::SQRT_PPCF128);
3538          break;
3539        case ISD::FSIN:
3540          LC = GetFPLibCall(VT, RTLIB::SIN_F32, RTLIB::SIN_F64,
3541                            RTLIB::SIN_F80, RTLIB::SIN_PPCF128);
3542          break;
3543        case ISD::FCOS:
3544          LC = GetFPLibCall(VT, RTLIB::COS_F32, RTLIB::COS_F64,
3545                            RTLIB::COS_F80, RTLIB::COS_PPCF128);
3546          break;
3547        default: assert(0 && "Unreachable!");
3548        }
3549        SDOperand Dummy;
3550        Result = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Dummy);
3551        break;
3552      }
3553      }
3554      break;
3555    }
3556    break;
3557  case ISD::FPOWI: {
3558    MVT VT = Node->getValueType(0);
3559
3560    // Expand unsupported unary vector operators by unrolling them.
3561    if (VT.isVector()) {
3562      Result = LegalizeOp(UnrollVectorOp(Op));
3563      break;
3564    }
3565
3566    // We always lower FPOWI into a libcall.  No target support for it yet.
3567    RTLIB::Libcall LC = GetFPLibCall(VT, RTLIB::POWI_F32, RTLIB::POWI_F64,
3568                                     RTLIB::POWI_F80, RTLIB::POWI_PPCF128);
3569    SDOperand Dummy;
3570    Result = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Dummy);
3571    break;
3572  }
3573  case ISD::BIT_CONVERT:
3574    if (!isTypeLegal(Node->getOperand(0).getValueType())) {
3575      Result = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
3576                                Node->getValueType(0));
3577    } else if (Op.getOperand(0).getValueType().isVector()) {
3578      // The input has to be a vector type, we have to either scalarize it, pack
3579      // it, or convert it based on whether the input vector type is legal.
3580      SDNode *InVal = Node->getOperand(0).Val;
3581      int InIx = Node->getOperand(0).ResNo;
3582      unsigned NumElems = InVal->getValueType(InIx).getVectorNumElements();
3583      MVT EVT = InVal->getValueType(InIx).getVectorElementType();
3584
3585      // Figure out if there is a simple type corresponding to this Vector
3586      // type.  If so, convert to the vector type.
3587      MVT TVT = MVT::getVectorVT(EVT, NumElems);
3588      if (TLI.isTypeLegal(TVT)) {
3589        // Turn this into a bit convert of the vector input.
3590        Result = DAG.getNode(ISD::BIT_CONVERT, Node->getValueType(0),
3591                             LegalizeOp(Node->getOperand(0)));
3592        break;
3593      } else if (NumElems == 1) {
3594        // Turn this into a bit convert of the scalar input.
3595        Result = DAG.getNode(ISD::BIT_CONVERT, Node->getValueType(0),
3596                             ScalarizeVectorOp(Node->getOperand(0)));
3597        break;
3598      } else {
3599        // FIXME: UNIMP!  Store then reload
3600        assert(0 && "Cast from unsupported vector type not implemented yet!");
3601      }
3602    } else {
3603      switch (TLI.getOperationAction(ISD::BIT_CONVERT,
3604                                     Node->getOperand(0).getValueType())) {
3605      default: assert(0 && "Unknown operation action!");
3606      case TargetLowering::Expand:
3607        Result = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
3608                                  Node->getValueType(0));
3609        break;
3610      case TargetLowering::Legal:
3611        Tmp1 = LegalizeOp(Node->getOperand(0));
3612        Result = DAG.UpdateNodeOperands(Result, Tmp1);
3613        break;
3614      }
3615    }
3616    break;
3617
3618    // Conversion operators.  The source and destination have different types.
3619  case ISD::SINT_TO_FP:
3620  case ISD::UINT_TO_FP: {
3621    bool isSigned = Node->getOpcode() == ISD::SINT_TO_FP;
3622    switch (getTypeAction(Node->getOperand(0).getValueType())) {
3623    case Legal:
3624      switch (TLI.getOperationAction(Node->getOpcode(),
3625                                     Node->getOperand(0).getValueType())) {
3626      default: assert(0 && "Unknown operation action!");
3627      case TargetLowering::Custom:
3628        isCustom = true;
3629        // FALLTHROUGH
3630      case TargetLowering::Legal:
3631        Tmp1 = LegalizeOp(Node->getOperand(0));
3632        Result = DAG.UpdateNodeOperands(Result, Tmp1);
3633        if (isCustom) {
3634          Tmp1 = TLI.LowerOperation(Result, DAG);
3635          if (Tmp1.Val) Result = Tmp1;
3636        }
3637        break;
3638      case TargetLowering::Expand:
3639        Result = ExpandLegalINT_TO_FP(isSigned,
3640                                      LegalizeOp(Node->getOperand(0)),
3641                                      Node->getValueType(0));
3642        break;
3643      case TargetLowering::Promote:
3644        Result = PromoteLegalINT_TO_FP(LegalizeOp(Node->getOperand(0)),
3645                                       Node->getValueType(0),
3646                                       isSigned);
3647        break;
3648      }
3649      break;
3650    case Expand:
3651      Result = ExpandIntToFP(Node->getOpcode() == ISD::SINT_TO_FP,
3652                             Node->getValueType(0), Node->getOperand(0));
3653      break;
3654    case Promote:
3655      Tmp1 = PromoteOp(Node->getOperand(0));
3656      if (isSigned) {
3657        Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, Tmp1.getValueType(),
3658                 Tmp1, DAG.getValueType(Node->getOperand(0).getValueType()));
3659      } else {
3660        Tmp1 = DAG.getZeroExtendInReg(Tmp1,
3661                                      Node->getOperand(0).getValueType());
3662      }
3663      Result = DAG.UpdateNodeOperands(Result, Tmp1);
3664      Result = LegalizeOp(Result);  // The 'op' is not necessarily legal!
3665      break;
3666    }
3667    break;
3668  }
3669  case ISD::TRUNCATE:
3670    switch (getTypeAction(Node->getOperand(0).getValueType())) {
3671    case Legal:
3672      Tmp1 = LegalizeOp(Node->getOperand(0));
3673      Result = DAG.UpdateNodeOperands(Result, Tmp1);
3674      break;
3675    case Expand:
3676      ExpandOp(Node->getOperand(0), Tmp1, Tmp2);
3677
3678      // Since the result is legal, we should just be able to truncate the low
3679      // part of the source.
3680      Result = DAG.getNode(ISD::TRUNCATE, Node->getValueType(0), Tmp1);
3681      break;
3682    case Promote:
3683      Result = PromoteOp(Node->getOperand(0));
3684      Result = DAG.getNode(ISD::TRUNCATE, Op.getValueType(), Result);
3685      break;
3686    }
3687    break;
3688
3689  case ISD::FP_TO_SINT:
3690  case ISD::FP_TO_UINT:
3691    switch (getTypeAction(Node->getOperand(0).getValueType())) {
3692    case Legal:
3693      Tmp1 = LegalizeOp(Node->getOperand(0));
3694
3695      switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))){
3696      default: assert(0 && "Unknown operation action!");
3697      case TargetLowering::Custom:
3698        isCustom = true;
3699        // FALLTHROUGH
3700      case TargetLowering::Legal:
3701        Result = DAG.UpdateNodeOperands(Result, Tmp1);
3702        if (isCustom) {
3703          Tmp1 = TLI.LowerOperation(Result, DAG);
3704          if (Tmp1.Val) Result = Tmp1;
3705        }
3706        break;
3707      case TargetLowering::Promote:
3708        Result = PromoteLegalFP_TO_INT(Tmp1, Node->getValueType(0),
3709                                       Node->getOpcode() == ISD::FP_TO_SINT);
3710        break;
3711      case TargetLowering::Expand:
3712        if (Node->getOpcode() == ISD::FP_TO_UINT) {
3713          SDOperand True, False;
3714          MVT VT =  Node->getOperand(0).getValueType();
3715          MVT NVT = Node->getValueType(0);
3716          const uint64_t zero[] = {0, 0};
3717          APFloat apf = APFloat(APInt(VT.getSizeInBits(), 2, zero));
3718          APInt x = APInt::getSignBit(NVT.getSizeInBits());
3719          (void)apf.convertFromAPInt(x, false, APFloat::rmNearestTiesToEven);
3720          Tmp2 = DAG.getConstantFP(apf, VT);
3721          Tmp3 = DAG.getSetCC(TLI.getSetCCResultType(Node->getOperand(0)),
3722                            Node->getOperand(0), Tmp2, ISD::SETLT);
3723          True = DAG.getNode(ISD::FP_TO_SINT, NVT, Node->getOperand(0));
3724          False = DAG.getNode(ISD::FP_TO_SINT, NVT,
3725                              DAG.getNode(ISD::FSUB, VT, Node->getOperand(0),
3726                                          Tmp2));
3727          False = DAG.getNode(ISD::XOR, NVT, False,
3728                              DAG.getConstant(x, NVT));
3729          Result = DAG.getNode(ISD::SELECT, NVT, Tmp3, True, False);
3730          break;
3731        } else {
3732          assert(0 && "Do not know how to expand FP_TO_SINT yet!");
3733        }
3734        break;
3735      }
3736      break;
3737    case Expand: {
3738      MVT VT = Op.getValueType();
3739      MVT OVT = Node->getOperand(0).getValueType();
3740      // Convert ppcf128 to i32
3741      if (OVT == MVT::ppcf128 && VT == MVT::i32) {
3742        if (Node->getOpcode() == ISD::FP_TO_SINT) {
3743          Result = DAG.getNode(ISD::FP_ROUND_INREG, MVT::ppcf128,
3744                               Node->getOperand(0), DAG.getValueType(MVT::f64));
3745          Result = DAG.getNode(ISD::FP_ROUND, MVT::f64, Result,
3746                               DAG.getIntPtrConstant(1));
3747          Result = DAG.getNode(ISD::FP_TO_SINT, VT, Result);
3748        } else {
3749          const uint64_t TwoE31[] = {0x41e0000000000000LL, 0};
3750          APFloat apf = APFloat(APInt(128, 2, TwoE31));
3751          Tmp2 = DAG.getConstantFP(apf, OVT);
3752          //  X>=2^31 ? (int)(X-2^31)+0x80000000 : (int)X
3753          // FIXME: generated code sucks.
3754          Result = DAG.getNode(ISD::SELECT_CC, VT, Node->getOperand(0), Tmp2,
3755                               DAG.getNode(ISD::ADD, MVT::i32,
3756                                 DAG.getNode(ISD::FP_TO_SINT, VT,
3757                                   DAG.getNode(ISD::FSUB, OVT,
3758                                                 Node->getOperand(0), Tmp2)),
3759                                 DAG.getConstant(0x80000000, MVT::i32)),
3760                               DAG.getNode(ISD::FP_TO_SINT, VT,
3761                                           Node->getOperand(0)),
3762                               DAG.getCondCode(ISD::SETGE));
3763        }
3764        break;
3765      }
3766      // Convert f32 / f64 to i32 / i64 / i128.
3767      RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3768      switch (Node->getOpcode()) {
3769      case ISD::FP_TO_SINT: {
3770        if (VT == MVT::i32) {
3771          if (OVT == MVT::f32)
3772            LC = RTLIB::FPTOSINT_F32_I32;
3773          else if (OVT == MVT::f64)
3774            LC = RTLIB::FPTOSINT_F64_I32;
3775          else if (OVT == MVT::f80)
3776            LC = RTLIB::FPTOSINT_F80_I32;
3777          else if (OVT == MVT::ppcf128)
3778            LC = RTLIB::FPTOSINT_PPCF128_I32;
3779          else
3780            assert(0 && "Unexpected i32-to-fp conversion!");
3781        } else if (VT == MVT::i64) {
3782          if (OVT == MVT::f32)
3783            LC = RTLIB::FPTOSINT_F32_I64;
3784          else if (OVT == MVT::f64)
3785            LC = RTLIB::FPTOSINT_F64_I64;
3786          else if (OVT == MVT::f80)
3787            LC = RTLIB::FPTOSINT_F80_I64;
3788          else if (OVT == MVT::ppcf128)
3789            LC = RTLIB::FPTOSINT_PPCF128_I64;
3790          else
3791            assert(0 && "Unexpected i64-to-fp conversion!");
3792        } else if (VT == MVT::i128) {
3793          if (OVT == MVT::f32)
3794            LC = RTLIB::FPTOSINT_F32_I128;
3795          else if (OVT == MVT::f64)
3796            LC = RTLIB::FPTOSINT_F64_I128;
3797          else if (OVT == MVT::f80)
3798            LC = RTLIB::FPTOSINT_F80_I128;
3799          else if (OVT == MVT::ppcf128)
3800            LC = RTLIB::FPTOSINT_PPCF128_I128;
3801          else
3802            assert(0 && "Unexpected i128-to-fp conversion!");
3803        } else {
3804          assert(0 && "Unexpectd int-to-fp conversion!");
3805        }
3806        break;
3807      }
3808      case ISD::FP_TO_UINT: {
3809        if (VT == MVT::i32) {
3810          if (OVT == MVT::f32)
3811            LC = RTLIB::FPTOUINT_F32_I32;
3812          else if (OVT == MVT::f64)
3813            LC = RTLIB::FPTOUINT_F64_I32;
3814          else if (OVT == MVT::f80)
3815            LC = RTLIB::FPTOUINT_F80_I32;
3816          else
3817            assert(0 && "Unexpected i32-to-fp conversion!");
3818        } else if (VT == MVT::i64) {
3819          if (OVT == MVT::f32)
3820            LC = RTLIB::FPTOUINT_F32_I64;
3821          else if (OVT == MVT::f64)
3822            LC = RTLIB::FPTOUINT_F64_I64;
3823          else if (OVT == MVT::f80)
3824            LC = RTLIB::FPTOUINT_F80_I64;
3825          else if (OVT == MVT::ppcf128)
3826            LC = RTLIB::FPTOUINT_PPCF128_I64;
3827          else
3828            assert(0 && "Unexpected i64-to-fp conversion!");
3829        } else if (VT == MVT::i128) {
3830          if (OVT == MVT::f32)
3831            LC = RTLIB::FPTOUINT_F32_I128;
3832          else if (OVT == MVT::f64)
3833            LC = RTLIB::FPTOUINT_F64_I128;
3834          else if (OVT == MVT::f80)
3835            LC = RTLIB::FPTOUINT_F80_I128;
3836          else if (OVT == MVT::ppcf128)
3837            LC = RTLIB::FPTOUINT_PPCF128_I128;
3838          else
3839            assert(0 && "Unexpected i128-to-fp conversion!");
3840        } else {
3841          assert(0 && "Unexpectd int-to-fp conversion!");
3842        }
3843        break;
3844      }
3845      default: assert(0 && "Unreachable!");
3846      }
3847      SDOperand Dummy;
3848      Result = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Dummy);
3849      break;
3850    }
3851    case Promote:
3852      Tmp1 = PromoteOp(Node->getOperand(0));
3853      Result = DAG.UpdateNodeOperands(Result, LegalizeOp(Tmp1));
3854      Result = LegalizeOp(Result);
3855      break;
3856    }
3857    break;
3858
3859  case ISD::FP_EXTEND: {
3860    MVT DstVT = Op.getValueType();
3861    MVT SrcVT = Op.getOperand(0).getValueType();
3862    if (TLI.getConvertAction(SrcVT, DstVT) == TargetLowering::Expand) {
3863      // The only other way we can lower this is to turn it into a STORE,
3864      // LOAD pair, targetting a temporary location (a stack slot).
3865      Result = EmitStackConvert(Node->getOperand(0), SrcVT, DstVT);
3866      break;
3867    }
3868    switch (getTypeAction(Node->getOperand(0).getValueType())) {
3869    case Expand: assert(0 && "Shouldn't need to expand other operators here!");
3870    case Legal:
3871      Tmp1 = LegalizeOp(Node->getOperand(0));
3872      Result = DAG.UpdateNodeOperands(Result, Tmp1);
3873      break;
3874    case Promote:
3875      Tmp1 = PromoteOp(Node->getOperand(0));
3876      Result = DAG.getNode(ISD::FP_EXTEND, Op.getValueType(), Tmp1);
3877      break;
3878    }
3879    break;
3880  }
3881  case ISD::FP_ROUND: {
3882    MVT DstVT = Op.getValueType();
3883    MVT SrcVT = Op.getOperand(0).getValueType();
3884    if (TLI.getConvertAction(SrcVT, DstVT) == TargetLowering::Expand) {
3885      if (SrcVT == MVT::ppcf128) {
3886        SDOperand Lo;
3887        ExpandOp(Node->getOperand(0), Lo, Result);
3888        // Round it the rest of the way (e.g. to f32) if needed.
3889        if (DstVT!=MVT::f64)
3890          Result = DAG.getNode(ISD::FP_ROUND, DstVT, Result, Op.getOperand(1));
3891        break;
3892      }
3893      // The only other way we can lower this is to turn it into a STORE,
3894      // LOAD pair, targetting a temporary location (a stack slot).
3895      Result = EmitStackConvert(Node->getOperand(0), DstVT, DstVT);
3896      break;
3897    }
3898    switch (getTypeAction(Node->getOperand(0).getValueType())) {
3899    case Expand: assert(0 && "Shouldn't need to expand other operators here!");
3900    case Legal:
3901      Tmp1 = LegalizeOp(Node->getOperand(0));
3902      Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1));
3903      break;
3904    case Promote:
3905      Tmp1 = PromoteOp(Node->getOperand(0));
3906      Result = DAG.getNode(ISD::FP_ROUND, Op.getValueType(), Tmp1,
3907                           Node->getOperand(1));
3908      break;
3909    }
3910    break;
3911  }
3912  case ISD::ANY_EXTEND:
3913  case ISD::ZERO_EXTEND:
3914  case ISD::SIGN_EXTEND:
3915    switch (getTypeAction(Node->getOperand(0).getValueType())) {
3916    case Expand: assert(0 && "Shouldn't need to expand other operators here!");
3917    case Legal:
3918      Tmp1 = LegalizeOp(Node->getOperand(0));
3919      Result = DAG.UpdateNodeOperands(Result, Tmp1);
3920      if (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)) ==
3921          TargetLowering::Custom) {
3922        Tmp1 = TLI.LowerOperation(Result, DAG);
3923        if (Tmp1.Val) Result = Tmp1;
3924      }
3925      break;
3926    case Promote:
3927      switch (Node->getOpcode()) {
3928      case ISD::ANY_EXTEND:
3929        Tmp1 = PromoteOp(Node->getOperand(0));
3930        Result = DAG.getNode(ISD::ANY_EXTEND, Op.getValueType(), Tmp1);
3931        break;
3932      case ISD::ZERO_EXTEND:
3933        Result = PromoteOp(Node->getOperand(0));
3934        Result = DAG.getNode(ISD::ANY_EXTEND, Op.getValueType(), Result);
3935        Result = DAG.getZeroExtendInReg(Result,
3936                                        Node->getOperand(0).getValueType());
3937        break;
3938      case ISD::SIGN_EXTEND:
3939        Result = PromoteOp(Node->getOperand(0));
3940        Result = DAG.getNode(ISD::ANY_EXTEND, Op.getValueType(), Result);
3941        Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, Result.getValueType(),
3942                             Result,
3943                          DAG.getValueType(Node->getOperand(0).getValueType()));
3944        break;
3945      }
3946    }
3947    break;
3948  case ISD::FP_ROUND_INREG:
3949  case ISD::SIGN_EXTEND_INREG: {
3950    Tmp1 = LegalizeOp(Node->getOperand(0));
3951    MVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
3952
3953    // If this operation is not supported, convert it to a shl/shr or load/store
3954    // pair.
3955    switch (TLI.getOperationAction(Node->getOpcode(), ExtraVT)) {
3956    default: assert(0 && "This action not supported for this op yet!");
3957    case TargetLowering::Legal:
3958      Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1));
3959      break;
3960    case TargetLowering::Expand:
3961      // If this is an integer extend and shifts are supported, do that.
3962      if (Node->getOpcode() == ISD::SIGN_EXTEND_INREG) {
3963        // NOTE: we could fall back on load/store here too for targets without
3964        // SAR.  However, it is doubtful that any exist.
3965        unsigned BitsDiff = Node->getValueType(0).getSizeInBits() -
3966                            ExtraVT.getSizeInBits();
3967        SDOperand ShiftCst = DAG.getConstant(BitsDiff, TLI.getShiftAmountTy());
3968        Result = DAG.getNode(ISD::SHL, Node->getValueType(0),
3969                             Node->getOperand(0), ShiftCst);
3970        Result = DAG.getNode(ISD::SRA, Node->getValueType(0),
3971                             Result, ShiftCst);
3972      } else if (Node->getOpcode() == ISD::FP_ROUND_INREG) {
3973        // The only way we can lower this is to turn it into a TRUNCSTORE,
3974        // EXTLOAD pair, targetting a temporary location (a stack slot).
3975
3976        // NOTE: there is a choice here between constantly creating new stack
3977        // slots and always reusing the same one.  We currently always create
3978        // new ones, as reuse may inhibit scheduling.
3979        Result = EmitStackConvert(Node->getOperand(0), ExtraVT,
3980                                  Node->getValueType(0));
3981      } else {
3982        assert(0 && "Unknown op");
3983      }
3984      break;
3985    }
3986    break;
3987  }
3988  case ISD::TRAMPOLINE: {
3989    SDOperand Ops[6];
3990    for (unsigned i = 0; i != 6; ++i)
3991      Ops[i] = LegalizeOp(Node->getOperand(i));
3992    Result = DAG.UpdateNodeOperands(Result, Ops, 6);
3993    // The only option for this node is to custom lower it.
3994    Result = TLI.LowerOperation(Result, DAG);
3995    assert(Result.Val && "Should always custom lower!");
3996
3997    // Since trampoline produces two values, make sure to remember that we
3998    // legalized both of them.
3999    Tmp1 = LegalizeOp(Result.getValue(1));
4000    Result = LegalizeOp(Result);
4001    AddLegalizedOperand(SDOperand(Node, 0), Result);
4002    AddLegalizedOperand(SDOperand(Node, 1), Tmp1);
4003    return Op.ResNo ? Tmp1 : Result;
4004  }
4005  case ISD::FLT_ROUNDS_: {
4006    MVT VT = Node->getValueType(0);
4007    switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
4008    default: assert(0 && "This action not supported for this op yet!");
4009    case TargetLowering::Custom:
4010      Result = TLI.LowerOperation(Op, DAG);
4011      if (Result.Val) break;
4012      // Fall Thru
4013    case TargetLowering::Legal:
4014      // If this operation is not supported, lower it to constant 1
4015      Result = DAG.getConstant(1, VT);
4016      break;
4017    }
4018    break;
4019  }
4020  case ISD::TRAP: {
4021    MVT VT = Node->getValueType(0);
4022    switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
4023    default: assert(0 && "This action not supported for this op yet!");
4024    case TargetLowering::Legal:
4025      Tmp1 = LegalizeOp(Node->getOperand(0));
4026      Result = DAG.UpdateNodeOperands(Result, Tmp1);
4027      break;
4028    case TargetLowering::Custom:
4029      Result = TLI.LowerOperation(Op, DAG);
4030      if (Result.Val) break;
4031      // Fall Thru
4032    case TargetLowering::Expand:
4033      // If this operation is not supported, lower it to 'abort()' call
4034      Tmp1 = LegalizeOp(Node->getOperand(0));
4035      TargetLowering::ArgListTy Args;
4036      std::pair<SDOperand,SDOperand> CallResult =
4037        TLI.LowerCallTo(Tmp1, Type::VoidTy,
4038                        false, false, false, CallingConv::C, false,
4039                        DAG.getExternalSymbol("abort", TLI.getPointerTy()),
4040                        Args, DAG);
4041      Result = CallResult.second;
4042      break;
4043    }
4044    break;
4045  }
4046  }
4047
4048  assert(Result.getValueType() == Op.getValueType() &&
4049         "Bad legalization!");
4050
4051  // Make sure that the generated code is itself legal.
4052  if (Result != Op)
4053    Result = LegalizeOp(Result);
4054
4055  // Note that LegalizeOp may be reentered even from single-use nodes, which
4056  // means that we always must cache transformed nodes.
4057  AddLegalizedOperand(Op, Result);
4058  return Result;
4059}
4060
4061/// PromoteOp - Given an operation that produces a value in an invalid type,
4062/// promote it to compute the value into a larger type.  The produced value will
4063/// have the correct bits for the low portion of the register, but no guarantee
4064/// is made about the top bits: it may be zero, sign-extended, or garbage.
4065SDOperand SelectionDAGLegalize::PromoteOp(SDOperand Op) {
4066  MVT VT = Op.getValueType();
4067  MVT NVT = TLI.getTypeToTransformTo(VT);
4068  assert(getTypeAction(VT) == Promote &&
4069         "Caller should expand or legalize operands that are not promotable!");
4070  assert(NVT.bitsGT(VT) && NVT.isInteger() == VT.isInteger() &&
4071         "Cannot promote to smaller type!");
4072
4073  SDOperand Tmp1, Tmp2, Tmp3;
4074  SDOperand Result;
4075  SDNode *Node = Op.Val;
4076
4077  DenseMap<SDOperand, SDOperand>::iterator I = PromotedNodes.find(Op);
4078  if (I != PromotedNodes.end()) return I->second;
4079
4080  switch (Node->getOpcode()) {
4081  case ISD::CopyFromReg:
4082    assert(0 && "CopyFromReg must be legal!");
4083  default:
4084#ifndef NDEBUG
4085    cerr << "NODE: "; Node->dump(&DAG); cerr << "\n";
4086#endif
4087    assert(0 && "Do not know how to promote this operator!");
4088    abort();
4089  case ISD::UNDEF:
4090    Result = DAG.getNode(ISD::UNDEF, NVT);
4091    break;
4092  case ISD::Constant:
4093    if (VT != MVT::i1)
4094      Result = DAG.getNode(ISD::SIGN_EXTEND, NVT, Op);
4095    else
4096      Result = DAG.getNode(ISD::ZERO_EXTEND, NVT, Op);
4097    assert(isa<ConstantSDNode>(Result) && "Didn't constant fold zext?");
4098    break;
4099  case ISD::ConstantFP:
4100    Result = DAG.getNode(ISD::FP_EXTEND, NVT, Op);
4101    assert(isa<ConstantFPSDNode>(Result) && "Didn't constant fold fp_extend?");
4102    break;
4103
4104  case ISD::SETCC:
4105    assert(isTypeLegal(TLI.getSetCCResultType(Node->getOperand(0)))
4106           && "SetCC type is not legal??");
4107    Result = DAG.getNode(ISD::SETCC,
4108                         TLI.getSetCCResultType(Node->getOperand(0)),
4109                         Node->getOperand(0), Node->getOperand(1),
4110                         Node->getOperand(2));
4111    break;
4112
4113  case ISD::TRUNCATE:
4114    switch (getTypeAction(Node->getOperand(0).getValueType())) {
4115    case Legal:
4116      Result = LegalizeOp(Node->getOperand(0));
4117      assert(Result.getValueType().bitsGE(NVT) &&
4118             "This truncation doesn't make sense!");
4119      if (Result.getValueType().bitsGT(NVT))    // Truncate to NVT instead of VT
4120        Result = DAG.getNode(ISD::TRUNCATE, NVT, Result);
4121      break;
4122    case Promote:
4123      // The truncation is not required, because we don't guarantee anything
4124      // about high bits anyway.
4125      Result = PromoteOp(Node->getOperand(0));
4126      break;
4127    case Expand:
4128      ExpandOp(Node->getOperand(0), Tmp1, Tmp2);
4129      // Truncate the low part of the expanded value to the result type
4130      Result = DAG.getNode(ISD::TRUNCATE, NVT, Tmp1);
4131    }
4132    break;
4133  case ISD::SIGN_EXTEND:
4134  case ISD::ZERO_EXTEND:
4135  case ISD::ANY_EXTEND:
4136    switch (getTypeAction(Node->getOperand(0).getValueType())) {
4137    case Expand: assert(0 && "BUG: Smaller reg should have been promoted!");
4138    case Legal:
4139      // Input is legal?  Just do extend all the way to the larger type.
4140      Result = DAG.getNode(Node->getOpcode(), NVT, Node->getOperand(0));
4141      break;
4142    case Promote:
4143      // Promote the reg if it's smaller.
4144      Result = PromoteOp(Node->getOperand(0));
4145      // The high bits are not guaranteed to be anything.  Insert an extend.
4146      if (Node->getOpcode() == ISD::SIGN_EXTEND)
4147        Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Result,
4148                         DAG.getValueType(Node->getOperand(0).getValueType()));
4149      else if (Node->getOpcode() == ISD::ZERO_EXTEND)
4150        Result = DAG.getZeroExtendInReg(Result,
4151                                        Node->getOperand(0).getValueType());
4152      break;
4153    }
4154    break;
4155  case ISD::BIT_CONVERT:
4156    Result = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
4157                              Node->getValueType(0));
4158    Result = PromoteOp(Result);
4159    break;
4160
4161  case ISD::FP_EXTEND:
4162    assert(0 && "Case not implemented.  Dynamically dead with 2 FP types!");
4163  case ISD::FP_ROUND:
4164    switch (getTypeAction(Node->getOperand(0).getValueType())) {
4165    case Expand: assert(0 && "BUG: Cannot expand FP regs!");
4166    case Promote:  assert(0 && "Unreachable with 2 FP types!");
4167    case Legal:
4168      if (Node->getConstantOperandVal(1) == 0) {
4169        // Input is legal?  Do an FP_ROUND_INREG.
4170        Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Node->getOperand(0),
4171                             DAG.getValueType(VT));
4172      } else {
4173        // Just remove the truncate, it isn't affecting the value.
4174        Result = DAG.getNode(ISD::FP_ROUND, NVT, Node->getOperand(0),
4175                             Node->getOperand(1));
4176      }
4177      break;
4178    }
4179    break;
4180  case ISD::SINT_TO_FP:
4181  case ISD::UINT_TO_FP:
4182    switch (getTypeAction(Node->getOperand(0).getValueType())) {
4183    case Legal:
4184      // No extra round required here.
4185      Result = DAG.getNode(Node->getOpcode(), NVT, Node->getOperand(0));
4186      break;
4187
4188    case Promote:
4189      Result = PromoteOp(Node->getOperand(0));
4190      if (Node->getOpcode() == ISD::SINT_TO_FP)
4191        Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, Result.getValueType(),
4192                             Result,
4193                         DAG.getValueType(Node->getOperand(0).getValueType()));
4194      else
4195        Result = DAG.getZeroExtendInReg(Result,
4196                                        Node->getOperand(0).getValueType());
4197      // No extra round required here.
4198      Result = DAG.getNode(Node->getOpcode(), NVT, Result);
4199      break;
4200    case Expand:
4201      Result = ExpandIntToFP(Node->getOpcode() == ISD::SINT_TO_FP, NVT,
4202                             Node->getOperand(0));
4203      // Round if we cannot tolerate excess precision.
4204      if (NoExcessFPPrecision)
4205        Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Result,
4206                             DAG.getValueType(VT));
4207      break;
4208    }
4209    break;
4210
4211  case ISD::SIGN_EXTEND_INREG:
4212    Result = PromoteOp(Node->getOperand(0));
4213    Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Result,
4214                         Node->getOperand(1));
4215    break;
4216  case ISD::FP_TO_SINT:
4217  case ISD::FP_TO_UINT:
4218    switch (getTypeAction(Node->getOperand(0).getValueType())) {
4219    case Legal:
4220    case Expand:
4221      Tmp1 = Node->getOperand(0);
4222      break;
4223    case Promote:
4224      // The input result is prerounded, so we don't have to do anything
4225      // special.
4226      Tmp1 = PromoteOp(Node->getOperand(0));
4227      break;
4228    }
4229    // If we're promoting a UINT to a larger size, check to see if the new node
4230    // will be legal.  If it isn't, check to see if FP_TO_SINT is legal, since
4231    // we can use that instead.  This allows us to generate better code for
4232    // FP_TO_UINT for small destination sizes on targets where FP_TO_UINT is not
4233    // legal, such as PowerPC.
4234    if (Node->getOpcode() == ISD::FP_TO_UINT &&
4235        !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
4236        (TLI.isOperationLegal(ISD::FP_TO_SINT, NVT) ||
4237         TLI.getOperationAction(ISD::FP_TO_SINT, NVT)==TargetLowering::Custom)){
4238      Result = DAG.getNode(ISD::FP_TO_SINT, NVT, Tmp1);
4239    } else {
4240      Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1);
4241    }
4242    break;
4243
4244  case ISD::FABS:
4245  case ISD::FNEG:
4246    Tmp1 = PromoteOp(Node->getOperand(0));
4247    assert(Tmp1.getValueType() == NVT);
4248    Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1);
4249    // NOTE: we do not have to do any extra rounding here for
4250    // NoExcessFPPrecision, because we know the input will have the appropriate
4251    // precision, and these operations don't modify precision at all.
4252    break;
4253
4254  case ISD::FSQRT:
4255  case ISD::FSIN:
4256  case ISD::FCOS:
4257    Tmp1 = PromoteOp(Node->getOperand(0));
4258    assert(Tmp1.getValueType() == NVT);
4259    Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1);
4260    if (NoExcessFPPrecision)
4261      Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Result,
4262                           DAG.getValueType(VT));
4263    break;
4264
4265  case ISD::FPOWI: {
4266    // Promote f32 powi to f64 powi.  Note that this could insert a libcall
4267    // directly as well, which may be better.
4268    Tmp1 = PromoteOp(Node->getOperand(0));
4269    assert(Tmp1.getValueType() == NVT);
4270    Result = DAG.getNode(ISD::FPOWI, NVT, Tmp1, Node->getOperand(1));
4271    if (NoExcessFPPrecision)
4272      Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Result,
4273                           DAG.getValueType(VT));
4274    break;
4275  }
4276
4277  case ISD::ATOMIC_CMP_SWAP: {
4278    AtomicSDNode* AtomNode = cast<AtomicSDNode>(Node);
4279    Tmp2 = PromoteOp(Node->getOperand(2));
4280    Tmp3 = PromoteOp(Node->getOperand(3));
4281    Result = DAG.getAtomic(Node->getOpcode(), AtomNode->getChain(),
4282                           AtomNode->getBasePtr(), Tmp2, Tmp3,
4283                           AtomNode->getSrcValue(),
4284                           AtomNode->getAlignment());
4285    // Remember that we legalized the chain.
4286    AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1)));
4287    break;
4288  }
4289  case ISD::ATOMIC_LOAD_ADD:
4290  case ISD::ATOMIC_LOAD_SUB:
4291  case ISD::ATOMIC_LOAD_AND:
4292  case ISD::ATOMIC_LOAD_OR:
4293  case ISD::ATOMIC_LOAD_XOR:
4294  case ISD::ATOMIC_LOAD_NAND:
4295  case ISD::ATOMIC_LOAD_MIN:
4296  case ISD::ATOMIC_LOAD_MAX:
4297  case ISD::ATOMIC_LOAD_UMIN:
4298  case ISD::ATOMIC_LOAD_UMAX:
4299  case ISD::ATOMIC_SWAP: {
4300    AtomicSDNode* AtomNode = cast<AtomicSDNode>(Node);
4301    Tmp2 = PromoteOp(Node->getOperand(2));
4302    Result = DAG.getAtomic(Node->getOpcode(), AtomNode->getChain(),
4303                           AtomNode->getBasePtr(), Tmp2,
4304                           AtomNode->getSrcValue(),
4305                           AtomNode->getAlignment());
4306    // Remember that we legalized the chain.
4307    AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1)));
4308    break;
4309  }
4310
4311  case ISD::AND:
4312  case ISD::OR:
4313  case ISD::XOR:
4314  case ISD::ADD:
4315  case ISD::SUB:
4316  case ISD::MUL:
4317    // The input may have strange things in the top bits of the registers, but
4318    // these operations don't care.  They may have weird bits going out, but
4319    // that too is okay if they are integer operations.
4320    Tmp1 = PromoteOp(Node->getOperand(0));
4321    Tmp2 = PromoteOp(Node->getOperand(1));
4322    assert(Tmp1.getValueType() == NVT && Tmp2.getValueType() == NVT);
4323    Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1, Tmp2);
4324    break;
4325  case ISD::FADD:
4326  case ISD::FSUB:
4327  case ISD::FMUL:
4328    Tmp1 = PromoteOp(Node->getOperand(0));
4329    Tmp2 = PromoteOp(Node->getOperand(1));
4330    assert(Tmp1.getValueType() == NVT && Tmp2.getValueType() == NVT);
4331    Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1, Tmp2);
4332
4333    // Floating point operations will give excess precision that we may not be
4334    // able to tolerate.  If we DO allow excess precision, just leave it,
4335    // otherwise excise it.
4336    // FIXME: Why would we need to round FP ops more than integer ones?
4337    //     Is Round(Add(Add(A,B),C)) != Round(Add(Round(Add(A,B)), C))
4338    if (NoExcessFPPrecision)
4339      Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Result,
4340                           DAG.getValueType(VT));
4341    break;
4342
4343  case ISD::SDIV:
4344  case ISD::SREM:
4345    // These operators require that their input be sign extended.
4346    Tmp1 = PromoteOp(Node->getOperand(0));
4347    Tmp2 = PromoteOp(Node->getOperand(1));
4348    if (NVT.isInteger()) {
4349      Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Tmp1,
4350                         DAG.getValueType(VT));
4351      Tmp2 = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Tmp2,
4352                         DAG.getValueType(VT));
4353    }
4354    Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1, Tmp2);
4355
4356    // Perform FP_ROUND: this is probably overly pessimistic.
4357    if (NVT.isFloatingPoint() && NoExcessFPPrecision)
4358      Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Result,
4359                           DAG.getValueType(VT));
4360    break;
4361  case ISD::FDIV:
4362  case ISD::FREM:
4363  case ISD::FCOPYSIGN:
4364    // These operators require that their input be fp extended.
4365    switch (getTypeAction(Node->getOperand(0).getValueType())) {
4366    case Expand: assert(0 && "not implemented");
4367    case Legal:   Tmp1 = LegalizeOp(Node->getOperand(0)); break;
4368    case Promote: Tmp1 = PromoteOp(Node->getOperand(0));  break;
4369    }
4370    switch (getTypeAction(Node->getOperand(1).getValueType())) {
4371    case Expand: assert(0 && "not implemented");
4372    case Legal:   Tmp2 = LegalizeOp(Node->getOperand(1)); break;
4373    case Promote: Tmp2 = PromoteOp(Node->getOperand(1)); break;
4374    }
4375    Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1, Tmp2);
4376
4377    // Perform FP_ROUND: this is probably overly pessimistic.
4378    if (NoExcessFPPrecision && Node->getOpcode() != ISD::FCOPYSIGN)
4379      Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Result,
4380                           DAG.getValueType(VT));
4381    break;
4382
4383  case ISD::UDIV:
4384  case ISD::UREM:
4385    // These operators require that their input be zero extended.
4386    Tmp1 = PromoteOp(Node->getOperand(0));
4387    Tmp2 = PromoteOp(Node->getOperand(1));
4388    assert(NVT.isInteger() && "Operators don't apply to FP!");
4389    Tmp1 = DAG.getZeroExtendInReg(Tmp1, VT);
4390    Tmp2 = DAG.getZeroExtendInReg(Tmp2, VT);
4391    Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1, Tmp2);
4392    break;
4393
4394  case ISD::SHL:
4395    Tmp1 = PromoteOp(Node->getOperand(0));
4396    Result = DAG.getNode(ISD::SHL, NVT, Tmp1, Node->getOperand(1));
4397    break;
4398  case ISD::SRA:
4399    // The input value must be properly sign extended.
4400    Tmp1 = PromoteOp(Node->getOperand(0));
4401    Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Tmp1,
4402                       DAG.getValueType(VT));
4403    Result = DAG.getNode(ISD::SRA, NVT, Tmp1, Node->getOperand(1));
4404    break;
4405  case ISD::SRL:
4406    // The input value must be properly zero extended.
4407    Tmp1 = PromoteOp(Node->getOperand(0));
4408    Tmp1 = DAG.getZeroExtendInReg(Tmp1, VT);
4409    Result = DAG.getNode(ISD::SRL, NVT, Tmp1, Node->getOperand(1));
4410    break;
4411
4412  case ISD::VAARG:
4413    Tmp1 = Node->getOperand(0);   // Get the chain.
4414    Tmp2 = Node->getOperand(1);   // Get the pointer.
4415    if (TLI.getOperationAction(ISD::VAARG, VT) == TargetLowering::Custom) {
4416      Tmp3 = DAG.getVAArg(VT, Tmp1, Tmp2, Node->getOperand(2));
4417      Result = TLI.LowerOperation(Tmp3, DAG);
4418    } else {
4419      const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
4420      SDOperand VAList = DAG.getLoad(TLI.getPointerTy(), Tmp1, Tmp2, V, 0);
4421      // Increment the pointer, VAList, to the next vaarg
4422      Tmp3 = DAG.getNode(ISD::ADD, TLI.getPointerTy(), VAList,
4423                         DAG.getConstant(VT.getSizeInBits()/8,
4424                                         TLI.getPointerTy()));
4425      // Store the incremented VAList to the legalized pointer
4426      Tmp3 = DAG.getStore(VAList.getValue(1), Tmp3, Tmp2, V, 0);
4427      // Load the actual argument out of the pointer VAList
4428      Result = DAG.getExtLoad(ISD::EXTLOAD, NVT, Tmp3, VAList, NULL, 0, VT);
4429    }
4430    // Remember that we legalized the chain.
4431    AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1)));
4432    break;
4433
4434  case ISD::LOAD: {
4435    LoadSDNode *LD = cast<LoadSDNode>(Node);
4436    ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(Node)
4437      ? ISD::EXTLOAD : LD->getExtensionType();
4438    Result = DAG.getExtLoad(ExtType, NVT,
4439                            LD->getChain(), LD->getBasePtr(),
4440                            LD->getSrcValue(), LD->getSrcValueOffset(),
4441                            LD->getMemoryVT(),
4442                            LD->isVolatile(),
4443                            LD->getAlignment());
4444    // Remember that we legalized the chain.
4445    AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1)));
4446    break;
4447  }
4448  case ISD::SELECT: {
4449    Tmp2 = PromoteOp(Node->getOperand(1));   // Legalize the op0
4450    Tmp3 = PromoteOp(Node->getOperand(2));   // Legalize the op1
4451
4452    MVT VT2 = Tmp2.getValueType();
4453    assert(VT2 == Tmp3.getValueType()
4454           && "PromoteOp SELECT: Operands 2 and 3 ValueTypes don't match");
4455    // Ensure that the resulting node is at least the same size as the operands'
4456    // value types, because we cannot assume that TLI.getSetCCValueType() is
4457    // constant.
4458    Result = DAG.getNode(ISD::SELECT, VT2, Node->getOperand(0), Tmp2, Tmp3);
4459    break;
4460  }
4461  case ISD::SELECT_CC:
4462    Tmp2 = PromoteOp(Node->getOperand(2));   // True
4463    Tmp3 = PromoteOp(Node->getOperand(3));   // False
4464    Result = DAG.getNode(ISD::SELECT_CC, NVT, Node->getOperand(0),
4465                         Node->getOperand(1), Tmp2, Tmp3, Node->getOperand(4));
4466    break;
4467  case ISD::BSWAP:
4468    Tmp1 = Node->getOperand(0);
4469    Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, NVT, Tmp1);
4470    Tmp1 = DAG.getNode(ISD::BSWAP, NVT, Tmp1);
4471    Result = DAG.getNode(ISD::SRL, NVT, Tmp1,
4472                         DAG.getConstant(NVT.getSizeInBits() -
4473                                         VT.getSizeInBits(),
4474                                         TLI.getShiftAmountTy()));
4475    break;
4476  case ISD::CTPOP:
4477  case ISD::CTTZ:
4478  case ISD::CTLZ:
4479    // Zero extend the argument
4480    Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, NVT, Node->getOperand(0));
4481    // Perform the larger operation, then subtract if needed.
4482    Tmp1 = DAG.getNode(Node->getOpcode(), NVT, Tmp1);
4483    switch(Node->getOpcode()) {
4484    case ISD::CTPOP:
4485      Result = Tmp1;
4486      break;
4487    case ISD::CTTZ:
4488      // if Tmp1 == sizeinbits(NVT) then Tmp1 = sizeinbits(Old VT)
4489      Tmp2 = DAG.getSetCC(TLI.getSetCCResultType(Tmp1), Tmp1,
4490                          DAG.getConstant(NVT.getSizeInBits(), NVT),
4491                          ISD::SETEQ);
4492      Result = DAG.getNode(ISD::SELECT, NVT, Tmp2,
4493                           DAG.getConstant(VT.getSizeInBits(), NVT), Tmp1);
4494      break;
4495    case ISD::CTLZ:
4496      //Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
4497      Result = DAG.getNode(ISD::SUB, NVT, Tmp1,
4498                           DAG.getConstant(NVT.getSizeInBits() -
4499                                           VT.getSizeInBits(), NVT));
4500      break;
4501    }
4502    break;
4503  case ISD::EXTRACT_SUBVECTOR:
4504    Result = PromoteOp(ExpandEXTRACT_SUBVECTOR(Op));
4505    break;
4506  case ISD::EXTRACT_VECTOR_ELT:
4507    Result = PromoteOp(ExpandEXTRACT_VECTOR_ELT(Op));
4508    break;
4509  }
4510
4511  assert(Result.Val && "Didn't set a result!");
4512
4513  // Make sure the result is itself legal.
4514  Result = LegalizeOp(Result);
4515
4516  // Remember that we promoted this!
4517  AddPromotedOperand(Op, Result);
4518  return Result;
4519}
4520
4521/// ExpandEXTRACT_VECTOR_ELT - Expand an EXTRACT_VECTOR_ELT operation into
4522/// a legal EXTRACT_VECTOR_ELT operation, scalar code, or memory traffic,
4523/// based on the vector type. The return type of this matches the element type
4524/// of the vector, which may not be legal for the target.
4525SDOperand SelectionDAGLegalize::ExpandEXTRACT_VECTOR_ELT(SDOperand Op) {
4526  // We know that operand #0 is the Vec vector.  If the index is a constant
4527  // or if the invec is a supported hardware type, we can use it.  Otherwise,
4528  // lower to a store then an indexed load.
4529  SDOperand Vec = Op.getOperand(0);
4530  SDOperand Idx = Op.getOperand(1);
4531
4532  MVT TVT = Vec.getValueType();
4533  unsigned NumElems = TVT.getVectorNumElements();
4534
4535  switch (TLI.getOperationAction(ISD::EXTRACT_VECTOR_ELT, TVT)) {
4536  default: assert(0 && "This action is not supported yet!");
4537  case TargetLowering::Custom: {
4538    Vec = LegalizeOp(Vec);
4539    Op = DAG.UpdateNodeOperands(Op, Vec, Idx);
4540    SDOperand Tmp3 = TLI.LowerOperation(Op, DAG);
4541    if (Tmp3.Val)
4542      return Tmp3;
4543    break;
4544  }
4545  case TargetLowering::Legal:
4546    if (isTypeLegal(TVT)) {
4547      Vec = LegalizeOp(Vec);
4548      Op = DAG.UpdateNodeOperands(Op, Vec, Idx);
4549      return Op;
4550    }
4551    break;
4552  case TargetLowering::Expand:
4553    break;
4554  }
4555
4556  if (NumElems == 1) {
4557    // This must be an access of the only element.  Return it.
4558    Op = ScalarizeVectorOp(Vec);
4559  } else if (!TLI.isTypeLegal(TVT) && isa<ConstantSDNode>(Idx)) {
4560    unsigned NumLoElts =  1 << Log2_32(NumElems-1);
4561    ConstantSDNode *CIdx = cast<ConstantSDNode>(Idx);
4562    SDOperand Lo, Hi;
4563    SplitVectorOp(Vec, Lo, Hi);
4564    if (CIdx->getValue() < NumLoElts) {
4565      Vec = Lo;
4566    } else {
4567      Vec = Hi;
4568      Idx = DAG.getConstant(CIdx->getValue() - NumLoElts,
4569                            Idx.getValueType());
4570    }
4571
4572    // It's now an extract from the appropriate high or low part.  Recurse.
4573    Op = DAG.UpdateNodeOperands(Op, Vec, Idx);
4574    Op = ExpandEXTRACT_VECTOR_ELT(Op);
4575  } else {
4576    // Store the value to a temporary stack slot, then LOAD the scalar
4577    // element back out.
4578    SDOperand StackPtr = DAG.CreateStackTemporary(Vec.getValueType());
4579    SDOperand Ch = DAG.getStore(DAG.getEntryNode(), Vec, StackPtr, NULL, 0);
4580
4581    // Add the offset to the index.
4582    unsigned EltSize = Op.getValueType().getSizeInBits()/8;
4583    Idx = DAG.getNode(ISD::MUL, Idx.getValueType(), Idx,
4584                      DAG.getConstant(EltSize, Idx.getValueType()));
4585
4586    if (Idx.getValueType().bitsGT(TLI.getPointerTy()))
4587      Idx = DAG.getNode(ISD::TRUNCATE, TLI.getPointerTy(), Idx);
4588    else
4589      Idx = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(), Idx);
4590
4591    StackPtr = DAG.getNode(ISD::ADD, Idx.getValueType(), Idx, StackPtr);
4592
4593    Op = DAG.getLoad(Op.getValueType(), Ch, StackPtr, NULL, 0);
4594  }
4595  return Op;
4596}
4597
4598/// ExpandEXTRACT_SUBVECTOR - Expand a EXTRACT_SUBVECTOR operation.  For now
4599/// we assume the operation can be split if it is not already legal.
4600SDOperand SelectionDAGLegalize::ExpandEXTRACT_SUBVECTOR(SDOperand Op) {
4601  // We know that operand #0 is the Vec vector.  For now we assume the index
4602  // is a constant and that the extracted result is a supported hardware type.
4603  SDOperand Vec = Op.getOperand(0);
4604  SDOperand Idx = LegalizeOp(Op.getOperand(1));
4605
4606  unsigned NumElems = Vec.getValueType().getVectorNumElements();
4607
4608  if (NumElems == Op.getValueType().getVectorNumElements()) {
4609    // This must be an access of the desired vector length.  Return it.
4610    return Vec;
4611  }
4612
4613  ConstantSDNode *CIdx = cast<ConstantSDNode>(Idx);
4614  SDOperand Lo, Hi;
4615  SplitVectorOp(Vec, Lo, Hi);
4616  if (CIdx->getValue() < NumElems/2) {
4617    Vec = Lo;
4618  } else {
4619    Vec = Hi;
4620    Idx = DAG.getConstant(CIdx->getValue() - NumElems/2, Idx.getValueType());
4621  }
4622
4623  // It's now an extract from the appropriate high or low part.  Recurse.
4624  Op = DAG.UpdateNodeOperands(Op, Vec, Idx);
4625  return ExpandEXTRACT_SUBVECTOR(Op);
4626}
4627
4628/// LegalizeSetCCOperands - Attempts to create a legal LHS and RHS for a SETCC
4629/// with condition CC on the current target.  This usually involves legalizing
4630/// or promoting the arguments.  In the case where LHS and RHS must be expanded,
4631/// there may be no choice but to create a new SetCC node to represent the
4632/// legalized value of setcc lhs, rhs.  In this case, the value is returned in
4633/// LHS, and the SDOperand returned in RHS has a nil SDNode value.
4634void SelectionDAGLegalize::LegalizeSetCCOperands(SDOperand &LHS,
4635                                                 SDOperand &RHS,
4636                                                 SDOperand &CC) {
4637  SDOperand Tmp1, Tmp2, Tmp3, Result;
4638
4639  switch (getTypeAction(LHS.getValueType())) {
4640  case Legal:
4641    Tmp1 = LegalizeOp(LHS);   // LHS
4642    Tmp2 = LegalizeOp(RHS);   // RHS
4643    break;
4644  case Promote:
4645    Tmp1 = PromoteOp(LHS);   // LHS
4646    Tmp2 = PromoteOp(RHS);   // RHS
4647
4648    // If this is an FP compare, the operands have already been extended.
4649    if (LHS.getValueType().isInteger()) {
4650      MVT VT = LHS.getValueType();
4651      MVT NVT = TLI.getTypeToTransformTo(VT);
4652
4653      // Otherwise, we have to insert explicit sign or zero extends.  Note
4654      // that we could insert sign extends for ALL conditions, but zero extend
4655      // is cheaper on many machines (an AND instead of two shifts), so prefer
4656      // it.
4657      switch (cast<CondCodeSDNode>(CC)->get()) {
4658      default: assert(0 && "Unknown integer comparison!");
4659      case ISD::SETEQ:
4660      case ISD::SETNE:
4661      case ISD::SETUGE:
4662      case ISD::SETUGT:
4663      case ISD::SETULE:
4664      case ISD::SETULT:
4665        // ALL of these operations will work if we either sign or zero extend
4666        // the operands (including the unsigned comparisons!).  Zero extend is
4667        // usually a simpler/cheaper operation, so prefer it.
4668        Tmp1 = DAG.getZeroExtendInReg(Tmp1, VT);
4669        Tmp2 = DAG.getZeroExtendInReg(Tmp2, VT);
4670        break;
4671      case ISD::SETGE:
4672      case ISD::SETGT:
4673      case ISD::SETLT:
4674      case ISD::SETLE:
4675        Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Tmp1,
4676                           DAG.getValueType(VT));
4677        Tmp2 = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Tmp2,
4678                           DAG.getValueType(VT));
4679        break;
4680      }
4681    }
4682    break;
4683  case Expand: {
4684    MVT VT = LHS.getValueType();
4685    if (VT == MVT::f32 || VT == MVT::f64) {
4686      // Expand into one or more soft-fp libcall(s).
4687      RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
4688      switch (cast<CondCodeSDNode>(CC)->get()) {
4689      case ISD::SETEQ:
4690      case ISD::SETOEQ:
4691        LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 : RTLIB::OEQ_F64;
4692        break;
4693      case ISD::SETNE:
4694      case ISD::SETUNE:
4695        LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 : RTLIB::UNE_F64;
4696        break;
4697      case ISD::SETGE:
4698      case ISD::SETOGE:
4699        LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 : RTLIB::OGE_F64;
4700        break;
4701      case ISD::SETLT:
4702      case ISD::SETOLT:
4703        LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 : RTLIB::OLT_F64;
4704        break;
4705      case ISD::SETLE:
4706      case ISD::SETOLE:
4707        LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 : RTLIB::OLE_F64;
4708        break;
4709      case ISD::SETGT:
4710      case ISD::SETOGT:
4711        LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 : RTLIB::OGT_F64;
4712        break;
4713      case ISD::SETUO:
4714        LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 : RTLIB::UO_F64;
4715        break;
4716      case ISD::SETO:
4717        LC1 = (VT == MVT::f32) ? RTLIB::O_F32 : RTLIB::O_F64;
4718        break;
4719      default:
4720        LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 : RTLIB::UO_F64;
4721        switch (cast<CondCodeSDNode>(CC)->get()) {
4722        case ISD::SETONE:
4723          // SETONE = SETOLT | SETOGT
4724          LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 : RTLIB::OLT_F64;
4725          // Fallthrough
4726        case ISD::SETUGT:
4727          LC2 = (VT == MVT::f32) ? RTLIB::OGT_F32 : RTLIB::OGT_F64;
4728          break;
4729        case ISD::SETUGE:
4730          LC2 = (VT == MVT::f32) ? RTLIB::OGE_F32 : RTLIB::OGE_F64;
4731          break;
4732        case ISD::SETULT:
4733          LC2 = (VT == MVT::f32) ? RTLIB::OLT_F32 : RTLIB::OLT_F64;
4734          break;
4735        case ISD::SETULE:
4736          LC2 = (VT == MVT::f32) ? RTLIB::OLE_F32 : RTLIB::OLE_F64;
4737          break;
4738        case ISD::SETUEQ:
4739          LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 : RTLIB::OEQ_F64;
4740          break;
4741        default: assert(0 && "Unsupported FP setcc!");
4742        }
4743      }
4744
4745      SDOperand Dummy;
4746      SDOperand Ops[2] = { LHS, RHS };
4747      Tmp1 = ExpandLibCall(LC1, DAG.getMergeValues(Ops, 2).Val,
4748                           false /*sign irrelevant*/, Dummy);
4749      Tmp2 = DAG.getConstant(0, MVT::i32);
4750      CC = DAG.getCondCode(TLI.getCmpLibcallCC(LC1));
4751      if (LC2 != RTLIB::UNKNOWN_LIBCALL) {
4752        Tmp1 = DAG.getNode(ISD::SETCC, TLI.getSetCCResultType(Tmp1), Tmp1, Tmp2,
4753                           CC);
4754        LHS = ExpandLibCall(LC2, DAG.getMergeValues(Ops, 2).Val,
4755                            false /*sign irrelevant*/, Dummy);
4756        Tmp2 = DAG.getNode(ISD::SETCC, TLI.getSetCCResultType(LHS), LHS, Tmp2,
4757                           DAG.getCondCode(TLI.getCmpLibcallCC(LC2)));
4758        Tmp1 = DAG.getNode(ISD::OR, Tmp1.getValueType(), Tmp1, Tmp2);
4759        Tmp2 = SDOperand();
4760      }
4761      LHS = LegalizeOp(Tmp1);
4762      RHS = Tmp2;
4763      return;
4764    }
4765
4766    SDOperand LHSLo, LHSHi, RHSLo, RHSHi;
4767    ExpandOp(LHS, LHSLo, LHSHi);
4768    ExpandOp(RHS, RHSLo, RHSHi);
4769    ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
4770
4771    if (VT==MVT::ppcf128) {
4772      // FIXME:  This generated code sucks.  We want to generate
4773      //         FCMP crN, hi1, hi2
4774      //         BNE crN, L:
4775      //         FCMP crN, lo1, lo2
4776      // The following can be improved, but not that much.
4777      Tmp1 = DAG.getSetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, ISD::SETEQ);
4778      Tmp2 = DAG.getSetCC(TLI.getSetCCResultType(LHSLo), LHSLo, RHSLo, CCCode);
4779      Tmp3 = DAG.getNode(ISD::AND, Tmp1.getValueType(), Tmp1, Tmp2);
4780      Tmp1 = DAG.getSetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, ISD::SETNE);
4781      Tmp2 = DAG.getSetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi, CCCode);
4782      Tmp1 = DAG.getNode(ISD::AND, Tmp1.getValueType(), Tmp1, Tmp2);
4783      Tmp1 = DAG.getNode(ISD::OR, Tmp1.getValueType(), Tmp1, Tmp3);
4784      Tmp2 = SDOperand();
4785      break;
4786    }
4787
4788    switch (CCCode) {
4789    case ISD::SETEQ:
4790    case ISD::SETNE:
4791      if (RHSLo == RHSHi)
4792        if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo))
4793          if (RHSCST->isAllOnesValue()) {
4794            // Comparison to -1.
4795            Tmp1 = DAG.getNode(ISD::AND, LHSLo.getValueType(), LHSLo, LHSHi);
4796            Tmp2 = RHSLo;
4797            break;
4798          }
4799
4800      Tmp1 = DAG.getNode(ISD::XOR, LHSLo.getValueType(), LHSLo, RHSLo);
4801      Tmp2 = DAG.getNode(ISD::XOR, LHSLo.getValueType(), LHSHi, RHSHi);
4802      Tmp1 = DAG.getNode(ISD::OR, Tmp1.getValueType(), Tmp1, Tmp2);
4803      Tmp2 = DAG.getConstant(0, Tmp1.getValueType());
4804      break;
4805    default:
4806      // If this is a comparison of the sign bit, just look at the top part.
4807      // X > -1,  x < 0
4808      if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(RHS))
4809        if ((cast<CondCodeSDNode>(CC)->get() == ISD::SETLT &&
4810             CST->isNullValue()) ||               // X < 0
4811            (cast<CondCodeSDNode>(CC)->get() == ISD::SETGT &&
4812             CST->isAllOnesValue())) {            // X > -1
4813          Tmp1 = LHSHi;
4814          Tmp2 = RHSHi;
4815          break;
4816        }
4817
4818      // FIXME: This generated code sucks.
4819      ISD::CondCode LowCC;
4820      switch (CCCode) {
4821      default: assert(0 && "Unknown integer setcc!");
4822      case ISD::SETLT:
4823      case ISD::SETULT: LowCC = ISD::SETULT; break;
4824      case ISD::SETGT:
4825      case ISD::SETUGT: LowCC = ISD::SETUGT; break;
4826      case ISD::SETLE:
4827      case ISD::SETULE: LowCC = ISD::SETULE; break;
4828      case ISD::SETGE:
4829      case ISD::SETUGE: LowCC = ISD::SETUGE; break;
4830      }
4831
4832      // Tmp1 = lo(op1) < lo(op2)   // Always unsigned comparison
4833      // Tmp2 = hi(op1) < hi(op2)   // Signedness depends on operands
4834      // dest = hi(op1) == hi(op2) ? Tmp1 : Tmp2;
4835
4836      // NOTE: on targets without efficient SELECT of bools, we can always use
4837      // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
4838      TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, false, true, NULL);
4839      Tmp1 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSLo), LHSLo, RHSLo,
4840                               LowCC, false, DagCombineInfo);
4841      if (!Tmp1.Val)
4842        Tmp1 = DAG.getSetCC(TLI.getSetCCResultType(LHSLo), LHSLo, RHSLo, LowCC);
4843      Tmp2 = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi,
4844                               CCCode, false, DagCombineInfo);
4845      if (!Tmp2.Val)
4846        Tmp2 = DAG.getNode(ISD::SETCC, TLI.getSetCCResultType(LHSHi), LHSHi,
4847                           RHSHi,CC);
4848
4849      ConstantSDNode *Tmp1C = dyn_cast<ConstantSDNode>(Tmp1.Val);
4850      ConstantSDNode *Tmp2C = dyn_cast<ConstantSDNode>(Tmp2.Val);
4851      if ((Tmp1C && Tmp1C->isNullValue()) ||
4852          (Tmp2C && Tmp2C->isNullValue() &&
4853           (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
4854            CCCode == ISD::SETUGE || CCCode == ISD::SETULE)) ||
4855          (Tmp2C && Tmp2C->getAPIntValue() == 1 &&
4856           (CCCode == ISD::SETLT || CCCode == ISD::SETGT ||
4857            CCCode == ISD::SETUGT || CCCode == ISD::SETULT))) {
4858        // low part is known false, returns high part.
4859        // For LE / GE, if high part is known false, ignore the low part.
4860        // For LT / GT, if high part is known true, ignore the low part.
4861        Tmp1 = Tmp2;
4862        Tmp2 = SDOperand();
4863      } else {
4864        Result = TLI.SimplifySetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi,
4865                                   ISD::SETEQ, false, DagCombineInfo);
4866        if (!Result.Val)
4867          Result=DAG.getSetCC(TLI.getSetCCResultType(LHSHi), LHSHi, RHSHi,
4868                              ISD::SETEQ);
4869        Result = LegalizeOp(DAG.getNode(ISD::SELECT, Tmp1.getValueType(),
4870                                        Result, Tmp1, Tmp2));
4871        Tmp1 = Result;
4872        Tmp2 = SDOperand();
4873      }
4874    }
4875  }
4876  }
4877  LHS = Tmp1;
4878  RHS = Tmp2;
4879}
4880
4881/// EmitStackConvert - Emit a store/load combination to the stack.  This stores
4882/// SrcOp to a stack slot of type SlotVT, truncating it if needed.  It then does
4883/// a load from the stack slot to DestVT, extending it if needed.
4884/// The resultant code need not be legal.
4885SDOperand SelectionDAGLegalize::EmitStackConvert(SDOperand SrcOp,
4886                                                 MVT SlotVT,
4887                                                 MVT DestVT) {
4888  // Create the stack frame object.
4889  unsigned SrcAlign = TLI.getTargetData()->getPrefTypeAlignment(
4890                                          SrcOp.getValueType().getTypeForMVT());
4891  SDOperand FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign);
4892
4893  FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
4894  int SPFI = StackPtrFI->getIndex();
4895
4896  unsigned SrcSize = SrcOp.getValueType().getSizeInBits();
4897  unsigned SlotSize = SlotVT.getSizeInBits();
4898  unsigned DestSize = DestVT.getSizeInBits();
4899  unsigned DestAlign = TLI.getTargetData()->getPrefTypeAlignment(
4900                                                        DestVT.getTypeForMVT());
4901
4902  // Emit a store to the stack slot.  Use a truncstore if the input value is
4903  // later than DestVT.
4904  SDOperand Store;
4905
4906  if (SrcSize > SlotSize)
4907    Store = DAG.getTruncStore(DAG.getEntryNode(), SrcOp, FIPtr,
4908                              PseudoSourceValue::getFixedStack(SPFI), 0,
4909                              SlotVT, false, SrcAlign);
4910  else {
4911    assert(SrcSize == SlotSize && "Invalid store");
4912    Store = DAG.getStore(DAG.getEntryNode(), SrcOp, FIPtr,
4913                         PseudoSourceValue::getFixedStack(SPFI), 0,
4914                         false, SrcAlign);
4915  }
4916
4917  // Result is a load from the stack slot.
4918  if (SlotSize == DestSize)
4919    return DAG.getLoad(DestVT, Store, FIPtr, NULL, 0, false, DestAlign);
4920
4921  assert(SlotSize < DestSize && "Unknown extension!");
4922  return DAG.getExtLoad(ISD::EXTLOAD, DestVT, Store, FIPtr, NULL, 0, SlotVT,
4923                        false, DestAlign);
4924}
4925
4926SDOperand SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
4927  // Create a vector sized/aligned stack slot, store the value to element #0,
4928  // then load the whole vector back out.
4929  SDOperand StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
4930
4931  FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
4932  int SPFI = StackPtrFI->getIndex();
4933
4934  SDOperand Ch = DAG.getStore(DAG.getEntryNode(), Node->getOperand(0), StackPtr,
4935                              PseudoSourceValue::getFixedStack(SPFI), 0);
4936  return DAG.getLoad(Node->getValueType(0), Ch, StackPtr,
4937                     PseudoSourceValue::getFixedStack(SPFI), 0);
4938}
4939
4940
4941/// ExpandBUILD_VECTOR - Expand a BUILD_VECTOR node on targets that don't
4942/// support the operation, but do support the resultant vector type.
4943SDOperand SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
4944
4945  // If the only non-undef value is the low element, turn this into a
4946  // SCALAR_TO_VECTOR node.  If this is { X, X, X, X }, determine X.
4947  unsigned NumElems = Node->getNumOperands();
4948  bool isOnlyLowElement = true;
4949  SDOperand SplatValue = Node->getOperand(0);
4950
4951  // FIXME: it would be far nicer to change this into map<SDOperand,uint64_t>
4952  // and use a bitmask instead of a list of elements.
4953  std::map<SDOperand, std::vector<unsigned> > Values;
4954  Values[SplatValue].push_back(0);
4955  bool isConstant = true;
4956  if (!isa<ConstantFPSDNode>(SplatValue) && !isa<ConstantSDNode>(SplatValue) &&
4957      SplatValue.getOpcode() != ISD::UNDEF)
4958    isConstant = false;
4959
4960  for (unsigned i = 1; i < NumElems; ++i) {
4961    SDOperand V = Node->getOperand(i);
4962    Values[V].push_back(i);
4963    if (V.getOpcode() != ISD::UNDEF)
4964      isOnlyLowElement = false;
4965    if (SplatValue != V)
4966      SplatValue = SDOperand(0,0);
4967
4968    // If this isn't a constant element or an undef, we can't use a constant
4969    // pool load.
4970    if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V) &&
4971        V.getOpcode() != ISD::UNDEF)
4972      isConstant = false;
4973  }
4974
4975  if (isOnlyLowElement) {
4976    // If the low element is an undef too, then this whole things is an undef.
4977    if (Node->getOperand(0).getOpcode() == ISD::UNDEF)
4978      return DAG.getNode(ISD::UNDEF, Node->getValueType(0));
4979    // Otherwise, turn this into a scalar_to_vector node.
4980    return DAG.getNode(ISD::SCALAR_TO_VECTOR, Node->getValueType(0),
4981                       Node->getOperand(0));
4982  }
4983
4984  // If all elements are constants, create a load from the constant pool.
4985  if (isConstant) {
4986    MVT VT = Node->getValueType(0);
4987    std::vector<Constant*> CV;
4988    for (unsigned i = 0, e = NumElems; i != e; ++i) {
4989      if (ConstantFPSDNode *V =
4990          dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
4991        CV.push_back(ConstantFP::get(V->getValueAPF()));
4992      } else if (ConstantSDNode *V =
4993                   dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
4994        CV.push_back(ConstantInt::get(V->getAPIntValue()));
4995      } else {
4996        assert(Node->getOperand(i).getOpcode() == ISD::UNDEF);
4997        const Type *OpNTy =
4998          Node->getOperand(0).getValueType().getTypeForMVT();
4999        CV.push_back(UndefValue::get(OpNTy));
5000      }
5001    }
5002    Constant *CP = ConstantVector::get(CV);
5003    SDOperand CPIdx = DAG.getConstantPool(CP, TLI.getPointerTy());
5004    return DAG.getLoad(VT, DAG.getEntryNode(), CPIdx,
5005                       PseudoSourceValue::getConstantPool(), 0);
5006  }
5007
5008  if (SplatValue.Val) {   // Splat of one value?
5009    // Build the shuffle constant vector: <0, 0, 0, 0>
5010    MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems);
5011    SDOperand Zero = DAG.getConstant(0, MaskVT.getVectorElementType());
5012    std::vector<SDOperand> ZeroVec(NumElems, Zero);
5013    SDOperand SplatMask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
5014                                      &ZeroVec[0], ZeroVec.size());
5015
5016    // If the target supports VECTOR_SHUFFLE and this shuffle mask, use it.
5017    if (isShuffleLegal(Node->getValueType(0), SplatMask)) {
5018      // Get the splatted value into the low element of a vector register.
5019      SDOperand LowValVec =
5020        DAG.getNode(ISD::SCALAR_TO_VECTOR, Node->getValueType(0), SplatValue);
5021
5022      // Return shuffle(LowValVec, undef, <0,0,0,0>)
5023      return DAG.getNode(ISD::VECTOR_SHUFFLE, Node->getValueType(0), LowValVec,
5024                         DAG.getNode(ISD::UNDEF, Node->getValueType(0)),
5025                         SplatMask);
5026    }
5027  }
5028
5029  // If there are only two unique elements, we may be able to turn this into a
5030  // vector shuffle.
5031  if (Values.size() == 2) {
5032    // Get the two values in deterministic order.
5033    SDOperand Val1 = Node->getOperand(1);
5034    SDOperand Val2;
5035    std::map<SDOperand, std::vector<unsigned> >::iterator MI = Values.begin();
5036    if (MI->first != Val1)
5037      Val2 = MI->first;
5038    else
5039      Val2 = (++MI)->first;
5040
5041    // If Val1 is an undef, make sure end ends up as Val2, to ensure that our
5042    // vector shuffle has the undef vector on the RHS.
5043    if (Val1.getOpcode() == ISD::UNDEF)
5044      std::swap(Val1, Val2);
5045
5046    // Build the shuffle constant vector: e.g. <0, 4, 0, 4>
5047    MVT MaskVT = MVT::getIntVectorWithNumElements(NumElems);
5048    MVT MaskEltVT = MaskVT.getVectorElementType();
5049    std::vector<SDOperand> MaskVec(NumElems);
5050
5051    // Set elements of the shuffle mask for Val1.
5052    std::vector<unsigned> &Val1Elts = Values[Val1];
5053    for (unsigned i = 0, e = Val1Elts.size(); i != e; ++i)
5054      MaskVec[Val1Elts[i]] = DAG.getConstant(0, MaskEltVT);
5055
5056    // Set elements of the shuffle mask for Val2.
5057    std::vector<unsigned> &Val2Elts = Values[Val2];
5058    for (unsigned i = 0, e = Val2Elts.size(); i != e; ++i)
5059      if (Val2.getOpcode() != ISD::UNDEF)
5060        MaskVec[Val2Elts[i]] = DAG.getConstant(NumElems, MaskEltVT);
5061      else
5062        MaskVec[Val2Elts[i]] = DAG.getNode(ISD::UNDEF, MaskEltVT);
5063
5064    SDOperand ShuffleMask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
5065                                        &MaskVec[0], MaskVec.size());
5066
5067    // If the target supports SCALAR_TO_VECTOR and this shuffle mask, use it.
5068    if (TLI.isOperationLegal(ISD::SCALAR_TO_VECTOR, Node->getValueType(0)) &&
5069        isShuffleLegal(Node->getValueType(0), ShuffleMask)) {
5070      Val1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, Node->getValueType(0), Val1);
5071      Val2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, Node->getValueType(0), Val2);
5072      SDOperand Ops[] = { Val1, Val2, ShuffleMask };
5073
5074      // Return shuffle(LoValVec, HiValVec, <0,1,0,1>)
5075      return DAG.getNode(ISD::VECTOR_SHUFFLE, Node->getValueType(0), Ops, 3);
5076    }
5077  }
5078
5079  // Otherwise, we can't handle this case efficiently.  Allocate a sufficiently
5080  // aligned object on the stack, store each element into it, then load
5081  // the result as a vector.
5082  MVT VT = Node->getValueType(0);
5083  // Create the stack frame object.
5084  SDOperand FIPtr = DAG.CreateStackTemporary(VT);
5085
5086  // Emit a store of each element to the stack slot.
5087  SmallVector<SDOperand, 8> Stores;
5088  unsigned TypeByteSize = Node->getOperand(0).getValueType().getSizeInBits()/8;
5089  // Store (in the right endianness) the elements to memory.
5090  for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
5091    // Ignore undef elements.
5092    if (Node->getOperand(i).getOpcode() == ISD::UNDEF) continue;
5093
5094    unsigned Offset = TypeByteSize*i;
5095
5096    SDOperand Idx = DAG.getConstant(Offset, FIPtr.getValueType());
5097    Idx = DAG.getNode(ISD::ADD, FIPtr.getValueType(), FIPtr, Idx);
5098
5099    Stores.push_back(DAG.getStore(DAG.getEntryNode(), Node->getOperand(i), Idx,
5100                                  NULL, 0));
5101  }
5102
5103  SDOperand StoreChain;
5104  if (!Stores.empty())    // Not all undef elements?
5105    StoreChain = DAG.getNode(ISD::TokenFactor, MVT::Other,
5106                             &Stores[0], Stores.size());
5107  else
5108    StoreChain = DAG.getEntryNode();
5109
5110  // Result is a load from the stack slot.
5111  return DAG.getLoad(VT, StoreChain, FIPtr, NULL, 0);
5112}
5113
5114void SelectionDAGLegalize::ExpandShiftParts(unsigned NodeOp,
5115                                            SDOperand Op, SDOperand Amt,
5116                                            SDOperand &Lo, SDOperand &Hi) {
5117  // Expand the subcomponents.
5118  SDOperand LHSL, LHSH;
5119  ExpandOp(Op, LHSL, LHSH);
5120
5121  SDOperand Ops[] = { LHSL, LHSH, Amt };
5122  MVT VT = LHSL.getValueType();
5123  Lo = DAG.getNode(NodeOp, DAG.getNodeValueTypes(VT, VT), 2, Ops, 3);
5124  Hi = Lo.getValue(1);
5125}
5126
5127
5128/// ExpandShift - Try to find a clever way to expand this shift operation out to
5129/// smaller elements.  If we can't find a way that is more efficient than a
5130/// libcall on this target, return false.  Otherwise, return true with the
5131/// low-parts expanded into Lo and Hi.
5132bool SelectionDAGLegalize::ExpandShift(unsigned Opc, SDOperand Op,SDOperand Amt,
5133                                       SDOperand &Lo, SDOperand &Hi) {
5134  assert((Opc == ISD::SHL || Opc == ISD::SRA || Opc == ISD::SRL) &&
5135         "This is not a shift!");
5136
5137  MVT NVT = TLI.getTypeToTransformTo(Op.getValueType());
5138  SDOperand ShAmt = LegalizeOp(Amt);
5139  MVT ShTy = ShAmt.getValueType();
5140  unsigned ShBits = ShTy.getSizeInBits();
5141  unsigned VTBits = Op.getValueType().getSizeInBits();
5142  unsigned NVTBits = NVT.getSizeInBits();
5143
5144  // Handle the case when Amt is an immediate.
5145  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Amt.Val)) {
5146    unsigned Cst = CN->getValue();
5147    // Expand the incoming operand to be shifted, so that we have its parts
5148    SDOperand InL, InH;
5149    ExpandOp(Op, InL, InH);
5150    switch(Opc) {
5151    case ISD::SHL:
5152      if (Cst > VTBits) {
5153        Lo = DAG.getConstant(0, NVT);
5154        Hi = DAG.getConstant(0, NVT);
5155      } else if (Cst > NVTBits) {
5156        Lo = DAG.getConstant(0, NVT);
5157        Hi = DAG.getNode(ISD::SHL, NVT, InL, DAG.getConstant(Cst-NVTBits,ShTy));
5158      } else if (Cst == NVTBits) {
5159        Lo = DAG.getConstant(0, NVT);
5160        Hi = InL;
5161      } else {
5162        Lo = DAG.getNode(ISD::SHL, NVT, InL, DAG.getConstant(Cst, ShTy));
5163        Hi = DAG.getNode(ISD::OR, NVT,
5164           DAG.getNode(ISD::SHL, NVT, InH, DAG.getConstant(Cst, ShTy)),
5165           DAG.getNode(ISD::SRL, NVT, InL, DAG.getConstant(NVTBits-Cst, ShTy)));
5166      }
5167      return true;
5168    case ISD::SRL:
5169      if (Cst > VTBits) {
5170        Lo = DAG.getConstant(0, NVT);
5171        Hi = DAG.getConstant(0, NVT);
5172      } else if (Cst > NVTBits) {
5173        Lo = DAG.getNode(ISD::SRL, NVT, InH, DAG.getConstant(Cst-NVTBits,ShTy));
5174        Hi = DAG.getConstant(0, NVT);
5175      } else if (Cst == NVTBits) {
5176        Lo = InH;
5177        Hi = DAG.getConstant(0, NVT);
5178      } else {
5179        Lo = DAG.getNode(ISD::OR, NVT,
5180           DAG.getNode(ISD::SRL, NVT, InL, DAG.getConstant(Cst, ShTy)),
5181           DAG.getNode(ISD::SHL, NVT, InH, DAG.getConstant(NVTBits-Cst, ShTy)));
5182        Hi = DAG.getNode(ISD::SRL, NVT, InH, DAG.getConstant(Cst, ShTy));
5183      }
5184      return true;
5185    case ISD::SRA:
5186      if (Cst > VTBits) {
5187        Hi = Lo = DAG.getNode(ISD::SRA, NVT, InH,
5188                              DAG.getConstant(NVTBits-1, ShTy));
5189      } else if (Cst > NVTBits) {
5190        Lo = DAG.getNode(ISD::SRA, NVT, InH,
5191                           DAG.getConstant(Cst-NVTBits, ShTy));
5192        Hi = DAG.getNode(ISD::SRA, NVT, InH,
5193                              DAG.getConstant(NVTBits-1, ShTy));
5194      } else if (Cst == NVTBits) {
5195        Lo = InH;
5196        Hi = DAG.getNode(ISD::SRA, NVT, InH,
5197                              DAG.getConstant(NVTBits-1, ShTy));
5198      } else {
5199        Lo = DAG.getNode(ISD::OR, NVT,
5200           DAG.getNode(ISD::SRL, NVT, InL, DAG.getConstant(Cst, ShTy)),
5201           DAG.getNode(ISD::SHL, NVT, InH, DAG.getConstant(NVTBits-Cst, ShTy)));
5202        Hi = DAG.getNode(ISD::SRA, NVT, InH, DAG.getConstant(Cst, ShTy));
5203      }
5204      return true;
5205    }
5206  }
5207
5208  // Okay, the shift amount isn't constant.  However, if we can tell that it is
5209  // >= 32 or < 32, we can still simplify it, without knowing the actual value.
5210  APInt Mask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
5211  APInt KnownZero, KnownOne;
5212  DAG.ComputeMaskedBits(Amt, Mask, KnownZero, KnownOne);
5213
5214  // If we know that if any of the high bits of the shift amount are one, then
5215  // we can do this as a couple of simple shifts.
5216  if (KnownOne.intersects(Mask)) {
5217    // Mask out the high bit, which we know is set.
5218    Amt = DAG.getNode(ISD::AND, Amt.getValueType(), Amt,
5219                      DAG.getConstant(~Mask, Amt.getValueType()));
5220
5221    // Expand the incoming operand to be shifted, so that we have its parts
5222    SDOperand InL, InH;
5223    ExpandOp(Op, InL, InH);
5224    switch(Opc) {
5225    case ISD::SHL:
5226      Lo = DAG.getConstant(0, NVT);              // Low part is zero.
5227      Hi = DAG.getNode(ISD::SHL, NVT, InL, Amt); // High part from Lo part.
5228      return true;
5229    case ISD::SRL:
5230      Hi = DAG.getConstant(0, NVT);              // Hi part is zero.
5231      Lo = DAG.getNode(ISD::SRL, NVT, InH, Amt); // Lo part from Hi part.
5232      return true;
5233    case ISD::SRA:
5234      Hi = DAG.getNode(ISD::SRA, NVT, InH,       // Sign extend high part.
5235                       DAG.getConstant(NVTBits-1, Amt.getValueType()));
5236      Lo = DAG.getNode(ISD::SRA, NVT, InH, Amt); // Lo part from Hi part.
5237      return true;
5238    }
5239  }
5240
5241  // If we know that the high bits of the shift amount are all zero, then we can
5242  // do this as a couple of simple shifts.
5243  if ((KnownZero & Mask) == Mask) {
5244    // Compute 32-amt.
5245    SDOperand Amt2 = DAG.getNode(ISD::SUB, Amt.getValueType(),
5246                                 DAG.getConstant(NVTBits, Amt.getValueType()),
5247                                 Amt);
5248
5249    // Expand the incoming operand to be shifted, so that we have its parts
5250    SDOperand InL, InH;
5251    ExpandOp(Op, InL, InH);
5252    switch(Opc) {
5253    case ISD::SHL:
5254      Lo = DAG.getNode(ISD::SHL, NVT, InL, Amt);
5255      Hi = DAG.getNode(ISD::OR, NVT,
5256                       DAG.getNode(ISD::SHL, NVT, InH, Amt),
5257                       DAG.getNode(ISD::SRL, NVT, InL, Amt2));
5258      return true;
5259    case ISD::SRL:
5260      Hi = DAG.getNode(ISD::SRL, NVT, InH, Amt);
5261      Lo = DAG.getNode(ISD::OR, NVT,
5262                       DAG.getNode(ISD::SRL, NVT, InL, Amt),
5263                       DAG.getNode(ISD::SHL, NVT, InH, Amt2));
5264      return true;
5265    case ISD::SRA:
5266      Hi = DAG.getNode(ISD::SRA, NVT, InH, Amt);
5267      Lo = DAG.getNode(ISD::OR, NVT,
5268                       DAG.getNode(ISD::SRL, NVT, InL, Amt),
5269                       DAG.getNode(ISD::SHL, NVT, InH, Amt2));
5270      return true;
5271    }
5272  }
5273
5274  return false;
5275}
5276
5277
5278// ExpandLibCall - Expand a node into a call to a libcall.  If the result value
5279// does not fit into a register, return the lo part and set the hi part to the
5280// by-reg argument.  If it does fit into a single register, return the result
5281// and leave the Hi part unset.
5282SDOperand SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
5283                                              bool isSigned, SDOperand &Hi) {
5284  assert(!IsLegalizingCall && "Cannot overlap legalization of calls!");
5285  // The input chain to this libcall is the entry node of the function.
5286  // Legalizing the call will automatically add the previous call to the
5287  // dependence.
5288  SDOperand InChain = DAG.getEntryNode();
5289
5290  TargetLowering::ArgListTy Args;
5291  TargetLowering::ArgListEntry Entry;
5292  for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
5293    MVT ArgVT = Node->getOperand(i).getValueType();
5294    const Type *ArgTy = ArgVT.getTypeForMVT();
5295    Entry.Node = Node->getOperand(i); Entry.Ty = ArgTy;
5296    Entry.isSExt = isSigned;
5297    Entry.isZExt = !isSigned;
5298    Args.push_back(Entry);
5299  }
5300  SDOperand Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
5301                                           TLI.getPointerTy());
5302
5303  // Splice the libcall in wherever FindInputOutputChains tells us to.
5304  const Type *RetTy = Node->getValueType(0).getTypeForMVT();
5305  std::pair<SDOperand,SDOperand> CallInfo =
5306    TLI.LowerCallTo(InChain, RetTy, isSigned, !isSigned, false, CallingConv::C,
5307                    false, Callee, Args, DAG);
5308
5309  // Legalize the call sequence, starting with the chain.  This will advance
5310  // the LastCALLSEQ_END to the legalized version of the CALLSEQ_END node that
5311  // was added by LowerCallTo (guaranteeing proper serialization of calls).
5312  LegalizeOp(CallInfo.second);
5313  SDOperand Result;
5314  switch (getTypeAction(CallInfo.first.getValueType())) {
5315  default: assert(0 && "Unknown thing");
5316  case Legal:
5317    Result = CallInfo.first;
5318    break;
5319  case Expand:
5320    ExpandOp(CallInfo.first, Result, Hi);
5321    break;
5322  }
5323  return Result;
5324}
5325
5326
5327/// ExpandIntToFP - Expand a [US]INT_TO_FP operation.
5328///
5329SDOperand SelectionDAGLegalize::
5330ExpandIntToFP(bool isSigned, MVT DestTy, SDOperand Source) {
5331  MVT SourceVT = Source.getValueType();
5332  bool ExpandSource = getTypeAction(SourceVT) == Expand;
5333
5334  // Special case for i32 source to take advantage of UINTTOFP_I32_F32, etc.
5335  if (!isSigned && SourceVT != MVT::i32) {
5336    // The integer value loaded will be incorrectly if the 'sign bit' of the
5337    // incoming integer is set.  To handle this, we dynamically test to see if
5338    // it is set, and, if so, add a fudge factor.
5339    SDOperand Hi;
5340    if (ExpandSource) {
5341      SDOperand Lo;
5342      ExpandOp(Source, Lo, Hi);
5343      Source = DAG.getNode(ISD::BUILD_PAIR, SourceVT, Lo, Hi);
5344    } else {
5345      // The comparison for the sign bit will use the entire operand.
5346      Hi = Source;
5347    }
5348
5349    // If this is unsigned, and not supported, first perform the conversion to
5350    // signed, then adjust the result if the sign bit is set.
5351    SDOperand SignedConv = ExpandIntToFP(true, DestTy, Source);
5352
5353    SDOperand SignSet = DAG.getSetCC(TLI.getSetCCResultType(Hi), Hi,
5354                                     DAG.getConstant(0, Hi.getValueType()),
5355                                     ISD::SETLT);
5356    SDOperand Zero = DAG.getIntPtrConstant(0), Four = DAG.getIntPtrConstant(4);
5357    SDOperand CstOffset = DAG.getNode(ISD::SELECT, Zero.getValueType(),
5358                                      SignSet, Four, Zero);
5359    uint64_t FF = 0x5f800000ULL;
5360    if (TLI.isLittleEndian()) FF <<= 32;
5361    static Constant *FudgeFactor = ConstantInt::get(Type::Int64Ty, FF);
5362
5363    SDOperand CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy());
5364    CPIdx = DAG.getNode(ISD::ADD, TLI.getPointerTy(), CPIdx, CstOffset);
5365    SDOperand FudgeInReg;
5366    if (DestTy == MVT::f32)
5367      FudgeInReg = DAG.getLoad(MVT::f32, DAG.getEntryNode(), CPIdx,
5368                               PseudoSourceValue::getConstantPool(), 0);
5369    else if (DestTy.bitsGT(MVT::f32))
5370      // FIXME: Avoid the extend by construction the right constantpool?
5371      FudgeInReg = DAG.getExtLoad(ISD::EXTLOAD, DestTy, DAG.getEntryNode(),
5372                                  CPIdx,
5373                                  PseudoSourceValue::getConstantPool(), 0,
5374                                  MVT::f32);
5375    else
5376      assert(0 && "Unexpected conversion");
5377
5378    MVT SCVT = SignedConv.getValueType();
5379    if (SCVT != DestTy) {
5380      // Destination type needs to be expanded as well. The FADD now we are
5381      // constructing will be expanded into a libcall.
5382      if (SCVT.getSizeInBits() != DestTy.getSizeInBits()) {
5383        assert(SCVT.getSizeInBits() * 2 == DestTy.getSizeInBits());
5384        SignedConv = DAG.getNode(ISD::BUILD_PAIR, DestTy,
5385                                 SignedConv, SignedConv.getValue(1));
5386      }
5387      SignedConv = DAG.getNode(ISD::BIT_CONVERT, DestTy, SignedConv);
5388    }
5389    return DAG.getNode(ISD::FADD, DestTy, SignedConv, FudgeInReg);
5390  }
5391
5392  // Check to see if the target has a custom way to lower this.  If so, use it.
5393  switch (TLI.getOperationAction(ISD::SINT_TO_FP, SourceVT)) {
5394  default: assert(0 && "This action not implemented for this operation!");
5395  case TargetLowering::Legal:
5396  case TargetLowering::Expand:
5397    break;   // This case is handled below.
5398  case TargetLowering::Custom: {
5399    SDOperand NV = TLI.LowerOperation(DAG.getNode(ISD::SINT_TO_FP, DestTy,
5400                                                  Source), DAG);
5401    if (NV.Val)
5402      return LegalizeOp(NV);
5403    break;   // The target decided this was legal after all
5404  }
5405  }
5406
5407  // Expand the source, then glue it back together for the call.  We must expand
5408  // the source in case it is shared (this pass of legalize must traverse it).
5409  if (ExpandSource) {
5410    SDOperand SrcLo, SrcHi;
5411    ExpandOp(Source, SrcLo, SrcHi);
5412    Source = DAG.getNode(ISD::BUILD_PAIR, SourceVT, SrcLo, SrcHi);
5413  }
5414
5415  RTLIB::Libcall LC;
5416  if (SourceVT == MVT::i32) {
5417    if (DestTy == MVT::f32)
5418      LC = isSigned ? RTLIB::SINTTOFP_I32_F32 : RTLIB::UINTTOFP_I32_F32;
5419    else {
5420      assert(DestTy == MVT::f64 && "Unknown fp value type!");
5421      LC = isSigned ? RTLIB::SINTTOFP_I32_F64 : RTLIB::UINTTOFP_I32_F64;
5422    }
5423  } else if (SourceVT == MVT::i64) {
5424    if (DestTy == MVT::f32)
5425      LC = RTLIB::SINTTOFP_I64_F32;
5426    else if (DestTy == MVT::f64)
5427      LC = RTLIB::SINTTOFP_I64_F64;
5428    else if (DestTy == MVT::f80)
5429      LC = RTLIB::SINTTOFP_I64_F80;
5430    else {
5431      assert(DestTy == MVT::ppcf128 && "Unknown fp value type!");
5432      LC = RTLIB::SINTTOFP_I64_PPCF128;
5433    }
5434  } else if (SourceVT == MVT::i128) {
5435    if (DestTy == MVT::f32)
5436      LC = RTLIB::SINTTOFP_I128_F32;
5437    else if (DestTy == MVT::f64)
5438      LC = RTLIB::SINTTOFP_I128_F64;
5439    else if (DestTy == MVT::f80)
5440      LC = RTLIB::SINTTOFP_I128_F80;
5441    else {
5442      assert(DestTy == MVT::ppcf128 && "Unknown fp value type!");
5443      LC = RTLIB::SINTTOFP_I128_PPCF128;
5444    }
5445  } else {
5446    assert(0 && "Unknown int value type");
5447  }
5448
5449  assert(TLI.getLibcallName(LC) && "Don't know how to expand this SINT_TO_FP!");
5450  Source = DAG.getNode(ISD::SINT_TO_FP, DestTy, Source);
5451  SDOperand HiPart;
5452  SDOperand Result = ExpandLibCall(LC, Source.Val, isSigned, HiPart);
5453  if (Result.getValueType() != DestTy && HiPart.Val)
5454    Result = DAG.getNode(ISD::BUILD_PAIR, DestTy, Result, HiPart);
5455  return Result;
5456}
5457
5458/// ExpandLegalINT_TO_FP - This function is responsible for legalizing a
5459/// INT_TO_FP operation of the specified operand when the target requests that
5460/// we expand it.  At this point, we know that the result and operand types are
5461/// legal for the target.
5462SDOperand SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned,
5463                                                     SDOperand Op0,
5464                                                     MVT DestVT) {
5465  if (Op0.getValueType() == MVT::i32) {
5466    // simple 32-bit [signed|unsigned] integer to float/double expansion
5467
5468    // Get the stack frame index of a 8 byte buffer.
5469    SDOperand StackSlot = DAG.CreateStackTemporary(MVT::f64);
5470
5471    // word offset constant for Hi/Lo address computation
5472    SDOperand WordOff = DAG.getConstant(sizeof(int), TLI.getPointerTy());
5473    // set up Hi and Lo (into buffer) address based on endian
5474    SDOperand Hi = StackSlot;
5475    SDOperand Lo = DAG.getNode(ISD::ADD, TLI.getPointerTy(), StackSlot,WordOff);
5476    if (TLI.isLittleEndian())
5477      std::swap(Hi, Lo);
5478
5479    // if signed map to unsigned space
5480    SDOperand Op0Mapped;
5481    if (isSigned) {
5482      // constant used to invert sign bit (signed to unsigned mapping)
5483      SDOperand SignBit = DAG.getConstant(0x80000000u, MVT::i32);
5484      Op0Mapped = DAG.getNode(ISD::XOR, MVT::i32, Op0, SignBit);
5485    } else {
5486      Op0Mapped = Op0;
5487    }
5488    // store the lo of the constructed double - based on integer input
5489    SDOperand Store1 = DAG.getStore(DAG.getEntryNode(),
5490                                    Op0Mapped, Lo, NULL, 0);
5491    // initial hi portion of constructed double
5492    SDOperand InitialHi = DAG.getConstant(0x43300000u, MVT::i32);
5493    // store the hi of the constructed double - biased exponent
5494    SDOperand Store2=DAG.getStore(Store1, InitialHi, Hi, NULL, 0);
5495    // load the constructed double
5496    SDOperand Load = DAG.getLoad(MVT::f64, Store2, StackSlot, NULL, 0);
5497    // FP constant to bias correct the final result
5498    SDOperand Bias = DAG.getConstantFP(isSigned ?
5499                                            BitsToDouble(0x4330000080000000ULL)
5500                                          : BitsToDouble(0x4330000000000000ULL),
5501                                     MVT::f64);
5502    // subtract the bias
5503    SDOperand Sub = DAG.getNode(ISD::FSUB, MVT::f64, Load, Bias);
5504    // final result
5505    SDOperand Result;
5506    // handle final rounding
5507    if (DestVT == MVT::f64) {
5508      // do nothing
5509      Result = Sub;
5510    } else if (DestVT.bitsLT(MVT::f64)) {
5511      Result = DAG.getNode(ISD::FP_ROUND, DestVT, Sub,
5512                           DAG.getIntPtrConstant(0));
5513    } else if (DestVT.bitsGT(MVT::f64)) {
5514      Result = DAG.getNode(ISD::FP_EXTEND, DestVT, Sub);
5515    }
5516    return Result;
5517  }
5518  assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet");
5519  SDOperand Tmp1 = DAG.getNode(ISD::SINT_TO_FP, DestVT, Op0);
5520
5521  SDOperand SignSet = DAG.getSetCC(TLI.getSetCCResultType(Op0), Op0,
5522                                   DAG.getConstant(0, Op0.getValueType()),
5523                                   ISD::SETLT);
5524  SDOperand Zero = DAG.getIntPtrConstant(0), Four = DAG.getIntPtrConstant(4);
5525  SDOperand CstOffset = DAG.getNode(ISD::SELECT, Zero.getValueType(),
5526                                    SignSet, Four, Zero);
5527
5528  // If the sign bit of the integer is set, the large number will be treated
5529  // as a negative number.  To counteract this, the dynamic code adds an
5530  // offset depending on the data type.
5531  uint64_t FF;
5532  switch (Op0.getValueType().getSimpleVT()) {
5533  default: assert(0 && "Unsupported integer type!");
5534  case MVT::i8 : FF = 0x43800000ULL; break;  // 2^8  (as a float)
5535  case MVT::i16: FF = 0x47800000ULL; break;  // 2^16 (as a float)
5536  case MVT::i32: FF = 0x4F800000ULL; break;  // 2^32 (as a float)
5537  case MVT::i64: FF = 0x5F800000ULL; break;  // 2^64 (as a float)
5538  }
5539  if (TLI.isLittleEndian()) FF <<= 32;
5540  static Constant *FudgeFactor = ConstantInt::get(Type::Int64Ty, FF);
5541
5542  SDOperand CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy());
5543  CPIdx = DAG.getNode(ISD::ADD, TLI.getPointerTy(), CPIdx, CstOffset);
5544  SDOperand FudgeInReg;
5545  if (DestVT == MVT::f32)
5546    FudgeInReg = DAG.getLoad(MVT::f32, DAG.getEntryNode(), CPIdx,
5547                             PseudoSourceValue::getConstantPool(), 0);
5548  else {
5549    FudgeInReg =
5550      LegalizeOp(DAG.getExtLoad(ISD::EXTLOAD, DestVT,
5551                                DAG.getEntryNode(), CPIdx,
5552                                PseudoSourceValue::getConstantPool(), 0,
5553                                MVT::f32));
5554  }
5555
5556  return DAG.getNode(ISD::FADD, DestVT, Tmp1, FudgeInReg);
5557}
5558
5559/// PromoteLegalINT_TO_FP - This function is responsible for legalizing a
5560/// *INT_TO_FP operation of the specified operand when the target requests that
5561/// we promote it.  At this point, we know that the result and operand types are
5562/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
5563/// operation that takes a larger input.
5564SDOperand SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDOperand LegalOp,
5565                                                      MVT DestVT,
5566                                                      bool isSigned) {
5567  // First step, figure out the appropriate *INT_TO_FP operation to use.
5568  MVT NewInTy = LegalOp.getValueType();
5569
5570  unsigned OpToUse = 0;
5571
5572  // Scan for the appropriate larger type to use.
5573  while (1) {
5574    NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT()+1);
5575    assert(NewInTy.isInteger() && "Ran out of possibilities!");
5576
5577    // If the target supports SINT_TO_FP of this type, use it.
5578    switch (TLI.getOperationAction(ISD::SINT_TO_FP, NewInTy)) {
5579      default: break;
5580      case TargetLowering::Legal:
5581        if (!TLI.isTypeLegal(NewInTy))
5582          break;  // Can't use this datatype.
5583        // FALL THROUGH.
5584      case TargetLowering::Custom:
5585        OpToUse = ISD::SINT_TO_FP;
5586        break;
5587    }
5588    if (OpToUse) break;
5589    if (isSigned) continue;
5590
5591    // If the target supports UINT_TO_FP of this type, use it.
5592    switch (TLI.getOperationAction(ISD::UINT_TO_FP, NewInTy)) {
5593      default: break;
5594      case TargetLowering::Legal:
5595        if (!TLI.isTypeLegal(NewInTy))
5596          break;  // Can't use this datatype.
5597        // FALL THROUGH.
5598      case TargetLowering::Custom:
5599        OpToUse = ISD::UINT_TO_FP;
5600        break;
5601    }
5602    if (OpToUse) break;
5603
5604    // Otherwise, try a larger type.
5605  }
5606
5607  // Okay, we found the operation and type to use.  Zero extend our input to the
5608  // desired type then run the operation on it.
5609  return DAG.getNode(OpToUse, DestVT,
5610                     DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
5611                                 NewInTy, LegalOp));
5612}
5613
5614/// PromoteLegalFP_TO_INT - This function is responsible for legalizing a
5615/// FP_TO_*INT operation of the specified operand when the target requests that
5616/// we promote it.  At this point, we know that the result and operand types are
5617/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
5618/// operation that returns a larger result.
5619SDOperand SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDOperand LegalOp,
5620                                                      MVT DestVT,
5621                                                      bool isSigned) {
5622  // First step, figure out the appropriate FP_TO*INT operation to use.
5623  MVT NewOutTy = DestVT;
5624
5625  unsigned OpToUse = 0;
5626
5627  // Scan for the appropriate larger type to use.
5628  while (1) {
5629    NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT()+1);
5630    assert(NewOutTy.isInteger() && "Ran out of possibilities!");
5631
5632    // If the target supports FP_TO_SINT returning this type, use it.
5633    switch (TLI.getOperationAction(ISD::FP_TO_SINT, NewOutTy)) {
5634    default: break;
5635    case TargetLowering::Legal:
5636      if (!TLI.isTypeLegal(NewOutTy))
5637        break;  // Can't use this datatype.
5638      // FALL THROUGH.
5639    case TargetLowering::Custom:
5640      OpToUse = ISD::FP_TO_SINT;
5641      break;
5642    }
5643    if (OpToUse) break;
5644
5645    // If the target supports FP_TO_UINT of this type, use it.
5646    switch (TLI.getOperationAction(ISD::FP_TO_UINT, NewOutTy)) {
5647    default: break;
5648    case TargetLowering::Legal:
5649      if (!TLI.isTypeLegal(NewOutTy))
5650        break;  // Can't use this datatype.
5651      // FALL THROUGH.
5652    case TargetLowering::Custom:
5653      OpToUse = ISD::FP_TO_UINT;
5654      break;
5655    }
5656    if (OpToUse) break;
5657
5658    // Otherwise, try a larger type.
5659  }
5660
5661
5662  // Okay, we found the operation and type to use.
5663  SDOperand Operation = DAG.getNode(OpToUse, NewOutTy, LegalOp);
5664
5665  // If the operation produces an invalid type, it must be custom lowered.  Use
5666  // the target lowering hooks to expand it.  Just keep the low part of the
5667  // expanded operation, we know that we're truncating anyway.
5668  if (getTypeAction(NewOutTy) == Expand) {
5669    Operation = SDOperand(TLI.ReplaceNodeResults(Operation.Val, DAG), 0);
5670    assert(Operation.Val && "Didn't return anything");
5671  }
5672
5673  // Truncate the result of the extended FP_TO_*INT operation to the desired
5674  // size.
5675  return DAG.getNode(ISD::TRUNCATE, DestVT, Operation);
5676}
5677
5678/// ExpandBSWAP - Open code the operations for BSWAP of the specified operation.
5679///
5680SDOperand SelectionDAGLegalize::ExpandBSWAP(SDOperand Op) {
5681  MVT VT = Op.getValueType();
5682  MVT SHVT = TLI.getShiftAmountTy();
5683  SDOperand Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
5684  switch (VT.getSimpleVT()) {
5685  default: assert(0 && "Unhandled Expand type in BSWAP!"); abort();
5686  case MVT::i16:
5687    Tmp2 = DAG.getNode(ISD::SHL, VT, Op, DAG.getConstant(8, SHVT));
5688    Tmp1 = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(8, SHVT));
5689    return DAG.getNode(ISD::OR, VT, Tmp1, Tmp2);
5690  case MVT::i32:
5691    Tmp4 = DAG.getNode(ISD::SHL, VT, Op, DAG.getConstant(24, SHVT));
5692    Tmp3 = DAG.getNode(ISD::SHL, VT, Op, DAG.getConstant(8, SHVT));
5693    Tmp2 = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(8, SHVT));
5694    Tmp1 = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(24, SHVT));
5695    Tmp3 = DAG.getNode(ISD::AND, VT, Tmp3, DAG.getConstant(0xFF0000, VT));
5696    Tmp2 = DAG.getNode(ISD::AND, VT, Tmp2, DAG.getConstant(0xFF00, VT));
5697    Tmp4 = DAG.getNode(ISD::OR, VT, Tmp4, Tmp3);
5698    Tmp2 = DAG.getNode(ISD::OR, VT, Tmp2, Tmp1);
5699    return DAG.getNode(ISD::OR, VT, Tmp4, Tmp2);
5700  case MVT::i64:
5701    Tmp8 = DAG.getNode(ISD::SHL, VT, Op, DAG.getConstant(56, SHVT));
5702    Tmp7 = DAG.getNode(ISD::SHL, VT, Op, DAG.getConstant(40, SHVT));
5703    Tmp6 = DAG.getNode(ISD::SHL, VT, Op, DAG.getConstant(24, SHVT));
5704    Tmp5 = DAG.getNode(ISD::SHL, VT, Op, DAG.getConstant(8, SHVT));
5705    Tmp4 = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(8, SHVT));
5706    Tmp3 = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(24, SHVT));
5707    Tmp2 = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(40, SHVT));
5708    Tmp1 = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(56, SHVT));
5709    Tmp7 = DAG.getNode(ISD::AND, VT, Tmp7, DAG.getConstant(255ULL<<48, VT));
5710    Tmp6 = DAG.getNode(ISD::AND, VT, Tmp6, DAG.getConstant(255ULL<<40, VT));
5711    Tmp5 = DAG.getNode(ISD::AND, VT, Tmp5, DAG.getConstant(255ULL<<32, VT));
5712    Tmp4 = DAG.getNode(ISD::AND, VT, Tmp4, DAG.getConstant(255ULL<<24, VT));
5713    Tmp3 = DAG.getNode(ISD::AND, VT, Tmp3, DAG.getConstant(255ULL<<16, VT));
5714    Tmp2 = DAG.getNode(ISD::AND, VT, Tmp2, DAG.getConstant(255ULL<<8 , VT));
5715    Tmp8 = DAG.getNode(ISD::OR, VT, Tmp8, Tmp7);
5716    Tmp6 = DAG.getNode(ISD::OR, VT, Tmp6, Tmp5);
5717    Tmp4 = DAG.getNode(ISD::OR, VT, Tmp4, Tmp3);
5718    Tmp2 = DAG.getNode(ISD::OR, VT, Tmp2, Tmp1);
5719    Tmp8 = DAG.getNode(ISD::OR, VT, Tmp8, Tmp6);
5720    Tmp4 = DAG.getNode(ISD::OR, VT, Tmp4, Tmp2);
5721    return DAG.getNode(ISD::OR, VT, Tmp8, Tmp4);
5722  }
5723}
5724
5725/// ExpandBitCount - Expand the specified bitcount instruction into operations.
5726///
5727SDOperand SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDOperand Op) {
5728  switch (Opc) {
5729  default: assert(0 && "Cannot expand this yet!");
5730  case ISD::CTPOP: {
5731    static const uint64_t mask[6] = {
5732      0x5555555555555555ULL, 0x3333333333333333ULL,
5733      0x0F0F0F0F0F0F0F0FULL, 0x00FF00FF00FF00FFULL,
5734      0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL
5735    };
5736    MVT VT = Op.getValueType();
5737    MVT ShVT = TLI.getShiftAmountTy();
5738    unsigned len = VT.getSizeInBits();
5739    for (unsigned i = 0; (1U << i) <= (len / 2); ++i) {
5740      //x = (x & mask[i][len/8]) + (x >> (1 << i) & mask[i][len/8])
5741      SDOperand Tmp2 = DAG.getConstant(mask[i], VT);
5742      SDOperand Tmp3 = DAG.getConstant(1ULL << i, ShVT);
5743      Op = DAG.getNode(ISD::ADD, VT, DAG.getNode(ISD::AND, VT, Op, Tmp2),
5744                       DAG.getNode(ISD::AND, VT,
5745                                   DAG.getNode(ISD::SRL, VT, Op, Tmp3),Tmp2));
5746    }
5747    return Op;
5748  }
5749  case ISD::CTLZ: {
5750    // for now, we do this:
5751    // x = x | (x >> 1);
5752    // x = x | (x >> 2);
5753    // ...
5754    // x = x | (x >>16);
5755    // x = x | (x >>32); // for 64-bit input
5756    // return popcount(~x);
5757    //
5758    // but see also: http://www.hackersdelight.org/HDcode/nlz.cc
5759    MVT VT = Op.getValueType();
5760    MVT ShVT = TLI.getShiftAmountTy();
5761    unsigned len = VT.getSizeInBits();
5762    for (unsigned i = 0; (1U << i) <= (len / 2); ++i) {
5763      SDOperand Tmp3 = DAG.getConstant(1ULL << i, ShVT);
5764      Op = DAG.getNode(ISD::OR, VT, Op, DAG.getNode(ISD::SRL, VT, Op, Tmp3));
5765    }
5766    Op = DAG.getNode(ISD::XOR, VT, Op, DAG.getConstant(~0ULL, VT));
5767    return DAG.getNode(ISD::CTPOP, VT, Op);
5768  }
5769  case ISD::CTTZ: {
5770    // for now, we use: { return popcount(~x & (x - 1)); }
5771    // unless the target has ctlz but not ctpop, in which case we use:
5772    // { return 32 - nlz(~x & (x-1)); }
5773    // see also http://www.hackersdelight.org/HDcode/ntz.cc
5774    MVT VT = Op.getValueType();
5775    SDOperand Tmp2 = DAG.getConstant(~0ULL, VT);
5776    SDOperand Tmp3 = DAG.getNode(ISD::AND, VT,
5777                       DAG.getNode(ISD::XOR, VT, Op, Tmp2),
5778                       DAG.getNode(ISD::SUB, VT, Op, DAG.getConstant(1, VT)));
5779    // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
5780    if (!TLI.isOperationLegal(ISD::CTPOP, VT) &&
5781        TLI.isOperationLegal(ISD::CTLZ, VT))
5782      return DAG.getNode(ISD::SUB, VT,
5783                         DAG.getConstant(VT.getSizeInBits(), VT),
5784                         DAG.getNode(ISD::CTLZ, VT, Tmp3));
5785    return DAG.getNode(ISD::CTPOP, VT, Tmp3);
5786  }
5787  }
5788}
5789
5790/// ExpandOp - Expand the specified SDOperand into its two component pieces
5791/// Lo&Hi.  Note that the Op MUST be an expanded type.  As a result of this, the
5792/// LegalizeNodes map is filled in for any results that are not expanded, the
5793/// ExpandedNodes map is filled in for any results that are expanded, and the
5794/// Lo/Hi values are returned.
5795void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){
5796  MVT VT = Op.getValueType();
5797  MVT NVT = TLI.getTypeToTransformTo(VT);
5798  SDNode *Node = Op.Val;
5799  assert(getTypeAction(VT) == Expand && "Not an expanded type!");
5800  assert(((NVT.isInteger() && NVT.bitsLT(VT)) || VT.isFloatingPoint() ||
5801         VT.isVector()) && "Cannot expand to FP value or to larger int value!");
5802
5803  // See if we already expanded it.
5804  DenseMap<SDOperand, std::pair<SDOperand, SDOperand> >::iterator I
5805    = ExpandedNodes.find(Op);
5806  if (I != ExpandedNodes.end()) {
5807    Lo = I->second.first;
5808    Hi = I->second.second;
5809    return;
5810  }
5811
5812  switch (Node->getOpcode()) {
5813  case ISD::CopyFromReg:
5814    assert(0 && "CopyFromReg must be legal!");
5815  case ISD::FP_ROUND_INREG:
5816    if (VT == MVT::ppcf128 &&
5817        TLI.getOperationAction(ISD::FP_ROUND_INREG, VT) ==
5818            TargetLowering::Custom) {
5819      SDOperand SrcLo, SrcHi, Src;
5820      ExpandOp(Op.getOperand(0), SrcLo, SrcHi);
5821      Src = DAG.getNode(ISD::BUILD_PAIR, VT, SrcLo, SrcHi);
5822      SDOperand Result = TLI.LowerOperation(
5823        DAG.getNode(ISD::FP_ROUND_INREG, VT, Src, Op.getOperand(1)), DAG);
5824      assert(Result.Val->getOpcode() == ISD::BUILD_PAIR);
5825      Lo = Result.Val->getOperand(0);
5826      Hi = Result.Val->getOperand(1);
5827      break;
5828    }
5829    // fall through
5830  default:
5831#ifndef NDEBUG
5832    cerr << "NODE: "; Node->dump(&DAG); cerr << "\n";
5833#endif
5834    assert(0 && "Do not know how to expand this operator!");
5835    abort();
5836  case ISD::EXTRACT_ELEMENT:
5837    ExpandOp(Node->getOperand(0), Lo, Hi);
5838    if (cast<ConstantSDNode>(Node->getOperand(1))->getValue())
5839      return ExpandOp(Hi, Lo, Hi);
5840    return ExpandOp(Lo, Lo, Hi);
5841  case ISD::EXTRACT_VECTOR_ELT:
5842    assert(VT==MVT::i64 && "Do not know how to expand this operator!");
5843    // ExpandEXTRACT_VECTOR_ELT tolerates invalid result types.
5844    Lo  = ExpandEXTRACT_VECTOR_ELT(Op);
5845    return ExpandOp(Lo, Lo, Hi);
5846  case ISD::UNDEF:
5847    Lo = DAG.getNode(ISD::UNDEF, NVT);
5848    Hi = DAG.getNode(ISD::UNDEF, NVT);
5849    break;
5850  case ISD::Constant: {
5851    unsigned NVTBits = NVT.getSizeInBits();
5852    const APInt &Cst = cast<ConstantSDNode>(Node)->getAPIntValue();
5853    Lo = DAG.getConstant(APInt(Cst).trunc(NVTBits), NVT);
5854    Hi = DAG.getConstant(Cst.lshr(NVTBits).trunc(NVTBits), NVT);
5855    break;
5856  }
5857  case ISD::ConstantFP: {
5858    ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
5859    if (CFP->getValueType(0) == MVT::ppcf128) {
5860      APInt api = CFP->getValueAPF().convertToAPInt();
5861      Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &api.getRawData()[1])),
5862                             MVT::f64);
5863      Hi = DAG.getConstantFP(APFloat(APInt(64, 1, &api.getRawData()[0])),
5864                             MVT::f64);
5865      break;
5866    }
5867    Lo = ExpandConstantFP(CFP, false, DAG, TLI);
5868    if (getTypeAction(Lo.getValueType()) == Expand)
5869      ExpandOp(Lo, Lo, Hi);
5870    break;
5871  }
5872  case ISD::BUILD_PAIR:
5873    // Return the operands.
5874    Lo = Node->getOperand(0);
5875    Hi = Node->getOperand(1);
5876    break;
5877
5878  case ISD::MERGE_VALUES:
5879    if (Node->getNumValues() == 1) {
5880      ExpandOp(Op.getOperand(0), Lo, Hi);
5881      break;
5882    }
5883    // FIXME: For now only expand i64,chain = MERGE_VALUES (x, y)
5884    assert(Op.ResNo == 0 && Node->getNumValues() == 2 &&
5885           Op.getValue(1).getValueType() == MVT::Other &&
5886           "unhandled MERGE_VALUES");
5887    ExpandOp(Op.getOperand(0), Lo, Hi);
5888    // Remember that we legalized the chain.
5889    AddLegalizedOperand(Op.getValue(1), LegalizeOp(Op.getOperand(1)));
5890    break;
5891
5892  case ISD::SIGN_EXTEND_INREG:
5893    ExpandOp(Node->getOperand(0), Lo, Hi);
5894    // sext_inreg the low part if needed.
5895    Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Lo, Node->getOperand(1));
5896
5897    // The high part gets the sign extension from the lo-part.  This handles
5898    // things like sextinreg V:i64 from i8.
5899    Hi = DAG.getNode(ISD::SRA, NVT, Lo,
5900                     DAG.getConstant(NVT.getSizeInBits()-1,
5901                                     TLI.getShiftAmountTy()));
5902    break;
5903
5904  case ISD::BSWAP: {
5905    ExpandOp(Node->getOperand(0), Lo, Hi);
5906    SDOperand TempLo = DAG.getNode(ISD::BSWAP, NVT, Hi);
5907    Hi = DAG.getNode(ISD::BSWAP, NVT, Lo);
5908    Lo = TempLo;
5909    break;
5910  }
5911
5912  case ISD::CTPOP:
5913    ExpandOp(Node->getOperand(0), Lo, Hi);
5914    Lo = DAG.getNode(ISD::ADD, NVT,          // ctpop(HL) -> ctpop(H)+ctpop(L)
5915                     DAG.getNode(ISD::CTPOP, NVT, Lo),
5916                     DAG.getNode(ISD::CTPOP, NVT, Hi));
5917    Hi = DAG.getConstant(0, NVT);
5918    break;
5919
5920  case ISD::CTLZ: {
5921    // ctlz (HL) -> ctlz(H) != 32 ? ctlz(H) : (ctlz(L)+32)
5922    ExpandOp(Node->getOperand(0), Lo, Hi);
5923    SDOperand BitsC = DAG.getConstant(NVT.getSizeInBits(), NVT);
5924    SDOperand HLZ = DAG.getNode(ISD::CTLZ, NVT, Hi);
5925    SDOperand TopNotZero = DAG.getSetCC(TLI.getSetCCResultType(HLZ), HLZ, BitsC,
5926                                        ISD::SETNE);
5927    SDOperand LowPart = DAG.getNode(ISD::CTLZ, NVT, Lo);
5928    LowPart = DAG.getNode(ISD::ADD, NVT, LowPart, BitsC);
5929
5930    Lo = DAG.getNode(ISD::SELECT, NVT, TopNotZero, HLZ, LowPart);
5931    Hi = DAG.getConstant(0, NVT);
5932    break;
5933  }
5934
5935  case ISD::CTTZ: {
5936    // cttz (HL) -> cttz(L) != 32 ? cttz(L) : (cttz(H)+32)
5937    ExpandOp(Node->getOperand(0), Lo, Hi);
5938    SDOperand BitsC = DAG.getConstant(NVT.getSizeInBits(), NVT);
5939    SDOperand LTZ = DAG.getNode(ISD::CTTZ, NVT, Lo);
5940    SDOperand BotNotZero = DAG.getSetCC(TLI.getSetCCResultType(LTZ), LTZ, BitsC,
5941                                        ISD::SETNE);
5942    SDOperand HiPart = DAG.getNode(ISD::CTTZ, NVT, Hi);
5943    HiPart = DAG.getNode(ISD::ADD, NVT, HiPart, BitsC);
5944
5945    Lo = DAG.getNode(ISD::SELECT, NVT, BotNotZero, LTZ, HiPart);
5946    Hi = DAG.getConstant(0, NVT);
5947    break;
5948  }
5949
5950  case ISD::VAARG: {
5951    SDOperand Ch = Node->getOperand(0);   // Legalize the chain.
5952    SDOperand Ptr = Node->getOperand(1);  // Legalize the pointer.
5953    Lo = DAG.getVAArg(NVT, Ch, Ptr, Node->getOperand(2));
5954    Hi = DAG.getVAArg(NVT, Lo.getValue(1), Ptr, Node->getOperand(2));
5955
5956    // Remember that we legalized the chain.
5957    Hi = LegalizeOp(Hi);
5958    AddLegalizedOperand(Op.getValue(1), Hi.getValue(1));
5959    if (TLI.isBigEndian())
5960      std::swap(Lo, Hi);
5961    break;
5962  }
5963
5964  case ISD::LOAD: {
5965    LoadSDNode *LD = cast<LoadSDNode>(Node);
5966    SDOperand Ch  = LD->getChain();    // Legalize the chain.
5967    SDOperand Ptr = LD->getBasePtr();  // Legalize the pointer.
5968    ISD::LoadExtType ExtType = LD->getExtensionType();
5969    int SVOffset = LD->getSrcValueOffset();
5970    unsigned Alignment = LD->getAlignment();
5971    bool isVolatile = LD->isVolatile();
5972
5973    if (ExtType == ISD::NON_EXTLOAD) {
5974      Lo = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(), SVOffset,
5975                       isVolatile, Alignment);
5976      if (VT == MVT::f32 || VT == MVT::f64) {
5977        // f32->i32 or f64->i64 one to one expansion.
5978        // Remember that we legalized the chain.
5979        AddLegalizedOperand(SDOperand(Node, 1), LegalizeOp(Lo.getValue(1)));
5980        // Recursively expand the new load.
5981        if (getTypeAction(NVT) == Expand)
5982          ExpandOp(Lo, Lo, Hi);
5983        break;
5984      }
5985
5986      // Increment the pointer to the other half.
5987      unsigned IncrementSize = Lo.getValueType().getSizeInBits()/8;
5988      Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
5989                        DAG.getIntPtrConstant(IncrementSize));
5990      SVOffset += IncrementSize;
5991      Alignment = MinAlign(Alignment, IncrementSize);
5992      Hi = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(), SVOffset,
5993                       isVolatile, Alignment);
5994
5995      // Build a factor node to remember that this load is independent of the
5996      // other one.
5997      SDOperand TF = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1),
5998                                 Hi.getValue(1));
5999
6000      // Remember that we legalized the chain.
6001      AddLegalizedOperand(Op.getValue(1), LegalizeOp(TF));
6002      if (TLI.isBigEndian())
6003        std::swap(Lo, Hi);
6004    } else {
6005      MVT EVT = LD->getMemoryVT();
6006
6007      if ((VT == MVT::f64 && EVT == MVT::f32) ||
6008          (VT == MVT::ppcf128 && (EVT==MVT::f64 || EVT==MVT::f32))) {
6009        // f64 = EXTLOAD f32 should expand to LOAD, FP_EXTEND
6010        SDOperand Load = DAG.getLoad(EVT, Ch, Ptr, LD->getSrcValue(),
6011                                     SVOffset, isVolatile, Alignment);
6012        // Remember that we legalized the chain.
6013        AddLegalizedOperand(SDOperand(Node, 1), LegalizeOp(Load.getValue(1)));
6014        ExpandOp(DAG.getNode(ISD::FP_EXTEND, VT, Load), Lo, Hi);
6015        break;
6016      }
6017
6018      if (EVT == NVT)
6019        Lo = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(),
6020                         SVOffset, isVolatile, Alignment);
6021      else
6022        Lo = DAG.getExtLoad(ExtType, NVT, Ch, Ptr, LD->getSrcValue(),
6023                            SVOffset, EVT, isVolatile,
6024                            Alignment);
6025
6026      // Remember that we legalized the chain.
6027      AddLegalizedOperand(SDOperand(Node, 1), LegalizeOp(Lo.getValue(1)));
6028
6029      if (ExtType == ISD::SEXTLOAD) {
6030        // The high part is obtained by SRA'ing all but one of the bits of the
6031        // lo part.
6032        unsigned LoSize = Lo.getValueType().getSizeInBits();
6033        Hi = DAG.getNode(ISD::SRA, NVT, Lo,
6034                         DAG.getConstant(LoSize-1, TLI.getShiftAmountTy()));
6035      } else if (ExtType == ISD::ZEXTLOAD) {
6036        // The high part is just a zero.
6037        Hi = DAG.getConstant(0, NVT);
6038      } else /* if (ExtType == ISD::EXTLOAD) */ {
6039        // The high part is undefined.
6040        Hi = DAG.getNode(ISD::UNDEF, NVT);
6041      }
6042    }
6043    break;
6044  }
6045  case ISD::AND:
6046  case ISD::OR:
6047  case ISD::XOR: {   // Simple logical operators -> two trivial pieces.
6048    SDOperand LL, LH, RL, RH;
6049    ExpandOp(Node->getOperand(0), LL, LH);
6050    ExpandOp(Node->getOperand(1), RL, RH);
6051    Lo = DAG.getNode(Node->getOpcode(), NVT, LL, RL);
6052    Hi = DAG.getNode(Node->getOpcode(), NVT, LH, RH);
6053    break;
6054  }
6055  case ISD::SELECT: {
6056    SDOperand LL, LH, RL, RH;
6057    ExpandOp(Node->getOperand(1), LL, LH);
6058    ExpandOp(Node->getOperand(2), RL, RH);
6059    if (getTypeAction(NVT) == Expand)
6060      NVT = TLI.getTypeToExpandTo(NVT);
6061    Lo = DAG.getNode(ISD::SELECT, NVT, Node->getOperand(0), LL, RL);
6062    if (VT != MVT::f32)
6063      Hi = DAG.getNode(ISD::SELECT, NVT, Node->getOperand(0), LH, RH);
6064    break;
6065  }
6066  case ISD::SELECT_CC: {
6067    SDOperand TL, TH, FL, FH;
6068    ExpandOp(Node->getOperand(2), TL, TH);
6069    ExpandOp(Node->getOperand(3), FL, FH);
6070    if (getTypeAction(NVT) == Expand)
6071      NVT = TLI.getTypeToExpandTo(NVT);
6072    Lo = DAG.getNode(ISD::SELECT_CC, NVT, Node->getOperand(0),
6073                     Node->getOperand(1), TL, FL, Node->getOperand(4));
6074    if (VT != MVT::f32)
6075      Hi = DAG.getNode(ISD::SELECT_CC, NVT, Node->getOperand(0),
6076                       Node->getOperand(1), TH, FH, Node->getOperand(4));
6077    break;
6078  }
6079  case ISD::ANY_EXTEND:
6080    // The low part is any extension of the input (which degenerates to a copy).
6081    Lo = DAG.getNode(ISD::ANY_EXTEND, NVT, Node->getOperand(0));
6082    // The high part is undefined.
6083    Hi = DAG.getNode(ISD::UNDEF, NVT);
6084    break;
6085  case ISD::SIGN_EXTEND: {
6086    // The low part is just a sign extension of the input (which degenerates to
6087    // a copy).
6088    Lo = DAG.getNode(ISD::SIGN_EXTEND, NVT, Node->getOperand(0));
6089
6090    // The high part is obtained by SRA'ing all but one of the bits of the lo
6091    // part.
6092    unsigned LoSize = Lo.getValueType().getSizeInBits();
6093    Hi = DAG.getNode(ISD::SRA, NVT, Lo,
6094                     DAG.getConstant(LoSize-1, TLI.getShiftAmountTy()));
6095    break;
6096  }
6097  case ISD::ZERO_EXTEND:
6098    // The low part is just a zero extension of the input (which degenerates to
6099    // a copy).
6100    Lo = DAG.getNode(ISD::ZERO_EXTEND, NVT, Node->getOperand(0));
6101
6102    // The high part is just a zero.
6103    Hi = DAG.getConstant(0, NVT);
6104    break;
6105
6106  case ISD::TRUNCATE: {
6107    // The input value must be larger than this value.  Expand *it*.
6108    SDOperand NewLo;
6109    ExpandOp(Node->getOperand(0), NewLo, Hi);
6110
6111    // The low part is now either the right size, or it is closer.  If not the
6112    // right size, make an illegal truncate so we recursively expand it.
6113    if (NewLo.getValueType() != Node->getValueType(0))
6114      NewLo = DAG.getNode(ISD::TRUNCATE, Node->getValueType(0), NewLo);
6115    ExpandOp(NewLo, Lo, Hi);
6116    break;
6117  }
6118
6119  case ISD::BIT_CONVERT: {
6120    SDOperand Tmp;
6121    if (TLI.getOperationAction(ISD::BIT_CONVERT, VT) == TargetLowering::Custom){
6122      // If the target wants to, allow it to lower this itself.
6123      switch (getTypeAction(Node->getOperand(0).getValueType())) {
6124      case Expand: assert(0 && "cannot expand FP!");
6125      case Legal:   Tmp = LegalizeOp(Node->getOperand(0)); break;
6126      case Promote: Tmp = PromoteOp (Node->getOperand(0)); break;
6127      }
6128      Tmp = TLI.LowerOperation(DAG.getNode(ISD::BIT_CONVERT, VT, Tmp), DAG);
6129    }
6130
6131    // f32 / f64 must be expanded to i32 / i64.
6132    if (VT == MVT::f32 || VT == MVT::f64) {
6133      Lo = DAG.getNode(ISD::BIT_CONVERT, NVT, Node->getOperand(0));
6134      if (getTypeAction(NVT) == Expand)
6135        ExpandOp(Lo, Lo, Hi);
6136      break;
6137    }
6138
6139    // If source operand will be expanded to the same type as VT, i.e.
6140    // i64 <- f64, i32 <- f32, expand the source operand instead.
6141    MVT VT0 = Node->getOperand(0).getValueType();
6142    if (getTypeAction(VT0) == Expand && TLI.getTypeToTransformTo(VT0) == VT) {
6143      ExpandOp(Node->getOperand(0), Lo, Hi);
6144      break;
6145    }
6146
6147    // Turn this into a load/store pair by default.
6148    if (Tmp.Val == 0)
6149      Tmp = EmitStackConvert(Node->getOperand(0), VT, VT);
6150
6151    ExpandOp(Tmp, Lo, Hi);
6152    break;
6153  }
6154
6155  case ISD::READCYCLECOUNTER: {
6156    assert(TLI.getOperationAction(ISD::READCYCLECOUNTER, VT) ==
6157                 TargetLowering::Custom &&
6158           "Must custom expand ReadCycleCounter");
6159    SDOperand Tmp = TLI.LowerOperation(Op, DAG);
6160    assert(Tmp.Val && "Node must be custom expanded!");
6161    ExpandOp(Tmp.getValue(0), Lo, Hi);
6162    AddLegalizedOperand(SDOperand(Node, 1), // Remember we legalized the chain.
6163                        LegalizeOp(Tmp.getValue(1)));
6164    break;
6165  }
6166
6167  case ISD::ATOMIC_CMP_SWAP: {
6168    SDOperand Tmp = TLI.LowerOperation(Op, DAG);
6169    assert(Tmp.Val && "Node must be custom expanded!");
6170    ExpandOp(Tmp.getValue(0), Lo, Hi);
6171    AddLegalizedOperand(SDOperand(Node, 1), // Remember we legalized the chain.
6172                        LegalizeOp(Tmp.getValue(1)));
6173    break;
6174  }
6175
6176
6177
6178    // These operators cannot be expanded directly, emit them as calls to
6179    // library functions.
6180  case ISD::FP_TO_SINT: {
6181    if (TLI.getOperationAction(ISD::FP_TO_SINT, VT) == TargetLowering::Custom) {
6182      SDOperand Op;
6183      switch (getTypeAction(Node->getOperand(0).getValueType())) {
6184      case Expand: assert(0 && "cannot expand FP!");
6185      case Legal:   Op = LegalizeOp(Node->getOperand(0)); break;
6186      case Promote: Op = PromoteOp (Node->getOperand(0)); break;
6187      }
6188
6189      Op = TLI.LowerOperation(DAG.getNode(ISD::FP_TO_SINT, VT, Op), DAG);
6190
6191      // Now that the custom expander is done, expand the result, which is still
6192      // VT.
6193      if (Op.Val) {
6194        ExpandOp(Op, Lo, Hi);
6195        break;
6196      }
6197    }
6198
6199    RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
6200    if (VT == MVT::i64) {
6201      if (Node->getOperand(0).getValueType() == MVT::f32)
6202        LC = RTLIB::FPTOSINT_F32_I64;
6203      else if (Node->getOperand(0).getValueType() == MVT::f64)
6204        LC = RTLIB::FPTOSINT_F64_I64;
6205      else if (Node->getOperand(0).getValueType() == MVT::f80)
6206        LC = RTLIB::FPTOSINT_F80_I64;
6207      else if (Node->getOperand(0).getValueType() == MVT::ppcf128)
6208        LC = RTLIB::FPTOSINT_PPCF128_I64;
6209      Lo = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Hi);
6210    } else if (VT == MVT::i128) {
6211      if (Node->getOperand(0).getValueType() == MVT::f32)
6212        LC = RTLIB::FPTOSINT_F32_I128;
6213      else if (Node->getOperand(0).getValueType() == MVT::f64)
6214        LC = RTLIB::FPTOSINT_F64_I128;
6215      else if (Node->getOperand(0).getValueType() == MVT::f80)
6216        LC = RTLIB::FPTOSINT_F80_I128;
6217      else if (Node->getOperand(0).getValueType() == MVT::ppcf128)
6218        LC = RTLIB::FPTOSINT_PPCF128_I128;
6219      Lo = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Hi);
6220    } else {
6221      assert(0 && "Unexpected uint-to-fp conversion!");
6222    }
6223    break;
6224  }
6225
6226  case ISD::FP_TO_UINT: {
6227    if (TLI.getOperationAction(ISD::FP_TO_UINT, VT) == TargetLowering::Custom) {
6228      SDOperand Op;
6229      switch (getTypeAction(Node->getOperand(0).getValueType())) {
6230        case Expand: assert(0 && "cannot expand FP!");
6231        case Legal:   Op = LegalizeOp(Node->getOperand(0)); break;
6232        case Promote: Op = PromoteOp (Node->getOperand(0)); break;
6233      }
6234
6235      Op = TLI.LowerOperation(DAG.getNode(ISD::FP_TO_UINT, VT, Op), DAG);
6236
6237      // Now that the custom expander is done, expand the result.
6238      if (Op.Val) {
6239        ExpandOp(Op, Lo, Hi);
6240        break;
6241      }
6242    }
6243
6244    RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
6245    if (VT == MVT::i64) {
6246      if (Node->getOperand(0).getValueType() == MVT::f32)
6247        LC = RTLIB::FPTOUINT_F32_I64;
6248      else if (Node->getOperand(0).getValueType() == MVT::f64)
6249        LC = RTLIB::FPTOUINT_F64_I64;
6250      else if (Node->getOperand(0).getValueType() == MVT::f80)
6251        LC = RTLIB::FPTOUINT_F80_I64;
6252      else if (Node->getOperand(0).getValueType() == MVT::ppcf128)
6253        LC = RTLIB::FPTOUINT_PPCF128_I64;
6254      Lo = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Hi);
6255    } else if (VT == MVT::i128) {
6256      if (Node->getOperand(0).getValueType() == MVT::f32)
6257        LC = RTLIB::FPTOUINT_F32_I128;
6258      else if (Node->getOperand(0).getValueType() == MVT::f64)
6259        LC = RTLIB::FPTOUINT_F64_I128;
6260      else if (Node->getOperand(0).getValueType() == MVT::f80)
6261        LC = RTLIB::FPTOUINT_F80_I128;
6262      else if (Node->getOperand(0).getValueType() == MVT::ppcf128)
6263        LC = RTLIB::FPTOUINT_PPCF128_I128;
6264      Lo = ExpandLibCall(LC, Node, false/*sign irrelevant*/, Hi);
6265    } else {
6266      assert(0 && "Unexpected uint-to-fp conversion!");
6267    }
6268    break;
6269  }
6270
6271  case ISD::SHL: {
6272    // If the target wants custom lowering, do so.
6273    SDOperand ShiftAmt = LegalizeOp(Node->getOperand(1));
6274    if (TLI.getOperationAction(ISD::SHL, VT) == TargetLowering::Custom) {
6275      SDOperand Op = DAG.getNode(ISD::SHL, VT, Node->getOperand(0), ShiftAmt);
6276      Op = TLI.LowerOperation(Op, DAG);
6277      if (Op.Val) {
6278        // Now that the custom expander is done, expand the result, which is
6279        // still VT.
6280        ExpandOp(Op, Lo, Hi);
6281        break;
6282      }
6283    }
6284
6285    // If ADDC/ADDE are supported and if the shift amount is a constant 1, emit
6286    // this X << 1 as X+X.
6287    if (ConstantSDNode *ShAmt = dyn_cast<ConstantSDNode>(ShiftAmt)) {
6288      if (ShAmt->getAPIntValue() == 1 && TLI.isOperationLegal(ISD::ADDC, NVT) &&
6289          TLI.isOperationLegal(ISD::ADDE, NVT)) {
6290        SDOperand LoOps[2], HiOps[3];
6291        ExpandOp(Node->getOperand(0), LoOps[0], HiOps[0]);
6292        SDVTList VTList = DAG.getVTList(LoOps[0].getValueType(), MVT::Flag);
6293        LoOps[1] = LoOps[0];
6294        Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2);
6295
6296        HiOps[1] = HiOps[0];
6297        HiOps[2] = Lo.getValue(1);
6298        Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3);
6299        break;
6300      }
6301    }
6302
6303    // If we can emit an efficient shift operation, do so now.
6304    if (ExpandShift(ISD::SHL, Node->getOperand(0), ShiftAmt, Lo, Hi))
6305      break;
6306
6307    // If this target supports SHL_PARTS, use it.
6308    TargetLowering::LegalizeAction Action =
6309      TLI.getOperationAction(ISD::SHL_PARTS, NVT);
6310    if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
6311        Action == TargetLowering::Custom) {
6312      ExpandShiftParts(ISD::SHL_PARTS, Node->getOperand(0), ShiftAmt, Lo, Hi);
6313      break;
6314    }
6315
6316    // Otherwise, emit a libcall.
6317    Lo = ExpandLibCall(RTLIB::SHL_I64, Node, false/*left shift=unsigned*/, Hi);
6318    break;
6319  }
6320
6321  case ISD::SRA: {
6322    // If the target wants custom lowering, do so.
6323    SDOperand ShiftAmt = LegalizeOp(Node->getOperand(1));
6324    if (TLI.getOperationAction(ISD::SRA, VT) == TargetLowering::Custom) {
6325      SDOperand Op = DAG.getNode(ISD::SRA, VT, Node->getOperand(0), ShiftAmt);
6326      Op = TLI.LowerOperation(Op, DAG);
6327      if (Op.Val) {
6328        // Now that the custom expander is done, expand the result, which is
6329        // still VT.
6330        ExpandOp(Op, Lo, Hi);
6331        break;
6332      }
6333    }
6334
6335    // If we can emit an efficient shift operation, do so now.
6336    if (ExpandShift(ISD::SRA, Node->getOperand(0), ShiftAmt, Lo, Hi))
6337      break;
6338
6339    // If this target supports SRA_PARTS, use it.
6340    TargetLowering::LegalizeAction Action =
6341      TLI.getOperationAction(ISD::SRA_PARTS, NVT);
6342    if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
6343        Action == TargetLowering::Custom) {
6344      ExpandShiftParts(ISD::SRA_PARTS, Node->getOperand(0), ShiftAmt, Lo, Hi);
6345      break;
6346    }
6347
6348    // Otherwise, emit a libcall.
6349    Lo = ExpandLibCall(RTLIB::SRA_I64, Node, true/*ashr is signed*/, Hi);
6350    break;
6351  }
6352
6353  case ISD::SRL: {
6354    // If the target wants custom lowering, do so.
6355    SDOperand ShiftAmt = LegalizeOp(Node->getOperand(1));
6356    if (TLI.getOperationAction(ISD::SRL, VT) == TargetLowering::Custom) {
6357      SDOperand Op = DAG.getNode(ISD::SRL, VT, Node->getOperand(0), ShiftAmt);
6358      Op = TLI.LowerOperation(Op, DAG);
6359      if (Op.Val) {
6360        // Now that the custom expander is done, expand the result, which is
6361        // still VT.
6362        ExpandOp(Op, Lo, Hi);
6363        break;
6364      }
6365    }
6366
6367    // If we can emit an efficient shift operation, do so now.
6368    if (ExpandShift(ISD::SRL, Node->getOperand(0), ShiftAmt, Lo, Hi))
6369      break;
6370
6371    // If this target supports SRL_PARTS, use it.
6372    TargetLowering::LegalizeAction Action =
6373      TLI.getOperationAction(ISD::SRL_PARTS, NVT);
6374    if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
6375        Action == TargetLowering::Custom) {
6376      ExpandShiftParts(ISD::SRL_PARTS, Node->getOperand(0), ShiftAmt, Lo, Hi);
6377      break;
6378    }
6379
6380    // Otherwise, emit a libcall.
6381    Lo = ExpandLibCall(RTLIB::SRL_I64, Node, false/*lshr is unsigned*/, Hi);
6382    break;
6383  }
6384
6385  case ISD::ADD:
6386  case ISD::SUB: {
6387    // If the target wants to custom expand this, let them.
6388    if (TLI.getOperationAction(Node->getOpcode(), VT) ==
6389            TargetLowering::Custom) {
6390      SDOperand Result = TLI.LowerOperation(Op, DAG);
6391      if (Result.Val) {
6392        ExpandOp(Result, Lo, Hi);
6393        break;
6394      }
6395    }
6396
6397    // Expand the subcomponents.
6398    SDOperand LHSL, LHSH, RHSL, RHSH;
6399    ExpandOp(Node->getOperand(0), LHSL, LHSH);
6400    ExpandOp(Node->getOperand(1), RHSL, RHSH);
6401    SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag);
6402    SDOperand LoOps[2], HiOps[3];
6403    LoOps[0] = LHSL;
6404    LoOps[1] = RHSL;
6405    HiOps[0] = LHSH;
6406    HiOps[1] = RHSH;
6407    if (Node->getOpcode() == ISD::ADD) {
6408      Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2);
6409      HiOps[2] = Lo.getValue(1);
6410      Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3);
6411    } else {
6412      Lo = DAG.getNode(ISD::SUBC, VTList, LoOps, 2);
6413      HiOps[2] = Lo.getValue(1);
6414      Hi = DAG.getNode(ISD::SUBE, VTList, HiOps, 3);
6415    }
6416    break;
6417  }
6418
6419  case ISD::ADDC:
6420  case ISD::SUBC: {
6421    // Expand the subcomponents.
6422    SDOperand LHSL, LHSH, RHSL, RHSH;
6423    ExpandOp(Node->getOperand(0), LHSL, LHSH);
6424    ExpandOp(Node->getOperand(1), RHSL, RHSH);
6425    SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag);
6426    SDOperand LoOps[2] = { LHSL, RHSL };
6427    SDOperand HiOps[3] = { LHSH, RHSH };
6428
6429    if (Node->getOpcode() == ISD::ADDC) {
6430      Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2);
6431      HiOps[2] = Lo.getValue(1);
6432      Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3);
6433    } else {
6434      Lo = DAG.getNode(ISD::SUBC, VTList, LoOps, 2);
6435      HiOps[2] = Lo.getValue(1);
6436      Hi = DAG.getNode(ISD::SUBE, VTList, HiOps, 3);
6437    }
6438    // Remember that we legalized the flag.
6439    AddLegalizedOperand(Op.getValue(1), LegalizeOp(Hi.getValue(1)));
6440    break;
6441  }
6442  case ISD::ADDE:
6443  case ISD::SUBE: {
6444    // Expand the subcomponents.
6445    SDOperand LHSL, LHSH, RHSL, RHSH;
6446    ExpandOp(Node->getOperand(0), LHSL, LHSH);
6447    ExpandOp(Node->getOperand(1), RHSL, RHSH);
6448    SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag);
6449    SDOperand LoOps[3] = { LHSL, RHSL, Node->getOperand(2) };
6450    SDOperand HiOps[3] = { LHSH, RHSH };
6451
6452    Lo = DAG.getNode(Node->getOpcode(), VTList, LoOps, 3);
6453    HiOps[2] = Lo.getValue(1);
6454    Hi = DAG.getNode(Node->getOpcode(), VTList, HiOps, 3);
6455
6456    // Remember that we legalized the flag.
6457    AddLegalizedOperand(Op.getValue(1), LegalizeOp(Hi.getValue(1)));
6458    break;
6459  }
6460  case ISD::MUL: {
6461    // If the target wants to custom expand this, let them.
6462    if (TLI.getOperationAction(ISD::MUL, VT) == TargetLowering::Custom) {
6463      SDOperand New = TLI.LowerOperation(Op, DAG);
6464      if (New.Val) {
6465        ExpandOp(New, Lo, Hi);
6466        break;
6467      }
6468    }
6469
6470    bool HasMULHS = TLI.isOperationLegal(ISD::MULHS, NVT);
6471    bool HasMULHU = TLI.isOperationLegal(ISD::MULHU, NVT);
6472    bool HasSMUL_LOHI = TLI.isOperationLegal(ISD::SMUL_LOHI, NVT);
6473    bool HasUMUL_LOHI = TLI.isOperationLegal(ISD::UMUL_LOHI, NVT);
6474    if (HasMULHU || HasMULHS || HasUMUL_LOHI || HasSMUL_LOHI) {
6475      SDOperand LL, LH, RL, RH;
6476      ExpandOp(Node->getOperand(0), LL, LH);
6477      ExpandOp(Node->getOperand(1), RL, RH);
6478      unsigned OuterBitSize = Op.getValueSizeInBits();
6479      unsigned InnerBitSize = RH.getValueSizeInBits();
6480      unsigned LHSSB = DAG.ComputeNumSignBits(Op.getOperand(0));
6481      unsigned RHSSB = DAG.ComputeNumSignBits(Op.getOperand(1));
6482      APInt HighMask = APInt::getHighBitsSet(OuterBitSize, InnerBitSize);
6483      if (DAG.MaskedValueIsZero(Node->getOperand(0), HighMask) &&
6484          DAG.MaskedValueIsZero(Node->getOperand(1), HighMask)) {
6485        // The inputs are both zero-extended.
6486        if (HasUMUL_LOHI) {
6487          // We can emit a umul_lohi.
6488          Lo = DAG.getNode(ISD::UMUL_LOHI, DAG.getVTList(NVT, NVT), LL, RL);
6489          Hi = SDOperand(Lo.Val, 1);
6490          break;
6491        }
6492        if (HasMULHU) {
6493          // We can emit a mulhu+mul.
6494          Lo = DAG.getNode(ISD::MUL, NVT, LL, RL);
6495          Hi = DAG.getNode(ISD::MULHU, NVT, LL, RL);
6496          break;
6497        }
6498      }
6499      if (LHSSB > InnerBitSize && RHSSB > InnerBitSize) {
6500        // The input values are both sign-extended.
6501        if (HasSMUL_LOHI) {
6502          // We can emit a smul_lohi.
6503          Lo = DAG.getNode(ISD::SMUL_LOHI, DAG.getVTList(NVT, NVT), LL, RL);
6504          Hi = SDOperand(Lo.Val, 1);
6505          break;
6506        }
6507        if (HasMULHS) {
6508          // We can emit a mulhs+mul.
6509          Lo = DAG.getNode(ISD::MUL, NVT, LL, RL);
6510          Hi = DAG.getNode(ISD::MULHS, NVT, LL, RL);
6511          break;
6512        }
6513      }
6514      if (HasUMUL_LOHI) {
6515        // Lo,Hi = umul LHS, RHS.
6516        SDOperand UMulLOHI = DAG.getNode(ISD::UMUL_LOHI,
6517                                         DAG.getVTList(NVT, NVT), LL, RL);
6518        Lo = UMulLOHI;
6519        Hi = UMulLOHI.getValue(1);
6520        RH = DAG.getNode(ISD::MUL, NVT, LL, RH);
6521        LH = DAG.getNode(ISD::MUL, NVT, LH, RL);
6522        Hi = DAG.getNode(ISD::ADD, NVT, Hi, RH);
6523        Hi = DAG.getNode(ISD::ADD, NVT, Hi, LH);
6524        break;
6525      }
6526      if (HasMULHU) {
6527        Lo = DAG.getNode(ISD::MUL, NVT, LL, RL);
6528        Hi = DAG.getNode(ISD::MULHU, NVT, LL, RL);
6529        RH = DAG.getNode(ISD::MUL, NVT, LL, RH);
6530        LH = DAG.getNode(ISD::MUL, NVT, LH, RL);
6531        Hi = DAG.getNode(ISD::ADD, NVT, Hi, RH);
6532        Hi = DAG.getNode(ISD::ADD, NVT, Hi, LH);
6533        break;
6534      }
6535    }
6536
6537    // If nothing else, we can make a libcall.
6538    Lo = ExpandLibCall(RTLIB::MUL_I64, Node, false/*sign irrelevant*/, Hi);
6539    break;
6540  }
6541  case ISD::SDIV:
6542    Lo = ExpandLibCall(RTLIB::SDIV_I64, Node, true, Hi);
6543    break;
6544  case ISD::UDIV:
6545    Lo = ExpandLibCall(RTLIB::UDIV_I64, Node, true, Hi);
6546    break;
6547  case ISD::SREM:
6548    Lo = ExpandLibCall(RTLIB::SREM_I64, Node, true, Hi);
6549    break;
6550  case ISD::UREM:
6551    Lo = ExpandLibCall(RTLIB::UREM_I64, Node, true, Hi);
6552    break;
6553
6554  case ISD::FADD:
6555    Lo = ExpandLibCall(GetFPLibCall(VT, RTLIB::ADD_F32,
6556                                        RTLIB::ADD_F64,
6557                                        RTLIB::ADD_F80,
6558                                        RTLIB::ADD_PPCF128),
6559                       Node, false, Hi);
6560    break;
6561  case ISD::FSUB:
6562    Lo = ExpandLibCall(GetFPLibCall(VT, RTLIB::SUB_F32,
6563                                        RTLIB::SUB_F64,
6564                                        RTLIB::SUB_F80,
6565                                        RTLIB::SUB_PPCF128),
6566                       Node, false, Hi);
6567    break;
6568  case ISD::FMUL:
6569    Lo = ExpandLibCall(GetFPLibCall(VT, RTLIB::MUL_F32,
6570                                        RTLIB::MUL_F64,
6571                                        RTLIB::MUL_F80,
6572                                        RTLIB::MUL_PPCF128),
6573                       Node, false, Hi);
6574    break;
6575  case ISD::FDIV:
6576    Lo = ExpandLibCall(GetFPLibCall(VT, RTLIB::DIV_F32,
6577                                        RTLIB::DIV_F64,
6578                                        RTLIB::DIV_F80,
6579                                        RTLIB::DIV_PPCF128),
6580                       Node, false, Hi);
6581    break;
6582  case ISD::FP_EXTEND:
6583    if (VT == MVT::ppcf128) {
6584      assert(Node->getOperand(0).getValueType()==MVT::f32 ||
6585             Node->getOperand(0).getValueType()==MVT::f64);
6586      const uint64_t zero = 0;
6587      if (Node->getOperand(0).getValueType()==MVT::f32)
6588        Hi = DAG.getNode(ISD::FP_EXTEND, MVT::f64, Node->getOperand(0));
6589      else
6590        Hi = Node->getOperand(0);
6591      Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &zero)), MVT::f64);
6592      break;
6593    }
6594    Lo = ExpandLibCall(RTLIB::FPEXT_F32_F64, Node, true, Hi);
6595    break;
6596  case ISD::FP_ROUND:
6597    Lo = ExpandLibCall(RTLIB::FPROUND_F64_F32, Node, true, Hi);
6598    break;
6599  case ISD::FPOWI:
6600    Lo = ExpandLibCall(GetFPLibCall(VT, RTLIB::POWI_F32,
6601                                        RTLIB::POWI_F64,
6602                                        RTLIB::POWI_F80,
6603                                        RTLIB::POWI_PPCF128),
6604                       Node, false, Hi);
6605    break;
6606  case ISD::FSQRT:
6607  case ISD::FSIN:
6608  case ISD::FCOS: {
6609    RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
6610    switch(Node->getOpcode()) {
6611    case ISD::FSQRT:
6612      LC = GetFPLibCall(VT, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
6613                        RTLIB::SQRT_F80, RTLIB::SQRT_PPCF128);
6614      break;
6615    case ISD::FSIN:
6616      LC = GetFPLibCall(VT, RTLIB::SIN_F32, RTLIB::SIN_F64,
6617                        RTLIB::SIN_F80, RTLIB::SIN_PPCF128);
6618      break;
6619    case ISD::FCOS:
6620      LC = GetFPLibCall(VT, RTLIB::COS_F32, RTLIB::COS_F64,
6621                        RTLIB::COS_F80, RTLIB::COS_PPCF128);
6622      break;
6623    default: assert(0 && "Unreachable!");
6624    }
6625    Lo = ExpandLibCall(LC, Node, false, Hi);
6626    break;
6627  }
6628  case ISD::FABS: {
6629    if (VT == MVT::ppcf128) {
6630      SDOperand Tmp;
6631      ExpandOp(Node->getOperand(0), Lo, Tmp);
6632      Hi = DAG.getNode(ISD::FABS, NVT, Tmp);
6633      // lo = hi==fabs(hi) ? lo : -lo;
6634      Lo = DAG.getNode(ISD::SELECT_CC, NVT, Hi, Tmp,
6635                    Lo, DAG.getNode(ISD::FNEG, NVT, Lo),
6636                    DAG.getCondCode(ISD::SETEQ));
6637      break;
6638    }
6639    SDOperand Mask = (VT == MVT::f64)
6640      ? DAG.getConstantFP(BitsToDouble(~(1ULL << 63)), VT)
6641      : DAG.getConstantFP(BitsToFloat(~(1U << 31)), VT);
6642    Mask = DAG.getNode(ISD::BIT_CONVERT, NVT, Mask);
6643    Lo = DAG.getNode(ISD::BIT_CONVERT, NVT, Node->getOperand(0));
6644    Lo = DAG.getNode(ISD::AND, NVT, Lo, Mask);
6645    if (getTypeAction(NVT) == Expand)
6646      ExpandOp(Lo, Lo, Hi);
6647    break;
6648  }
6649  case ISD::FNEG: {
6650    if (VT == MVT::ppcf128) {
6651      ExpandOp(Node->getOperand(0), Lo, Hi);
6652      Lo = DAG.getNode(ISD::FNEG, MVT::f64, Lo);
6653      Hi = DAG.getNode(ISD::FNEG, MVT::f64, Hi);
6654      break;
6655    }
6656    SDOperand Mask = (VT == MVT::f64)
6657      ? DAG.getConstantFP(BitsToDouble(1ULL << 63), VT)
6658      : DAG.getConstantFP(BitsToFloat(1U << 31), VT);
6659    Mask = DAG.getNode(ISD::BIT_CONVERT, NVT, Mask);
6660    Lo = DAG.getNode(ISD::BIT_CONVERT, NVT, Node->getOperand(0));
6661    Lo = DAG.getNode(ISD::XOR, NVT, Lo, Mask);
6662    if (getTypeAction(NVT) == Expand)
6663      ExpandOp(Lo, Lo, Hi);
6664    break;
6665  }
6666  case ISD::FCOPYSIGN: {
6667    Lo = ExpandFCOPYSIGNToBitwiseOps(Node, NVT, DAG, TLI);
6668    if (getTypeAction(NVT) == Expand)
6669      ExpandOp(Lo, Lo, Hi);
6670    break;
6671  }
6672  case ISD::SINT_TO_FP:
6673  case ISD::UINT_TO_FP: {
6674    bool isSigned = Node->getOpcode() == ISD::SINT_TO_FP;
6675    MVT SrcVT = Node->getOperand(0).getValueType();
6676
6677    // Promote the operand if needed.  Do this before checking for
6678    // ppcf128 so conversions of i16 and i8 work.
6679    if (getTypeAction(SrcVT) == Promote) {
6680      SDOperand Tmp = PromoteOp(Node->getOperand(0));
6681      Tmp = isSigned
6682        ? DAG.getNode(ISD::SIGN_EXTEND_INREG, Tmp.getValueType(), Tmp,
6683                      DAG.getValueType(SrcVT))
6684        : DAG.getZeroExtendInReg(Tmp, SrcVT);
6685      Node = DAG.UpdateNodeOperands(Op, Tmp).Val;
6686      SrcVT = Node->getOperand(0).getValueType();
6687    }
6688
6689    if (VT == MVT::ppcf128 && SrcVT == MVT::i32) {
6690      static const uint64_t zero = 0;
6691      if (isSigned) {
6692        Hi = LegalizeOp(DAG.getNode(ISD::SINT_TO_FP, MVT::f64,
6693                                    Node->getOperand(0)));
6694        Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &zero)), MVT::f64);
6695      } else {
6696        static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
6697        Hi = LegalizeOp(DAG.getNode(ISD::SINT_TO_FP, MVT::f64,
6698                                    Node->getOperand(0)));
6699        Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &zero)), MVT::f64);
6700        Hi = DAG.getNode(ISD::BUILD_PAIR, VT, Lo, Hi);
6701        // X>=0 ? {(f64)x, 0} : {(f64)x, 0} + 2^32
6702        ExpandOp(DAG.getNode(ISD::SELECT_CC, MVT::ppcf128, Node->getOperand(0),
6703                             DAG.getConstant(0, MVT::i32),
6704                             DAG.getNode(ISD::FADD, MVT::ppcf128, Hi,
6705                                         DAG.getConstantFP(
6706                                            APFloat(APInt(128, 2, TwoE32)),
6707                                            MVT::ppcf128)),
6708                             Hi,
6709                             DAG.getCondCode(ISD::SETLT)),
6710                 Lo, Hi);
6711      }
6712      break;
6713    }
6714    if (VT == MVT::ppcf128 && SrcVT == MVT::i64 && !isSigned) {
6715      // si64->ppcf128 done by libcall, below
6716      static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
6717      ExpandOp(DAG.getNode(ISD::SINT_TO_FP, MVT::ppcf128, Node->getOperand(0)),
6718               Lo, Hi);
6719      Hi = DAG.getNode(ISD::BUILD_PAIR, VT, Lo, Hi);
6720      // x>=0 ? (ppcf128)(i64)x : (ppcf128)(i64)x + 2^64
6721      ExpandOp(DAG.getNode(ISD::SELECT_CC, MVT::ppcf128, Node->getOperand(0),
6722                           DAG.getConstant(0, MVT::i64),
6723                           DAG.getNode(ISD::FADD, MVT::ppcf128, Hi,
6724                                       DAG.getConstantFP(
6725                                          APFloat(APInt(128, 2, TwoE64)),
6726                                          MVT::ppcf128)),
6727                           Hi,
6728                           DAG.getCondCode(ISD::SETLT)),
6729               Lo, Hi);
6730      break;
6731    }
6732
6733    Lo = ExpandIntToFP(Node->getOpcode() == ISD::SINT_TO_FP, VT,
6734                       Node->getOperand(0));
6735    if (getTypeAction(Lo.getValueType()) == Expand)
6736      // float to i32 etc. can be 'expanded' to a single node.
6737      ExpandOp(Lo, Lo, Hi);
6738    break;
6739  }
6740  }
6741
6742  // Make sure the resultant values have been legalized themselves, unless this
6743  // is a type that requires multi-step expansion.
6744  if (getTypeAction(NVT) != Expand && NVT != MVT::isVoid) {
6745    Lo = LegalizeOp(Lo);
6746    if (Hi.Val)
6747      // Don't legalize the high part if it is expanded to a single node.
6748      Hi = LegalizeOp(Hi);
6749  }
6750
6751  // Remember in a map if the values will be reused later.
6752  bool isNew =
6753    ExpandedNodes.insert(std::make_pair(Op, std::make_pair(Lo, Hi))).second;
6754  assert(isNew && "Value already expanded?!?");
6755}
6756
6757/// SplitVectorOp - Given an operand of vector type, break it down into
6758/// two smaller values, still of vector type.
6759void SelectionDAGLegalize::SplitVectorOp(SDOperand Op, SDOperand &Lo,
6760                                         SDOperand &Hi) {
6761  assert(Op.getValueType().isVector() && "Cannot split non-vector type!");
6762  SDNode *Node = Op.Val;
6763  unsigned NumElements = Op.getValueType().getVectorNumElements();
6764  assert(NumElements > 1 && "Cannot split a single element vector!");
6765
6766  MVT NewEltVT = Op.getValueType().getVectorElementType();
6767
6768  unsigned NewNumElts_Lo = 1 << Log2_32(NumElements-1);
6769  unsigned NewNumElts_Hi = NumElements - NewNumElts_Lo;
6770
6771  MVT NewVT_Lo = MVT::getVectorVT(NewEltVT, NewNumElts_Lo);
6772  MVT NewVT_Hi = MVT::getVectorVT(NewEltVT, NewNumElts_Hi);
6773
6774  // See if we already split it.
6775  std::map<SDOperand, std::pair<SDOperand, SDOperand> >::iterator I
6776    = SplitNodes.find(Op);
6777  if (I != SplitNodes.end()) {
6778    Lo = I->second.first;
6779    Hi = I->second.second;
6780    return;
6781  }
6782
6783  switch (Node->getOpcode()) {
6784  default:
6785#ifndef NDEBUG
6786    Node->dump(&DAG);
6787#endif
6788    assert(0 && "Unhandled operation in SplitVectorOp!");
6789  case ISD::UNDEF:
6790    Lo = DAG.getNode(ISD::UNDEF, NewVT_Lo);
6791    Hi = DAG.getNode(ISD::UNDEF, NewVT_Hi);
6792    break;
6793  case ISD::BUILD_PAIR:
6794    Lo = Node->getOperand(0);
6795    Hi = Node->getOperand(1);
6796    break;
6797  case ISD::INSERT_VECTOR_ELT: {
6798    if (ConstantSDNode *Idx = dyn_cast<ConstantSDNode>(Node->getOperand(2))) {
6799      SplitVectorOp(Node->getOperand(0), Lo, Hi);
6800      unsigned Index = Idx->getValue();
6801      SDOperand ScalarOp = Node->getOperand(1);
6802      if (Index < NewNumElts_Lo)
6803        Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, NewVT_Lo, Lo, ScalarOp,
6804                         DAG.getIntPtrConstant(Index));
6805      else
6806        Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, NewVT_Hi, Hi, ScalarOp,
6807                         DAG.getIntPtrConstant(Index - NewNumElts_Lo));
6808      break;
6809    }
6810    SDOperand Tmp = PerformInsertVectorEltInMemory(Node->getOperand(0),
6811                                                   Node->getOperand(1),
6812                                                   Node->getOperand(2));
6813    SplitVectorOp(Tmp, Lo, Hi);
6814    break;
6815  }
6816  case ISD::VECTOR_SHUFFLE: {
6817    // Build the low part.
6818    SDOperand Mask = Node->getOperand(2);
6819    SmallVector<SDOperand, 8> Ops;
6820    MVT PtrVT = TLI.getPointerTy();
6821
6822    // Insert all of the elements from the input that are needed.  We use
6823    // buildvector of extractelement here because the input vectors will have
6824    // to be legalized, so this makes the code simpler.
6825    for (unsigned i = 0; i != NewNumElts_Lo; ++i) {
6826      SDOperand IdxNode = Mask.getOperand(i);
6827      if (IdxNode.getOpcode() == ISD::UNDEF) {
6828        Ops.push_back(DAG.getNode(ISD::UNDEF, NewEltVT));
6829        continue;
6830      }
6831      unsigned Idx = cast<ConstantSDNode>(IdxNode)->getValue();
6832      SDOperand InVec = Node->getOperand(0);
6833      if (Idx >= NumElements) {
6834        InVec = Node->getOperand(1);
6835        Idx -= NumElements;
6836      }
6837      Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, NewEltVT, InVec,
6838                                DAG.getConstant(Idx, PtrVT)));
6839    }
6840    Lo = DAG.getNode(ISD::BUILD_VECTOR, NewVT_Lo, &Ops[0], Ops.size());
6841    Ops.clear();
6842
6843    for (unsigned i = NewNumElts_Lo; i != NumElements; ++i) {
6844      SDOperand IdxNode = Mask.getOperand(i);
6845      if (IdxNode.getOpcode() == ISD::UNDEF) {
6846        Ops.push_back(DAG.getNode(ISD::UNDEF, NewEltVT));
6847        continue;
6848      }
6849      unsigned Idx = cast<ConstantSDNode>(IdxNode)->getValue();
6850      SDOperand InVec = Node->getOperand(0);
6851      if (Idx >= NumElements) {
6852        InVec = Node->getOperand(1);
6853        Idx -= NumElements;
6854      }
6855      Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, NewEltVT, InVec,
6856                                DAG.getConstant(Idx, PtrVT)));
6857    }
6858    Hi = DAG.getNode(ISD::BUILD_VECTOR, NewVT_Lo, &Ops[0], Ops.size());
6859    break;
6860  }
6861  case ISD::BUILD_VECTOR: {
6862    SmallVector<SDOperand, 8> LoOps(Node->op_begin(),
6863                                    Node->op_begin()+NewNumElts_Lo);
6864    Lo = DAG.getNode(ISD::BUILD_VECTOR, NewVT_Lo, &LoOps[0], LoOps.size());
6865
6866    SmallVector<SDOperand, 8> HiOps(Node->op_begin()+NewNumElts_Lo,
6867                                    Node->op_end());
6868    Hi = DAG.getNode(ISD::BUILD_VECTOR, NewVT_Hi, &HiOps[0], HiOps.size());
6869    break;
6870  }
6871  case ISD::CONCAT_VECTORS: {
6872    // FIXME: Handle non-power-of-two vectors?
6873    unsigned NewNumSubvectors = Node->getNumOperands() / 2;
6874    if (NewNumSubvectors == 1) {
6875      Lo = Node->getOperand(0);
6876      Hi = Node->getOperand(1);
6877    } else {
6878      SmallVector<SDOperand, 8> LoOps(Node->op_begin(),
6879                                      Node->op_begin()+NewNumSubvectors);
6880      Lo = DAG.getNode(ISD::CONCAT_VECTORS, NewVT_Lo, &LoOps[0], LoOps.size());
6881
6882      SmallVector<SDOperand, 8> HiOps(Node->op_begin()+NewNumSubvectors,
6883                                      Node->op_end());
6884      Hi = DAG.getNode(ISD::CONCAT_VECTORS, NewVT_Hi, &HiOps[0], HiOps.size());
6885    }
6886    break;
6887  }
6888  case ISD::SELECT: {
6889    SDOperand Cond = Node->getOperand(0);
6890
6891    SDOperand LL, LH, RL, RH;
6892    SplitVectorOp(Node->getOperand(1), LL, LH);
6893    SplitVectorOp(Node->getOperand(2), RL, RH);
6894
6895    if (Cond.getValueType().isVector()) {
6896      // Handle a vector merge.
6897      SDOperand CL, CH;
6898      SplitVectorOp(Cond, CL, CH);
6899      Lo = DAG.getNode(Node->getOpcode(), NewVT_Lo, CL, LL, RL);
6900      Hi = DAG.getNode(Node->getOpcode(), NewVT_Hi, CH, LH, RH);
6901    } else {
6902      // Handle a simple select with vector operands.
6903      Lo = DAG.getNode(Node->getOpcode(), NewVT_Lo, Cond, LL, RL);
6904      Hi = DAG.getNode(Node->getOpcode(), NewVT_Hi, Cond, LH, RH);
6905    }
6906    break;
6907  }
6908  case ISD::SELECT_CC: {
6909    SDOperand CondLHS = Node->getOperand(0);
6910    SDOperand CondRHS = Node->getOperand(1);
6911    SDOperand CondCode = Node->getOperand(4);
6912
6913    SDOperand LL, LH, RL, RH;
6914    SplitVectorOp(Node->getOperand(2), LL, LH);
6915    SplitVectorOp(Node->getOperand(3), RL, RH);
6916
6917    // Handle a simple select with vector operands.
6918    Lo = DAG.getNode(ISD::SELECT_CC, NewVT_Lo, CondLHS, CondRHS,
6919                     LL, RL, CondCode);
6920    Hi = DAG.getNode(ISD::SELECT_CC, NewVT_Hi, CondLHS, CondRHS,
6921                     LH, RH, CondCode);
6922    break;
6923  }
6924  case ISD::VSETCC: {
6925    SDOperand LL, LH, RL, RH;
6926    SplitVectorOp(Node->getOperand(0), LL, LH);
6927    SplitVectorOp(Node->getOperand(1), RL, RH);
6928    Lo = DAG.getNode(ISD::VSETCC, NewVT_Lo, LL, RL, Node->getOperand(2));
6929    Hi = DAG.getNode(ISD::VSETCC, NewVT_Hi, LH, RH, Node->getOperand(2));
6930    break;
6931  }
6932  case ISD::ADD:
6933  case ISD::SUB:
6934  case ISD::MUL:
6935  case ISD::FADD:
6936  case ISD::FSUB:
6937  case ISD::FMUL:
6938  case ISD::SDIV:
6939  case ISD::UDIV:
6940  case ISD::FDIV:
6941  case ISD::FPOW:
6942  case ISD::AND:
6943  case ISD::OR:
6944  case ISD::XOR:
6945  case ISD::UREM:
6946  case ISD::SREM:
6947  case ISD::FREM: {
6948    SDOperand LL, LH, RL, RH;
6949    SplitVectorOp(Node->getOperand(0), LL, LH);
6950    SplitVectorOp(Node->getOperand(1), RL, RH);
6951
6952    Lo = DAG.getNode(Node->getOpcode(), NewVT_Lo, LL, RL);
6953    Hi = DAG.getNode(Node->getOpcode(), NewVT_Hi, LH, RH);
6954    break;
6955  }
6956  case ISD::FPOWI: {
6957    SDOperand L, H;
6958    SplitVectorOp(Node->getOperand(0), L, H);
6959
6960    Lo = DAG.getNode(Node->getOpcode(), NewVT_Lo, L, Node->getOperand(1));
6961    Hi = DAG.getNode(Node->getOpcode(), NewVT_Hi, H, Node->getOperand(1));
6962    break;
6963  }
6964  case ISD::CTTZ:
6965  case ISD::CTLZ:
6966  case ISD::CTPOP:
6967  case ISD::FNEG:
6968  case ISD::FABS:
6969  case ISD::FSQRT:
6970  case ISD::FSIN:
6971  case ISD::FCOS:
6972  case ISD::FP_TO_SINT:
6973  case ISD::FP_TO_UINT:
6974  case ISD::SINT_TO_FP:
6975  case ISD::UINT_TO_FP: {
6976    SDOperand L, H;
6977    SplitVectorOp(Node->getOperand(0), L, H);
6978
6979    Lo = DAG.getNode(Node->getOpcode(), NewVT_Lo, L);
6980    Hi = DAG.getNode(Node->getOpcode(), NewVT_Hi, H);
6981    break;
6982  }
6983  case ISD::LOAD: {
6984    LoadSDNode *LD = cast<LoadSDNode>(Node);
6985    SDOperand Ch = LD->getChain();
6986    SDOperand Ptr = LD->getBasePtr();
6987    const Value *SV = LD->getSrcValue();
6988    int SVOffset = LD->getSrcValueOffset();
6989    unsigned Alignment = LD->getAlignment();
6990    bool isVolatile = LD->isVolatile();
6991
6992    Lo = DAG.getLoad(NewVT_Lo, Ch, Ptr, SV, SVOffset, isVolatile, Alignment);
6993    unsigned IncrementSize = NewNumElts_Lo * NewEltVT.getSizeInBits()/8;
6994    Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
6995                      DAG.getIntPtrConstant(IncrementSize));
6996    SVOffset += IncrementSize;
6997    Alignment = MinAlign(Alignment, IncrementSize);
6998    Hi = DAG.getLoad(NewVT_Hi, Ch, Ptr, SV, SVOffset, isVolatile, Alignment);
6999
7000    // Build a factor node to remember that this load is independent of the
7001    // other one.
7002    SDOperand TF = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1),
7003                               Hi.getValue(1));
7004
7005    // Remember that we legalized the chain.
7006    AddLegalizedOperand(Op.getValue(1), LegalizeOp(TF));
7007    break;
7008  }
7009  case ISD::BIT_CONVERT: {
7010    // We know the result is a vector.  The input may be either a vector or a
7011    // scalar value.
7012    SDOperand InOp = Node->getOperand(0);
7013    if (!InOp.getValueType().isVector() ||
7014        InOp.getValueType().getVectorNumElements() == 1) {
7015      // The input is a scalar or single-element vector.
7016      // Lower to a store/load so that it can be split.
7017      // FIXME: this could be improved probably.
7018      unsigned LdAlign = TLI.getTargetData()->getPrefTypeAlignment(
7019                                            Op.getValueType().getTypeForMVT());
7020      SDOperand Ptr = DAG.CreateStackTemporary(InOp.getValueType(), LdAlign);
7021      int FI = cast<FrameIndexSDNode>(Ptr.Val)->getIndex();
7022
7023      SDOperand St = DAG.getStore(DAG.getEntryNode(),
7024                                  InOp, Ptr,
7025                                  PseudoSourceValue::getFixedStack(FI), 0);
7026      InOp = DAG.getLoad(Op.getValueType(), St, Ptr,
7027                         PseudoSourceValue::getFixedStack(FI), 0);
7028    }
7029    // Split the vector and convert each of the pieces now.
7030    SplitVectorOp(InOp, Lo, Hi);
7031    Lo = DAG.getNode(ISD::BIT_CONVERT, NewVT_Lo, Lo);
7032    Hi = DAG.getNode(ISD::BIT_CONVERT, NewVT_Hi, Hi);
7033    break;
7034  }
7035  }
7036
7037  // Remember in a map if the values will be reused later.
7038  bool isNew =
7039    SplitNodes.insert(std::make_pair(Op, std::make_pair(Lo, Hi))).second;
7040  assert(isNew && "Value already split?!?");
7041}
7042
7043
7044/// ScalarizeVectorOp - Given an operand of single-element vector type
7045/// (e.g. v1f32), convert it into the equivalent operation that returns a
7046/// scalar (e.g. f32) value.
7047SDOperand SelectionDAGLegalize::ScalarizeVectorOp(SDOperand Op) {
7048  assert(Op.getValueType().isVector() && "Bad ScalarizeVectorOp invocation!");
7049  SDNode *Node = Op.Val;
7050  MVT NewVT = Op.getValueType().getVectorElementType();
7051  assert(Op.getValueType().getVectorNumElements() == 1);
7052
7053  // See if we already scalarized it.
7054  std::map<SDOperand, SDOperand>::iterator I = ScalarizedNodes.find(Op);
7055  if (I != ScalarizedNodes.end()) return I->second;
7056
7057  SDOperand Result;
7058  switch (Node->getOpcode()) {
7059  default:
7060#ifndef NDEBUG
7061    Node->dump(&DAG); cerr << "\n";
7062#endif
7063    assert(0 && "Unknown vector operation in ScalarizeVectorOp!");
7064  case ISD::ADD:
7065  case ISD::FADD:
7066  case ISD::SUB:
7067  case ISD::FSUB:
7068  case ISD::MUL:
7069  case ISD::FMUL:
7070  case ISD::SDIV:
7071  case ISD::UDIV:
7072  case ISD::FDIV:
7073  case ISD::SREM:
7074  case ISD::UREM:
7075  case ISD::FREM:
7076  case ISD::FPOW:
7077  case ISD::AND:
7078  case ISD::OR:
7079  case ISD::XOR:
7080    Result = DAG.getNode(Node->getOpcode(),
7081                         NewVT,
7082                         ScalarizeVectorOp(Node->getOperand(0)),
7083                         ScalarizeVectorOp(Node->getOperand(1)));
7084    break;
7085  case ISD::FNEG:
7086  case ISD::FABS:
7087  case ISD::FSQRT:
7088  case ISD::FSIN:
7089  case ISD::FCOS:
7090    Result = DAG.getNode(Node->getOpcode(),
7091                         NewVT,
7092                         ScalarizeVectorOp(Node->getOperand(0)));
7093    break;
7094  case ISD::FPOWI:
7095    Result = DAG.getNode(Node->getOpcode(),
7096                         NewVT,
7097                         ScalarizeVectorOp(Node->getOperand(0)),
7098                         Node->getOperand(1));
7099    break;
7100  case ISD::LOAD: {
7101    LoadSDNode *LD = cast<LoadSDNode>(Node);
7102    SDOperand Ch = LegalizeOp(LD->getChain());     // Legalize the chain.
7103    SDOperand Ptr = LegalizeOp(LD->getBasePtr());  // Legalize the pointer.
7104
7105    const Value *SV = LD->getSrcValue();
7106    int SVOffset = LD->getSrcValueOffset();
7107    Result = DAG.getLoad(NewVT, Ch, Ptr, SV, SVOffset,
7108                         LD->isVolatile(), LD->getAlignment());
7109
7110    // Remember that we legalized the chain.
7111    AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1)));
7112    break;
7113  }
7114  case ISD::BUILD_VECTOR:
7115    Result = Node->getOperand(0);
7116    break;
7117  case ISD::INSERT_VECTOR_ELT:
7118    // Returning the inserted scalar element.
7119    Result = Node->getOperand(1);
7120    break;
7121  case ISD::CONCAT_VECTORS:
7122    assert(Node->getOperand(0).getValueType() == NewVT &&
7123           "Concat of non-legal vectors not yet supported!");
7124    Result = Node->getOperand(0);
7125    break;
7126  case ISD::VECTOR_SHUFFLE: {
7127    // Figure out if the scalar is the LHS or RHS and return it.
7128    SDOperand EltNum = Node->getOperand(2).getOperand(0);
7129    if (cast<ConstantSDNode>(EltNum)->getValue())
7130      Result = ScalarizeVectorOp(Node->getOperand(1));
7131    else
7132      Result = ScalarizeVectorOp(Node->getOperand(0));
7133    break;
7134  }
7135  case ISD::EXTRACT_SUBVECTOR:
7136    Result = Node->getOperand(0);
7137    assert(Result.getValueType() == NewVT);
7138    break;
7139  case ISD::BIT_CONVERT: {
7140    SDOperand Op0 = Op.getOperand(0);
7141    if (Op0.getValueType().getVectorNumElements() == 1)
7142      Op0 = ScalarizeVectorOp(Op0);
7143    Result = DAG.getNode(ISD::BIT_CONVERT, NewVT, Op0);
7144    break;
7145  }
7146  case ISD::SELECT:
7147    Result = DAG.getNode(ISD::SELECT, NewVT, Op.getOperand(0),
7148                         ScalarizeVectorOp(Op.getOperand(1)),
7149                         ScalarizeVectorOp(Op.getOperand(2)));
7150    break;
7151  case ISD::SELECT_CC:
7152    Result = DAG.getNode(ISD::SELECT_CC, NewVT, Node->getOperand(0),
7153                         Node->getOperand(1),
7154                         ScalarizeVectorOp(Op.getOperand(2)),
7155                         ScalarizeVectorOp(Op.getOperand(3)),
7156                         Node->getOperand(4));
7157    break;
7158  case ISD::VSETCC: {
7159    SDOperand Op0 = ScalarizeVectorOp(Op.getOperand(0));
7160    SDOperand Op1 = ScalarizeVectorOp(Op.getOperand(1));
7161    Result = DAG.getNode(ISD::SETCC, TLI.getSetCCResultType(Op0), Op0, Op1,
7162                         Op.getOperand(2));
7163    Result = DAG.getNode(ISD::SELECT, NewVT, Result,
7164                         DAG.getConstant(-1ULL, NewVT),
7165                         DAG.getConstant(0ULL, NewVT));
7166    break;
7167  }
7168  }
7169
7170  if (TLI.isTypeLegal(NewVT))
7171    Result = LegalizeOp(Result);
7172  bool isNew = ScalarizedNodes.insert(std::make_pair(Op, Result)).second;
7173  assert(isNew && "Value already scalarized?");
7174  return Result;
7175}
7176
7177
7178// SelectionDAG::Legalize - This is the entry point for the file.
7179//
7180void SelectionDAG::Legalize() {
7181  if (ViewLegalizeDAGs) viewGraph();
7182
7183  /// run - This is the main entry point to this class.
7184  ///
7185  SelectionDAGLegalize(*this).LegalizeDAG();
7186}
7187
7188