LegalizeDAG.cpp revision f411b83c8c6853c2a922b692e782566353153f08
1//===-- LegalizeDAG.cpp - Implement SelectionDAG::Legalize ----------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file was developed by the LLVM research group and is distributed under
6// the University of Illinois Open Source License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the SelectionDAG::Legalize method.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/CodeGen/SelectionDAG.h"
15#include "llvm/CodeGen/MachineFunction.h"
16#include "llvm/CodeGen/MachineFrameInfo.h"
17#include "llvm/CodeGen/MachineJumpTableInfo.h"
18#include "llvm/Target/TargetFrameInfo.h"
19#include "llvm/Target/TargetLowering.h"
20#include "llvm/Target/TargetData.h"
21#include "llvm/Target/TargetMachine.h"
22#include "llvm/Target/TargetOptions.h"
23#include "llvm/CallingConv.h"
24#include "llvm/Constants.h"
25#include "llvm/DerivedTypes.h"
26#include "llvm/Support/MathExtras.h"
27#include "llvm/Support/CommandLine.h"
28#include "llvm/Support/Compiler.h"
29#include "llvm/ADT/DenseMap.h"
30#include "llvm/ADT/SmallVector.h"
31#include "llvm/ADT/SmallPtrSet.h"
32#include <map>
33using namespace llvm;
34
35#ifndef NDEBUG
36static cl::opt<bool>
37ViewLegalizeDAGs("view-legalize-dags", cl::Hidden,
38                 cl::desc("Pop up a window to show dags before legalize"));
39#else
40static const bool ViewLegalizeDAGs = 0;
41#endif
42
43//===----------------------------------------------------------------------===//
44/// SelectionDAGLegalize - This takes an arbitrary SelectionDAG as input and
45/// hacks on it until the target machine can handle it.  This involves
46/// eliminating value sizes the machine cannot handle (promoting small sizes to
47/// large sizes or splitting up large values into small values) as well as
48/// eliminating operations the machine cannot handle.
49///
50/// This code also does a small amount of optimization and recognition of idioms
51/// as part of its processing.  For example, if a target does not support a
52/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
53/// will attempt merge setcc and brc instructions into brcc's.
54///
55namespace {
56class VISIBILITY_HIDDEN SelectionDAGLegalize {
57  TargetLowering &TLI;
58  SelectionDAG &DAG;
59
60  // Libcall insertion helpers.
61
62  /// LastCALLSEQ_END - This keeps track of the CALLSEQ_END node that has been
63  /// legalized.  We use this to ensure that calls are properly serialized
64  /// against each other, including inserted libcalls.
65  SDOperand LastCALLSEQ_END;
66
67  /// IsLegalizingCall - This member is used *only* for purposes of providing
68  /// helpful assertions that a libcall isn't created while another call is
69  /// being legalized (which could lead to non-serialized call sequences).
70  bool IsLegalizingCall;
71
72  enum LegalizeAction {
73    Legal,      // The target natively supports this operation.
74    Promote,    // This operation should be executed in a larger type.
75    Expand      // Try to expand this to other ops, otherwise use a libcall.
76  };
77
78  /// ValueTypeActions - This is a bitvector that contains two bits for each
79  /// value type, where the two bits correspond to the LegalizeAction enum.
80  /// This can be queried with "getTypeAction(VT)".
81  TargetLowering::ValueTypeActionImpl ValueTypeActions;
82
83  /// LegalizedNodes - For nodes that are of legal width, and that have more
84  /// than one use, this map indicates what regularized operand to use.  This
85  /// allows us to avoid legalizing the same thing more than once.
86  DenseMap<SDOperand, SDOperand> LegalizedNodes;
87
88  /// PromotedNodes - For nodes that are below legal width, and that have more
89  /// than one use, this map indicates what promoted value to use.  This allows
90  /// us to avoid promoting the same thing more than once.
91  DenseMap<SDOperand, SDOperand> PromotedNodes;
92
93  /// ExpandedNodes - For nodes that need to be expanded this map indicates
94  /// which which operands are the expanded version of the input.  This allows
95  /// us to avoid expanding the same node more than once.
96  DenseMap<SDOperand, std::pair<SDOperand, SDOperand> > ExpandedNodes;
97
98  /// SplitNodes - For vector nodes that need to be split, this map indicates
99  /// which which operands are the split version of the input.  This allows us
100  /// to avoid splitting the same node more than once.
101  std::map<SDOperand, std::pair<SDOperand, SDOperand> > SplitNodes;
102
103  /// ScalarizedNodes - For nodes that need to be converted from vector types to
104  /// scalar types, this contains the mapping of ones we have already
105  /// processed to the result.
106  std::map<SDOperand, SDOperand> ScalarizedNodes;
107
108  void AddLegalizedOperand(SDOperand From, SDOperand To) {
109    LegalizedNodes.insert(std::make_pair(From, To));
110    // If someone requests legalization of the new node, return itself.
111    if (From != To)
112      LegalizedNodes.insert(std::make_pair(To, To));
113  }
114  void AddPromotedOperand(SDOperand From, SDOperand To) {
115    bool isNew = PromotedNodes.insert(std::make_pair(From, To));
116    assert(isNew && "Got into the map somehow?");
117    // If someone requests legalization of the new node, return itself.
118    LegalizedNodes.insert(std::make_pair(To, To));
119  }
120
121public:
122
123  SelectionDAGLegalize(SelectionDAG &DAG);
124
125  /// getTypeAction - Return how we should legalize values of this type, either
126  /// it is already legal or we need to expand it into multiple registers of
127  /// smaller integer type, or we need to promote it to a larger type.
128  LegalizeAction getTypeAction(MVT::ValueType VT) const {
129    return (LegalizeAction)ValueTypeActions.getTypeAction(VT);
130  }
131
132  /// isTypeLegal - Return true if this type is legal on this target.
133  ///
134  bool isTypeLegal(MVT::ValueType VT) const {
135    return getTypeAction(VT) == Legal;
136  }
137
138  void LegalizeDAG();
139
140private:
141  /// HandleOp - Legalize, Promote, or Expand the specified operand as
142  /// appropriate for its type.
143  void HandleOp(SDOperand Op);
144
145  /// LegalizeOp - We know that the specified value has a legal type.
146  /// Recursively ensure that the operands have legal types, then return the
147  /// result.
148  SDOperand LegalizeOp(SDOperand O);
149
150  /// UnrollVectorOp - We know that the given vector has a legal type, however
151  /// the operation it performs is not legal and is an operation that we have
152  /// no way of lowering.  "Unroll" the vector, splitting out the scalars and
153  /// operating on each element individually.
154  SDOperand UnrollVectorOp(SDOperand O);
155
156  /// PromoteOp - Given an operation that produces a value in an invalid type,
157  /// promote it to compute the value into a larger type.  The produced value
158  /// will have the correct bits for the low portion of the register, but no
159  /// guarantee is made about the top bits: it may be zero, sign-extended, or
160  /// garbage.
161  SDOperand PromoteOp(SDOperand O);
162
163  /// ExpandOp - Expand the specified SDOperand into its two component pieces
164  /// Lo&Hi.  Note that the Op MUST be an expanded type.  As a result of this,
165  /// the LegalizeNodes map is filled in for any results that are not expanded,
166  /// the ExpandedNodes map is filled in for any results that are expanded, and
167  /// the Lo/Hi values are returned.   This applies to integer types and Vector
168  /// types.
169  void ExpandOp(SDOperand O, SDOperand &Lo, SDOperand &Hi);
170
171  /// SplitVectorOp - Given an operand of vector type, break it down into
172  /// two smaller values.
173  void SplitVectorOp(SDOperand O, SDOperand &Lo, SDOperand &Hi);
174
175  /// ScalarizeVectorOp - Given an operand of single-element vector type
176  /// (e.g. v1f32), convert it into the equivalent operation that returns a
177  /// scalar (e.g. f32) value.
178  SDOperand ScalarizeVectorOp(SDOperand O);
179
180  /// isShuffleLegal - Return true if a vector shuffle is legal with the
181  /// specified mask and type.  Targets can specify exactly which masks they
182  /// support and the code generator is tasked with not creating illegal masks.
183  ///
184  /// Note that this will also return true for shuffles that are promoted to a
185  /// different type.
186  ///
187  /// If this is a legal shuffle, this method returns the (possibly promoted)
188  /// build_vector Mask.  If it's not a legal shuffle, it returns null.
189  SDNode *isShuffleLegal(MVT::ValueType VT, SDOperand Mask) const;
190
191  bool LegalizeAllNodesNotLeadingTo(SDNode *N, SDNode *Dest,
192                                    SmallPtrSet<SDNode*, 32> &NodesLeadingTo);
193
194  void LegalizeSetCCOperands(SDOperand &LHS, SDOperand &RHS, SDOperand &CC);
195
196  SDOperand ExpandLibCall(const char *Name, SDNode *Node, bool isSigned,
197                          SDOperand &Hi);
198  SDOperand ExpandIntToFP(bool isSigned, MVT::ValueType DestTy,
199                          SDOperand Source);
200
201  SDOperand ExpandBIT_CONVERT(MVT::ValueType DestVT, SDOperand SrcOp);
202  SDOperand ExpandBUILD_VECTOR(SDNode *Node);
203  SDOperand ExpandSCALAR_TO_VECTOR(SDNode *Node);
204  SDOperand ExpandLegalINT_TO_FP(bool isSigned,
205                                 SDOperand LegalOp,
206                                 MVT::ValueType DestVT);
207  SDOperand PromoteLegalINT_TO_FP(SDOperand LegalOp, MVT::ValueType DestVT,
208                                  bool isSigned);
209  SDOperand PromoteLegalFP_TO_INT(SDOperand LegalOp, MVT::ValueType DestVT,
210                                  bool isSigned);
211
212  SDOperand ExpandBSWAP(SDOperand Op);
213  SDOperand ExpandBitCount(unsigned Opc, SDOperand Op);
214  bool ExpandShift(unsigned Opc, SDOperand Op, SDOperand Amt,
215                   SDOperand &Lo, SDOperand &Hi);
216  void ExpandShiftParts(unsigned NodeOp, SDOperand Op, SDOperand Amt,
217                        SDOperand &Lo, SDOperand &Hi);
218
219  SDOperand ExpandEXTRACT_SUBVECTOR(SDOperand Op);
220  SDOperand ExpandEXTRACT_VECTOR_ELT(SDOperand Op);
221
222  SDOperand getIntPtrConstant(uint64_t Val) {
223    return DAG.getConstant(Val, TLI.getPointerTy());
224  }
225};
226}
227
228/// isVectorShuffleLegal - Return true if a vector shuffle is legal with the
229/// specified mask and type.  Targets can specify exactly which masks they
230/// support and the code generator is tasked with not creating illegal masks.
231///
232/// Note that this will also return true for shuffles that are promoted to a
233/// different type.
234SDNode *SelectionDAGLegalize::isShuffleLegal(MVT::ValueType VT,
235                                             SDOperand Mask) const {
236  switch (TLI.getOperationAction(ISD::VECTOR_SHUFFLE, VT)) {
237  default: return 0;
238  case TargetLowering::Legal:
239  case TargetLowering::Custom:
240    break;
241  case TargetLowering::Promote: {
242    // If this is promoted to a different type, convert the shuffle mask and
243    // ask if it is legal in the promoted type!
244    MVT::ValueType NVT = TLI.getTypeToPromoteTo(ISD::VECTOR_SHUFFLE, VT);
245
246    // If we changed # elements, change the shuffle mask.
247    unsigned NumEltsGrowth =
248      MVT::getVectorNumElements(NVT) / MVT::getVectorNumElements(VT);
249    assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
250    if (NumEltsGrowth > 1) {
251      // Renumber the elements.
252      SmallVector<SDOperand, 8> Ops;
253      for (unsigned i = 0, e = Mask.getNumOperands(); i != e; ++i) {
254        SDOperand InOp = Mask.getOperand(i);
255        for (unsigned j = 0; j != NumEltsGrowth; ++j) {
256          if (InOp.getOpcode() == ISD::UNDEF)
257            Ops.push_back(DAG.getNode(ISD::UNDEF, MVT::i32));
258          else {
259            unsigned InEltNo = cast<ConstantSDNode>(InOp)->getValue();
260            Ops.push_back(DAG.getConstant(InEltNo*NumEltsGrowth+j, MVT::i32));
261          }
262        }
263      }
264      Mask = DAG.getNode(ISD::BUILD_VECTOR, NVT, &Ops[0], Ops.size());
265    }
266    VT = NVT;
267    break;
268  }
269  }
270  return TLI.isShuffleMaskLegal(Mask, VT) ? Mask.Val : 0;
271}
272
273SelectionDAGLegalize::SelectionDAGLegalize(SelectionDAG &dag)
274  : TLI(dag.getTargetLoweringInfo()), DAG(dag),
275    ValueTypeActions(TLI.getValueTypeActions()) {
276  assert(MVT::LAST_VALUETYPE <= 32 &&
277         "Too many value types for ValueTypeActions to hold!");
278}
279
280/// ComputeTopDownOrdering - Compute a top-down ordering of the dag, where Order
281/// contains all of a nodes operands before it contains the node.
282static void ComputeTopDownOrdering(SelectionDAG &DAG,
283                                   SmallVector<SDNode*, 64> &Order) {
284
285  DenseMap<SDNode*, unsigned> Visited;
286  std::vector<SDNode*> Worklist;
287  Worklist.reserve(128);
288
289  // Compute ordering from all of the leaves in the graphs, those (like the
290  // entry node) that have no operands.
291  for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
292       E = DAG.allnodes_end(); I != E; ++I) {
293    if (I->getNumOperands() == 0) {
294      Visited[I] = 0 - 1U;
295      Worklist.push_back(I);
296    }
297  }
298
299  while (!Worklist.empty()) {
300    SDNode *N = Worklist.back();
301    Worklist.pop_back();
302
303    if (++Visited[N] != N->getNumOperands())
304      continue;  // Haven't visited all operands yet
305
306    Order.push_back(N);
307
308    // Now that we have N in, add anything that uses it if all of their operands
309    // are now done.
310    for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end();
311         UI != E; ++UI)
312      Worklist.push_back(*UI);
313  }
314
315  assert(Order.size() == Visited.size() &&
316         Order.size() ==
317         (unsigned)std::distance(DAG.allnodes_begin(), DAG.allnodes_end()) &&
318         "Error: DAG is cyclic!");
319}
320
321
322void SelectionDAGLegalize::LegalizeDAG() {
323  LastCALLSEQ_END = DAG.getEntryNode();
324  IsLegalizingCall = false;
325
326  // The legalize process is inherently a bottom-up recursive process (users
327  // legalize their uses before themselves).  Given infinite stack space, we
328  // could just start legalizing on the root and traverse the whole graph.  In
329  // practice however, this causes us to run out of stack space on large basic
330  // blocks.  To avoid this problem, compute an ordering of the nodes where each
331  // node is only legalized after all of its operands are legalized.
332  SmallVector<SDNode*, 64> Order;
333  ComputeTopDownOrdering(DAG, Order);
334
335  for (unsigned i = 0, e = Order.size(); i != e; ++i)
336    HandleOp(SDOperand(Order[i], 0));
337
338  // Finally, it's possible the root changed.  Get the new root.
339  SDOperand OldRoot = DAG.getRoot();
340  assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?");
341  DAG.setRoot(LegalizedNodes[OldRoot]);
342
343  ExpandedNodes.clear();
344  LegalizedNodes.clear();
345  PromotedNodes.clear();
346  SplitNodes.clear();
347  ScalarizedNodes.clear();
348
349  // Remove dead nodes now.
350  DAG.RemoveDeadNodes();
351}
352
353
354/// FindCallEndFromCallStart - Given a chained node that is part of a call
355/// sequence, find the CALLSEQ_END node that terminates the call sequence.
356static SDNode *FindCallEndFromCallStart(SDNode *Node) {
357  if (Node->getOpcode() == ISD::CALLSEQ_END)
358    return Node;
359  if (Node->use_empty())
360    return 0;   // No CallSeqEnd
361
362  // The chain is usually at the end.
363  SDOperand TheChain(Node, Node->getNumValues()-1);
364  if (TheChain.getValueType() != MVT::Other) {
365    // Sometimes it's at the beginning.
366    TheChain = SDOperand(Node, 0);
367    if (TheChain.getValueType() != MVT::Other) {
368      // Otherwise, hunt for it.
369      for (unsigned i = 1, e = Node->getNumValues(); i != e; ++i)
370        if (Node->getValueType(i) == MVT::Other) {
371          TheChain = SDOperand(Node, i);
372          break;
373        }
374
375      // Otherwise, we walked into a node without a chain.
376      if (TheChain.getValueType() != MVT::Other)
377        return 0;
378    }
379  }
380
381  for (SDNode::use_iterator UI = Node->use_begin(),
382       E = Node->use_end(); UI != E; ++UI) {
383
384    // Make sure to only follow users of our token chain.
385    SDNode *User = *UI;
386    for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i)
387      if (User->getOperand(i) == TheChain)
388        if (SDNode *Result = FindCallEndFromCallStart(User))
389          return Result;
390  }
391  return 0;
392}
393
394/// FindCallStartFromCallEnd - Given a chained node that is part of a call
395/// sequence, find the CALLSEQ_START node that initiates the call sequence.
396static SDNode *FindCallStartFromCallEnd(SDNode *Node) {
397  assert(Node && "Didn't find callseq_start for a call??");
398  if (Node->getOpcode() == ISD::CALLSEQ_START) return Node;
399
400  assert(Node->getOperand(0).getValueType() == MVT::Other &&
401         "Node doesn't have a token chain argument!");
402  return FindCallStartFromCallEnd(Node->getOperand(0).Val);
403}
404
405/// LegalizeAllNodesNotLeadingTo - Recursively walk the uses of N, looking to
406/// see if any uses can reach Dest.  If no dest operands can get to dest,
407/// legalize them, legalize ourself, and return false, otherwise, return true.
408///
409/// Keep track of the nodes we fine that actually do lead to Dest in
410/// NodesLeadingTo.  This avoids retraversing them exponential number of times.
411///
412bool SelectionDAGLegalize::LegalizeAllNodesNotLeadingTo(SDNode *N, SDNode *Dest,
413                                     SmallPtrSet<SDNode*, 32> &NodesLeadingTo) {
414  if (N == Dest) return true;  // N certainly leads to Dest :)
415
416  // If we've already processed this node and it does lead to Dest, there is no
417  // need to reprocess it.
418  if (NodesLeadingTo.count(N)) return true;
419
420  // If the first result of this node has been already legalized, then it cannot
421  // reach N.
422  switch (getTypeAction(N->getValueType(0))) {
423  case Legal:
424    if (LegalizedNodes.count(SDOperand(N, 0))) return false;
425    break;
426  case Promote:
427    if (PromotedNodes.count(SDOperand(N, 0))) return false;
428    break;
429  case Expand:
430    if (ExpandedNodes.count(SDOperand(N, 0))) return false;
431    break;
432  }
433
434  // Okay, this node has not already been legalized.  Check and legalize all
435  // operands.  If none lead to Dest, then we can legalize this node.
436  bool OperandsLeadToDest = false;
437  for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
438    OperandsLeadToDest |=     // If an operand leads to Dest, so do we.
439      LegalizeAllNodesNotLeadingTo(N->getOperand(i).Val, Dest, NodesLeadingTo);
440
441  if (OperandsLeadToDest) {
442    NodesLeadingTo.insert(N);
443    return true;
444  }
445
446  // Okay, this node looks safe, legalize it and return false.
447  HandleOp(SDOperand(N, 0));
448  return false;
449}
450
451/// HandleOp - Legalize, Promote, or Expand the specified operand as
452/// appropriate for its type.
453void SelectionDAGLegalize::HandleOp(SDOperand Op) {
454  MVT::ValueType VT = Op.getValueType();
455  switch (getTypeAction(VT)) {
456  default: assert(0 && "Bad type action!");
457  case Legal:   (void)LegalizeOp(Op); break;
458  case Promote: (void)PromoteOp(Op); break;
459  case Expand:
460    if (!MVT::isVector(VT)) {
461      // If this is an illegal scalar, expand it into its two component
462      // pieces.
463      SDOperand X, Y;
464      if (Op.getOpcode() == ISD::TargetConstant)
465        break;  // Allow illegal target nodes.
466      ExpandOp(Op, X, Y);
467    } else if (MVT::getVectorNumElements(VT) == 1) {
468      // If this is an illegal single element vector, convert it to a
469      // scalar operation.
470      (void)ScalarizeVectorOp(Op);
471    } else {
472      // Otherwise, this is an illegal multiple element vector.
473      // Split it in half and legalize both parts.
474      SDOperand X, Y;
475      SplitVectorOp(Op, X, Y);
476    }
477    break;
478  }
479}
480
481/// ExpandConstantFP - Expands the ConstantFP node to an integer constant or
482/// a load from the constant pool.
483static SDOperand ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP,
484                                  SelectionDAG &DAG, TargetLowering &TLI) {
485  bool Extend = false;
486
487  // If a FP immediate is precise when represented as a float and if the
488  // target can do an extending load from float to double, we put it into
489  // the constant pool as a float, even if it's is statically typed as a
490  // double.
491  MVT::ValueType VT = CFP->getValueType(0);
492  bool isDouble = VT == MVT::f64;
493  ConstantFP *LLVMC = ConstantFP::get(MVT::getTypeForValueType(VT),
494                                      CFP->getValueAPF());
495  if (!UseCP) {
496    if (VT!=MVT::f64 && VT!=MVT::f32)
497      assert(0 && "Invalid type expansion");
498    return DAG.getConstant(LLVMC->getValueAPF().convertToAPInt().getZExtValue(),
499                           isDouble ? MVT::i64 : MVT::i32);
500  }
501
502  if (isDouble && CFP->isValueValidForType(MVT::f32, CFP->getValueAPF()) &&
503      // Only do this if the target has a native EXTLOAD instruction from f32.
504      // Do not try to be clever about long doubles (so far)
505      TLI.isLoadXLegal(ISD::EXTLOAD, MVT::f32)) {
506    LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC,Type::FloatTy));
507    VT = MVT::f32;
508    Extend = true;
509  }
510
511  SDOperand CPIdx = DAG.getConstantPool(LLVMC, TLI.getPointerTy());
512  if (Extend) {
513    return DAG.getExtLoad(ISD::EXTLOAD, MVT::f64, DAG.getEntryNode(),
514                          CPIdx, NULL, 0, MVT::f32);
515  } else {
516    return DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, NULL, 0);
517  }
518}
519
520
521/// ExpandFCOPYSIGNToBitwiseOps - Expands fcopysign to a series of bitwise
522/// operations.
523static
524SDOperand ExpandFCOPYSIGNToBitwiseOps(SDNode *Node, MVT::ValueType NVT,
525                                      SelectionDAG &DAG, TargetLowering &TLI) {
526  MVT::ValueType VT = Node->getValueType(0);
527  MVT::ValueType SrcVT = Node->getOperand(1).getValueType();
528  assert((SrcVT == MVT::f32 || SrcVT == MVT::f64) &&
529         "fcopysign expansion only supported for f32 and f64");
530  MVT::ValueType SrcNVT = (SrcVT == MVT::f64) ? MVT::i64 : MVT::i32;
531
532  // First get the sign bit of second operand.
533  SDOperand Mask1 = (SrcVT == MVT::f64)
534    ? DAG.getConstantFP(BitsToDouble(1ULL << 63), SrcVT)
535    : DAG.getConstantFP(BitsToFloat(1U << 31), SrcVT);
536  Mask1 = DAG.getNode(ISD::BIT_CONVERT, SrcNVT, Mask1);
537  SDOperand SignBit= DAG.getNode(ISD::BIT_CONVERT, SrcNVT, Node->getOperand(1));
538  SignBit = DAG.getNode(ISD::AND, SrcNVT, SignBit, Mask1);
539  // Shift right or sign-extend it if the two operands have different types.
540  int SizeDiff = MVT::getSizeInBits(SrcNVT) - MVT::getSizeInBits(NVT);
541  if (SizeDiff > 0) {
542    SignBit = DAG.getNode(ISD::SRL, SrcNVT, SignBit,
543                          DAG.getConstant(SizeDiff, TLI.getShiftAmountTy()));
544    SignBit = DAG.getNode(ISD::TRUNCATE, NVT, SignBit);
545  } else if (SizeDiff < 0)
546    SignBit = DAG.getNode(ISD::SIGN_EXTEND, NVT, SignBit);
547
548  // Clear the sign bit of first operand.
549  SDOperand Mask2 = (VT == MVT::f64)
550    ? DAG.getConstantFP(BitsToDouble(~(1ULL << 63)), VT)
551    : DAG.getConstantFP(BitsToFloat(~(1U << 31)), VT);
552  Mask2 = DAG.getNode(ISD::BIT_CONVERT, NVT, Mask2);
553  SDOperand Result = DAG.getNode(ISD::BIT_CONVERT, NVT, Node->getOperand(0));
554  Result = DAG.getNode(ISD::AND, NVT, Result, Mask2);
555
556  // Or the value with the sign bit.
557  Result = DAG.getNode(ISD::OR, NVT, Result, SignBit);
558  return Result;
559}
560
561/// ExpandUnalignedStore - Expands an unaligned store to 2 half-size stores.
562static
563SDOperand ExpandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG,
564                               TargetLowering &TLI) {
565  SDOperand Chain = ST->getChain();
566  SDOperand Ptr = ST->getBasePtr();
567  SDOperand Val = ST->getValue();
568  MVT::ValueType VT = Val.getValueType();
569  int Alignment = ST->getAlignment();
570  int SVOffset = ST->getSrcValueOffset();
571  if (MVT::isFloatingPoint(ST->getStoredVT())) {
572    // Expand to a bitconvert of the value to the integer type of the
573    // same size, then a (misaligned) int store.
574    MVT::ValueType intVT;
575    if (VT==MVT::f64)
576      intVT = MVT::i64;
577    else if (VT==MVT::f32)
578      intVT = MVT::i32;
579    else
580      assert(0 && "Unaligned load of unsupported floating point type");
581
582    SDOperand Result = DAG.getNode(ISD::BIT_CONVERT, intVT, Val);
583    return DAG.getStore(Chain, Result, Ptr, ST->getSrcValue(),
584                        SVOffset, ST->isVolatile(), Alignment);
585  }
586  assert(MVT::isInteger(ST->getStoredVT()) &&
587         "Unaligned store of unknown type.");
588  // Get the half-size VT
589  MVT::ValueType NewStoredVT = ST->getStoredVT() - 1;
590  int NumBits = MVT::getSizeInBits(NewStoredVT);
591  int IncrementSize = NumBits / 8;
592
593  // Divide the stored value in two parts.
594  SDOperand ShiftAmount = DAG.getConstant(NumBits, TLI.getShiftAmountTy());
595  SDOperand Lo = Val;
596  SDOperand Hi = DAG.getNode(ISD::SRL, VT, Val, ShiftAmount);
597
598  // Store the two parts
599  SDOperand Store1, Store2;
600  Store1 = DAG.getTruncStore(Chain, TLI.isLittleEndian()?Lo:Hi, Ptr,
601                             ST->getSrcValue(), SVOffset, NewStoredVT,
602                             ST->isVolatile(), Alignment);
603  Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
604                    DAG.getConstant(IncrementSize, TLI.getPointerTy()));
605  Store2 = DAG.getTruncStore(Chain, TLI.isLittleEndian()?Hi:Lo, Ptr,
606                             ST->getSrcValue(), SVOffset + IncrementSize,
607                             NewStoredVT, ST->isVolatile(), Alignment);
608
609  return DAG.getNode(ISD::TokenFactor, MVT::Other, Store1, Store2);
610}
611
612/// ExpandUnalignedLoad - Expands an unaligned load to 2 half-size loads.
613static
614SDOperand ExpandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG,
615                              TargetLowering &TLI) {
616  int SVOffset = LD->getSrcValueOffset();
617  SDOperand Chain = LD->getChain();
618  SDOperand Ptr = LD->getBasePtr();
619  MVT::ValueType VT = LD->getValueType(0);
620  MVT::ValueType LoadedVT = LD->getLoadedVT();
621  if (MVT::isFloatingPoint(VT)) {
622    // Expand to a (misaligned) integer load of the same size,
623    // then bitconvert to floating point.
624    MVT::ValueType intVT;
625    if (LoadedVT==MVT::f64)
626      intVT = MVT::i64;
627    else if (LoadedVT==MVT::f32)
628      intVT = MVT::i32;
629    else
630      assert(0 && "Unaligned load of unsupported floating point type");
631
632    SDOperand newLoad = DAG.getLoad(intVT, Chain, Ptr, LD->getSrcValue(),
633                                    SVOffset, LD->isVolatile(),
634                                    LD->getAlignment());
635    SDOperand Result = DAG.getNode(ISD::BIT_CONVERT, LoadedVT, newLoad);
636    if (LoadedVT != VT)
637      Result = DAG.getNode(ISD::FP_EXTEND, VT, Result);
638
639    SDOperand Ops[] = { Result, Chain };
640    return DAG.getNode(ISD::MERGE_VALUES, DAG.getVTList(VT, MVT::Other),
641                       Ops, 2);
642  }
643  assert(MVT::isInteger(LoadedVT) && "Unaligned load of unsupported type.");
644  MVT::ValueType NewLoadedVT = LoadedVT - 1;
645  int NumBits = MVT::getSizeInBits(NewLoadedVT);
646  int Alignment = LD->getAlignment();
647  int IncrementSize = NumBits / 8;
648  ISD::LoadExtType HiExtType = LD->getExtensionType();
649
650  // If the original load is NON_EXTLOAD, the hi part load must be ZEXTLOAD.
651  if (HiExtType == ISD::NON_EXTLOAD)
652    HiExtType = ISD::ZEXTLOAD;
653
654  // Load the value in two parts
655  SDOperand Lo, Hi;
656  if (TLI.isLittleEndian()) {
657    Lo = DAG.getExtLoad(ISD::ZEXTLOAD, VT, Chain, Ptr, LD->getSrcValue(),
658                        SVOffset, NewLoadedVT, LD->isVolatile(), Alignment);
659    Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
660                      DAG.getConstant(IncrementSize, TLI.getPointerTy()));
661    Hi = DAG.getExtLoad(HiExtType, VT, Chain, Ptr, LD->getSrcValue(),
662                        SVOffset + IncrementSize, NewLoadedVT, LD->isVolatile(),
663                        Alignment);
664  } else {
665    Hi = DAG.getExtLoad(HiExtType, VT, Chain, Ptr, LD->getSrcValue(), SVOffset,
666                        NewLoadedVT,LD->isVolatile(), Alignment);
667    Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
668                      DAG.getConstant(IncrementSize, TLI.getPointerTy()));
669    Lo = DAG.getExtLoad(ISD::ZEXTLOAD, VT, Chain, Ptr, LD->getSrcValue(),
670                        SVOffset + IncrementSize, NewLoadedVT, LD->isVolatile(),
671                        Alignment);
672  }
673
674  // aggregate the two parts
675  SDOperand ShiftAmount = DAG.getConstant(NumBits, TLI.getShiftAmountTy());
676  SDOperand Result = DAG.getNode(ISD::SHL, VT, Hi, ShiftAmount);
677  Result = DAG.getNode(ISD::OR, VT, Result, Lo);
678
679  SDOperand TF = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1),
680                             Hi.getValue(1));
681
682  SDOperand Ops[] = { Result, TF };
683  return DAG.getNode(ISD::MERGE_VALUES, DAG.getVTList(VT, MVT::Other), Ops, 2);
684}
685
686/// UnrollVectorOp - We know that the given vector has a legal type, however
687/// the operation it performs is not legal and is an operation that we have
688/// no way of lowering.  "Unroll" the vector, splitting out the scalars and
689/// operating on each element individually.
690SDOperand SelectionDAGLegalize::UnrollVectorOp(SDOperand Op) {
691  MVT::ValueType VT = Op.getValueType();
692  assert(isTypeLegal(VT) &&
693         "Caller should expand or promote operands that are not legal!");
694  assert(Op.Val->getNumValues() == 1 &&
695         "Can't unroll a vector with multiple results!");
696  unsigned NE = MVT::getVectorNumElements(VT);
697  MVT::ValueType EltVT = MVT::getVectorElementType(VT);
698
699  SmallVector<SDOperand, 8> Scalars;
700  SmallVector<SDOperand, 4> Operands(Op.getNumOperands());
701  for (unsigned i = 0; i != NE; ++i) {
702    for (unsigned j = 0; j != Op.getNumOperands(); ++j) {
703      SDOperand Operand = Op.getOperand(j);
704      MVT::ValueType OperandVT = Operand.getValueType();
705      if (MVT::isVector(OperandVT)) {
706        // A vector operand; extract a single element.
707        MVT::ValueType OperandEltVT = MVT::getVectorElementType(OperandVT);
708        Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT,
709                                  OperandEltVT,
710                                  Operand,
711                                  DAG.getConstant(i, MVT::i32));
712      } else {
713        // A scalar operand; just use it as is.
714        Operands[j] = Operand;
715      }
716    }
717    Scalars.push_back(DAG.getNode(Op.getOpcode(), EltVT,
718                                  &Operands[0], Operands.size()));
719  }
720
721  return DAG.getNode(ISD::BUILD_VECTOR, VT, &Scalars[0], Scalars.size());
722}
723
724/// LegalizeOp - We know that the specified value has a legal type, and
725/// that its operands are legal.  Now ensure that the operation itself
726/// is legal, recursively ensuring that the operands' operations remain
727/// legal.
728SDOperand SelectionDAGLegalize::LegalizeOp(SDOperand Op) {
729  if (Op.getOpcode() == ISD::TargetConstant) // Allow illegal target nodes.
730    return Op;
731
732  assert(isTypeLegal(Op.getValueType()) &&
733         "Caller should expand or promote operands that are not legal!");
734  SDNode *Node = Op.Val;
735
736  // If this operation defines any values that cannot be represented in a
737  // register on this target, make sure to expand or promote them.
738  if (Node->getNumValues() > 1) {
739    for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
740      if (getTypeAction(Node->getValueType(i)) != Legal) {
741        HandleOp(Op.getValue(i));
742        assert(LegalizedNodes.count(Op) &&
743               "Handling didn't add legal operands!");
744        return LegalizedNodes[Op];
745      }
746  }
747
748  // Note that LegalizeOp may be reentered even from single-use nodes, which
749  // means that we always must cache transformed nodes.
750  DenseMap<SDOperand, SDOperand>::iterator I = LegalizedNodes.find(Op);
751  if (I != LegalizedNodes.end()) return I->second;
752
753  SDOperand Tmp1, Tmp2, Tmp3, Tmp4;
754  SDOperand Result = Op;
755  bool isCustom = false;
756
757  switch (Node->getOpcode()) {
758  case ISD::FrameIndex:
759  case ISD::EntryToken:
760  case ISD::Register:
761  case ISD::BasicBlock:
762  case ISD::TargetFrameIndex:
763  case ISD::TargetJumpTable:
764  case ISD::TargetConstant:
765  case ISD::TargetConstantFP:
766  case ISD::TargetConstantPool:
767  case ISD::TargetGlobalAddress:
768  case ISD::TargetGlobalTLSAddress:
769  case ISD::TargetExternalSymbol:
770  case ISD::VALUETYPE:
771  case ISD::SRCVALUE:
772  case ISD::STRING:
773  case ISD::CONDCODE:
774    // Primitives must all be legal.
775    assert(TLI.isOperationLegal(Node->getOpcode(), Node->getValueType(0)) &&
776           "This must be legal!");
777    break;
778  default:
779    if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
780      // If this is a target node, legalize it by legalizing the operands then
781      // passing it through.
782      SmallVector<SDOperand, 8> Ops;
783      for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i)
784        Ops.push_back(LegalizeOp(Node->getOperand(i)));
785
786      Result = DAG.UpdateNodeOperands(Result.getValue(0), &Ops[0], Ops.size());
787
788      for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
789        AddLegalizedOperand(Op.getValue(i), Result.getValue(i));
790      return Result.getValue(Op.ResNo);
791    }
792    // Otherwise this is an unhandled builtin node.  splat.
793#ifndef NDEBUG
794    cerr << "NODE: "; Node->dump(&DAG); cerr << "\n";
795#endif
796    assert(0 && "Do not know how to legalize this operator!");
797    abort();
798  case ISD::GLOBAL_OFFSET_TABLE:
799  case ISD::GlobalAddress:
800  case ISD::GlobalTLSAddress:
801  case ISD::ExternalSymbol:
802  case ISD::ConstantPool:
803  case ISD::JumpTable: // Nothing to do.
804    switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
805    default: assert(0 && "This action is not supported yet!");
806    case TargetLowering::Custom:
807      Tmp1 = TLI.LowerOperation(Op, DAG);
808      if (Tmp1.Val) Result = Tmp1;
809      // FALLTHROUGH if the target doesn't want to lower this op after all.
810    case TargetLowering::Legal:
811      break;
812    }
813    break;
814  case ISD::FRAMEADDR:
815  case ISD::RETURNADDR:
816    // The only option for these nodes is to custom lower them.  If the target
817    // does not custom lower them, then return zero.
818    Tmp1 = TLI.LowerOperation(Op, DAG);
819    if (Tmp1.Val)
820      Result = Tmp1;
821    else
822      Result = DAG.getConstant(0, TLI.getPointerTy());
823    break;
824  case ISD::FRAME_TO_ARGS_OFFSET: {
825    MVT::ValueType VT = Node->getValueType(0);
826    switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
827    default: assert(0 && "This action is not supported yet!");
828    case TargetLowering::Custom:
829      Result = TLI.LowerOperation(Op, DAG);
830      if (Result.Val) break;
831      // Fall Thru
832    case TargetLowering::Legal:
833      Result = DAG.getConstant(0, VT);
834      break;
835    }
836    }
837    break;
838  case ISD::EXCEPTIONADDR: {
839    Tmp1 = LegalizeOp(Node->getOperand(0));
840    MVT::ValueType VT = Node->getValueType(0);
841    switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
842    default: assert(0 && "This action is not supported yet!");
843    case TargetLowering::Expand: {
844        unsigned Reg = TLI.getExceptionAddressRegister();
845        Result = DAG.getCopyFromReg(Tmp1, Reg, VT).getValue(Op.ResNo);
846      }
847      break;
848    case TargetLowering::Custom:
849      Result = TLI.LowerOperation(Op, DAG);
850      if (Result.Val) break;
851      // Fall Thru
852    case TargetLowering::Legal: {
853      SDOperand Ops[] = { DAG.getConstant(0, VT), Tmp1 };
854      Result = DAG.getNode(ISD::MERGE_VALUES, DAG.getVTList(VT, MVT::Other),
855                           Ops, 2).getValue(Op.ResNo);
856      break;
857    }
858    }
859    }
860    break;
861  case ISD::EHSELECTION: {
862    Tmp1 = LegalizeOp(Node->getOperand(0));
863    Tmp2 = LegalizeOp(Node->getOperand(1));
864    MVT::ValueType VT = Node->getValueType(0);
865    switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
866    default: assert(0 && "This action is not supported yet!");
867    case TargetLowering::Expand: {
868        unsigned Reg = TLI.getExceptionSelectorRegister();
869        Result = DAG.getCopyFromReg(Tmp2, Reg, VT).getValue(Op.ResNo);
870      }
871      break;
872    case TargetLowering::Custom:
873      Result = TLI.LowerOperation(Op, DAG);
874      if (Result.Val) break;
875      // Fall Thru
876    case TargetLowering::Legal: {
877      SDOperand Ops[] = { DAG.getConstant(0, VT), Tmp2 };
878      Result = DAG.getNode(ISD::MERGE_VALUES, DAG.getVTList(VT, MVT::Other),
879                           Ops, 2).getValue(Op.ResNo);
880      break;
881    }
882    }
883    }
884    break;
885  case ISD::EH_RETURN: {
886    MVT::ValueType VT = Node->getValueType(0);
887    // The only "good" option for this node is to custom lower it.
888    switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
889    default: assert(0 && "This action is not supported at all!");
890    case TargetLowering::Custom:
891      Result = TLI.LowerOperation(Op, DAG);
892      if (Result.Val) break;
893      // Fall Thru
894    case TargetLowering::Legal:
895      // Target does not know, how to lower this, lower to noop
896      Result = LegalizeOp(Node->getOperand(0));
897      break;
898    }
899    }
900    break;
901  case ISD::AssertSext:
902  case ISD::AssertZext:
903    Tmp1 = LegalizeOp(Node->getOperand(0));
904    Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1));
905    break;
906  case ISD::MERGE_VALUES:
907    // Legalize eliminates MERGE_VALUES nodes.
908    Result = Node->getOperand(Op.ResNo);
909    break;
910  case ISD::CopyFromReg:
911    Tmp1 = LegalizeOp(Node->getOperand(0));
912    Result = Op.getValue(0);
913    if (Node->getNumValues() == 2) {
914      Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1));
915    } else {
916      assert(Node->getNumValues() == 3 && "Invalid copyfromreg!");
917      if (Node->getNumOperands() == 3) {
918        Tmp2 = LegalizeOp(Node->getOperand(2));
919        Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1),Tmp2);
920      } else {
921        Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1));
922      }
923      AddLegalizedOperand(Op.getValue(2), Result.getValue(2));
924    }
925    // Since CopyFromReg produces two values, make sure to remember that we
926    // legalized both of them.
927    AddLegalizedOperand(Op.getValue(0), Result);
928    AddLegalizedOperand(Op.getValue(1), Result.getValue(1));
929    return Result.getValue(Op.ResNo);
930  case ISD::UNDEF: {
931    MVT::ValueType VT = Op.getValueType();
932    switch (TLI.getOperationAction(ISD::UNDEF, VT)) {
933    default: assert(0 && "This action is not supported yet!");
934    case TargetLowering::Expand:
935      if (MVT::isInteger(VT))
936        Result = DAG.getConstant(0, VT);
937      else if (MVT::isFloatingPoint(VT))
938        Result = DAG.getConstantFP(APFloat(APInt(MVT::getSizeInBits(VT), 0)),
939                                   VT);
940      else
941        assert(0 && "Unknown value type!");
942      break;
943    case TargetLowering::Legal:
944      break;
945    }
946    break;
947  }
948
949  case ISD::INTRINSIC_W_CHAIN:
950  case ISD::INTRINSIC_WO_CHAIN:
951  case ISD::INTRINSIC_VOID: {
952    SmallVector<SDOperand, 8> Ops;
953    for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i)
954      Ops.push_back(LegalizeOp(Node->getOperand(i)));
955    Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size());
956
957    // Allow the target to custom lower its intrinsics if it wants to.
958    if (TLI.getOperationAction(Node->getOpcode(), MVT::Other) ==
959        TargetLowering::Custom) {
960      Tmp3 = TLI.LowerOperation(Result, DAG);
961      if (Tmp3.Val) Result = Tmp3;
962    }
963
964    if (Result.Val->getNumValues() == 1) break;
965
966    // Must have return value and chain result.
967    assert(Result.Val->getNumValues() == 2 &&
968           "Cannot return more than two values!");
969
970    // Since loads produce two values, make sure to remember that we
971    // legalized both of them.
972    AddLegalizedOperand(SDOperand(Node, 0), Result.getValue(0));
973    AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1));
974    return Result.getValue(Op.ResNo);
975  }
976
977  case ISD::LOCATION:
978    assert(Node->getNumOperands() == 5 && "Invalid LOCATION node!");
979    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the input chain.
980
981    switch (TLI.getOperationAction(ISD::LOCATION, MVT::Other)) {
982    case TargetLowering::Promote:
983    default: assert(0 && "This action is not supported yet!");
984    case TargetLowering::Expand: {
985      MachineModuleInfo *MMI = DAG.getMachineModuleInfo();
986      bool useDEBUG_LOC = TLI.isOperationLegal(ISD::DEBUG_LOC, MVT::Other);
987      bool useLABEL = TLI.isOperationLegal(ISD::LABEL, MVT::Other);
988
989      if (MMI && (useDEBUG_LOC || useLABEL)) {
990        const std::string &FName =
991          cast<StringSDNode>(Node->getOperand(3))->getValue();
992        const std::string &DirName =
993          cast<StringSDNode>(Node->getOperand(4))->getValue();
994        unsigned SrcFile = MMI->RecordSource(DirName, FName);
995
996        SmallVector<SDOperand, 8> Ops;
997        Ops.push_back(Tmp1);  // chain
998        SDOperand LineOp = Node->getOperand(1);
999        SDOperand ColOp = Node->getOperand(2);
1000
1001        if (useDEBUG_LOC) {
1002          Ops.push_back(LineOp);  // line #
1003          Ops.push_back(ColOp);  // col #
1004          Ops.push_back(DAG.getConstant(SrcFile, MVT::i32));  // source file id
1005          Result = DAG.getNode(ISD::DEBUG_LOC, MVT::Other, &Ops[0], Ops.size());
1006        } else {
1007          unsigned Line = cast<ConstantSDNode>(LineOp)->getValue();
1008          unsigned Col = cast<ConstantSDNode>(ColOp)->getValue();
1009          unsigned ID = MMI->RecordLabel(Line, Col, SrcFile);
1010          Ops.push_back(DAG.getConstant(ID, MVT::i32));
1011          Result = DAG.getNode(ISD::LABEL, MVT::Other,&Ops[0],Ops.size());
1012        }
1013      } else {
1014        Result = Tmp1;  // chain
1015      }
1016      break;
1017    }
1018    case TargetLowering::Legal:
1019      if (Tmp1 != Node->getOperand(0) ||
1020          getTypeAction(Node->getOperand(1).getValueType()) == Promote) {
1021        SmallVector<SDOperand, 8> Ops;
1022        Ops.push_back(Tmp1);
1023        if (getTypeAction(Node->getOperand(1).getValueType()) == Legal) {
1024          Ops.push_back(Node->getOperand(1));  // line # must be legal.
1025          Ops.push_back(Node->getOperand(2));  // col # must be legal.
1026        } else {
1027          // Otherwise promote them.
1028          Ops.push_back(PromoteOp(Node->getOperand(1)));
1029          Ops.push_back(PromoteOp(Node->getOperand(2)));
1030        }
1031        Ops.push_back(Node->getOperand(3));  // filename must be legal.
1032        Ops.push_back(Node->getOperand(4));  // working dir # must be legal.
1033        Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size());
1034      }
1035      break;
1036    }
1037    break;
1038
1039  case ISD::DEBUG_LOC:
1040    assert(Node->getNumOperands() == 4 && "Invalid DEBUG_LOC node!");
1041    switch (TLI.getOperationAction(ISD::DEBUG_LOC, MVT::Other)) {
1042    default: assert(0 && "This action is not supported yet!");
1043    case TargetLowering::Legal:
1044      Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
1045      Tmp2 = LegalizeOp(Node->getOperand(1));  // Legalize the line #.
1046      Tmp3 = LegalizeOp(Node->getOperand(2));  // Legalize the col #.
1047      Tmp4 = LegalizeOp(Node->getOperand(3));  // Legalize the source file id.
1048      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3, Tmp4);
1049      break;
1050    }
1051    break;
1052
1053  case ISD::LABEL:
1054    assert(Node->getNumOperands() == 2 && "Invalid LABEL node!");
1055    switch (TLI.getOperationAction(ISD::LABEL, MVT::Other)) {
1056    default: assert(0 && "This action is not supported yet!");
1057    case TargetLowering::Legal:
1058      Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
1059      Tmp2 = LegalizeOp(Node->getOperand(1));  // Legalize the label id.
1060      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
1061      break;
1062    case TargetLowering::Expand:
1063      Result = LegalizeOp(Node->getOperand(0));
1064      break;
1065    }
1066    break;
1067
1068  case ISD::Constant: {
1069    ConstantSDNode *CN = cast<ConstantSDNode>(Node);
1070    unsigned opAction =
1071      TLI.getOperationAction(ISD::Constant, CN->getValueType(0));
1072
1073    // We know we don't need to expand constants here, constants only have one
1074    // value and we check that it is fine above.
1075
1076    if (opAction == TargetLowering::Custom) {
1077      Tmp1 = TLI.LowerOperation(Result, DAG);
1078      if (Tmp1.Val)
1079        Result = Tmp1;
1080    }
1081    break;
1082  }
1083  case ISD::ConstantFP: {
1084    // Spill FP immediates to the constant pool if the target cannot directly
1085    // codegen them.  Targets often have some immediate values that can be
1086    // efficiently generated into an FP register without a load.  We explicitly
1087    // leave these constants as ConstantFP nodes for the target to deal with.
1088    ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
1089
1090    // Check to see if this FP immediate is already legal.
1091    bool isLegal = false;
1092    for (TargetLowering::legal_fpimm_iterator I = TLI.legal_fpimm_begin(),
1093           E = TLI.legal_fpimm_end(); I != E; ++I)
1094      if (CFP->isExactlyValue(*I)) {
1095        isLegal = true;
1096        break;
1097      }
1098
1099    // If this is a legal constant, turn it into a TargetConstantFP node.
1100    if (isLegal) {
1101      Result = DAG.getTargetConstantFP(CFP->getValueAPF(),
1102                                       CFP->getValueType(0));
1103      break;
1104    }
1105
1106    switch (TLI.getOperationAction(ISD::ConstantFP, CFP->getValueType(0))) {
1107    default: assert(0 && "This action is not supported yet!");
1108    case TargetLowering::Custom:
1109      Tmp3 = TLI.LowerOperation(Result, DAG);
1110      if (Tmp3.Val) {
1111        Result = Tmp3;
1112        break;
1113      }
1114      // FALLTHROUGH
1115    case TargetLowering::Expand:
1116      Result = ExpandConstantFP(CFP, true, DAG, TLI);
1117    }
1118    break;
1119  }
1120  case ISD::TokenFactor:
1121    if (Node->getNumOperands() == 2) {
1122      Tmp1 = LegalizeOp(Node->getOperand(0));
1123      Tmp2 = LegalizeOp(Node->getOperand(1));
1124      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
1125    } else if (Node->getNumOperands() == 3) {
1126      Tmp1 = LegalizeOp(Node->getOperand(0));
1127      Tmp2 = LegalizeOp(Node->getOperand(1));
1128      Tmp3 = LegalizeOp(Node->getOperand(2));
1129      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
1130    } else {
1131      SmallVector<SDOperand, 8> Ops;
1132      // Legalize the operands.
1133      for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i)
1134        Ops.push_back(LegalizeOp(Node->getOperand(i)));
1135      Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size());
1136    }
1137    break;
1138
1139  case ISD::FORMAL_ARGUMENTS:
1140  case ISD::CALL:
1141    // The only option for this is to custom lower it.
1142    Tmp3 = TLI.LowerOperation(Result.getValue(0), DAG);
1143    assert(Tmp3.Val && "Target didn't custom lower this node!");
1144    assert(Tmp3.Val->getNumValues() == Result.Val->getNumValues() &&
1145           "Lowering call/formal_arguments produced unexpected # results!");
1146
1147    // Since CALL/FORMAL_ARGUMENTS nodes produce multiple values, make sure to
1148    // remember that we legalized all of them, so it doesn't get relegalized.
1149    for (unsigned i = 0, e = Tmp3.Val->getNumValues(); i != e; ++i) {
1150      Tmp1 = LegalizeOp(Tmp3.getValue(i));
1151      if (Op.ResNo == i)
1152        Tmp2 = Tmp1;
1153      AddLegalizedOperand(SDOperand(Node, i), Tmp1);
1154    }
1155    return Tmp2;
1156   case ISD::EXTRACT_SUBREG: {
1157      Tmp1 = LegalizeOp(Node->getOperand(0));
1158      ConstantSDNode *idx = dyn_cast<ConstantSDNode>(Node->getOperand(1));
1159      assert(idx && "Operand must be a constant");
1160      Tmp2 = DAG.getTargetConstant(idx->getValue(), idx->getValueType(0));
1161      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
1162    }
1163    break;
1164  case ISD::INSERT_SUBREG: {
1165      Tmp1 = LegalizeOp(Node->getOperand(0));
1166      Tmp2 = LegalizeOp(Node->getOperand(1));
1167      ConstantSDNode *idx = dyn_cast<ConstantSDNode>(Node->getOperand(2));
1168      assert(idx && "Operand must be a constant");
1169      Tmp3 = DAG.getTargetConstant(idx->getValue(), idx->getValueType(0));
1170      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
1171    }
1172    break;
1173  case ISD::BUILD_VECTOR:
1174    switch (TLI.getOperationAction(ISD::BUILD_VECTOR, Node->getValueType(0))) {
1175    default: assert(0 && "This action is not supported yet!");
1176    case TargetLowering::Custom:
1177      Tmp3 = TLI.LowerOperation(Result, DAG);
1178      if (Tmp3.Val) {
1179        Result = Tmp3;
1180        break;
1181      }
1182      // FALLTHROUGH
1183    case TargetLowering::Expand:
1184      Result = ExpandBUILD_VECTOR(Result.Val);
1185      break;
1186    }
1187    break;
1188  case ISD::INSERT_VECTOR_ELT:
1189    Tmp1 = LegalizeOp(Node->getOperand(0));  // InVec
1190    Tmp2 = LegalizeOp(Node->getOperand(1));  // InVal
1191    Tmp3 = LegalizeOp(Node->getOperand(2));  // InEltNo
1192    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
1193
1194    switch (TLI.getOperationAction(ISD::INSERT_VECTOR_ELT,
1195                                   Node->getValueType(0))) {
1196    default: assert(0 && "This action is not supported yet!");
1197    case TargetLowering::Legal:
1198      break;
1199    case TargetLowering::Custom:
1200      Tmp3 = TLI.LowerOperation(Result, DAG);
1201      if (Tmp3.Val) {
1202        Result = Tmp3;
1203        break;
1204      }
1205      // FALLTHROUGH
1206    case TargetLowering::Expand: {
1207      // If the insert index is a constant, codegen this as a scalar_to_vector,
1208      // then a shuffle that inserts it into the right position in the vector.
1209      if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Tmp3)) {
1210        SDOperand ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR,
1211                                      Tmp1.getValueType(), Tmp2);
1212
1213        unsigned NumElts = MVT::getVectorNumElements(Tmp1.getValueType());
1214        MVT::ValueType ShufMaskVT = MVT::getIntVectorWithNumElements(NumElts);
1215        MVT::ValueType ShufMaskEltVT = MVT::getVectorElementType(ShufMaskVT);
1216
1217        // We generate a shuffle of InVec and ScVec, so the shuffle mask should
1218        // be 0,1,2,3,4,5... with the appropriate element replaced with elt 0 of
1219        // the RHS.
1220        SmallVector<SDOperand, 8> ShufOps;
1221        for (unsigned i = 0; i != NumElts; ++i) {
1222          if (i != InsertPos->getValue())
1223            ShufOps.push_back(DAG.getConstant(i, ShufMaskEltVT));
1224          else
1225            ShufOps.push_back(DAG.getConstant(NumElts, ShufMaskEltVT));
1226        }
1227        SDOperand ShufMask = DAG.getNode(ISD::BUILD_VECTOR, ShufMaskVT,
1228                                         &ShufOps[0], ShufOps.size());
1229
1230        Result = DAG.getNode(ISD::VECTOR_SHUFFLE, Tmp1.getValueType(),
1231                             Tmp1, ScVec, ShufMask);
1232        Result = LegalizeOp(Result);
1233        break;
1234      }
1235
1236      // If the target doesn't support this, we have to spill the input vector
1237      // to a temporary stack slot, update the element, then reload it.  This is
1238      // badness.  We could also load the value into a vector register (either
1239      // with a "move to register" or "extload into register" instruction, then
1240      // permute it into place, if the idx is a constant and if the idx is
1241      // supported by the target.
1242      MVT::ValueType VT    = Tmp1.getValueType();
1243      MVT::ValueType EltVT = Tmp2.getValueType();
1244      MVT::ValueType IdxVT = Tmp3.getValueType();
1245      MVT::ValueType PtrVT = TLI.getPointerTy();
1246      SDOperand StackPtr = DAG.CreateStackTemporary(VT);
1247      // Store the vector.
1248      SDOperand Ch = DAG.getStore(DAG.getEntryNode(), Tmp1, StackPtr, NULL, 0);
1249
1250      // Truncate or zero extend offset to target pointer type.
1251      unsigned CastOpc = (IdxVT > PtrVT) ? ISD::TRUNCATE : ISD::ZERO_EXTEND;
1252      Tmp3 = DAG.getNode(CastOpc, PtrVT, Tmp3);
1253      // Add the offset to the index.
1254      unsigned EltSize = MVT::getSizeInBits(EltVT)/8;
1255      Tmp3 = DAG.getNode(ISD::MUL, IdxVT, Tmp3,DAG.getConstant(EltSize, IdxVT));
1256      SDOperand StackPtr2 = DAG.getNode(ISD::ADD, IdxVT, Tmp3, StackPtr);
1257      // Store the scalar value.
1258      Ch = DAG.getStore(Ch, Tmp2, StackPtr2, NULL, 0);
1259      // Load the updated vector.
1260      Result = DAG.getLoad(VT, Ch, StackPtr, NULL, 0);
1261      break;
1262    }
1263    }
1264    break;
1265  case ISD::SCALAR_TO_VECTOR:
1266    if (!TLI.isTypeLegal(Node->getOperand(0).getValueType())) {
1267      Result = LegalizeOp(ExpandSCALAR_TO_VECTOR(Node));
1268      break;
1269    }
1270
1271    Tmp1 = LegalizeOp(Node->getOperand(0));  // InVal
1272    Result = DAG.UpdateNodeOperands(Result, Tmp1);
1273    switch (TLI.getOperationAction(ISD::SCALAR_TO_VECTOR,
1274                                   Node->getValueType(0))) {
1275    default: assert(0 && "This action is not supported yet!");
1276    case TargetLowering::Legal:
1277      break;
1278    case TargetLowering::Custom:
1279      Tmp3 = TLI.LowerOperation(Result, DAG);
1280      if (Tmp3.Val) {
1281        Result = Tmp3;
1282        break;
1283      }
1284      // FALLTHROUGH
1285    case TargetLowering::Expand:
1286      Result = LegalizeOp(ExpandSCALAR_TO_VECTOR(Node));
1287      break;
1288    }
1289    break;
1290  case ISD::VECTOR_SHUFFLE:
1291    Tmp1 = LegalizeOp(Node->getOperand(0));   // Legalize the input vectors,
1292    Tmp2 = LegalizeOp(Node->getOperand(1));   // but not the shuffle mask.
1293    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2));
1294
1295    // Allow targets to custom lower the SHUFFLEs they support.
1296    switch (TLI.getOperationAction(ISD::VECTOR_SHUFFLE,Result.getValueType())) {
1297    default: assert(0 && "Unknown operation action!");
1298    case TargetLowering::Legal:
1299      assert(isShuffleLegal(Result.getValueType(), Node->getOperand(2)) &&
1300             "vector shuffle should not be created if not legal!");
1301      break;
1302    case TargetLowering::Custom:
1303      Tmp3 = TLI.LowerOperation(Result, DAG);
1304      if (Tmp3.Val) {
1305        Result = Tmp3;
1306        break;
1307      }
1308      // FALLTHROUGH
1309    case TargetLowering::Expand: {
1310      MVT::ValueType VT = Node->getValueType(0);
1311      MVT::ValueType EltVT = MVT::getVectorElementType(VT);
1312      MVT::ValueType PtrVT = TLI.getPointerTy();
1313      SDOperand Mask = Node->getOperand(2);
1314      unsigned NumElems = Mask.getNumOperands();
1315      SmallVector<SDOperand,8> Ops;
1316      for (unsigned i = 0; i != NumElems; ++i) {
1317        SDOperand Arg = Mask.getOperand(i);
1318        if (Arg.getOpcode() == ISD::UNDEF) {
1319          Ops.push_back(DAG.getNode(ISD::UNDEF, EltVT));
1320        } else {
1321          assert(isa<ConstantSDNode>(Arg) && "Invalid VECTOR_SHUFFLE mask!");
1322          unsigned Idx = cast<ConstantSDNode>(Arg)->getValue();
1323          if (Idx < NumElems)
1324            Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EltVT, Tmp1,
1325                                      DAG.getConstant(Idx, PtrVT)));
1326          else
1327            Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, EltVT, Tmp2,
1328                                      DAG.getConstant(Idx - NumElems, PtrVT)));
1329        }
1330      }
1331      Result = DAG.getNode(ISD::BUILD_VECTOR, VT, &Ops[0], Ops.size());
1332      break;
1333    }
1334    case TargetLowering::Promote: {
1335      // Change base type to a different vector type.
1336      MVT::ValueType OVT = Node->getValueType(0);
1337      MVT::ValueType NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
1338
1339      // Cast the two input vectors.
1340      Tmp1 = DAG.getNode(ISD::BIT_CONVERT, NVT, Tmp1);
1341      Tmp2 = DAG.getNode(ISD::BIT_CONVERT, NVT, Tmp2);
1342
1343      // Convert the shuffle mask to the right # elements.
1344      Tmp3 = SDOperand(isShuffleLegal(OVT, Node->getOperand(2)), 0);
1345      assert(Tmp3.Val && "Shuffle not legal?");
1346      Result = DAG.getNode(ISD::VECTOR_SHUFFLE, NVT, Tmp1, Tmp2, Tmp3);
1347      Result = DAG.getNode(ISD::BIT_CONVERT, OVT, Result);
1348      break;
1349    }
1350    }
1351    break;
1352
1353  case ISD::EXTRACT_VECTOR_ELT:
1354    Tmp1 = Node->getOperand(0);
1355    Tmp2 = LegalizeOp(Node->getOperand(1));
1356    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
1357    Result = ExpandEXTRACT_VECTOR_ELT(Result);
1358    break;
1359
1360  case ISD::EXTRACT_SUBVECTOR:
1361    Tmp1 = Node->getOperand(0);
1362    Tmp2 = LegalizeOp(Node->getOperand(1));
1363    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
1364    Result = ExpandEXTRACT_SUBVECTOR(Result);
1365    break;
1366
1367  case ISD::CALLSEQ_START: {
1368    SDNode *CallEnd = FindCallEndFromCallStart(Node);
1369
1370    // Recursively Legalize all of the inputs of the call end that do not lead
1371    // to this call start.  This ensures that any libcalls that need be inserted
1372    // are inserted *before* the CALLSEQ_START.
1373    {SmallPtrSet<SDNode*, 32> NodesLeadingTo;
1374    for (unsigned i = 0, e = CallEnd->getNumOperands(); i != e; ++i)
1375      LegalizeAllNodesNotLeadingTo(CallEnd->getOperand(i).Val, Node,
1376                                   NodesLeadingTo);
1377    }
1378
1379    // Now that we legalized all of the inputs (which may have inserted
1380    // libcalls) create the new CALLSEQ_START node.
1381    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
1382
1383    // Merge in the last call, to ensure that this call start after the last
1384    // call ended.
1385    if (LastCALLSEQ_END.getOpcode() != ISD::EntryToken) {
1386      Tmp1 = DAG.getNode(ISD::TokenFactor, MVT::Other, Tmp1, LastCALLSEQ_END);
1387      Tmp1 = LegalizeOp(Tmp1);
1388    }
1389
1390    // Do not try to legalize the target-specific arguments (#1+).
1391    if (Tmp1 != Node->getOperand(0)) {
1392      SmallVector<SDOperand, 8> Ops(Node->op_begin(), Node->op_end());
1393      Ops[0] = Tmp1;
1394      Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size());
1395    }
1396
1397    // Remember that the CALLSEQ_START is legalized.
1398    AddLegalizedOperand(Op.getValue(0), Result);
1399    if (Node->getNumValues() == 2)    // If this has a flag result, remember it.
1400      AddLegalizedOperand(Op.getValue(1), Result.getValue(1));
1401
1402    // Now that the callseq_start and all of the non-call nodes above this call
1403    // sequence have been legalized, legalize the call itself.  During this
1404    // process, no libcalls can/will be inserted, guaranteeing that no calls
1405    // can overlap.
1406    assert(!IsLegalizingCall && "Inconsistent sequentialization of calls!");
1407    SDOperand InCallSEQ = LastCALLSEQ_END;
1408    // Note that we are selecting this call!
1409    LastCALLSEQ_END = SDOperand(CallEnd, 0);
1410    IsLegalizingCall = true;
1411
1412    // Legalize the call, starting from the CALLSEQ_END.
1413    LegalizeOp(LastCALLSEQ_END);
1414    assert(!IsLegalizingCall && "CALLSEQ_END should have cleared this!");
1415    return Result;
1416  }
1417  case ISD::CALLSEQ_END:
1418    // If the CALLSEQ_START node hasn't been legalized first, legalize it.  This
1419    // will cause this node to be legalized as well as handling libcalls right.
1420    if (LastCALLSEQ_END.Val != Node) {
1421      LegalizeOp(SDOperand(FindCallStartFromCallEnd(Node), 0));
1422      DenseMap<SDOperand, SDOperand>::iterator I = LegalizedNodes.find(Op);
1423      assert(I != LegalizedNodes.end() &&
1424             "Legalizing the call start should have legalized this node!");
1425      return I->second;
1426    }
1427
1428    // Otherwise, the call start has been legalized and everything is going
1429    // according to plan.  Just legalize ourselves normally here.
1430    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
1431    // Do not try to legalize the target-specific arguments (#1+), except for
1432    // an optional flag input.
1433    if (Node->getOperand(Node->getNumOperands()-1).getValueType() != MVT::Flag){
1434      if (Tmp1 != Node->getOperand(0)) {
1435        SmallVector<SDOperand, 8> Ops(Node->op_begin(), Node->op_end());
1436        Ops[0] = Tmp1;
1437        Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size());
1438      }
1439    } else {
1440      Tmp2 = LegalizeOp(Node->getOperand(Node->getNumOperands()-1));
1441      if (Tmp1 != Node->getOperand(0) ||
1442          Tmp2 != Node->getOperand(Node->getNumOperands()-1)) {
1443        SmallVector<SDOperand, 8> Ops(Node->op_begin(), Node->op_end());
1444        Ops[0] = Tmp1;
1445        Ops.back() = Tmp2;
1446        Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size());
1447      }
1448    }
1449    assert(IsLegalizingCall && "Call sequence imbalance between start/end?");
1450    // This finishes up call legalization.
1451    IsLegalizingCall = false;
1452
1453    // If the CALLSEQ_END node has a flag, remember that we legalized it.
1454    AddLegalizedOperand(SDOperand(Node, 0), Result.getValue(0));
1455    if (Node->getNumValues() == 2)
1456      AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1));
1457    return Result.getValue(Op.ResNo);
1458  case ISD::DYNAMIC_STACKALLOC: {
1459    MVT::ValueType VT = Node->getValueType(0);
1460    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
1461    Tmp2 = LegalizeOp(Node->getOperand(1));  // Legalize the size.
1462    Tmp3 = LegalizeOp(Node->getOperand(2));  // Legalize the alignment.
1463    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
1464
1465    Tmp1 = Result.getValue(0);
1466    Tmp2 = Result.getValue(1);
1467    switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
1468    default: assert(0 && "This action is not supported yet!");
1469    case TargetLowering::Expand: {
1470      unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore();
1471      assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1472             " not tell us which reg is the stack pointer!");
1473      SDOperand Chain = Tmp1.getOperand(0);
1474      SDOperand Size  = Tmp2.getOperand(1);
1475      SDOperand SP = DAG.getCopyFromReg(Chain, SPReg, VT);
1476      Chain = SP.getValue(1);
1477      unsigned Align = cast<ConstantSDNode>(Tmp3)->getValue();
1478      unsigned StackAlign =
1479        TLI.getTargetMachine().getFrameInfo()->getStackAlignment();
1480      if (Align > StackAlign)
1481        SP = DAG.getNode(ISD::AND, VT, SP,
1482                         DAG.getConstant(-(uint64_t)Align, VT));
1483      Tmp1 = DAG.getNode(ISD::SUB, VT, SP, Size);       // Value
1484      Tmp2 = DAG.getCopyToReg(Chain, SPReg, Tmp1);      // Output chain
1485      Tmp1 = LegalizeOp(Tmp1);
1486      Tmp2 = LegalizeOp(Tmp2);
1487      break;
1488    }
1489    case TargetLowering::Custom:
1490      Tmp3 = TLI.LowerOperation(Tmp1, DAG);
1491      if (Tmp3.Val) {
1492        Tmp1 = LegalizeOp(Tmp3);
1493        Tmp2 = LegalizeOp(Tmp3.getValue(1));
1494      }
1495      break;
1496    case TargetLowering::Legal:
1497      break;
1498    }
1499    // Since this op produce two values, make sure to remember that we
1500    // legalized both of them.
1501    AddLegalizedOperand(SDOperand(Node, 0), Tmp1);
1502    AddLegalizedOperand(SDOperand(Node, 1), Tmp2);
1503    return Op.ResNo ? Tmp2 : Tmp1;
1504  }
1505  case ISD::INLINEASM: {
1506    SmallVector<SDOperand, 8> Ops(Node->op_begin(), Node->op_end());
1507    bool Changed = false;
1508    // Legalize all of the operands of the inline asm, in case they are nodes
1509    // that need to be expanded or something.  Note we skip the asm string and
1510    // all of the TargetConstant flags.
1511    SDOperand Op = LegalizeOp(Ops[0]);
1512    Changed = Op != Ops[0];
1513    Ops[0] = Op;
1514
1515    bool HasInFlag = Ops.back().getValueType() == MVT::Flag;
1516    for (unsigned i = 2, e = Ops.size()-HasInFlag; i < e; ) {
1517      unsigned NumVals = cast<ConstantSDNode>(Ops[i])->getValue() >> 3;
1518      for (++i; NumVals; ++i, --NumVals) {
1519        SDOperand Op = LegalizeOp(Ops[i]);
1520        if (Op != Ops[i]) {
1521          Changed = true;
1522          Ops[i] = Op;
1523        }
1524      }
1525    }
1526
1527    if (HasInFlag) {
1528      Op = LegalizeOp(Ops.back());
1529      Changed |= Op != Ops.back();
1530      Ops.back() = Op;
1531    }
1532
1533    if (Changed)
1534      Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size());
1535
1536    // INLINE asm returns a chain and flag, make sure to add both to the map.
1537    AddLegalizedOperand(SDOperand(Node, 0), Result.getValue(0));
1538    AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1));
1539    return Result.getValue(Op.ResNo);
1540  }
1541  case ISD::BR:
1542    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
1543    // Ensure that libcalls are emitted before a branch.
1544    Tmp1 = DAG.getNode(ISD::TokenFactor, MVT::Other, Tmp1, LastCALLSEQ_END);
1545    Tmp1 = LegalizeOp(Tmp1);
1546    LastCALLSEQ_END = DAG.getEntryNode();
1547
1548    Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1));
1549    break;
1550  case ISD::BRIND:
1551    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
1552    // Ensure that libcalls are emitted before a branch.
1553    Tmp1 = DAG.getNode(ISD::TokenFactor, MVT::Other, Tmp1, LastCALLSEQ_END);
1554    Tmp1 = LegalizeOp(Tmp1);
1555    LastCALLSEQ_END = DAG.getEntryNode();
1556
1557    switch (getTypeAction(Node->getOperand(1).getValueType())) {
1558    default: assert(0 && "Indirect target must be legal type (pointer)!");
1559    case Legal:
1560      Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the condition.
1561      break;
1562    }
1563    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
1564    break;
1565  case ISD::BR_JT:
1566    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
1567    // Ensure that libcalls are emitted before a branch.
1568    Tmp1 = DAG.getNode(ISD::TokenFactor, MVT::Other, Tmp1, LastCALLSEQ_END);
1569    Tmp1 = LegalizeOp(Tmp1);
1570    LastCALLSEQ_END = DAG.getEntryNode();
1571
1572    Tmp2 = LegalizeOp(Node->getOperand(1));  // Legalize the jumptable node.
1573    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2));
1574
1575    switch (TLI.getOperationAction(ISD::BR_JT, MVT::Other)) {
1576    default: assert(0 && "This action is not supported yet!");
1577    case TargetLowering::Legal: break;
1578    case TargetLowering::Custom:
1579      Tmp1 = TLI.LowerOperation(Result, DAG);
1580      if (Tmp1.Val) Result = Tmp1;
1581      break;
1582    case TargetLowering::Expand: {
1583      SDOperand Chain = Result.getOperand(0);
1584      SDOperand Table = Result.getOperand(1);
1585      SDOperand Index = Result.getOperand(2);
1586
1587      MVT::ValueType PTy = TLI.getPointerTy();
1588      MachineFunction &MF = DAG.getMachineFunction();
1589      unsigned EntrySize = MF.getJumpTableInfo()->getEntrySize();
1590      Index= DAG.getNode(ISD::MUL, PTy, Index, DAG.getConstant(EntrySize, PTy));
1591      SDOperand Addr = DAG.getNode(ISD::ADD, PTy, Index, Table);
1592
1593      SDOperand LD;
1594      switch (EntrySize) {
1595      default: assert(0 && "Size of jump table not supported yet."); break;
1596      case 4: LD = DAG.getLoad(MVT::i32, Chain, Addr, NULL, 0); break;
1597      case 8: LD = DAG.getLoad(MVT::i64, Chain, Addr, NULL, 0); break;
1598      }
1599
1600      if (TLI.getTargetMachine().getRelocationModel() == Reloc::PIC_) {
1601        // For PIC, the sequence is:
1602        // BRIND(load(Jumptable + index) + RelocBase)
1603        // RelocBase is the JumpTable on PPC and X86, GOT on Alpha
1604        SDOperand Reloc;
1605        if (TLI.usesGlobalOffsetTable())
1606          Reloc = DAG.getNode(ISD::GLOBAL_OFFSET_TABLE, PTy);
1607        else
1608          Reloc = Table;
1609        Addr = (PTy != MVT::i32) ? DAG.getNode(ISD::SIGN_EXTEND, PTy, LD) : LD;
1610        Addr = DAG.getNode(ISD::ADD, PTy, Addr, Reloc);
1611        Result = DAG.getNode(ISD::BRIND, MVT::Other, LD.getValue(1), Addr);
1612      } else {
1613        Result = DAG.getNode(ISD::BRIND, MVT::Other, LD.getValue(1), LD);
1614      }
1615    }
1616    }
1617    break;
1618  case ISD::BRCOND:
1619    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
1620    // Ensure that libcalls are emitted before a return.
1621    Tmp1 = DAG.getNode(ISD::TokenFactor, MVT::Other, Tmp1, LastCALLSEQ_END);
1622    Tmp1 = LegalizeOp(Tmp1);
1623    LastCALLSEQ_END = DAG.getEntryNode();
1624
1625    switch (getTypeAction(Node->getOperand(1).getValueType())) {
1626    case Expand: assert(0 && "It's impossible to expand bools");
1627    case Legal:
1628      Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the condition.
1629      break;
1630    case Promote:
1631      Tmp2 = PromoteOp(Node->getOperand(1));  // Promote the condition.
1632
1633      // The top bits of the promoted condition are not necessarily zero, ensure
1634      // that the value is properly zero extended.
1635      if (!DAG.MaskedValueIsZero(Tmp2,
1636                                 MVT::getIntVTBitMask(Tmp2.getValueType())^1))
1637        Tmp2 = DAG.getZeroExtendInReg(Tmp2, MVT::i1);
1638      break;
1639    }
1640
1641    // Basic block destination (Op#2) is always legal.
1642    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2));
1643
1644    switch (TLI.getOperationAction(ISD::BRCOND, MVT::Other)) {
1645    default: assert(0 && "This action is not supported yet!");
1646    case TargetLowering::Legal: break;
1647    case TargetLowering::Custom:
1648      Tmp1 = TLI.LowerOperation(Result, DAG);
1649      if (Tmp1.Val) Result = Tmp1;
1650      break;
1651    case TargetLowering::Expand:
1652      // Expand brcond's setcc into its constituent parts and create a BR_CC
1653      // Node.
1654      if (Tmp2.getOpcode() == ISD::SETCC) {
1655        Result = DAG.getNode(ISD::BR_CC, MVT::Other, Tmp1, Tmp2.getOperand(2),
1656                             Tmp2.getOperand(0), Tmp2.getOperand(1),
1657                             Node->getOperand(2));
1658      } else {
1659        Result = DAG.getNode(ISD::BR_CC, MVT::Other, Tmp1,
1660                             DAG.getCondCode(ISD::SETNE), Tmp2,
1661                             DAG.getConstant(0, Tmp2.getValueType()),
1662                             Node->getOperand(2));
1663      }
1664      break;
1665    }
1666    break;
1667  case ISD::BR_CC:
1668    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
1669    // Ensure that libcalls are emitted before a branch.
1670    Tmp1 = DAG.getNode(ISD::TokenFactor, MVT::Other, Tmp1, LastCALLSEQ_END);
1671    Tmp1 = LegalizeOp(Tmp1);
1672    Tmp2 = Node->getOperand(2);              // LHS
1673    Tmp3 = Node->getOperand(3);              // RHS
1674    Tmp4 = Node->getOperand(1);              // CC
1675
1676    LegalizeSetCCOperands(Tmp2, Tmp3, Tmp4);
1677    LastCALLSEQ_END = DAG.getEntryNode();
1678
1679    // If we didn't get both a LHS and RHS back from LegalizeSetCCOperands,
1680    // the LHS is a legal SETCC itself.  In this case, we need to compare
1681    // the result against zero to select between true and false values.
1682    if (Tmp3.Val == 0) {
1683      Tmp3 = DAG.getConstant(0, Tmp2.getValueType());
1684      Tmp4 = DAG.getCondCode(ISD::SETNE);
1685    }
1686
1687    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp4, Tmp2, Tmp3,
1688                                    Node->getOperand(4));
1689
1690    switch (TLI.getOperationAction(ISD::BR_CC, Tmp3.getValueType())) {
1691    default: assert(0 && "Unexpected action for BR_CC!");
1692    case TargetLowering::Legal: break;
1693    case TargetLowering::Custom:
1694      Tmp4 = TLI.LowerOperation(Result, DAG);
1695      if (Tmp4.Val) Result = Tmp4;
1696      break;
1697    }
1698    break;
1699  case ISD::LOAD: {
1700    LoadSDNode *LD = cast<LoadSDNode>(Node);
1701    Tmp1 = LegalizeOp(LD->getChain());   // Legalize the chain.
1702    Tmp2 = LegalizeOp(LD->getBasePtr()); // Legalize the base pointer.
1703
1704    ISD::LoadExtType ExtType = LD->getExtensionType();
1705    if (ExtType == ISD::NON_EXTLOAD) {
1706      MVT::ValueType VT = Node->getValueType(0);
1707      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, LD->getOffset());
1708      Tmp3 = Result.getValue(0);
1709      Tmp4 = Result.getValue(1);
1710
1711      switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
1712      default: assert(0 && "This action is not supported yet!");
1713      case TargetLowering::Legal:
1714        // If this is an unaligned load and the target doesn't support it,
1715        // expand it.
1716        if (!TLI.allowsUnalignedMemoryAccesses()) {
1717          unsigned ABIAlignment = TLI.getTargetData()->
1718            getABITypeAlignment(MVT::getTypeForValueType(LD->getLoadedVT()));
1719          if (LD->getAlignment() < ABIAlignment){
1720            Result = ExpandUnalignedLoad(cast<LoadSDNode>(Result.Val), DAG,
1721                                         TLI);
1722            Tmp3 = Result.getOperand(0);
1723            Tmp4 = Result.getOperand(1);
1724            Tmp3 = LegalizeOp(Tmp3);
1725            Tmp4 = LegalizeOp(Tmp4);
1726          }
1727        }
1728        break;
1729      case TargetLowering::Custom:
1730        Tmp1 = TLI.LowerOperation(Tmp3, DAG);
1731        if (Tmp1.Val) {
1732          Tmp3 = LegalizeOp(Tmp1);
1733          Tmp4 = LegalizeOp(Tmp1.getValue(1));
1734        }
1735        break;
1736      case TargetLowering::Promote: {
1737        // Only promote a load of vector type to another.
1738        assert(MVT::isVector(VT) && "Cannot promote this load!");
1739        // Change base type to a different vector type.
1740        MVT::ValueType NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
1741
1742        Tmp1 = DAG.getLoad(NVT, Tmp1, Tmp2, LD->getSrcValue(),
1743                           LD->getSrcValueOffset(),
1744                           LD->isVolatile(), LD->getAlignment());
1745        Tmp3 = LegalizeOp(DAG.getNode(ISD::BIT_CONVERT, VT, Tmp1));
1746        Tmp4 = LegalizeOp(Tmp1.getValue(1));
1747        break;
1748      }
1749      }
1750      // Since loads produce two values, make sure to remember that we
1751      // legalized both of them.
1752      AddLegalizedOperand(SDOperand(Node, 0), Tmp3);
1753      AddLegalizedOperand(SDOperand(Node, 1), Tmp4);
1754      return Op.ResNo ? Tmp4 : Tmp3;
1755    } else {
1756      MVT::ValueType SrcVT = LD->getLoadedVT();
1757      switch (TLI.getLoadXAction(ExtType, SrcVT)) {
1758      default: assert(0 && "This action is not supported yet!");
1759      case TargetLowering::Promote:
1760        assert(SrcVT == MVT::i1 &&
1761               "Can only promote extending LOAD from i1 -> i8!");
1762        Result = DAG.getExtLoad(ExtType, Node->getValueType(0), Tmp1, Tmp2,
1763                                LD->getSrcValue(), LD->getSrcValueOffset(),
1764                                MVT::i8, LD->isVolatile(), LD->getAlignment());
1765        Tmp1 = Result.getValue(0);
1766        Tmp2 = Result.getValue(1);
1767      break;
1768      case TargetLowering::Custom:
1769        isCustom = true;
1770        // FALLTHROUGH
1771      case TargetLowering::Legal:
1772        Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, LD->getOffset());
1773        Tmp1 = Result.getValue(0);
1774        Tmp2 = Result.getValue(1);
1775
1776        if (isCustom) {
1777          Tmp3 = TLI.LowerOperation(Result, DAG);
1778          if (Tmp3.Val) {
1779            Tmp1 = LegalizeOp(Tmp3);
1780            Tmp2 = LegalizeOp(Tmp3.getValue(1));
1781          }
1782        } else {
1783          // If this is an unaligned load and the target doesn't support it,
1784          // expand it.
1785          if (!TLI.allowsUnalignedMemoryAccesses()) {
1786            unsigned ABIAlignment = TLI.getTargetData()->
1787              getABITypeAlignment(MVT::getTypeForValueType(LD->getLoadedVT()));
1788            if (LD->getAlignment() < ABIAlignment){
1789              Result = ExpandUnalignedLoad(cast<LoadSDNode>(Result.Val), DAG,
1790                                           TLI);
1791              Tmp1 = Result.getOperand(0);
1792              Tmp2 = Result.getOperand(1);
1793              Tmp1 = LegalizeOp(Tmp1);
1794              Tmp2 = LegalizeOp(Tmp2);
1795            }
1796          }
1797        }
1798        break;
1799      case TargetLowering::Expand:
1800        // f64 = EXTLOAD f32 should expand to LOAD, FP_EXTEND
1801        if (SrcVT == MVT::f32 && Node->getValueType(0) == MVT::f64) {
1802          SDOperand Load = DAG.getLoad(SrcVT, Tmp1, Tmp2, LD->getSrcValue(),
1803                                       LD->getSrcValueOffset(),
1804                                       LD->isVolatile(), LD->getAlignment());
1805          Result = DAG.getNode(ISD::FP_EXTEND, Node->getValueType(0), Load);
1806          Tmp1 = LegalizeOp(Result);  // Relegalize new nodes.
1807          Tmp2 = LegalizeOp(Load.getValue(1));
1808          break;
1809        }
1810        assert(ExtType != ISD::EXTLOAD &&"EXTLOAD should always be supported!");
1811        // Turn the unsupported load into an EXTLOAD followed by an explicit
1812        // zero/sign extend inreg.
1813        Result = DAG.getExtLoad(ISD::EXTLOAD, Node->getValueType(0),
1814                                Tmp1, Tmp2, LD->getSrcValue(),
1815                                LD->getSrcValueOffset(), SrcVT,
1816                                LD->isVolatile(), LD->getAlignment());
1817        SDOperand ValRes;
1818        if (ExtType == ISD::SEXTLOAD)
1819          ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, Result.getValueType(),
1820                               Result, DAG.getValueType(SrcVT));
1821        else
1822          ValRes = DAG.getZeroExtendInReg(Result, SrcVT);
1823        Tmp1 = LegalizeOp(ValRes);  // Relegalize new nodes.
1824        Tmp2 = LegalizeOp(Result.getValue(1));  // Relegalize new nodes.
1825        break;
1826      }
1827      // Since loads produce two values, make sure to remember that we legalized
1828      // both of them.
1829      AddLegalizedOperand(SDOperand(Node, 0), Tmp1);
1830      AddLegalizedOperand(SDOperand(Node, 1), Tmp2);
1831      return Op.ResNo ? Tmp2 : Tmp1;
1832    }
1833  }
1834  case ISD::EXTRACT_ELEMENT: {
1835    MVT::ValueType OpTy = Node->getOperand(0).getValueType();
1836    switch (getTypeAction(OpTy)) {
1837    default: assert(0 && "EXTRACT_ELEMENT action for type unimplemented!");
1838    case Legal:
1839      if (cast<ConstantSDNode>(Node->getOperand(1))->getValue()) {
1840        // 1 -> Hi
1841        Result = DAG.getNode(ISD::SRL, OpTy, Node->getOperand(0),
1842                             DAG.getConstant(MVT::getSizeInBits(OpTy)/2,
1843                                             TLI.getShiftAmountTy()));
1844        Result = DAG.getNode(ISD::TRUNCATE, Node->getValueType(0), Result);
1845      } else {
1846        // 0 -> Lo
1847        Result = DAG.getNode(ISD::TRUNCATE, Node->getValueType(0),
1848                             Node->getOperand(0));
1849      }
1850      break;
1851    case Expand:
1852      // Get both the low and high parts.
1853      ExpandOp(Node->getOperand(0), Tmp1, Tmp2);
1854      if (cast<ConstantSDNode>(Node->getOperand(1))->getValue())
1855        Result = Tmp2;  // 1 -> Hi
1856      else
1857        Result = Tmp1;  // 0 -> Lo
1858      break;
1859    }
1860    break;
1861  }
1862
1863  case ISD::CopyToReg:
1864    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
1865
1866    assert(isTypeLegal(Node->getOperand(2).getValueType()) &&
1867           "Register type must be legal!");
1868    // Legalize the incoming value (must be a legal type).
1869    Tmp2 = LegalizeOp(Node->getOperand(2));
1870    if (Node->getNumValues() == 1) {
1871      Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1), Tmp2);
1872    } else {
1873      assert(Node->getNumValues() == 2 && "Unknown CopyToReg");
1874      if (Node->getNumOperands() == 4) {
1875        Tmp3 = LegalizeOp(Node->getOperand(3));
1876        Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1), Tmp2,
1877                                        Tmp3);
1878      } else {
1879        Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1),Tmp2);
1880      }
1881
1882      // Since this produces two values, make sure to remember that we legalized
1883      // both of them.
1884      AddLegalizedOperand(SDOperand(Node, 0), Result.getValue(0));
1885      AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1));
1886      return Result;
1887    }
1888    break;
1889
1890  case ISD::RET:
1891    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
1892
1893    // Ensure that libcalls are emitted before a return.
1894    Tmp1 = DAG.getNode(ISD::TokenFactor, MVT::Other, Tmp1, LastCALLSEQ_END);
1895    Tmp1 = LegalizeOp(Tmp1);
1896    LastCALLSEQ_END = DAG.getEntryNode();
1897
1898    switch (Node->getNumOperands()) {
1899    case 3:  // ret val
1900      Tmp2 = Node->getOperand(1);
1901      Tmp3 = Node->getOperand(2);  // Signness
1902      switch (getTypeAction(Tmp2.getValueType())) {
1903      case Legal:
1904        Result = DAG.UpdateNodeOperands(Result, Tmp1, LegalizeOp(Tmp2), Tmp3);
1905        break;
1906      case Expand:
1907        if (!MVT::isVector(Tmp2.getValueType())) {
1908          SDOperand Lo, Hi;
1909          ExpandOp(Tmp2, Lo, Hi);
1910
1911          // Big endian systems want the hi reg first.
1912          if (!TLI.isLittleEndian())
1913            std::swap(Lo, Hi);
1914
1915          if (Hi.Val)
1916            Result = DAG.getNode(ISD::RET, MVT::Other, Tmp1, Lo, Tmp3, Hi,Tmp3);
1917          else
1918            Result = DAG.getNode(ISD::RET, MVT::Other, Tmp1, Lo, Tmp3);
1919          Result = LegalizeOp(Result);
1920        } else {
1921          SDNode *InVal = Tmp2.Val;
1922          unsigned NumElems = MVT::getVectorNumElements(InVal->getValueType(0));
1923          MVT::ValueType EVT = MVT::getVectorElementType(InVal->getValueType(0));
1924
1925          // Figure out if there is a simple type corresponding to this Vector
1926          // type.  If so, convert to the vector type.
1927          MVT::ValueType TVT = MVT::getVectorType(EVT, NumElems);
1928          if (TLI.isTypeLegal(TVT)) {
1929            // Turn this into a return of the vector type.
1930            Tmp2 = LegalizeOp(Tmp2);
1931            Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
1932          } else if (NumElems == 1) {
1933            // Turn this into a return of the scalar type.
1934            Tmp2 = ScalarizeVectorOp(Tmp2);
1935            Tmp2 = LegalizeOp(Tmp2);
1936            Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
1937
1938            // FIXME: Returns of gcc generic vectors smaller than a legal type
1939            // should be returned in integer registers!
1940
1941            // The scalarized value type may not be legal, e.g. it might require
1942            // promotion or expansion.  Relegalize the return.
1943            Result = LegalizeOp(Result);
1944          } else {
1945            // FIXME: Returns of gcc generic vectors larger than a legal vector
1946            // type should be returned by reference!
1947            SDOperand Lo, Hi;
1948            SplitVectorOp(Tmp2, Lo, Hi);
1949            Result = DAG.getNode(ISD::RET, MVT::Other, Tmp1, Lo, Tmp3, Hi,Tmp3);
1950            Result = LegalizeOp(Result);
1951          }
1952        }
1953        break;
1954      case Promote:
1955        Tmp2 = PromoteOp(Node->getOperand(1));
1956        Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
1957        Result = LegalizeOp(Result);
1958        break;
1959      }
1960      break;
1961    case 1:  // ret void
1962      Result = DAG.UpdateNodeOperands(Result, Tmp1);
1963      break;
1964    default: { // ret <values>
1965      SmallVector<SDOperand, 8> NewValues;
1966      NewValues.push_back(Tmp1);
1967      for (unsigned i = 1, e = Node->getNumOperands(); i < e; i += 2)
1968        switch (getTypeAction(Node->getOperand(i).getValueType())) {
1969        case Legal:
1970          NewValues.push_back(LegalizeOp(Node->getOperand(i)));
1971          NewValues.push_back(Node->getOperand(i+1));
1972          break;
1973        case Expand: {
1974          SDOperand Lo, Hi;
1975          assert(!MVT::isExtendedVT(Node->getOperand(i).getValueType()) &&
1976                 "FIXME: TODO: implement returning non-legal vector types!");
1977          ExpandOp(Node->getOperand(i), Lo, Hi);
1978          NewValues.push_back(Lo);
1979          NewValues.push_back(Node->getOperand(i+1));
1980          if (Hi.Val) {
1981            NewValues.push_back(Hi);
1982            NewValues.push_back(Node->getOperand(i+1));
1983          }
1984          break;
1985        }
1986        case Promote:
1987          assert(0 && "Can't promote multiple return value yet!");
1988        }
1989
1990      if (NewValues.size() == Node->getNumOperands())
1991        Result = DAG.UpdateNodeOperands(Result, &NewValues[0],NewValues.size());
1992      else
1993        Result = DAG.getNode(ISD::RET, MVT::Other,
1994                             &NewValues[0], NewValues.size());
1995      break;
1996    }
1997    }
1998
1999    if (Result.getOpcode() == ISD::RET) {
2000      switch (TLI.getOperationAction(Result.getOpcode(), MVT::Other)) {
2001      default: assert(0 && "This action is not supported yet!");
2002      case TargetLowering::Legal: break;
2003      case TargetLowering::Custom:
2004        Tmp1 = TLI.LowerOperation(Result, DAG);
2005        if (Tmp1.Val) Result = Tmp1;
2006        break;
2007      }
2008    }
2009    break;
2010  case ISD::STORE: {
2011    StoreSDNode *ST = cast<StoreSDNode>(Node);
2012    Tmp1 = LegalizeOp(ST->getChain());    // Legalize the chain.
2013    Tmp2 = LegalizeOp(ST->getBasePtr());  // Legalize the pointer.
2014    int SVOffset = ST->getSrcValueOffset();
2015    unsigned Alignment = ST->getAlignment();
2016    bool isVolatile = ST->isVolatile();
2017
2018    if (!ST->isTruncatingStore()) {
2019      // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
2020      // FIXME: We shouldn't do this for TargetConstantFP's.
2021      // FIXME: move this to the DAG Combiner!  Note that we can't regress due
2022      // to phase ordering between legalized code and the dag combiner.  This
2023      // probably means that we need to integrate dag combiner and legalizer
2024      // together.
2025      // We generally can't do this one for long doubles.
2026      if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) {
2027        if (CFP->getValueType(0) == MVT::f32 &&
2028            getTypeAction(MVT::i32) == Legal) {
2029          Tmp3 = DAG.getConstant((uint32_t)CFP->getValueAPF().
2030                                          convertToAPInt().getZExtValue(),
2031                                  MVT::i32);
2032          Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
2033                                SVOffset, isVolatile, Alignment);
2034          break;
2035        } else if (CFP->getValueType(0) == MVT::f64) {
2036          // If this target supports 64-bit registers, do a single 64-bit store.
2037          if (getTypeAction(MVT::i64) == Legal) {
2038            Tmp3 = DAG.getConstant(CFP->getValueAPF().convertToAPInt().
2039                                     getZExtValue(), MVT::i64);
2040            Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
2041                                  SVOffset, isVolatile, Alignment);
2042            break;
2043          } else if (getTypeAction(MVT::i32) == Legal) {
2044            // Otherwise, if the target supports 32-bit registers, use 2 32-bit
2045            // stores.  If the target supports neither 32- nor 64-bits, this
2046            // xform is certainly not worth it.
2047            uint64_t IntVal =CFP->getValueAPF().convertToAPInt().getZExtValue();
2048            SDOperand Lo = DAG.getConstant(uint32_t(IntVal), MVT::i32);
2049            SDOperand Hi = DAG.getConstant(uint32_t(IntVal >>32), MVT::i32);
2050            if (!TLI.isLittleEndian()) std::swap(Lo, Hi);
2051
2052            Lo = DAG.getStore(Tmp1, Lo, Tmp2, ST->getSrcValue(),
2053                              SVOffset, isVolatile, Alignment);
2054            Tmp2 = DAG.getNode(ISD::ADD, Tmp2.getValueType(), Tmp2,
2055                               getIntPtrConstant(4));
2056            Hi = DAG.getStore(Tmp1, Hi, Tmp2, ST->getSrcValue(), SVOffset+4,
2057                              isVolatile, std::max(Alignment, 4U));
2058
2059            Result = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi);
2060            break;
2061          }
2062        }
2063      }
2064
2065      switch (getTypeAction(ST->getStoredVT())) {
2066      case Legal: {
2067        Tmp3 = LegalizeOp(ST->getValue());
2068        Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp3, Tmp2,
2069                                        ST->getOffset());
2070
2071        MVT::ValueType VT = Tmp3.getValueType();
2072        switch (TLI.getOperationAction(ISD::STORE, VT)) {
2073        default: assert(0 && "This action is not supported yet!");
2074        case TargetLowering::Legal:
2075          // If this is an unaligned store and the target doesn't support it,
2076          // expand it.
2077          if (!TLI.allowsUnalignedMemoryAccesses()) {
2078            unsigned ABIAlignment = TLI.getTargetData()->
2079              getABITypeAlignment(MVT::getTypeForValueType(ST->getStoredVT()));
2080            if (ST->getAlignment() < ABIAlignment)
2081              Result = ExpandUnalignedStore(cast<StoreSDNode>(Result.Val), DAG,
2082                                            TLI);
2083          }
2084          break;
2085        case TargetLowering::Custom:
2086          Tmp1 = TLI.LowerOperation(Result, DAG);
2087          if (Tmp1.Val) Result = Tmp1;
2088          break;
2089        case TargetLowering::Promote:
2090          assert(MVT::isVector(VT) && "Unknown legal promote case!");
2091          Tmp3 = DAG.getNode(ISD::BIT_CONVERT,
2092                             TLI.getTypeToPromoteTo(ISD::STORE, VT), Tmp3);
2093          Result = DAG.getStore(Tmp1, Tmp3, Tmp2,
2094                                ST->getSrcValue(), SVOffset, isVolatile,
2095                                Alignment);
2096          break;
2097        }
2098        break;
2099      }
2100      case Promote:
2101        // Truncate the value and store the result.
2102        Tmp3 = PromoteOp(ST->getValue());
2103        Result = DAG.getTruncStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
2104                                   SVOffset, ST->getStoredVT(),
2105                                   isVolatile, Alignment);
2106        break;
2107
2108      case Expand:
2109        unsigned IncrementSize = 0;
2110        SDOperand Lo, Hi;
2111
2112        // If this is a vector type, then we have to calculate the increment as
2113        // the product of the element size in bytes, and the number of elements
2114        // in the high half of the vector.
2115        if (MVT::isVector(ST->getValue().getValueType())) {
2116          SDNode *InVal = ST->getValue().Val;
2117          unsigned NumElems = MVT::getVectorNumElements(InVal->getValueType(0));
2118          MVT::ValueType EVT = MVT::getVectorElementType(InVal->getValueType(0));
2119
2120          // Figure out if there is a simple type corresponding to this Vector
2121          // type.  If so, convert to the vector type.
2122          MVT::ValueType TVT = MVT::getVectorType(EVT, NumElems);
2123          if (TLI.isTypeLegal(TVT)) {
2124            // Turn this into a normal store of the vector type.
2125            Tmp3 = LegalizeOp(Node->getOperand(1));
2126            Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
2127                                  SVOffset, isVolatile, Alignment);
2128            Result = LegalizeOp(Result);
2129            break;
2130          } else if (NumElems == 1) {
2131            // Turn this into a normal store of the scalar type.
2132            Tmp3 = ScalarizeVectorOp(Node->getOperand(1));
2133            Result = DAG.getStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
2134                                  SVOffset, isVolatile, Alignment);
2135            // The scalarized value type may not be legal, e.g. it might require
2136            // promotion or expansion.  Relegalize the scalar store.
2137            Result = LegalizeOp(Result);
2138            break;
2139          } else {
2140            SplitVectorOp(Node->getOperand(1), Lo, Hi);
2141            IncrementSize = NumElems/2 * MVT::getSizeInBits(EVT)/8;
2142          }
2143        } else {
2144          ExpandOp(Node->getOperand(1), Lo, Hi);
2145          IncrementSize = Hi.Val ? MVT::getSizeInBits(Hi.getValueType())/8 : 0;
2146
2147          if (!TLI.isLittleEndian())
2148            std::swap(Lo, Hi);
2149        }
2150
2151        Lo = DAG.getStore(Tmp1, Lo, Tmp2, ST->getSrcValue(),
2152                          SVOffset, isVolatile, Alignment);
2153
2154        if (Hi.Val == NULL) {
2155          // Must be int <-> float one-to-one expansion.
2156          Result = Lo;
2157          break;
2158        }
2159
2160        Tmp2 = DAG.getNode(ISD::ADD, Tmp2.getValueType(), Tmp2,
2161                           getIntPtrConstant(IncrementSize));
2162        assert(isTypeLegal(Tmp2.getValueType()) &&
2163               "Pointers must be legal!");
2164        SVOffset += IncrementSize;
2165        if (Alignment > IncrementSize)
2166          Alignment = IncrementSize;
2167        Hi = DAG.getStore(Tmp1, Hi, Tmp2, ST->getSrcValue(),
2168                          SVOffset, isVolatile, Alignment);
2169        Result = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo, Hi);
2170        break;
2171      }
2172    } else {
2173      // Truncating store
2174      assert(isTypeLegal(ST->getValue().getValueType()) &&
2175             "Cannot handle illegal TRUNCSTORE yet!");
2176      Tmp3 = LegalizeOp(ST->getValue());
2177
2178      // The only promote case we handle is TRUNCSTORE:i1 X into
2179      //   -> TRUNCSTORE:i8 (and X, 1)
2180      if (ST->getStoredVT() == MVT::i1 &&
2181          TLI.getStoreXAction(MVT::i1) == TargetLowering::Promote) {
2182        // Promote the bool to a mask then store.
2183        Tmp3 = DAG.getNode(ISD::AND, Tmp3.getValueType(), Tmp3,
2184                           DAG.getConstant(1, Tmp3.getValueType()));
2185        Result = DAG.getTruncStore(Tmp1, Tmp3, Tmp2, ST->getSrcValue(),
2186                                   SVOffset, MVT::i8,
2187                                   isVolatile, Alignment);
2188      } else if (Tmp1 != ST->getChain() || Tmp3 != ST->getValue() ||
2189                 Tmp2 != ST->getBasePtr()) {
2190        Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp3, Tmp2,
2191                                        ST->getOffset());
2192      }
2193
2194      MVT::ValueType StVT = cast<StoreSDNode>(Result.Val)->getStoredVT();
2195      switch (TLI.getStoreXAction(StVT)) {
2196      default: assert(0 && "This action is not supported yet!");
2197      case TargetLowering::Legal:
2198        // If this is an unaligned store and the target doesn't support it,
2199        // expand it.
2200        if (!TLI.allowsUnalignedMemoryAccesses()) {
2201          unsigned ABIAlignment = TLI.getTargetData()->
2202            getABITypeAlignment(MVT::getTypeForValueType(ST->getStoredVT()));
2203          if (ST->getAlignment() < ABIAlignment)
2204            Result = ExpandUnalignedStore(cast<StoreSDNode>(Result.Val), DAG,
2205                                          TLI);
2206        }
2207        break;
2208      case TargetLowering::Custom:
2209        Tmp1 = TLI.LowerOperation(Result, DAG);
2210        if (Tmp1.Val) Result = Tmp1;
2211        break;
2212      }
2213    }
2214    break;
2215  }
2216  case ISD::PCMARKER:
2217    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
2218    Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1));
2219    break;
2220  case ISD::STACKSAVE:
2221    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
2222    Result = DAG.UpdateNodeOperands(Result, Tmp1);
2223    Tmp1 = Result.getValue(0);
2224    Tmp2 = Result.getValue(1);
2225
2226    switch (TLI.getOperationAction(ISD::STACKSAVE, MVT::Other)) {
2227    default: assert(0 && "This action is not supported yet!");
2228    case TargetLowering::Legal: break;
2229    case TargetLowering::Custom:
2230      Tmp3 = TLI.LowerOperation(Result, DAG);
2231      if (Tmp3.Val) {
2232        Tmp1 = LegalizeOp(Tmp3);
2233        Tmp2 = LegalizeOp(Tmp3.getValue(1));
2234      }
2235      break;
2236    case TargetLowering::Expand:
2237      // Expand to CopyFromReg if the target set
2238      // StackPointerRegisterToSaveRestore.
2239      if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
2240        Tmp1 = DAG.getCopyFromReg(Result.getOperand(0), SP,
2241                                  Node->getValueType(0));
2242        Tmp2 = Tmp1.getValue(1);
2243      } else {
2244        Tmp1 = DAG.getNode(ISD::UNDEF, Node->getValueType(0));
2245        Tmp2 = Node->getOperand(0);
2246      }
2247      break;
2248    }
2249
2250    // Since stacksave produce two values, make sure to remember that we
2251    // legalized both of them.
2252    AddLegalizedOperand(SDOperand(Node, 0), Tmp1);
2253    AddLegalizedOperand(SDOperand(Node, 1), Tmp2);
2254    return Op.ResNo ? Tmp2 : Tmp1;
2255
2256  case ISD::STACKRESTORE:
2257    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
2258    Tmp2 = LegalizeOp(Node->getOperand(1));  // Legalize the pointer.
2259    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
2260
2261    switch (TLI.getOperationAction(ISD::STACKRESTORE, MVT::Other)) {
2262    default: assert(0 && "This action is not supported yet!");
2263    case TargetLowering::Legal: break;
2264    case TargetLowering::Custom:
2265      Tmp1 = TLI.LowerOperation(Result, DAG);
2266      if (Tmp1.Val) Result = Tmp1;
2267      break;
2268    case TargetLowering::Expand:
2269      // Expand to CopyToReg if the target set
2270      // StackPointerRegisterToSaveRestore.
2271      if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
2272        Result = DAG.getCopyToReg(Tmp1, SP, Tmp2);
2273      } else {
2274        Result = Tmp1;
2275      }
2276      break;
2277    }
2278    break;
2279
2280  case ISD::READCYCLECOUNTER:
2281    Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the chain
2282    Result = DAG.UpdateNodeOperands(Result, Tmp1);
2283    switch (TLI.getOperationAction(ISD::READCYCLECOUNTER,
2284                                   Node->getValueType(0))) {
2285    default: assert(0 && "This action is not supported yet!");
2286    case TargetLowering::Legal:
2287      Tmp1 = Result.getValue(0);
2288      Tmp2 = Result.getValue(1);
2289      break;
2290    case TargetLowering::Custom:
2291      Result = TLI.LowerOperation(Result, DAG);
2292      Tmp1 = LegalizeOp(Result.getValue(0));
2293      Tmp2 = LegalizeOp(Result.getValue(1));
2294      break;
2295    }
2296
2297    // Since rdcc produce two values, make sure to remember that we legalized
2298    // both of them.
2299    AddLegalizedOperand(SDOperand(Node, 0), Tmp1);
2300    AddLegalizedOperand(SDOperand(Node, 1), Tmp2);
2301    return Result;
2302
2303  case ISD::SELECT:
2304    switch (getTypeAction(Node->getOperand(0).getValueType())) {
2305    case Expand: assert(0 && "It's impossible to expand bools");
2306    case Legal:
2307      Tmp1 = LegalizeOp(Node->getOperand(0)); // Legalize the condition.
2308      break;
2309    case Promote:
2310      Tmp1 = PromoteOp(Node->getOperand(0));  // Promote the condition.
2311      // Make sure the condition is either zero or one.
2312      if (!DAG.MaskedValueIsZero(Tmp1,
2313                                 MVT::getIntVTBitMask(Tmp1.getValueType())^1))
2314        Tmp1 = DAG.getZeroExtendInReg(Tmp1, MVT::i1);
2315      break;
2316    }
2317    Tmp2 = LegalizeOp(Node->getOperand(1));   // TrueVal
2318    Tmp3 = LegalizeOp(Node->getOperand(2));   // FalseVal
2319
2320    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
2321
2322    switch (TLI.getOperationAction(ISD::SELECT, Tmp2.getValueType())) {
2323    default: assert(0 && "This action is not supported yet!");
2324    case TargetLowering::Legal: break;
2325    case TargetLowering::Custom: {
2326      Tmp1 = TLI.LowerOperation(Result, DAG);
2327      if (Tmp1.Val) Result = Tmp1;
2328      break;
2329    }
2330    case TargetLowering::Expand:
2331      if (Tmp1.getOpcode() == ISD::SETCC) {
2332        Result = DAG.getSelectCC(Tmp1.getOperand(0), Tmp1.getOperand(1),
2333                              Tmp2, Tmp3,
2334                              cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
2335      } else {
2336        Result = DAG.getSelectCC(Tmp1,
2337                                 DAG.getConstant(0, Tmp1.getValueType()),
2338                                 Tmp2, Tmp3, ISD::SETNE);
2339      }
2340      break;
2341    case TargetLowering::Promote: {
2342      MVT::ValueType NVT =
2343        TLI.getTypeToPromoteTo(ISD::SELECT, Tmp2.getValueType());
2344      unsigned ExtOp, TruncOp;
2345      if (MVT::isVector(Tmp2.getValueType())) {
2346        ExtOp   = ISD::BIT_CONVERT;
2347        TruncOp = ISD::BIT_CONVERT;
2348      } else if (MVT::isInteger(Tmp2.getValueType())) {
2349        ExtOp   = ISD::ANY_EXTEND;
2350        TruncOp = ISD::TRUNCATE;
2351      } else {
2352        ExtOp   = ISD::FP_EXTEND;
2353        TruncOp = ISD::FP_ROUND;
2354      }
2355      // Promote each of the values to the new type.
2356      Tmp2 = DAG.getNode(ExtOp, NVT, Tmp2);
2357      Tmp3 = DAG.getNode(ExtOp, NVT, Tmp3);
2358      // Perform the larger operation, then round down.
2359      Result = DAG.getNode(ISD::SELECT, NVT, Tmp1, Tmp2,Tmp3);
2360      Result = DAG.getNode(TruncOp, Node->getValueType(0), Result);
2361      break;
2362    }
2363    }
2364    break;
2365  case ISD::SELECT_CC: {
2366    Tmp1 = Node->getOperand(0);               // LHS
2367    Tmp2 = Node->getOperand(1);               // RHS
2368    Tmp3 = LegalizeOp(Node->getOperand(2));   // True
2369    Tmp4 = LegalizeOp(Node->getOperand(3));   // False
2370    SDOperand CC = Node->getOperand(4);
2371
2372    LegalizeSetCCOperands(Tmp1, Tmp2, CC);
2373
2374    // If we didn't get both a LHS and RHS back from LegalizeSetCCOperands,
2375    // the LHS is a legal SETCC itself.  In this case, we need to compare
2376    // the result against zero to select between true and false values.
2377    if (Tmp2.Val == 0) {
2378      Tmp2 = DAG.getConstant(0, Tmp1.getValueType());
2379      CC = DAG.getCondCode(ISD::SETNE);
2380    }
2381    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3, Tmp4, CC);
2382
2383    // Everything is legal, see if we should expand this op or something.
2384    switch (TLI.getOperationAction(ISD::SELECT_CC, Tmp3.getValueType())) {
2385    default: assert(0 && "This action is not supported yet!");
2386    case TargetLowering::Legal: break;
2387    case TargetLowering::Custom:
2388      Tmp1 = TLI.LowerOperation(Result, DAG);
2389      if (Tmp1.Val) Result = Tmp1;
2390      break;
2391    }
2392    break;
2393  }
2394  case ISD::SETCC:
2395    Tmp1 = Node->getOperand(0);
2396    Tmp2 = Node->getOperand(1);
2397    Tmp3 = Node->getOperand(2);
2398    LegalizeSetCCOperands(Tmp1, Tmp2, Tmp3);
2399
2400    // If we had to Expand the SetCC operands into a SELECT node, then it may
2401    // not always be possible to return a true LHS & RHS.  In this case, just
2402    // return the value we legalized, returned in the LHS
2403    if (Tmp2.Val == 0) {
2404      Result = Tmp1;
2405      break;
2406    }
2407
2408    switch (TLI.getOperationAction(ISD::SETCC, Tmp1.getValueType())) {
2409    default: assert(0 && "Cannot handle this action for SETCC yet!");
2410    case TargetLowering::Custom:
2411      isCustom = true;
2412      // FALLTHROUGH.
2413    case TargetLowering::Legal:
2414      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
2415      if (isCustom) {
2416        Tmp4 = TLI.LowerOperation(Result, DAG);
2417        if (Tmp4.Val) Result = Tmp4;
2418      }
2419      break;
2420    case TargetLowering::Promote: {
2421      // First step, figure out the appropriate operation to use.
2422      // Allow SETCC to not be supported for all legal data types
2423      // Mostly this targets FP
2424      MVT::ValueType NewInTy = Node->getOperand(0).getValueType();
2425      MVT::ValueType OldVT = NewInTy; OldVT = OldVT;
2426
2427      // Scan for the appropriate larger type to use.
2428      while (1) {
2429        NewInTy = (MVT::ValueType)(NewInTy+1);
2430
2431        assert(MVT::isInteger(NewInTy) == MVT::isInteger(OldVT) &&
2432               "Fell off of the edge of the integer world");
2433        assert(MVT::isFloatingPoint(NewInTy) == MVT::isFloatingPoint(OldVT) &&
2434               "Fell off of the edge of the floating point world");
2435
2436        // If the target supports SETCC of this type, use it.
2437        if (TLI.isOperationLegal(ISD::SETCC, NewInTy))
2438          break;
2439      }
2440      if (MVT::isInteger(NewInTy))
2441        assert(0 && "Cannot promote Legal Integer SETCC yet");
2442      else {
2443        Tmp1 = DAG.getNode(ISD::FP_EXTEND, NewInTy, Tmp1);
2444        Tmp2 = DAG.getNode(ISD::FP_EXTEND, NewInTy, Tmp2);
2445      }
2446      Tmp1 = LegalizeOp(Tmp1);
2447      Tmp2 = LegalizeOp(Tmp2);
2448      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
2449      Result = LegalizeOp(Result);
2450      break;
2451    }
2452    case TargetLowering::Expand:
2453      // Expand a setcc node into a select_cc of the same condition, lhs, and
2454      // rhs that selects between const 1 (true) and const 0 (false).
2455      MVT::ValueType VT = Node->getValueType(0);
2456      Result = DAG.getNode(ISD::SELECT_CC, VT, Tmp1, Tmp2,
2457                           DAG.getConstant(1, VT), DAG.getConstant(0, VT),
2458                           Tmp3);
2459      break;
2460    }
2461    break;
2462  case ISD::MEMSET:
2463  case ISD::MEMCPY:
2464  case ISD::MEMMOVE: {
2465    Tmp1 = LegalizeOp(Node->getOperand(0));      // Chain
2466    Tmp2 = LegalizeOp(Node->getOperand(1));      // Pointer
2467
2468    if (Node->getOpcode() == ISD::MEMSET) {      // memset = ubyte
2469      switch (getTypeAction(Node->getOperand(2).getValueType())) {
2470      case Expand: assert(0 && "Cannot expand a byte!");
2471      case Legal:
2472        Tmp3 = LegalizeOp(Node->getOperand(2));
2473        break;
2474      case Promote:
2475        Tmp3 = PromoteOp(Node->getOperand(2));
2476        break;
2477      }
2478    } else {
2479      Tmp3 = LegalizeOp(Node->getOperand(2));    // memcpy/move = pointer,
2480    }
2481
2482    SDOperand Tmp4;
2483    switch (getTypeAction(Node->getOperand(3).getValueType())) {
2484    case Expand: {
2485      // Length is too big, just take the lo-part of the length.
2486      SDOperand HiPart;
2487      ExpandOp(Node->getOperand(3), Tmp4, HiPart);
2488      break;
2489    }
2490    case Legal:
2491      Tmp4 = LegalizeOp(Node->getOperand(3));
2492      break;
2493    case Promote:
2494      Tmp4 = PromoteOp(Node->getOperand(3));
2495      break;
2496    }
2497
2498    SDOperand Tmp5;
2499    switch (getTypeAction(Node->getOperand(4).getValueType())) {  // uint
2500    case Expand: assert(0 && "Cannot expand this yet!");
2501    case Legal:
2502      Tmp5 = LegalizeOp(Node->getOperand(4));
2503      break;
2504    case Promote:
2505      Tmp5 = PromoteOp(Node->getOperand(4));
2506      break;
2507    }
2508
2509    switch (TLI.getOperationAction(Node->getOpcode(), MVT::Other)) {
2510    default: assert(0 && "This action not implemented for this operation!");
2511    case TargetLowering::Custom:
2512      isCustom = true;
2513      // FALLTHROUGH
2514    case TargetLowering::Legal:
2515      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3, Tmp4, Tmp5);
2516      if (isCustom) {
2517        Tmp1 = TLI.LowerOperation(Result, DAG);
2518        if (Tmp1.Val) Result = Tmp1;
2519      }
2520      break;
2521    case TargetLowering::Expand: {
2522      // Otherwise, the target does not support this operation.  Lower the
2523      // operation to an explicit libcall as appropriate.
2524      MVT::ValueType IntPtr = TLI.getPointerTy();
2525      const Type *IntPtrTy = TLI.getTargetData()->getIntPtrType();
2526      TargetLowering::ArgListTy Args;
2527      TargetLowering::ArgListEntry Entry;
2528
2529      const char *FnName = 0;
2530      if (Node->getOpcode() == ISD::MEMSET) {
2531        Entry.Node = Tmp2; Entry.Ty = IntPtrTy;
2532        Args.push_back(Entry);
2533        // Extend the (previously legalized) ubyte argument to be an int value
2534        // for the call.
2535        if (Tmp3.getValueType() > MVT::i32)
2536          Tmp3 = DAG.getNode(ISD::TRUNCATE, MVT::i32, Tmp3);
2537        else
2538          Tmp3 = DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, Tmp3);
2539        Entry.Node = Tmp3; Entry.Ty = Type::Int32Ty; Entry.isSExt = true;
2540        Args.push_back(Entry);
2541        Entry.Node = Tmp4; Entry.Ty = IntPtrTy; Entry.isSExt = false;
2542        Args.push_back(Entry);
2543
2544        FnName = "memset";
2545      } else if (Node->getOpcode() == ISD::MEMCPY ||
2546                 Node->getOpcode() == ISD::MEMMOVE) {
2547        Entry.Ty = IntPtrTy;
2548        Entry.Node = Tmp2; Args.push_back(Entry);
2549        Entry.Node = Tmp3; Args.push_back(Entry);
2550        Entry.Node = Tmp4; Args.push_back(Entry);
2551        FnName = Node->getOpcode() == ISD::MEMMOVE ? "memmove" : "memcpy";
2552      } else {
2553        assert(0 && "Unknown op!");
2554      }
2555
2556      std::pair<SDOperand,SDOperand> CallResult =
2557        TLI.LowerCallTo(Tmp1, Type::VoidTy, false, false, CallingConv::C, false,
2558                        DAG.getExternalSymbol(FnName, IntPtr), Args, DAG);
2559      Result = CallResult.second;
2560      break;
2561    }
2562    }
2563    break;
2564  }
2565
2566  case ISD::SHL_PARTS:
2567  case ISD::SRA_PARTS:
2568  case ISD::SRL_PARTS: {
2569    SmallVector<SDOperand, 8> Ops;
2570    bool Changed = false;
2571    for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
2572      Ops.push_back(LegalizeOp(Node->getOperand(i)));
2573      Changed |= Ops.back() != Node->getOperand(i);
2574    }
2575    if (Changed)
2576      Result = DAG.UpdateNodeOperands(Result, &Ops[0], Ops.size());
2577
2578    switch (TLI.getOperationAction(Node->getOpcode(),
2579                                   Node->getValueType(0))) {
2580    default: assert(0 && "This action is not supported yet!");
2581    case TargetLowering::Legal: break;
2582    case TargetLowering::Custom:
2583      Tmp1 = TLI.LowerOperation(Result, DAG);
2584      if (Tmp1.Val) {
2585        SDOperand Tmp2, RetVal(0, 0);
2586        for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
2587          Tmp2 = LegalizeOp(Tmp1.getValue(i));
2588          AddLegalizedOperand(SDOperand(Node, i), Tmp2);
2589          if (i == Op.ResNo)
2590            RetVal = Tmp2;
2591        }
2592        assert(RetVal.Val && "Illegal result number");
2593        return RetVal;
2594      }
2595      break;
2596    }
2597
2598    // Since these produce multiple values, make sure to remember that we
2599    // legalized all of them.
2600    for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
2601      AddLegalizedOperand(SDOperand(Node, i), Result.getValue(i));
2602    return Result.getValue(Op.ResNo);
2603  }
2604
2605    // Binary operators
2606  case ISD::ADD:
2607  case ISD::SUB:
2608  case ISD::MUL:
2609  case ISD::MULHS:
2610  case ISD::MULHU:
2611  case ISD::UDIV:
2612  case ISD::SDIV:
2613  case ISD::AND:
2614  case ISD::OR:
2615  case ISD::XOR:
2616  case ISD::SHL:
2617  case ISD::SRL:
2618  case ISD::SRA:
2619  case ISD::FADD:
2620  case ISD::FSUB:
2621  case ISD::FMUL:
2622  case ISD::FDIV:
2623  case ISD::FPOW:
2624    Tmp1 = LegalizeOp(Node->getOperand(0));   // LHS
2625    switch (getTypeAction(Node->getOperand(1).getValueType())) {
2626    case Expand: assert(0 && "Not possible");
2627    case Legal:
2628      Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the RHS.
2629      break;
2630    case Promote:
2631      Tmp2 = PromoteOp(Node->getOperand(1));  // Promote the RHS.
2632      break;
2633    }
2634
2635    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
2636
2637    switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
2638    default: assert(0 && "BinOp legalize operation not supported");
2639    case TargetLowering::Legal: break;
2640    case TargetLowering::Custom:
2641      Tmp1 = TLI.LowerOperation(Result, DAG);
2642      if (Tmp1.Val) Result = Tmp1;
2643      break;
2644    case TargetLowering::Expand: {
2645      MVT::ValueType VT = Op.getValueType();
2646
2647      // See if multiply or divide can be lowered using two-result operations.
2648      SDVTList VTs = DAG.getVTList(VT, VT);
2649      if (Node->getOpcode() == ISD::MUL) {
2650        // We just need the low half of the multiply; try both the signed
2651        // and unsigned forms. If the target supports both SMUL_LOHI and
2652        // UMUL_LOHI, form a preference by checking which forms of plain
2653        // MULH it supports.
2654        bool HasSMUL_LOHI = TLI.isOperationLegal(ISD::SMUL_LOHI, VT);
2655        bool HasUMUL_LOHI = TLI.isOperationLegal(ISD::UMUL_LOHI, VT);
2656        bool HasMULHS = TLI.isOperationLegal(ISD::MULHS, VT);
2657        bool HasMULHU = TLI.isOperationLegal(ISD::MULHU, VT);
2658        unsigned OpToUse = 0;
2659        if (HasSMUL_LOHI && !HasMULHS) {
2660          OpToUse = ISD::SMUL_LOHI;
2661        } else if (HasUMUL_LOHI && !HasMULHU) {
2662          OpToUse = ISD::UMUL_LOHI;
2663        } else if (HasSMUL_LOHI) {
2664          OpToUse = ISD::SMUL_LOHI;
2665        } else if (HasUMUL_LOHI) {
2666          OpToUse = ISD::UMUL_LOHI;
2667        }
2668        if (OpToUse) {
2669          Result = SDOperand(DAG.getNode(OpToUse, VTs, Tmp1, Tmp2).Val, 0);
2670          break;
2671        }
2672      }
2673      if (Node->getOpcode() == ISD::MULHS &&
2674          TLI.isOperationLegal(ISD::SMUL_LOHI, VT)) {
2675        Result = SDOperand(DAG.getNode(ISD::SMUL_LOHI, VTs, Tmp1, Tmp2).Val, 1);
2676        break;
2677      }
2678      if (Node->getOpcode() == ISD::MULHU &&
2679          TLI.isOperationLegal(ISD::UMUL_LOHI, VT)) {
2680        Result = SDOperand(DAG.getNode(ISD::UMUL_LOHI, VTs, Tmp1, Tmp2).Val, 1);
2681        break;
2682      }
2683      if (Node->getOpcode() == ISD::SDIV &&
2684          TLI.isOperationLegal(ISD::SDIVREM, VT)) {
2685        Result = SDOperand(DAG.getNode(ISD::SDIVREM, VTs, Tmp1, Tmp2).Val, 0);
2686        break;
2687      }
2688      if (Node->getOpcode() == ISD::UDIV &&
2689          TLI.isOperationLegal(ISD::UDIVREM, VT)) {
2690        Result = SDOperand(DAG.getNode(ISD::UDIVREM, VTs, Tmp1, Tmp2).Val, 0);
2691        break;
2692      }
2693
2694      // Check to see if we have a libcall for this operator.
2695      RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2696      bool isSigned = false;
2697      switch (Node->getOpcode()) {
2698      case ISD::UDIV:
2699      case ISD::SDIV:
2700        if (VT == MVT::i32) {
2701          LC = Node->getOpcode() == ISD::UDIV
2702            ? RTLIB::UDIV_I32 : RTLIB::SDIV_I32;
2703          isSigned = Node->getOpcode() == ISD::SDIV;
2704        }
2705        break;
2706      case ISD::FPOW:
2707        LC = VT == MVT::f32 ? RTLIB::POW_F32 :
2708             VT == MVT::f64 ? RTLIB::POW_F64 :
2709             VT == MVT::f80 ? RTLIB::POW_F80 :
2710             VT == MVT::ppcf128 ? RTLIB::POW_PPCF128 :
2711             RTLIB::UNKNOWN_LIBCALL;
2712        break;
2713      default: break;
2714      }
2715      if (LC != RTLIB::UNKNOWN_LIBCALL) {
2716        SDOperand Dummy;
2717        Result = ExpandLibCall(TLI.getLibcallName(LC), Node, isSigned, Dummy);
2718        break;
2719      }
2720
2721      assert(MVT::isVector(Node->getValueType(0)) &&
2722             "Cannot expand this binary operator!");
2723      // Expand the operation into a bunch of nasty scalar code.
2724      Result = LegalizeOp(UnrollVectorOp(Op));
2725      break;
2726    }
2727    case TargetLowering::Promote: {
2728      switch (Node->getOpcode()) {
2729      default:  assert(0 && "Do not know how to promote this BinOp!");
2730      case ISD::AND:
2731      case ISD::OR:
2732      case ISD::XOR: {
2733        MVT::ValueType OVT = Node->getValueType(0);
2734        MVT::ValueType NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
2735        assert(MVT::isVector(OVT) && "Cannot promote this BinOp!");
2736        // Bit convert each of the values to the new type.
2737        Tmp1 = DAG.getNode(ISD::BIT_CONVERT, NVT, Tmp1);
2738        Tmp2 = DAG.getNode(ISD::BIT_CONVERT, NVT, Tmp2);
2739        Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1, Tmp2);
2740        // Bit convert the result back the original type.
2741        Result = DAG.getNode(ISD::BIT_CONVERT, OVT, Result);
2742        break;
2743      }
2744      }
2745    }
2746    }
2747    break;
2748
2749  case ISD::SMUL_LOHI:
2750  case ISD::UMUL_LOHI:
2751  case ISD::SDIVREM:
2752  case ISD::UDIVREM:
2753    // These nodes will only be produced by target-specific lowering, so
2754    // they shouldn't be here if they aren't legal.
2755    assert(TLI.isOperationLegal(Node->getOpcode(), Node->getValueType(0)) &&
2756           "This must be legal!");
2757
2758    Tmp1 = LegalizeOp(Node->getOperand(0));   // LHS
2759    Tmp2 = LegalizeOp(Node->getOperand(1));   // RHS
2760    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
2761    break;
2762
2763  case ISD::FCOPYSIGN:  // FCOPYSIGN does not require LHS/RHS to match type!
2764    Tmp1 = LegalizeOp(Node->getOperand(0));   // LHS
2765    switch (getTypeAction(Node->getOperand(1).getValueType())) {
2766      case Expand: assert(0 && "Not possible");
2767      case Legal:
2768        Tmp2 = LegalizeOp(Node->getOperand(1)); // Legalize the RHS.
2769        break;
2770      case Promote:
2771        Tmp2 = PromoteOp(Node->getOperand(1));  // Promote the RHS.
2772        break;
2773    }
2774
2775    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
2776
2777    switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
2778    default: assert(0 && "Operation not supported");
2779    case TargetLowering::Custom:
2780      Tmp1 = TLI.LowerOperation(Result, DAG);
2781      if (Tmp1.Val) Result = Tmp1;
2782      break;
2783    case TargetLowering::Legal: break;
2784    case TargetLowering::Expand: {
2785      // If this target supports fabs/fneg natively and select is cheap,
2786      // do this efficiently.
2787      if (!TLI.isSelectExpensive() &&
2788          TLI.getOperationAction(ISD::FABS, Tmp1.getValueType()) ==
2789          TargetLowering::Legal &&
2790          TLI.getOperationAction(ISD::FNEG, Tmp1.getValueType()) ==
2791          TargetLowering::Legal) {
2792        // Get the sign bit of the RHS.
2793        MVT::ValueType IVT =
2794          Tmp2.getValueType() == MVT::f32 ? MVT::i32 : MVT::i64;
2795        SDOperand SignBit = DAG.getNode(ISD::BIT_CONVERT, IVT, Tmp2);
2796        SignBit = DAG.getSetCC(TLI.getSetCCResultTy(),
2797                               SignBit, DAG.getConstant(0, IVT), ISD::SETLT);
2798        // Get the absolute value of the result.
2799        SDOperand AbsVal = DAG.getNode(ISD::FABS, Tmp1.getValueType(), Tmp1);
2800        // Select between the nabs and abs value based on the sign bit of
2801        // the input.
2802        Result = DAG.getNode(ISD::SELECT, AbsVal.getValueType(), SignBit,
2803                             DAG.getNode(ISD::FNEG, AbsVal.getValueType(),
2804                                         AbsVal),
2805                             AbsVal);
2806        Result = LegalizeOp(Result);
2807        break;
2808      }
2809
2810      // Otherwise, do bitwise ops!
2811      MVT::ValueType NVT =
2812        Node->getValueType(0) == MVT::f32 ? MVT::i32 : MVT::i64;
2813      Result = ExpandFCOPYSIGNToBitwiseOps(Node, NVT, DAG, TLI);
2814      Result = DAG.getNode(ISD::BIT_CONVERT, Node->getValueType(0), Result);
2815      Result = LegalizeOp(Result);
2816      break;
2817    }
2818    }
2819    break;
2820
2821  case ISD::ADDC:
2822  case ISD::SUBC:
2823    Tmp1 = LegalizeOp(Node->getOperand(0));
2824    Tmp2 = LegalizeOp(Node->getOperand(1));
2825    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
2826    // Since this produces two values, make sure to remember that we legalized
2827    // both of them.
2828    AddLegalizedOperand(SDOperand(Node, 0), Result.getValue(0));
2829    AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1));
2830    return Result;
2831
2832  case ISD::ADDE:
2833  case ISD::SUBE:
2834    Tmp1 = LegalizeOp(Node->getOperand(0));
2835    Tmp2 = LegalizeOp(Node->getOperand(1));
2836    Tmp3 = LegalizeOp(Node->getOperand(2));
2837    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3);
2838    // Since this produces two values, make sure to remember that we legalized
2839    // both of them.
2840    AddLegalizedOperand(SDOperand(Node, 0), Result.getValue(0));
2841    AddLegalizedOperand(SDOperand(Node, 1), Result.getValue(1));
2842    return Result;
2843
2844  case ISD::BUILD_PAIR: {
2845    MVT::ValueType PairTy = Node->getValueType(0);
2846    // TODO: handle the case where the Lo and Hi operands are not of legal type
2847    Tmp1 = LegalizeOp(Node->getOperand(0));   // Lo
2848    Tmp2 = LegalizeOp(Node->getOperand(1));   // Hi
2849    switch (TLI.getOperationAction(ISD::BUILD_PAIR, PairTy)) {
2850    case TargetLowering::Promote:
2851    case TargetLowering::Custom:
2852      assert(0 && "Cannot promote/custom this yet!");
2853    case TargetLowering::Legal:
2854      if (Tmp1 != Node->getOperand(0) || Tmp2 != Node->getOperand(1))
2855        Result = DAG.getNode(ISD::BUILD_PAIR, PairTy, Tmp1, Tmp2);
2856      break;
2857    case TargetLowering::Expand:
2858      Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, PairTy, Tmp1);
2859      Tmp2 = DAG.getNode(ISD::ANY_EXTEND, PairTy, Tmp2);
2860      Tmp2 = DAG.getNode(ISD::SHL, PairTy, Tmp2,
2861                         DAG.getConstant(MVT::getSizeInBits(PairTy)/2,
2862                                         TLI.getShiftAmountTy()));
2863      Result = DAG.getNode(ISD::OR, PairTy, Tmp1, Tmp2);
2864      break;
2865    }
2866    break;
2867  }
2868
2869  case ISD::UREM:
2870  case ISD::SREM:
2871  case ISD::FREM:
2872    Tmp1 = LegalizeOp(Node->getOperand(0));   // LHS
2873    Tmp2 = LegalizeOp(Node->getOperand(1));   // RHS
2874
2875    switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
2876    case TargetLowering::Promote: assert(0 && "Cannot promote this yet!");
2877    case TargetLowering::Custom:
2878      isCustom = true;
2879      // FALLTHROUGH
2880    case TargetLowering::Legal:
2881      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
2882      if (isCustom) {
2883        Tmp1 = TLI.LowerOperation(Result, DAG);
2884        if (Tmp1.Val) Result = Tmp1;
2885      }
2886      break;
2887    case TargetLowering::Expand: {
2888      unsigned DivOpc= (Node->getOpcode() == ISD::UREM) ? ISD::UDIV : ISD::SDIV;
2889      bool isSigned = DivOpc == ISD::SDIV;
2890      MVT::ValueType VT = Node->getValueType(0);
2891
2892      // See if remainder can be lowered using two-result operations.
2893      SDVTList VTs = DAG.getVTList(VT, VT);
2894      if (Node->getOpcode() == ISD::SREM &&
2895          TLI.isOperationLegal(ISD::SDIVREM, VT)) {
2896        Result = SDOperand(DAG.getNode(ISD::SDIVREM, VTs, Tmp1, Tmp2).Val, 1);
2897        break;
2898      }
2899      if (Node->getOpcode() == ISD::UREM &&
2900          TLI.isOperationLegal(ISD::UDIVREM, VT)) {
2901        Result = SDOperand(DAG.getNode(ISD::UDIVREM, VTs, Tmp1, Tmp2).Val, 1);
2902        break;
2903      }
2904
2905      if (MVT::isInteger(VT)) {
2906        if (TLI.getOperationAction(DivOpc, VT) ==
2907            TargetLowering::Legal) {
2908          // X % Y -> X-X/Y*Y
2909          Result = DAG.getNode(DivOpc, VT, Tmp1, Tmp2);
2910          Result = DAG.getNode(ISD::MUL, VT, Result, Tmp2);
2911          Result = DAG.getNode(ISD::SUB, VT, Tmp1, Result);
2912        } else {
2913          assert(VT == MVT::i32 &&
2914                 "Cannot expand this binary operator!");
2915          RTLIB::Libcall LC = Node->getOpcode() == ISD::UREM
2916            ? RTLIB::UREM_I32 : RTLIB::SREM_I32;
2917          SDOperand Dummy;
2918          Result = ExpandLibCall(TLI.getLibcallName(LC), Node, isSigned, Dummy);
2919        }
2920      } else {
2921        // Floating point mod -> fmod libcall.
2922        RTLIB::Libcall LC = VT == MVT::f32
2923          ? RTLIB::REM_F32 : RTLIB::REM_F64;
2924        SDOperand Dummy;
2925        Result = ExpandLibCall(TLI.getLibcallName(LC), Node,
2926                               false/*sign irrelevant*/, Dummy);
2927      }
2928      break;
2929    }
2930    }
2931    break;
2932  case ISD::VAARG: {
2933    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
2934    Tmp2 = LegalizeOp(Node->getOperand(1));  // Legalize the pointer.
2935
2936    MVT::ValueType VT = Node->getValueType(0);
2937    switch (TLI.getOperationAction(Node->getOpcode(), MVT::Other)) {
2938    default: assert(0 && "This action is not supported yet!");
2939    case TargetLowering::Custom:
2940      isCustom = true;
2941      // FALLTHROUGH
2942    case TargetLowering::Legal:
2943      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2));
2944      Result = Result.getValue(0);
2945      Tmp1 = Result.getValue(1);
2946
2947      if (isCustom) {
2948        Tmp2 = TLI.LowerOperation(Result, DAG);
2949        if (Tmp2.Val) {
2950          Result = LegalizeOp(Tmp2);
2951          Tmp1 = LegalizeOp(Tmp2.getValue(1));
2952        }
2953      }
2954      break;
2955    case TargetLowering::Expand: {
2956      SrcValueSDNode *SV = cast<SrcValueSDNode>(Node->getOperand(2));
2957      SDOperand VAList = DAG.getLoad(TLI.getPointerTy(), Tmp1, Tmp2,
2958                                     SV->getValue(), SV->getOffset());
2959      // Increment the pointer, VAList, to the next vaarg
2960      Tmp3 = DAG.getNode(ISD::ADD, TLI.getPointerTy(), VAList,
2961                         DAG.getConstant(MVT::getSizeInBits(VT)/8,
2962                                         TLI.getPointerTy()));
2963      // Store the incremented VAList to the legalized pointer
2964      Tmp3 = DAG.getStore(VAList.getValue(1), Tmp3, Tmp2, SV->getValue(),
2965                          SV->getOffset());
2966      // Load the actual argument out of the pointer VAList
2967      Result = DAG.getLoad(VT, Tmp3, VAList, NULL, 0);
2968      Tmp1 = LegalizeOp(Result.getValue(1));
2969      Result = LegalizeOp(Result);
2970      break;
2971    }
2972    }
2973    // Since VAARG produces two values, make sure to remember that we
2974    // legalized both of them.
2975    AddLegalizedOperand(SDOperand(Node, 0), Result);
2976    AddLegalizedOperand(SDOperand(Node, 1), Tmp1);
2977    return Op.ResNo ? Tmp1 : Result;
2978  }
2979
2980  case ISD::VACOPY:
2981    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
2982    Tmp2 = LegalizeOp(Node->getOperand(1));  // Legalize the dest pointer.
2983    Tmp3 = LegalizeOp(Node->getOperand(2));  // Legalize the source pointer.
2984
2985    switch (TLI.getOperationAction(ISD::VACOPY, MVT::Other)) {
2986    default: assert(0 && "This action is not supported yet!");
2987    case TargetLowering::Custom:
2988      isCustom = true;
2989      // FALLTHROUGH
2990    case TargetLowering::Legal:
2991      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Tmp3,
2992                                      Node->getOperand(3), Node->getOperand(4));
2993      if (isCustom) {
2994        Tmp1 = TLI.LowerOperation(Result, DAG);
2995        if (Tmp1.Val) Result = Tmp1;
2996      }
2997      break;
2998    case TargetLowering::Expand:
2999      // This defaults to loading a pointer from the input and storing it to the
3000      // output, returning the chain.
3001      SrcValueSDNode *SVD = cast<SrcValueSDNode>(Node->getOperand(3));
3002      SrcValueSDNode *SVS = cast<SrcValueSDNode>(Node->getOperand(4));
3003      Tmp4 = DAG.getLoad(TLI.getPointerTy(), Tmp1, Tmp3, SVD->getValue(),
3004                         SVD->getOffset());
3005      Result = DAG.getStore(Tmp4.getValue(1), Tmp4, Tmp2, SVS->getValue(),
3006                            SVS->getOffset());
3007      break;
3008    }
3009    break;
3010
3011  case ISD::VAEND:
3012    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
3013    Tmp2 = LegalizeOp(Node->getOperand(1));  // Legalize the pointer.
3014
3015    switch (TLI.getOperationAction(ISD::VAEND, MVT::Other)) {
3016    default: assert(0 && "This action is not supported yet!");
3017    case TargetLowering::Custom:
3018      isCustom = true;
3019      // FALLTHROUGH
3020    case TargetLowering::Legal:
3021      Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2));
3022      if (isCustom) {
3023        Tmp1 = TLI.LowerOperation(Tmp1, DAG);
3024        if (Tmp1.Val) Result = Tmp1;
3025      }
3026      break;
3027    case TargetLowering::Expand:
3028      Result = Tmp1; // Default to a no-op, return the chain
3029      break;
3030    }
3031    break;
3032
3033  case ISD::VASTART:
3034    Tmp1 = LegalizeOp(Node->getOperand(0));  // Legalize the chain.
3035    Tmp2 = LegalizeOp(Node->getOperand(1));  // Legalize the pointer.
3036
3037    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2, Node->getOperand(2));
3038
3039    switch (TLI.getOperationAction(ISD::VASTART, MVT::Other)) {
3040    default: assert(0 && "This action is not supported yet!");
3041    case TargetLowering::Legal: break;
3042    case TargetLowering::Custom:
3043      Tmp1 = TLI.LowerOperation(Result, DAG);
3044      if (Tmp1.Val) Result = Tmp1;
3045      break;
3046    }
3047    break;
3048
3049  case ISD::ROTL:
3050  case ISD::ROTR:
3051    Tmp1 = LegalizeOp(Node->getOperand(0));   // LHS
3052    Tmp2 = LegalizeOp(Node->getOperand(1));   // RHS
3053    Result = DAG.UpdateNodeOperands(Result, Tmp1, Tmp2);
3054    switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
3055    default:
3056      assert(0 && "ROTL/ROTR legalize operation not supported");
3057      break;
3058    case TargetLowering::Legal:
3059      break;
3060    case TargetLowering::Custom:
3061      Tmp1 = TLI.LowerOperation(Result, DAG);
3062      if (Tmp1.Val) Result = Tmp1;
3063      break;
3064    case TargetLowering::Promote:
3065      assert(0 && "Do not know how to promote ROTL/ROTR");
3066      break;
3067    case TargetLowering::Expand:
3068      assert(0 && "Do not know how to expand ROTL/ROTR");
3069      break;
3070    }
3071    break;
3072
3073  case ISD::BSWAP:
3074    Tmp1 = LegalizeOp(Node->getOperand(0));   // Op
3075    switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
3076    case TargetLowering::Custom:
3077      assert(0 && "Cannot custom legalize this yet!");
3078    case TargetLowering::Legal:
3079      Result = DAG.UpdateNodeOperands(Result, Tmp1);
3080      break;
3081    case TargetLowering::Promote: {
3082      MVT::ValueType OVT = Tmp1.getValueType();
3083      MVT::ValueType NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
3084      unsigned DiffBits = MVT::getSizeInBits(NVT) - MVT::getSizeInBits(OVT);
3085
3086      Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, NVT, Tmp1);
3087      Tmp1 = DAG.getNode(ISD::BSWAP, NVT, Tmp1);
3088      Result = DAG.getNode(ISD::SRL, NVT, Tmp1,
3089                           DAG.getConstant(DiffBits, TLI.getShiftAmountTy()));
3090      break;
3091    }
3092    case TargetLowering::Expand:
3093      Result = ExpandBSWAP(Tmp1);
3094      break;
3095    }
3096    break;
3097
3098  case ISD::CTPOP:
3099  case ISD::CTTZ:
3100  case ISD::CTLZ:
3101    Tmp1 = LegalizeOp(Node->getOperand(0));   // Op
3102    switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
3103    case TargetLowering::Custom:
3104    case TargetLowering::Legal:
3105      Result = DAG.UpdateNodeOperands(Result, Tmp1);
3106      if (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)) ==
3107          TargetLowering::Custom) {
3108        Tmp1 = TLI.LowerOperation(Result, DAG);
3109        if (Tmp1.Val) {
3110          Result = Tmp1;
3111        }
3112      }
3113      break;
3114    case TargetLowering::Promote: {
3115      MVT::ValueType OVT = Tmp1.getValueType();
3116      MVT::ValueType NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
3117
3118      // Zero extend the argument.
3119      Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, NVT, Tmp1);
3120      // Perform the larger operation, then subtract if needed.
3121      Tmp1 = DAG.getNode(Node->getOpcode(), Node->getValueType(0), Tmp1);
3122      switch (Node->getOpcode()) {
3123      case ISD::CTPOP:
3124        Result = Tmp1;
3125        break;
3126      case ISD::CTTZ:
3127        //if Tmp1 == sizeinbits(NVT) then Tmp1 = sizeinbits(Old VT)
3128        Tmp2 = DAG.getSetCC(TLI.getSetCCResultTy(), Tmp1,
3129                            DAG.getConstant(MVT::getSizeInBits(NVT), NVT),
3130                            ISD::SETEQ);
3131        Result = DAG.getNode(ISD::SELECT, NVT, Tmp2,
3132                             DAG.getConstant(MVT::getSizeInBits(OVT),NVT), Tmp1);
3133        break;
3134      case ISD::CTLZ:
3135        // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
3136        Result = DAG.getNode(ISD::SUB, NVT, Tmp1,
3137                             DAG.getConstant(MVT::getSizeInBits(NVT) -
3138                                             MVT::getSizeInBits(OVT), NVT));
3139        break;
3140      }
3141      break;
3142    }
3143    case TargetLowering::Expand:
3144      Result = ExpandBitCount(Node->getOpcode(), Tmp1);
3145      break;
3146    }
3147    break;
3148
3149    // Unary operators
3150  case ISD::FABS:
3151  case ISD::FNEG:
3152  case ISD::FSQRT:
3153  case ISD::FSIN:
3154  case ISD::FCOS:
3155    Tmp1 = LegalizeOp(Node->getOperand(0));
3156    switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))) {
3157    case TargetLowering::Promote:
3158    case TargetLowering::Custom:
3159     isCustom = true;
3160     // FALLTHROUGH
3161    case TargetLowering::Legal:
3162      Result = DAG.UpdateNodeOperands(Result, Tmp1);
3163      if (isCustom) {
3164        Tmp1 = TLI.LowerOperation(Result, DAG);
3165        if (Tmp1.Val) Result = Tmp1;
3166      }
3167      break;
3168    case TargetLowering::Expand:
3169      switch (Node->getOpcode()) {
3170      default: assert(0 && "Unreachable!");
3171      case ISD::FNEG:
3172        // Expand Y = FNEG(X) ->  Y = SUB -0.0, X
3173        Tmp2 = DAG.getConstantFP(-0.0, Node->getValueType(0));
3174        Result = DAG.getNode(ISD::FSUB, Node->getValueType(0), Tmp2, Tmp1);
3175        break;
3176      case ISD::FABS: {
3177        // Expand Y = FABS(X) -> Y = (X >u 0.0) ? X : fneg(X).
3178        MVT::ValueType VT = Node->getValueType(0);
3179        Tmp2 = DAG.getConstantFP(0.0, VT);
3180        Tmp2 = DAG.getSetCC(TLI.getSetCCResultTy(), Tmp1, Tmp2, ISD::SETUGT);
3181        Tmp3 = DAG.getNode(ISD::FNEG, VT, Tmp1);
3182        Result = DAG.getNode(ISD::SELECT, VT, Tmp2, Tmp1, Tmp3);
3183        break;
3184      }
3185      case ISD::FSQRT:
3186      case ISD::FSIN:
3187      case ISD::FCOS: {
3188        MVT::ValueType VT = Node->getValueType(0);
3189
3190        // Expand unsupported unary vector operators by unrolling them.
3191        if (MVT::isVector(VT)) {
3192          Result = LegalizeOp(UnrollVectorOp(Op));
3193          break;
3194        }
3195
3196        RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3197        switch(Node->getOpcode()) {
3198        case ISD::FSQRT:
3199          LC = VT == MVT::f32 ? RTLIB::SQRT_F32 :
3200               VT == MVT::f64 ? RTLIB::SQRT_F64 :
3201               VT == MVT::f80 ? RTLIB::SQRT_F80 :
3202               VT == MVT::ppcf128 ? RTLIB::SQRT_PPCF128 :
3203               RTLIB::UNKNOWN_LIBCALL;
3204          break;
3205        case ISD::FSIN:
3206          LC = VT == MVT::f32 ? RTLIB::SIN_F32 : RTLIB::SIN_F64;
3207          break;
3208        case ISD::FCOS:
3209          LC = VT == MVT::f32 ? RTLIB::COS_F32 : RTLIB::COS_F64;
3210          break;
3211        default: assert(0 && "Unreachable!");
3212        }
3213        SDOperand Dummy;
3214        Result = ExpandLibCall(TLI.getLibcallName(LC), Node,
3215                               false/*sign irrelevant*/, Dummy);
3216        break;
3217      }
3218      }
3219      break;
3220    }
3221    break;
3222  case ISD::FPOWI: {
3223    MVT::ValueType VT = Node->getValueType(0);
3224
3225    // Expand unsupported unary vector operators by unrolling them.
3226    if (MVT::isVector(VT)) {
3227      Result = LegalizeOp(UnrollVectorOp(Op));
3228      break;
3229    }
3230
3231    // We always lower FPOWI into a libcall.  No target support for it yet.
3232    RTLIB::Libcall LC =
3233      VT == MVT::f32 ? RTLIB::POWI_F32 :
3234      VT == MVT::f64 ? RTLIB::POWI_F64 :
3235      VT == MVT::f80 ? RTLIB::POWI_F80 :
3236      VT == MVT::ppcf128 ? RTLIB::POWI_PPCF128 :
3237      RTLIB::UNKNOWN_LIBCALL;
3238    SDOperand Dummy;
3239    Result = ExpandLibCall(TLI.getLibcallName(LC), Node,
3240                           false/*sign irrelevant*/, Dummy);
3241    break;
3242  }
3243  case ISD::BIT_CONVERT:
3244    if (!isTypeLegal(Node->getOperand(0).getValueType())) {
3245      Result = ExpandBIT_CONVERT(Node->getValueType(0), Node->getOperand(0));
3246    } else if (MVT::isVector(Op.getOperand(0).getValueType())) {
3247      // The input has to be a vector type, we have to either scalarize it, pack
3248      // it, or convert it based on whether the input vector type is legal.
3249      SDNode *InVal = Node->getOperand(0).Val;
3250      unsigned NumElems = MVT::getVectorNumElements(InVal->getValueType(0));
3251      MVT::ValueType EVT = MVT::getVectorElementType(InVal->getValueType(0));
3252
3253      // Figure out if there is a simple type corresponding to this Vector
3254      // type.  If so, convert to the vector type.
3255      MVT::ValueType TVT = MVT::getVectorType(EVT, NumElems);
3256      if (TLI.isTypeLegal(TVT)) {
3257        // Turn this into a bit convert of the vector input.
3258        Result = DAG.getNode(ISD::BIT_CONVERT, Node->getValueType(0),
3259                             LegalizeOp(Node->getOperand(0)));
3260        break;
3261      } else if (NumElems == 1) {
3262        // Turn this into a bit convert of the scalar input.
3263        Result = DAG.getNode(ISD::BIT_CONVERT, Node->getValueType(0),
3264                             ScalarizeVectorOp(Node->getOperand(0)));
3265        break;
3266      } else {
3267        // FIXME: UNIMP!  Store then reload
3268        assert(0 && "Cast from unsupported vector type not implemented yet!");
3269      }
3270    } else {
3271      switch (TLI.getOperationAction(ISD::BIT_CONVERT,
3272                                     Node->getOperand(0).getValueType())) {
3273      default: assert(0 && "Unknown operation action!");
3274      case TargetLowering::Expand:
3275        Result = ExpandBIT_CONVERT(Node->getValueType(0), Node->getOperand(0));
3276        break;
3277      case TargetLowering::Legal:
3278        Tmp1 = LegalizeOp(Node->getOperand(0));
3279        Result = DAG.UpdateNodeOperands(Result, Tmp1);
3280        break;
3281      }
3282    }
3283    break;
3284
3285    // Conversion operators.  The source and destination have different types.
3286  case ISD::SINT_TO_FP:
3287  case ISD::UINT_TO_FP: {
3288    bool isSigned = Node->getOpcode() == ISD::SINT_TO_FP;
3289    switch (getTypeAction(Node->getOperand(0).getValueType())) {
3290    case Legal:
3291      switch (TLI.getOperationAction(Node->getOpcode(),
3292                                     Node->getOperand(0).getValueType())) {
3293      default: assert(0 && "Unknown operation action!");
3294      case TargetLowering::Custom:
3295        isCustom = true;
3296        // FALLTHROUGH
3297      case TargetLowering::Legal:
3298        Tmp1 = LegalizeOp(Node->getOperand(0));
3299        Result = DAG.UpdateNodeOperands(Result, Tmp1);
3300        if (isCustom) {
3301          Tmp1 = TLI.LowerOperation(Result, DAG);
3302          if (Tmp1.Val) Result = Tmp1;
3303        }
3304        break;
3305      case TargetLowering::Expand:
3306        Result = ExpandLegalINT_TO_FP(isSigned,
3307                                      LegalizeOp(Node->getOperand(0)),
3308                                      Node->getValueType(0));
3309        break;
3310      case TargetLowering::Promote:
3311        Result = PromoteLegalINT_TO_FP(LegalizeOp(Node->getOperand(0)),
3312                                       Node->getValueType(0),
3313                                       isSigned);
3314        break;
3315      }
3316      break;
3317    case Expand:
3318      Result = ExpandIntToFP(Node->getOpcode() == ISD::SINT_TO_FP,
3319                             Node->getValueType(0), Node->getOperand(0));
3320      break;
3321    case Promote:
3322      Tmp1 = PromoteOp(Node->getOperand(0));
3323      if (isSigned) {
3324        Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, Tmp1.getValueType(),
3325                 Tmp1, DAG.getValueType(Node->getOperand(0).getValueType()));
3326      } else {
3327        Tmp1 = DAG.getZeroExtendInReg(Tmp1,
3328                                      Node->getOperand(0).getValueType());
3329      }
3330      Result = DAG.UpdateNodeOperands(Result, Tmp1);
3331      Result = LegalizeOp(Result);  // The 'op' is not necessarily legal!
3332      break;
3333    }
3334    break;
3335  }
3336  case ISD::TRUNCATE:
3337    switch (getTypeAction(Node->getOperand(0).getValueType())) {
3338    case Legal:
3339      Tmp1 = LegalizeOp(Node->getOperand(0));
3340      Result = DAG.UpdateNodeOperands(Result, Tmp1);
3341      break;
3342    case Expand:
3343      ExpandOp(Node->getOperand(0), Tmp1, Tmp2);
3344
3345      // Since the result is legal, we should just be able to truncate the low
3346      // part of the source.
3347      Result = DAG.getNode(ISD::TRUNCATE, Node->getValueType(0), Tmp1);
3348      break;
3349    case Promote:
3350      Result = PromoteOp(Node->getOperand(0));
3351      Result = DAG.getNode(ISD::TRUNCATE, Op.getValueType(), Result);
3352      break;
3353    }
3354    break;
3355
3356  case ISD::FP_TO_SINT:
3357  case ISD::FP_TO_UINT:
3358    switch (getTypeAction(Node->getOperand(0).getValueType())) {
3359    case Legal:
3360      Tmp1 = LegalizeOp(Node->getOperand(0));
3361
3362      switch (TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0))){
3363      default: assert(0 && "Unknown operation action!");
3364      case TargetLowering::Custom:
3365        isCustom = true;
3366        // FALLTHROUGH
3367      case TargetLowering::Legal:
3368        Result = DAG.UpdateNodeOperands(Result, Tmp1);
3369        if (isCustom) {
3370          Tmp1 = TLI.LowerOperation(Result, DAG);
3371          if (Tmp1.Val) Result = Tmp1;
3372        }
3373        break;
3374      case TargetLowering::Promote:
3375        Result = PromoteLegalFP_TO_INT(Tmp1, Node->getValueType(0),
3376                                       Node->getOpcode() == ISD::FP_TO_SINT);
3377        break;
3378      case TargetLowering::Expand:
3379        if (Node->getOpcode() == ISD::FP_TO_UINT) {
3380          SDOperand True, False;
3381          MVT::ValueType VT =  Node->getOperand(0).getValueType();
3382          MVT::ValueType NVT = Node->getValueType(0);
3383          unsigned ShiftAmt = MVT::getSizeInBits(NVT)-1;
3384          const uint64_t zero[] = {0, 0};
3385          APFloat apf = APFloat(APInt(MVT::getSizeInBits(VT), 2, zero));
3386          uint64_t x = 1ULL << ShiftAmt;
3387          (void)apf.convertFromZeroExtendedInteger
3388            (&x, MVT::getSizeInBits(NVT), false, APFloat::rmNearestTiesToEven);
3389          Tmp2 = DAG.getConstantFP(apf, VT);
3390          Tmp3 = DAG.getSetCC(TLI.getSetCCResultTy(),
3391                            Node->getOperand(0), Tmp2, ISD::SETLT);
3392          True = DAG.getNode(ISD::FP_TO_SINT, NVT, Node->getOperand(0));
3393          False = DAG.getNode(ISD::FP_TO_SINT, NVT,
3394                              DAG.getNode(ISD::FSUB, VT, Node->getOperand(0),
3395                                          Tmp2));
3396          False = DAG.getNode(ISD::XOR, NVT, False,
3397                              DAG.getConstant(1ULL << ShiftAmt, NVT));
3398          Result = DAG.getNode(ISD::SELECT, NVT, Tmp3, True, False);
3399          break;
3400        } else {
3401          assert(0 && "Do not know how to expand FP_TO_SINT yet!");
3402        }
3403        break;
3404      }
3405      break;
3406    case Expand: {
3407      MVT::ValueType VT = Op.getValueType();
3408      MVT::ValueType OVT = Node->getOperand(0).getValueType();
3409      // Convert ppcf128 to i32
3410      if (OVT == MVT::ppcf128 && VT == MVT::i32) {
3411        if (Node->getOpcode()==ISD::FP_TO_SINT)
3412          Result = DAG.getNode(ISD::FP_TO_SINT, VT,
3413                             DAG.getNode(ISD::FP_ROUND, MVT::f64,
3414                                         (DAG.getNode(ISD::FP_ROUND_INREG,
3415                                          MVT::ppcf128, Node->getOperand(0),
3416                                          DAG.getValueType(MVT::f64)))));
3417        else {
3418          const uint64_t TwoE31[] = {0x41e0000000000000LL, 0};
3419          APFloat apf = APFloat(APInt(128, 2, TwoE31));
3420          Tmp2 = DAG.getConstantFP(apf, OVT);
3421          //  X>=2^31 ? (int)(X-2^31)+0x80000000 : (int)X
3422          // FIXME: generated code sucks.
3423          Result = DAG.getNode(ISD::SELECT_CC, VT, Node->getOperand(0), Tmp2,
3424                               DAG.getNode(ISD::ADD, MVT::i32,
3425                                 DAG.getNode(ISD::FP_TO_SINT, VT,
3426                                   DAG.getNode(ISD::FSUB, OVT,
3427                                                 Node->getOperand(0), Tmp2)),
3428                                 DAG.getConstant(0x80000000, MVT::i32)),
3429                               DAG.getNode(ISD::FP_TO_SINT, VT,
3430                                           Node->getOperand(0)),
3431                               DAG.getCondCode(ISD::SETGE));
3432        }
3433        break;
3434      }
3435      // Convert f32 / f64 to i32 / i64.
3436      RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3437      switch (Node->getOpcode()) {
3438      case ISD::FP_TO_SINT: {
3439        if (OVT == MVT::f32)
3440          LC = (VT == MVT::i32)
3441            ? RTLIB::FPTOSINT_F32_I32 : RTLIB::FPTOSINT_F32_I64;
3442        else if (OVT == MVT::f64)
3443          LC = (VT == MVT::i32)
3444            ? RTLIB::FPTOSINT_F64_I32 : RTLIB::FPTOSINT_F64_I64;
3445        else if (OVT == MVT::f80) {
3446          assert(VT == MVT::i64);
3447          LC = RTLIB::FPTOSINT_F80_I64;
3448        }
3449        else if (OVT == MVT::ppcf128) {
3450          assert(VT == MVT::i64);
3451          LC = RTLIB::FPTOSINT_PPCF128_I64;
3452        }
3453        break;
3454      }
3455      case ISD::FP_TO_UINT: {
3456        if (OVT == MVT::f32)
3457          LC = (VT == MVT::i32)
3458            ? RTLIB::FPTOUINT_F32_I32 : RTLIB::FPTOSINT_F32_I64;
3459        else if (OVT == MVT::f64)
3460          LC = (VT == MVT::i32)
3461            ? RTLIB::FPTOUINT_F64_I32 : RTLIB::FPTOSINT_F64_I64;
3462        else if (OVT == MVT::f80) {
3463          LC = (VT == MVT::i32)
3464            ? RTLIB::FPTOUINT_F80_I32 : RTLIB::FPTOUINT_F80_I64;
3465        }
3466        else if (OVT ==  MVT::ppcf128) {
3467          assert(VT == MVT::i64);
3468          LC = RTLIB::FPTOUINT_PPCF128_I64;
3469        }
3470        break;
3471      }
3472      default: assert(0 && "Unreachable!");
3473      }
3474      SDOperand Dummy;
3475      Result = ExpandLibCall(TLI.getLibcallName(LC), Node,
3476                             false/*sign irrelevant*/, Dummy);
3477      break;
3478    }
3479    case Promote:
3480      Tmp1 = PromoteOp(Node->getOperand(0));
3481      Result = DAG.UpdateNodeOperands(Result, LegalizeOp(Tmp1));
3482      Result = LegalizeOp(Result);
3483      break;
3484    }
3485    break;
3486
3487  case ISD::FP_EXTEND:
3488  case ISD::FP_ROUND: {
3489      MVT::ValueType newVT = Op.getValueType();
3490      MVT::ValueType oldVT = Op.getOperand(0).getValueType();
3491      if (TLI.getConvertAction(oldVT, newVT) == TargetLowering::Expand) {
3492        if (Node->getOpcode() == ISD::FP_ROUND && oldVT == MVT::ppcf128) {
3493          SDOperand Lo, Hi;
3494          ExpandOp(Node->getOperand(0), Lo, Hi);
3495          if (newVT == MVT::f64)
3496            Result = Hi;
3497          else
3498            Result = DAG.getNode(ISD::FP_ROUND, newVT, Hi);
3499          break;
3500        } else {
3501          // The only other way we can lower this is to turn it into a STORE,
3502          // LOAD pair, targetting a temporary location (a stack slot).
3503
3504          // NOTE: there is a choice here between constantly creating new stack
3505          // slots and always reusing the same one.  We currently always create
3506          // new ones, as reuse may inhibit scheduling.
3507          MVT::ValueType slotVT =
3508                  (Node->getOpcode() == ISD::FP_EXTEND) ? oldVT : newVT;
3509          const Type *Ty = MVT::getTypeForValueType(slotVT);
3510          uint64_t TySize = TLI.getTargetData()->getTypeSize(Ty);
3511          unsigned Align  = TLI.getTargetData()->getPrefTypeAlignment(Ty);
3512          MachineFunction &MF = DAG.getMachineFunction();
3513          int SSFI =
3514            MF.getFrameInfo()->CreateStackObject(TySize, Align);
3515          SDOperand StackSlot = DAG.getFrameIndex(SSFI, TLI.getPointerTy());
3516          if (Node->getOpcode() == ISD::FP_EXTEND) {
3517            Result = DAG.getStore(DAG.getEntryNode(), Node->getOperand(0),
3518                                       StackSlot, NULL, 0);
3519            Result = DAG.getExtLoad(ISD::EXTLOAD, newVT,
3520                                       Result, StackSlot, NULL, 0, oldVT);
3521          } else {
3522            Result = DAG.getTruncStore(DAG.getEntryNode(), Node->getOperand(0),
3523                                       StackSlot, NULL, 0, newVT);
3524            Result = DAG.getLoad(newVT, Result, StackSlot, NULL, 0);
3525          }
3526          break;
3527        }
3528      }
3529    }
3530    // FALL THROUGH
3531  case ISD::ANY_EXTEND:
3532  case ISD::ZERO_EXTEND:
3533  case ISD::SIGN_EXTEND:
3534    switch (getTypeAction(Node->getOperand(0).getValueType())) {
3535    case Expand: assert(0 && "Shouldn't need to expand other operators here!");
3536    case Legal:
3537      Tmp1 = LegalizeOp(Node->getOperand(0));
3538      Result = DAG.UpdateNodeOperands(Result, Tmp1);
3539      break;
3540    case Promote:
3541      switch (Node->getOpcode()) {
3542      case ISD::ANY_EXTEND:
3543        Tmp1 = PromoteOp(Node->getOperand(0));
3544        Result = DAG.getNode(ISD::ANY_EXTEND, Op.getValueType(), Tmp1);
3545        break;
3546      case ISD::ZERO_EXTEND:
3547        Result = PromoteOp(Node->getOperand(0));
3548        Result = DAG.getNode(ISD::ANY_EXTEND, Op.getValueType(), Result);
3549        Result = DAG.getZeroExtendInReg(Result,
3550                                        Node->getOperand(0).getValueType());
3551        break;
3552      case ISD::SIGN_EXTEND:
3553        Result = PromoteOp(Node->getOperand(0));
3554        Result = DAG.getNode(ISD::ANY_EXTEND, Op.getValueType(), Result);
3555        Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, Result.getValueType(),
3556                             Result,
3557                          DAG.getValueType(Node->getOperand(0).getValueType()));
3558        break;
3559      case ISD::FP_EXTEND:
3560        Result = PromoteOp(Node->getOperand(0));
3561        if (Result.getValueType() != Op.getValueType())
3562          // Dynamically dead while we have only 2 FP types.
3563          Result = DAG.getNode(ISD::FP_EXTEND, Op.getValueType(), Result);
3564        break;
3565      case ISD::FP_ROUND:
3566        Result = PromoteOp(Node->getOperand(0));
3567        Result = DAG.getNode(Node->getOpcode(), Op.getValueType(), Result);
3568        break;
3569      }
3570    }
3571    break;
3572  case ISD::FP_ROUND_INREG:
3573  case ISD::SIGN_EXTEND_INREG: {
3574    Tmp1 = LegalizeOp(Node->getOperand(0));
3575    MVT::ValueType ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
3576
3577    // If this operation is not supported, convert it to a shl/shr or load/store
3578    // pair.
3579    switch (TLI.getOperationAction(Node->getOpcode(), ExtraVT)) {
3580    default: assert(0 && "This action not supported for this op yet!");
3581    case TargetLowering::Legal:
3582      Result = DAG.UpdateNodeOperands(Result, Tmp1, Node->getOperand(1));
3583      break;
3584    case TargetLowering::Expand:
3585      // If this is an integer extend and shifts are supported, do that.
3586      if (Node->getOpcode() == ISD::SIGN_EXTEND_INREG) {
3587        // NOTE: we could fall back on load/store here too for targets without
3588        // SAR.  However, it is doubtful that any exist.
3589        unsigned BitsDiff = MVT::getSizeInBits(Node->getValueType(0)) -
3590                            MVT::getSizeInBits(ExtraVT);
3591        SDOperand ShiftCst = DAG.getConstant(BitsDiff, TLI.getShiftAmountTy());
3592        Result = DAG.getNode(ISD::SHL, Node->getValueType(0),
3593                             Node->getOperand(0), ShiftCst);
3594        Result = DAG.getNode(ISD::SRA, Node->getValueType(0),
3595                             Result, ShiftCst);
3596      } else if (Node->getOpcode() == ISD::FP_ROUND_INREG) {
3597        // The only way we can lower this is to turn it into a TRUNCSTORE,
3598        // EXTLOAD pair, targetting a temporary location (a stack slot).
3599
3600        // NOTE: there is a choice here between constantly creating new stack
3601        // slots and always reusing the same one.  We currently always create
3602        // new ones, as reuse may inhibit scheduling.
3603        const Type *Ty = MVT::getTypeForValueType(ExtraVT);
3604        uint64_t TySize = TLI.getTargetData()->getTypeSize(Ty);
3605        unsigned Align  = TLI.getTargetData()->getPrefTypeAlignment(Ty);
3606        MachineFunction &MF = DAG.getMachineFunction();
3607        int SSFI =
3608          MF.getFrameInfo()->CreateStackObject(TySize, Align);
3609        SDOperand StackSlot = DAG.getFrameIndex(SSFI, TLI.getPointerTy());
3610        Result = DAG.getTruncStore(DAG.getEntryNode(), Node->getOperand(0),
3611                                   StackSlot, NULL, 0, ExtraVT);
3612        Result = DAG.getExtLoad(ISD::EXTLOAD, Node->getValueType(0),
3613                                Result, StackSlot, NULL, 0, ExtraVT);
3614      } else {
3615        assert(0 && "Unknown op");
3616      }
3617      break;
3618    }
3619    break;
3620  }
3621  case ISD::TRAMPOLINE: {
3622    SDOperand Ops[6];
3623    for (unsigned i = 0; i != 6; ++i)
3624      Ops[i] = LegalizeOp(Node->getOperand(i));
3625    Result = DAG.UpdateNodeOperands(Result, Ops, 6);
3626    // The only option for this node is to custom lower it.
3627    Result = TLI.LowerOperation(Result, DAG);
3628    assert(Result.Val && "Should always custom lower!");
3629
3630    // Since trampoline produces two values, make sure to remember that we
3631    // legalized both of them.
3632    Tmp1 = LegalizeOp(Result.getValue(1));
3633    Result = LegalizeOp(Result);
3634    AddLegalizedOperand(SDOperand(Node, 0), Result);
3635    AddLegalizedOperand(SDOperand(Node, 1), Tmp1);
3636    return Op.ResNo ? Tmp1 : Result;
3637  }
3638  }
3639
3640  assert(Result.getValueType() == Op.getValueType() &&
3641         "Bad legalization!");
3642
3643  // Make sure that the generated code is itself legal.
3644  if (Result != Op)
3645    Result = LegalizeOp(Result);
3646
3647  // Note that LegalizeOp may be reentered even from single-use nodes, which
3648  // means that we always must cache transformed nodes.
3649  AddLegalizedOperand(Op, Result);
3650  return Result;
3651}
3652
3653/// PromoteOp - Given an operation that produces a value in an invalid type,
3654/// promote it to compute the value into a larger type.  The produced value will
3655/// have the correct bits for the low portion of the register, but no guarantee
3656/// is made about the top bits: it may be zero, sign-extended, or garbage.
3657SDOperand SelectionDAGLegalize::PromoteOp(SDOperand Op) {
3658  MVT::ValueType VT = Op.getValueType();
3659  MVT::ValueType NVT = TLI.getTypeToTransformTo(VT);
3660  assert(getTypeAction(VT) == Promote &&
3661         "Caller should expand or legalize operands that are not promotable!");
3662  assert(NVT > VT && MVT::isInteger(NVT) == MVT::isInteger(VT) &&
3663         "Cannot promote to smaller type!");
3664
3665  SDOperand Tmp1, Tmp2, Tmp3;
3666  SDOperand Result;
3667  SDNode *Node = Op.Val;
3668
3669  DenseMap<SDOperand, SDOperand>::iterator I = PromotedNodes.find(Op);
3670  if (I != PromotedNodes.end()) return I->second;
3671
3672  switch (Node->getOpcode()) {
3673  case ISD::CopyFromReg:
3674    assert(0 && "CopyFromReg must be legal!");
3675  default:
3676#ifndef NDEBUG
3677    cerr << "NODE: "; Node->dump(&DAG); cerr << "\n";
3678#endif
3679    assert(0 && "Do not know how to promote this operator!");
3680    abort();
3681  case ISD::UNDEF:
3682    Result = DAG.getNode(ISD::UNDEF, NVT);
3683    break;
3684  case ISD::Constant:
3685    if (VT != MVT::i1)
3686      Result = DAG.getNode(ISD::SIGN_EXTEND, NVT, Op);
3687    else
3688      Result = DAG.getNode(ISD::ZERO_EXTEND, NVT, Op);
3689    assert(isa<ConstantSDNode>(Result) && "Didn't constant fold zext?");
3690    break;
3691  case ISD::ConstantFP:
3692    Result = DAG.getNode(ISD::FP_EXTEND, NVT, Op);
3693    assert(isa<ConstantFPSDNode>(Result) && "Didn't constant fold fp_extend?");
3694    break;
3695
3696  case ISD::SETCC:
3697    assert(isTypeLegal(TLI.getSetCCResultTy()) && "SetCC type is not legal??");
3698    Result = DAG.getNode(ISD::SETCC, TLI.getSetCCResultTy(),Node->getOperand(0),
3699                         Node->getOperand(1), Node->getOperand(2));
3700    break;
3701
3702  case ISD::TRUNCATE:
3703    switch (getTypeAction(Node->getOperand(0).getValueType())) {
3704    case Legal:
3705      Result = LegalizeOp(Node->getOperand(0));
3706      assert(Result.getValueType() >= NVT &&
3707             "This truncation doesn't make sense!");
3708      if (Result.getValueType() > NVT)    // Truncate to NVT instead of VT
3709        Result = DAG.getNode(ISD::TRUNCATE, NVT, Result);
3710      break;
3711    case Promote:
3712      // The truncation is not required, because we don't guarantee anything
3713      // about high bits anyway.
3714      Result = PromoteOp(Node->getOperand(0));
3715      break;
3716    case Expand:
3717      ExpandOp(Node->getOperand(0), Tmp1, Tmp2);
3718      // Truncate the low part of the expanded value to the result type
3719      Result = DAG.getNode(ISD::TRUNCATE, NVT, Tmp1);
3720    }
3721    break;
3722  case ISD::SIGN_EXTEND:
3723  case ISD::ZERO_EXTEND:
3724  case ISD::ANY_EXTEND:
3725    switch (getTypeAction(Node->getOperand(0).getValueType())) {
3726    case Expand: assert(0 && "BUG: Smaller reg should have been promoted!");
3727    case Legal:
3728      // Input is legal?  Just do extend all the way to the larger type.
3729      Result = DAG.getNode(Node->getOpcode(), NVT, Node->getOperand(0));
3730      break;
3731    case Promote:
3732      // Promote the reg if it's smaller.
3733      Result = PromoteOp(Node->getOperand(0));
3734      // The high bits are not guaranteed to be anything.  Insert an extend.
3735      if (Node->getOpcode() == ISD::SIGN_EXTEND)
3736        Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Result,
3737                         DAG.getValueType(Node->getOperand(0).getValueType()));
3738      else if (Node->getOpcode() == ISD::ZERO_EXTEND)
3739        Result = DAG.getZeroExtendInReg(Result,
3740                                        Node->getOperand(0).getValueType());
3741      break;
3742    }
3743    break;
3744  case ISD::BIT_CONVERT:
3745    Result = ExpandBIT_CONVERT(Node->getValueType(0), Node->getOperand(0));
3746    Result = PromoteOp(Result);
3747    break;
3748
3749  case ISD::FP_EXTEND:
3750    assert(0 && "Case not implemented.  Dynamically dead with 2 FP types!");
3751  case ISD::FP_ROUND:
3752    switch (getTypeAction(Node->getOperand(0).getValueType())) {
3753    case Expand: assert(0 && "BUG: Cannot expand FP regs!");
3754    case Promote:  assert(0 && "Unreachable with 2 FP types!");
3755    case Legal:
3756      // Input is legal?  Do an FP_ROUND_INREG.
3757      Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Node->getOperand(0),
3758                           DAG.getValueType(VT));
3759      break;
3760    }
3761    break;
3762
3763  case ISD::SINT_TO_FP:
3764  case ISD::UINT_TO_FP:
3765    switch (getTypeAction(Node->getOperand(0).getValueType())) {
3766    case Legal:
3767      // No extra round required here.
3768      Result = DAG.getNode(Node->getOpcode(), NVT, Node->getOperand(0));
3769      break;
3770
3771    case Promote:
3772      Result = PromoteOp(Node->getOperand(0));
3773      if (Node->getOpcode() == ISD::SINT_TO_FP)
3774        Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, Result.getValueType(),
3775                             Result,
3776                         DAG.getValueType(Node->getOperand(0).getValueType()));
3777      else
3778        Result = DAG.getZeroExtendInReg(Result,
3779                                        Node->getOperand(0).getValueType());
3780      // No extra round required here.
3781      Result = DAG.getNode(Node->getOpcode(), NVT, Result);
3782      break;
3783    case Expand:
3784      Result = ExpandIntToFP(Node->getOpcode() == ISD::SINT_TO_FP, NVT,
3785                             Node->getOperand(0));
3786      // Round if we cannot tolerate excess precision.
3787      if (NoExcessFPPrecision)
3788        Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Result,
3789                             DAG.getValueType(VT));
3790      break;
3791    }
3792    break;
3793
3794  case ISD::SIGN_EXTEND_INREG:
3795    Result = PromoteOp(Node->getOperand(0));
3796    Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Result,
3797                         Node->getOperand(1));
3798    break;
3799  case ISD::FP_TO_SINT:
3800  case ISD::FP_TO_UINT:
3801    switch (getTypeAction(Node->getOperand(0).getValueType())) {
3802    case Legal:
3803    case Expand:
3804      Tmp1 = Node->getOperand(0);
3805      break;
3806    case Promote:
3807      // The input result is prerounded, so we don't have to do anything
3808      // special.
3809      Tmp1 = PromoteOp(Node->getOperand(0));
3810      break;
3811    }
3812    // If we're promoting a UINT to a larger size, check to see if the new node
3813    // will be legal.  If it isn't, check to see if FP_TO_SINT is legal, since
3814    // we can use that instead.  This allows us to generate better code for
3815    // FP_TO_UINT for small destination sizes on targets where FP_TO_UINT is not
3816    // legal, such as PowerPC.
3817    if (Node->getOpcode() == ISD::FP_TO_UINT &&
3818        !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
3819        (TLI.isOperationLegal(ISD::FP_TO_SINT, NVT) ||
3820         TLI.getOperationAction(ISD::FP_TO_SINT, NVT)==TargetLowering::Custom)){
3821      Result = DAG.getNode(ISD::FP_TO_SINT, NVT, Tmp1);
3822    } else {
3823      Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1);
3824    }
3825    break;
3826
3827  case ISD::FABS:
3828  case ISD::FNEG:
3829    Tmp1 = PromoteOp(Node->getOperand(0));
3830    assert(Tmp1.getValueType() == NVT);
3831    Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1);
3832    // NOTE: we do not have to do any extra rounding here for
3833    // NoExcessFPPrecision, because we know the input will have the appropriate
3834    // precision, and these operations don't modify precision at all.
3835    break;
3836
3837  case ISD::FSQRT:
3838  case ISD::FSIN:
3839  case ISD::FCOS:
3840    Tmp1 = PromoteOp(Node->getOperand(0));
3841    assert(Tmp1.getValueType() == NVT);
3842    Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1);
3843    if (NoExcessFPPrecision)
3844      Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Result,
3845                           DAG.getValueType(VT));
3846    break;
3847
3848  case ISD::FPOWI: {
3849    // Promote f32 powi to f64 powi.  Note that this could insert a libcall
3850    // directly as well, which may be better.
3851    Tmp1 = PromoteOp(Node->getOperand(0));
3852    assert(Tmp1.getValueType() == NVT);
3853    Result = DAG.getNode(ISD::FPOWI, NVT, Tmp1, Node->getOperand(1));
3854    if (NoExcessFPPrecision)
3855      Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Result,
3856                           DAG.getValueType(VT));
3857    break;
3858  }
3859
3860  case ISD::AND:
3861  case ISD::OR:
3862  case ISD::XOR:
3863  case ISD::ADD:
3864  case ISD::SUB:
3865  case ISD::MUL:
3866    // The input may have strange things in the top bits of the registers, but
3867    // these operations don't care.  They may have weird bits going out, but
3868    // that too is okay if they are integer operations.
3869    Tmp1 = PromoteOp(Node->getOperand(0));
3870    Tmp2 = PromoteOp(Node->getOperand(1));
3871    assert(Tmp1.getValueType() == NVT && Tmp2.getValueType() == NVT);
3872    Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1, Tmp2);
3873    break;
3874  case ISD::FADD:
3875  case ISD::FSUB:
3876  case ISD::FMUL:
3877    Tmp1 = PromoteOp(Node->getOperand(0));
3878    Tmp2 = PromoteOp(Node->getOperand(1));
3879    assert(Tmp1.getValueType() == NVT && Tmp2.getValueType() == NVT);
3880    Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1, Tmp2);
3881
3882    // Floating point operations will give excess precision that we may not be
3883    // able to tolerate.  If we DO allow excess precision, just leave it,
3884    // otherwise excise it.
3885    // FIXME: Why would we need to round FP ops more than integer ones?
3886    //     Is Round(Add(Add(A,B),C)) != Round(Add(Round(Add(A,B)), C))
3887    if (NoExcessFPPrecision)
3888      Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Result,
3889                           DAG.getValueType(VT));
3890    break;
3891
3892  case ISD::SDIV:
3893  case ISD::SREM:
3894    // These operators require that their input be sign extended.
3895    Tmp1 = PromoteOp(Node->getOperand(0));
3896    Tmp2 = PromoteOp(Node->getOperand(1));
3897    if (MVT::isInteger(NVT)) {
3898      Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Tmp1,
3899                         DAG.getValueType(VT));
3900      Tmp2 = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Tmp2,
3901                         DAG.getValueType(VT));
3902    }
3903    Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1, Tmp2);
3904
3905    // Perform FP_ROUND: this is probably overly pessimistic.
3906    if (MVT::isFloatingPoint(NVT) && NoExcessFPPrecision)
3907      Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Result,
3908                           DAG.getValueType(VT));
3909    break;
3910  case ISD::FDIV:
3911  case ISD::FREM:
3912  case ISD::FCOPYSIGN:
3913    // These operators require that their input be fp extended.
3914    switch (getTypeAction(Node->getOperand(0).getValueType())) {
3915      case Legal:
3916        Tmp1 = LegalizeOp(Node->getOperand(0));
3917        break;
3918      case Promote:
3919        Tmp1 = PromoteOp(Node->getOperand(0));
3920        break;
3921      case Expand:
3922        assert(0 && "not implemented");
3923    }
3924    switch (getTypeAction(Node->getOperand(1).getValueType())) {
3925      case Legal:
3926        Tmp2 = LegalizeOp(Node->getOperand(1));
3927        break;
3928      case Promote:
3929        Tmp2 = PromoteOp(Node->getOperand(1));
3930        break;
3931      case Expand:
3932        assert(0 && "not implemented");
3933    }
3934    Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1, Tmp2);
3935
3936    // Perform FP_ROUND: this is probably overly pessimistic.
3937    if (NoExcessFPPrecision && Node->getOpcode() != ISD::FCOPYSIGN)
3938      Result = DAG.getNode(ISD::FP_ROUND_INREG, NVT, Result,
3939                           DAG.getValueType(VT));
3940    break;
3941
3942  case ISD::UDIV:
3943  case ISD::UREM:
3944    // These operators require that their input be zero extended.
3945    Tmp1 = PromoteOp(Node->getOperand(0));
3946    Tmp2 = PromoteOp(Node->getOperand(1));
3947    assert(MVT::isInteger(NVT) && "Operators don't apply to FP!");
3948    Tmp1 = DAG.getZeroExtendInReg(Tmp1, VT);
3949    Tmp2 = DAG.getZeroExtendInReg(Tmp2, VT);
3950    Result = DAG.getNode(Node->getOpcode(), NVT, Tmp1, Tmp2);
3951    break;
3952
3953  case ISD::SHL:
3954    Tmp1 = PromoteOp(Node->getOperand(0));
3955    Result = DAG.getNode(ISD::SHL, NVT, Tmp1, Node->getOperand(1));
3956    break;
3957  case ISD::SRA:
3958    // The input value must be properly sign extended.
3959    Tmp1 = PromoteOp(Node->getOperand(0));
3960    Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Tmp1,
3961                       DAG.getValueType(VT));
3962    Result = DAG.getNode(ISD::SRA, NVT, Tmp1, Node->getOperand(1));
3963    break;
3964  case ISD::SRL:
3965    // The input value must be properly zero extended.
3966    Tmp1 = PromoteOp(Node->getOperand(0));
3967    Tmp1 = DAG.getZeroExtendInReg(Tmp1, VT);
3968    Result = DAG.getNode(ISD::SRL, NVT, Tmp1, Node->getOperand(1));
3969    break;
3970
3971  case ISD::VAARG:
3972    Tmp1 = Node->getOperand(0);   // Get the chain.
3973    Tmp2 = Node->getOperand(1);   // Get the pointer.
3974    if (TLI.getOperationAction(ISD::VAARG, VT) == TargetLowering::Custom) {
3975      Tmp3 = DAG.getVAArg(VT, Tmp1, Tmp2, Node->getOperand(2));
3976      Result = TLI.CustomPromoteOperation(Tmp3, DAG);
3977    } else {
3978      SrcValueSDNode *SV = cast<SrcValueSDNode>(Node->getOperand(2));
3979      SDOperand VAList = DAG.getLoad(TLI.getPointerTy(), Tmp1, Tmp2,
3980                                     SV->getValue(), SV->getOffset());
3981      // Increment the pointer, VAList, to the next vaarg
3982      Tmp3 = DAG.getNode(ISD::ADD, TLI.getPointerTy(), VAList,
3983                         DAG.getConstant(MVT::getSizeInBits(VT)/8,
3984                                         TLI.getPointerTy()));
3985      // Store the incremented VAList to the legalized pointer
3986      Tmp3 = DAG.getStore(VAList.getValue(1), Tmp3, Tmp2, SV->getValue(),
3987                          SV->getOffset());
3988      // Load the actual argument out of the pointer VAList
3989      Result = DAG.getExtLoad(ISD::EXTLOAD, NVT, Tmp3, VAList, NULL, 0, VT);
3990    }
3991    // Remember that we legalized the chain.
3992    AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1)));
3993    break;
3994
3995  case ISD::LOAD: {
3996    LoadSDNode *LD = cast<LoadSDNode>(Node);
3997    ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(Node)
3998      ? ISD::EXTLOAD : LD->getExtensionType();
3999    Result = DAG.getExtLoad(ExtType, NVT,
4000                            LD->getChain(), LD->getBasePtr(),
4001                            LD->getSrcValue(), LD->getSrcValueOffset(),
4002                            LD->getLoadedVT(),
4003                            LD->isVolatile(),
4004                            LD->getAlignment());
4005    // Remember that we legalized the chain.
4006    AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1)));
4007    break;
4008  }
4009  case ISD::SELECT:
4010    Tmp2 = PromoteOp(Node->getOperand(1));   // Legalize the op0
4011    Tmp3 = PromoteOp(Node->getOperand(2));   // Legalize the op1
4012    Result = DAG.getNode(ISD::SELECT, NVT, Node->getOperand(0), Tmp2, Tmp3);
4013    break;
4014  case ISD::SELECT_CC:
4015    Tmp2 = PromoteOp(Node->getOperand(2));   // True
4016    Tmp3 = PromoteOp(Node->getOperand(3));   // False
4017    Result = DAG.getNode(ISD::SELECT_CC, NVT, Node->getOperand(0),
4018                         Node->getOperand(1), Tmp2, Tmp3, Node->getOperand(4));
4019    break;
4020  case ISD::BSWAP:
4021    Tmp1 = Node->getOperand(0);
4022    Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, NVT, Tmp1);
4023    Tmp1 = DAG.getNode(ISD::BSWAP, NVT, Tmp1);
4024    Result = DAG.getNode(ISD::SRL, NVT, Tmp1,
4025                         DAG.getConstant(MVT::getSizeInBits(NVT) -
4026                                         MVT::getSizeInBits(VT),
4027                                         TLI.getShiftAmountTy()));
4028    break;
4029  case ISD::CTPOP:
4030  case ISD::CTTZ:
4031  case ISD::CTLZ:
4032    // Zero extend the argument
4033    Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, NVT, Node->getOperand(0));
4034    // Perform the larger operation, then subtract if needed.
4035    Tmp1 = DAG.getNode(Node->getOpcode(), NVT, Tmp1);
4036    switch(Node->getOpcode()) {
4037    case ISD::CTPOP:
4038      Result = Tmp1;
4039      break;
4040    case ISD::CTTZ:
4041      // if Tmp1 == sizeinbits(NVT) then Tmp1 = sizeinbits(Old VT)
4042      Tmp2 = DAG.getSetCC(TLI.getSetCCResultTy(), Tmp1,
4043                          DAG.getConstant(MVT::getSizeInBits(NVT), NVT),
4044                          ISD::SETEQ);
4045      Result = DAG.getNode(ISD::SELECT, NVT, Tmp2,
4046                           DAG.getConstant(MVT::getSizeInBits(VT), NVT), Tmp1);
4047      break;
4048    case ISD::CTLZ:
4049      //Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
4050      Result = DAG.getNode(ISD::SUB, NVT, Tmp1,
4051                           DAG.getConstant(MVT::getSizeInBits(NVT) -
4052                                           MVT::getSizeInBits(VT), NVT));
4053      break;
4054    }
4055    break;
4056  case ISD::EXTRACT_SUBVECTOR:
4057    Result = PromoteOp(ExpandEXTRACT_SUBVECTOR(Op));
4058    break;
4059  case ISD::EXTRACT_VECTOR_ELT:
4060    Result = PromoteOp(ExpandEXTRACT_VECTOR_ELT(Op));
4061    break;
4062  }
4063
4064  assert(Result.Val && "Didn't set a result!");
4065
4066  // Make sure the result is itself legal.
4067  Result = LegalizeOp(Result);
4068
4069  // Remember that we promoted this!
4070  AddPromotedOperand(Op, Result);
4071  return Result;
4072}
4073
4074/// ExpandEXTRACT_VECTOR_ELT - Expand an EXTRACT_VECTOR_ELT operation into
4075/// a legal EXTRACT_VECTOR_ELT operation, scalar code, or memory traffic,
4076/// based on the vector type. The return type of this matches the element type
4077/// of the vector, which may not be legal for the target.
4078SDOperand SelectionDAGLegalize::ExpandEXTRACT_VECTOR_ELT(SDOperand Op) {
4079  // We know that operand #0 is the Vec vector.  If the index is a constant
4080  // or if the invec is a supported hardware type, we can use it.  Otherwise,
4081  // lower to a store then an indexed load.
4082  SDOperand Vec = Op.getOperand(0);
4083  SDOperand Idx = Op.getOperand(1);
4084
4085  MVT::ValueType TVT = Vec.getValueType();
4086  unsigned NumElems = MVT::getVectorNumElements(TVT);
4087
4088  switch (TLI.getOperationAction(ISD::EXTRACT_VECTOR_ELT, TVT)) {
4089  default: assert(0 && "This action is not supported yet!");
4090  case TargetLowering::Custom: {
4091    Vec = LegalizeOp(Vec);
4092    Op = DAG.UpdateNodeOperands(Op, Vec, Idx);
4093    SDOperand Tmp3 = TLI.LowerOperation(Op, DAG);
4094    if (Tmp3.Val)
4095      return Tmp3;
4096    break;
4097  }
4098  case TargetLowering::Legal:
4099    if (isTypeLegal(TVT)) {
4100      Vec = LegalizeOp(Vec);
4101      Op = DAG.UpdateNodeOperands(Op, Vec, Idx);
4102      return Op;
4103    }
4104    break;
4105  case TargetLowering::Expand:
4106    break;
4107  }
4108
4109  if (NumElems == 1) {
4110    // This must be an access of the only element.  Return it.
4111    Op = ScalarizeVectorOp(Vec);
4112  } else if (!TLI.isTypeLegal(TVT) && isa<ConstantSDNode>(Idx)) {
4113    ConstantSDNode *CIdx = cast<ConstantSDNode>(Idx);
4114    SDOperand Lo, Hi;
4115    SplitVectorOp(Vec, Lo, Hi);
4116    if (CIdx->getValue() < NumElems/2) {
4117      Vec = Lo;
4118    } else {
4119      Vec = Hi;
4120      Idx = DAG.getConstant(CIdx->getValue() - NumElems/2,
4121                            Idx.getValueType());
4122    }
4123
4124    // It's now an extract from the appropriate high or low part.  Recurse.
4125    Op = DAG.UpdateNodeOperands(Op, Vec, Idx);
4126    Op = ExpandEXTRACT_VECTOR_ELT(Op);
4127  } else {
4128    // Store the value to a temporary stack slot, then LOAD the scalar
4129    // element back out.
4130    SDOperand StackPtr = DAG.CreateStackTemporary(Vec.getValueType());
4131    SDOperand Ch = DAG.getStore(DAG.getEntryNode(), Vec, StackPtr, NULL, 0);
4132
4133    // Add the offset to the index.
4134    unsigned EltSize = MVT::getSizeInBits(Op.getValueType())/8;
4135    Idx = DAG.getNode(ISD::MUL, Idx.getValueType(), Idx,
4136                      DAG.getConstant(EltSize, Idx.getValueType()));
4137    StackPtr = DAG.getNode(ISD::ADD, Idx.getValueType(), Idx, StackPtr);
4138
4139    Op = DAG.getLoad(Op.getValueType(), Ch, StackPtr, NULL, 0);
4140  }
4141  return Op;
4142}
4143
4144/// ExpandEXTRACT_SUBVECTOR - Expand a EXTRACT_SUBVECTOR operation.  For now
4145/// we assume the operation can be split if it is not already legal.
4146SDOperand SelectionDAGLegalize::ExpandEXTRACT_SUBVECTOR(SDOperand Op) {
4147  // We know that operand #0 is the Vec vector.  For now we assume the index
4148  // is a constant and that the extracted result is a supported hardware type.
4149  SDOperand Vec = Op.getOperand(0);
4150  SDOperand Idx = LegalizeOp(Op.getOperand(1));
4151
4152  unsigned NumElems = MVT::getVectorNumElements(Vec.getValueType());
4153
4154  if (NumElems == MVT::getVectorNumElements(Op.getValueType())) {
4155    // This must be an access of the desired vector length.  Return it.
4156    return Vec;
4157  }
4158
4159  ConstantSDNode *CIdx = cast<ConstantSDNode>(Idx);
4160  SDOperand Lo, Hi;
4161  SplitVectorOp(Vec, Lo, Hi);
4162  if (CIdx->getValue() < NumElems/2) {
4163    Vec = Lo;
4164  } else {
4165    Vec = Hi;
4166    Idx = DAG.getConstant(CIdx->getValue() - NumElems/2, Idx.getValueType());
4167  }
4168
4169  // It's now an extract from the appropriate high or low part.  Recurse.
4170  Op = DAG.UpdateNodeOperands(Op, Vec, Idx);
4171  return ExpandEXTRACT_SUBVECTOR(Op);
4172}
4173
4174/// LegalizeSetCCOperands - Attempts to create a legal LHS and RHS for a SETCC
4175/// with condition CC on the current target.  This usually involves legalizing
4176/// or promoting the arguments.  In the case where LHS and RHS must be expanded,
4177/// there may be no choice but to create a new SetCC node to represent the
4178/// legalized value of setcc lhs, rhs.  In this case, the value is returned in
4179/// LHS, and the SDOperand returned in RHS has a nil SDNode value.
4180void SelectionDAGLegalize::LegalizeSetCCOperands(SDOperand &LHS,
4181                                                 SDOperand &RHS,
4182                                                 SDOperand &CC) {
4183  SDOperand Tmp1, Tmp2, Tmp3, Result;
4184
4185  switch (getTypeAction(LHS.getValueType())) {
4186  case Legal:
4187    Tmp1 = LegalizeOp(LHS);   // LHS
4188    Tmp2 = LegalizeOp(RHS);   // RHS
4189    break;
4190  case Promote:
4191    Tmp1 = PromoteOp(LHS);   // LHS
4192    Tmp2 = PromoteOp(RHS);   // RHS
4193
4194    // If this is an FP compare, the operands have already been extended.
4195    if (MVT::isInteger(LHS.getValueType())) {
4196      MVT::ValueType VT = LHS.getValueType();
4197      MVT::ValueType NVT = TLI.getTypeToTransformTo(VT);
4198
4199      // Otherwise, we have to insert explicit sign or zero extends.  Note
4200      // that we could insert sign extends for ALL conditions, but zero extend
4201      // is cheaper on many machines (an AND instead of two shifts), so prefer
4202      // it.
4203      switch (cast<CondCodeSDNode>(CC)->get()) {
4204      default: assert(0 && "Unknown integer comparison!");
4205      case ISD::SETEQ:
4206      case ISD::SETNE:
4207      case ISD::SETUGE:
4208      case ISD::SETUGT:
4209      case ISD::SETULE:
4210      case ISD::SETULT:
4211        // ALL of these operations will work if we either sign or zero extend
4212        // the operands (including the unsigned comparisons!).  Zero extend is
4213        // usually a simpler/cheaper operation, so prefer it.
4214        Tmp1 = DAG.getZeroExtendInReg(Tmp1, VT);
4215        Tmp2 = DAG.getZeroExtendInReg(Tmp2, VT);
4216        break;
4217      case ISD::SETGE:
4218      case ISD::SETGT:
4219      case ISD::SETLT:
4220      case ISD::SETLE:
4221        Tmp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Tmp1,
4222                           DAG.getValueType(VT));
4223        Tmp2 = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Tmp2,
4224                           DAG.getValueType(VT));
4225        break;
4226      }
4227    }
4228    break;
4229  case Expand: {
4230    MVT::ValueType VT = LHS.getValueType();
4231    if (VT == MVT::f32 || VT == MVT::f64) {
4232      // Expand into one or more soft-fp libcall(s).
4233      RTLIB::Libcall LC1, LC2 = RTLIB::UNKNOWN_LIBCALL;
4234      switch (cast<CondCodeSDNode>(CC)->get()) {
4235      case ISD::SETEQ:
4236      case ISD::SETOEQ:
4237        LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 : RTLIB::OEQ_F64;
4238        break;
4239      case ISD::SETNE:
4240      case ISD::SETUNE:
4241        LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 : RTLIB::UNE_F64;
4242        break;
4243      case ISD::SETGE:
4244      case ISD::SETOGE:
4245        LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 : RTLIB::OGE_F64;
4246        break;
4247      case ISD::SETLT:
4248      case ISD::SETOLT:
4249        LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 : RTLIB::OLT_F64;
4250        break;
4251      case ISD::SETLE:
4252      case ISD::SETOLE:
4253        LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 : RTLIB::OLE_F64;
4254        break;
4255      case ISD::SETGT:
4256      case ISD::SETOGT:
4257        LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 : RTLIB::OGT_F64;
4258        break;
4259      case ISD::SETUO:
4260        LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 : RTLIB::UO_F64;
4261        break;
4262      case ISD::SETO:
4263        LC1 = (VT == MVT::f32) ? RTLIB::O_F32 : RTLIB::O_F64;
4264        break;
4265      default:
4266        LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 : RTLIB::UO_F64;
4267        switch (cast<CondCodeSDNode>(CC)->get()) {
4268        case ISD::SETONE:
4269          // SETONE = SETOLT | SETOGT
4270          LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 : RTLIB::OLT_F64;
4271          // Fallthrough
4272        case ISD::SETUGT:
4273          LC2 = (VT == MVT::f32) ? RTLIB::OGT_F32 : RTLIB::OGT_F64;
4274          break;
4275        case ISD::SETUGE:
4276          LC2 = (VT == MVT::f32) ? RTLIB::OGE_F32 : RTLIB::OGE_F64;
4277          break;
4278        case ISD::SETULT:
4279          LC2 = (VT == MVT::f32) ? RTLIB::OLT_F32 : RTLIB::OLT_F64;
4280          break;
4281        case ISD::SETULE:
4282          LC2 = (VT == MVT::f32) ? RTLIB::OLE_F32 : RTLIB::OLE_F64;
4283          break;
4284        case ISD::SETUEQ:
4285          LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 : RTLIB::OEQ_F64;
4286          break;
4287        default: assert(0 && "Unsupported FP setcc!");
4288        }
4289      }
4290
4291      SDOperand Dummy;
4292      Tmp1 = ExpandLibCall(TLI.getLibcallName(LC1),
4293                           DAG.getNode(ISD::MERGE_VALUES, VT, LHS, RHS).Val,
4294                           false /*sign irrelevant*/, Dummy);
4295      Tmp2 = DAG.getConstant(0, MVT::i32);
4296      CC = DAG.getCondCode(TLI.getCmpLibcallCC(LC1));
4297      if (LC2 != RTLIB::UNKNOWN_LIBCALL) {
4298        Tmp1 = DAG.getNode(ISD::SETCC, TLI.getSetCCResultTy(), Tmp1, Tmp2, CC);
4299        LHS = ExpandLibCall(TLI.getLibcallName(LC2),
4300                            DAG.getNode(ISD::MERGE_VALUES, VT, LHS, RHS).Val,
4301                            false /*sign irrelevant*/, Dummy);
4302        Tmp2 = DAG.getNode(ISD::SETCC, TLI.getSetCCResultTy(), LHS, Tmp2,
4303                           DAG.getCondCode(TLI.getCmpLibcallCC(LC2)));
4304        Tmp1 = DAG.getNode(ISD::OR, Tmp1.getValueType(), Tmp1, Tmp2);
4305        Tmp2 = SDOperand();
4306      }
4307      LHS = Tmp1;
4308      RHS = Tmp2;
4309      return;
4310    }
4311
4312    SDOperand LHSLo, LHSHi, RHSLo, RHSHi;
4313    ExpandOp(LHS, LHSLo, LHSHi);
4314    ExpandOp(RHS, RHSLo, RHSHi);
4315    ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
4316
4317    if (VT==MVT::ppcf128) {
4318      // FIXME:  This generated code sucks.  We want to generate
4319      //         FCMP crN, hi1, hi2
4320      //         BNE crN, L:
4321      //         FCMP crN, lo1, lo2
4322      // The following can be improved, but not that much.
4323      Tmp1 = DAG.getSetCC(TLI.getSetCCResultTy(), LHSHi, RHSHi, ISD::SETEQ);
4324      Tmp2 = DAG.getSetCC(TLI.getSetCCResultTy(), LHSLo, RHSLo, CCCode);
4325      Tmp3 = DAG.getNode(ISD::AND, Tmp1.getValueType(), Tmp1, Tmp2);
4326      Tmp1 = DAG.getSetCC(TLI.getSetCCResultTy(), LHSHi, RHSHi, ISD::SETNE);
4327      Tmp2 = DAG.getSetCC(TLI.getSetCCResultTy(), LHSHi, RHSHi, CCCode);
4328      Tmp1 = DAG.getNode(ISD::AND, Tmp1.getValueType(), Tmp1, Tmp2);
4329      Tmp1 = DAG.getNode(ISD::OR, Tmp1.getValueType(), Tmp1, Tmp3);
4330      Tmp2 = SDOperand();
4331      break;
4332    }
4333
4334    switch (CCCode) {
4335    case ISD::SETEQ:
4336    case ISD::SETNE:
4337      if (RHSLo == RHSHi)
4338        if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo))
4339          if (RHSCST->isAllOnesValue()) {
4340            // Comparison to -1.
4341            Tmp1 = DAG.getNode(ISD::AND, LHSLo.getValueType(), LHSLo, LHSHi);
4342            Tmp2 = RHSLo;
4343            break;
4344          }
4345
4346      Tmp1 = DAG.getNode(ISD::XOR, LHSLo.getValueType(), LHSLo, RHSLo);
4347      Tmp2 = DAG.getNode(ISD::XOR, LHSLo.getValueType(), LHSHi, RHSHi);
4348      Tmp1 = DAG.getNode(ISD::OR, Tmp1.getValueType(), Tmp1, Tmp2);
4349      Tmp2 = DAG.getConstant(0, Tmp1.getValueType());
4350      break;
4351    default:
4352      // If this is a comparison of the sign bit, just look at the top part.
4353      // X > -1,  x < 0
4354      if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(RHS))
4355        if ((cast<CondCodeSDNode>(CC)->get() == ISD::SETLT &&
4356             CST->getValue() == 0) ||             // X < 0
4357            (cast<CondCodeSDNode>(CC)->get() == ISD::SETGT &&
4358             CST->isAllOnesValue())) {            // X > -1
4359          Tmp1 = LHSHi;
4360          Tmp2 = RHSHi;
4361          break;
4362        }
4363
4364      // FIXME: This generated code sucks.
4365      ISD::CondCode LowCC;
4366      switch (CCCode) {
4367      default: assert(0 && "Unknown integer setcc!");
4368      case ISD::SETLT:
4369      case ISD::SETULT: LowCC = ISD::SETULT; break;
4370      case ISD::SETGT:
4371      case ISD::SETUGT: LowCC = ISD::SETUGT; break;
4372      case ISD::SETLE:
4373      case ISD::SETULE: LowCC = ISD::SETULE; break;
4374      case ISD::SETGE:
4375      case ISD::SETUGE: LowCC = ISD::SETUGE; break;
4376      }
4377
4378      // Tmp1 = lo(op1) < lo(op2)   // Always unsigned comparison
4379      // Tmp2 = hi(op1) < hi(op2)   // Signedness depends on operands
4380      // dest = hi(op1) == hi(op2) ? Tmp1 : Tmp2;
4381
4382      // NOTE: on targets without efficient SELECT of bools, we can always use
4383      // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
4384      TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, false, true, NULL);
4385      Tmp1 = TLI.SimplifySetCC(TLI.getSetCCResultTy(), LHSLo, RHSLo, LowCC,
4386                               false, DagCombineInfo);
4387      if (!Tmp1.Val)
4388        Tmp1 = DAG.getSetCC(TLI.getSetCCResultTy(), LHSLo, RHSLo, LowCC);
4389      Tmp2 = TLI.SimplifySetCC(TLI.getSetCCResultTy(), LHSHi, RHSHi,
4390                               CCCode, false, DagCombineInfo);
4391      if (!Tmp2.Val)
4392        Tmp2 = DAG.getNode(ISD::SETCC, TLI.getSetCCResultTy(), LHSHi, RHSHi,CC);
4393
4394      ConstantSDNode *Tmp1C = dyn_cast<ConstantSDNode>(Tmp1.Val);
4395      ConstantSDNode *Tmp2C = dyn_cast<ConstantSDNode>(Tmp2.Val);
4396      if ((Tmp1C && Tmp1C->getValue() == 0) ||
4397          (Tmp2C && Tmp2C->getValue() == 0 &&
4398           (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
4399            CCCode == ISD::SETUGE || CCCode == ISD::SETULE)) ||
4400          (Tmp2C && Tmp2C->getValue() == 1 &&
4401           (CCCode == ISD::SETLT || CCCode == ISD::SETGT ||
4402            CCCode == ISD::SETUGT || CCCode == ISD::SETULT))) {
4403        // low part is known false, returns high part.
4404        // For LE / GE, if high part is known false, ignore the low part.
4405        // For LT / GT, if high part is known true, ignore the low part.
4406        Tmp1 = Tmp2;
4407        Tmp2 = SDOperand();
4408      } else {
4409        Result = TLI.SimplifySetCC(TLI.getSetCCResultTy(), LHSHi, RHSHi,
4410                                   ISD::SETEQ, false, DagCombineInfo);
4411        if (!Result.Val)
4412          Result=DAG.getSetCC(TLI.getSetCCResultTy(), LHSHi, RHSHi, ISD::SETEQ);
4413        Result = LegalizeOp(DAG.getNode(ISD::SELECT, Tmp1.getValueType(),
4414                                        Result, Tmp1, Tmp2));
4415        Tmp1 = Result;
4416        Tmp2 = SDOperand();
4417      }
4418    }
4419  }
4420  }
4421  LHS = Tmp1;
4422  RHS = Tmp2;
4423}
4424
4425/// ExpandBIT_CONVERT - Expand a BIT_CONVERT node into a store/load combination.
4426/// The resultant code need not be legal.  Note that SrcOp is the input operand
4427/// to the BIT_CONVERT, not the BIT_CONVERT node itself.
4428SDOperand SelectionDAGLegalize::ExpandBIT_CONVERT(MVT::ValueType DestVT,
4429                                                  SDOperand SrcOp) {
4430  // Create the stack frame object.
4431  SDOperand FIPtr = DAG.CreateStackTemporary(DestVT);
4432
4433  // Emit a store to the stack slot.
4434  SDOperand Store = DAG.getStore(DAG.getEntryNode(), SrcOp, FIPtr, NULL, 0);
4435  // Result is a load from the stack slot.
4436  return DAG.getLoad(DestVT, Store, FIPtr, NULL, 0);
4437}
4438
4439SDOperand SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
4440  // Create a vector sized/aligned stack slot, store the value to element #0,
4441  // then load the whole vector back out.
4442  SDOperand StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
4443  SDOperand Ch = DAG.getStore(DAG.getEntryNode(), Node->getOperand(0), StackPtr,
4444                              NULL, 0);
4445  return DAG.getLoad(Node->getValueType(0), Ch, StackPtr, NULL, 0);
4446}
4447
4448
4449/// ExpandBUILD_VECTOR - Expand a BUILD_VECTOR node on targets that don't
4450/// support the operation, but do support the resultant vector type.
4451SDOperand SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
4452
4453  // If the only non-undef value is the low element, turn this into a
4454  // SCALAR_TO_VECTOR node.  If this is { X, X, X, X }, determine X.
4455  unsigned NumElems = Node->getNumOperands();
4456  bool isOnlyLowElement = true;
4457  SDOperand SplatValue = Node->getOperand(0);
4458  std::map<SDOperand, std::vector<unsigned> > Values;
4459  Values[SplatValue].push_back(0);
4460  bool isConstant = true;
4461  if (!isa<ConstantFPSDNode>(SplatValue) && !isa<ConstantSDNode>(SplatValue) &&
4462      SplatValue.getOpcode() != ISD::UNDEF)
4463    isConstant = false;
4464
4465  for (unsigned i = 1; i < NumElems; ++i) {
4466    SDOperand V = Node->getOperand(i);
4467    Values[V].push_back(i);
4468    if (V.getOpcode() != ISD::UNDEF)
4469      isOnlyLowElement = false;
4470    if (SplatValue != V)
4471      SplatValue = SDOperand(0,0);
4472
4473    // If this isn't a constant element or an undef, we can't use a constant
4474    // pool load.
4475    if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V) &&
4476        V.getOpcode() != ISD::UNDEF)
4477      isConstant = false;
4478  }
4479
4480  if (isOnlyLowElement) {
4481    // If the low element is an undef too, then this whole things is an undef.
4482    if (Node->getOperand(0).getOpcode() == ISD::UNDEF)
4483      return DAG.getNode(ISD::UNDEF, Node->getValueType(0));
4484    // Otherwise, turn this into a scalar_to_vector node.
4485    return DAG.getNode(ISD::SCALAR_TO_VECTOR, Node->getValueType(0),
4486                       Node->getOperand(0));
4487  }
4488
4489  // If all elements are constants, create a load from the constant pool.
4490  if (isConstant) {
4491    MVT::ValueType VT = Node->getValueType(0);
4492    const Type *OpNTy =
4493      MVT::getTypeForValueType(Node->getOperand(0).getValueType());
4494    std::vector<Constant*> CV;
4495    for (unsigned i = 0, e = NumElems; i != e; ++i) {
4496      if (ConstantFPSDNode *V =
4497          dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
4498        CV.push_back(ConstantFP::get(OpNTy, V->getValueAPF()));
4499      } else if (ConstantSDNode *V =
4500                 dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
4501        CV.push_back(ConstantInt::get(OpNTy, V->getValue()));
4502      } else {
4503        assert(Node->getOperand(i).getOpcode() == ISD::UNDEF);
4504        CV.push_back(UndefValue::get(OpNTy));
4505      }
4506    }
4507    Constant *CP = ConstantVector::get(CV);
4508    SDOperand CPIdx = DAG.getConstantPool(CP, TLI.getPointerTy());
4509    return DAG.getLoad(VT, DAG.getEntryNode(), CPIdx, NULL, 0);
4510  }
4511
4512  if (SplatValue.Val) {   // Splat of one value?
4513    // Build the shuffle constant vector: <0, 0, 0, 0>
4514    MVT::ValueType MaskVT =
4515      MVT::getIntVectorWithNumElements(NumElems);
4516    SDOperand Zero = DAG.getConstant(0, MVT::getVectorElementType(MaskVT));
4517    std::vector<SDOperand> ZeroVec(NumElems, Zero);
4518    SDOperand SplatMask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
4519                                      &ZeroVec[0], ZeroVec.size());
4520
4521    // If the target supports VECTOR_SHUFFLE and this shuffle mask, use it.
4522    if (isShuffleLegal(Node->getValueType(0), SplatMask)) {
4523      // Get the splatted value into the low element of a vector register.
4524      SDOperand LowValVec =
4525        DAG.getNode(ISD::SCALAR_TO_VECTOR, Node->getValueType(0), SplatValue);
4526
4527      // Return shuffle(LowValVec, undef, <0,0,0,0>)
4528      return DAG.getNode(ISD::VECTOR_SHUFFLE, Node->getValueType(0), LowValVec,
4529                         DAG.getNode(ISD::UNDEF, Node->getValueType(0)),
4530                         SplatMask);
4531    }
4532  }
4533
4534  // If there are only two unique elements, we may be able to turn this into a
4535  // vector shuffle.
4536  if (Values.size() == 2) {
4537    // Build the shuffle constant vector: e.g. <0, 4, 0, 4>
4538    MVT::ValueType MaskVT =
4539      MVT::getIntVectorWithNumElements(NumElems);
4540    std::vector<SDOperand> MaskVec(NumElems);
4541    unsigned i = 0;
4542    for (std::map<SDOperand,std::vector<unsigned> >::iterator I=Values.begin(),
4543           E = Values.end(); I != E; ++I) {
4544      for (std::vector<unsigned>::iterator II = I->second.begin(),
4545             EE = I->second.end(); II != EE; ++II)
4546        MaskVec[*II] = DAG.getConstant(i, MVT::getVectorElementType(MaskVT));
4547      i += NumElems;
4548    }
4549    SDOperand ShuffleMask = DAG.getNode(ISD::BUILD_VECTOR, MaskVT,
4550                                        &MaskVec[0], MaskVec.size());
4551
4552    // If the target supports VECTOR_SHUFFLE and this shuffle mask, use it.
4553    if (TLI.isOperationLegal(ISD::SCALAR_TO_VECTOR, Node->getValueType(0)) &&
4554        isShuffleLegal(Node->getValueType(0), ShuffleMask)) {
4555      SmallVector<SDOperand, 8> Ops;
4556      for(std::map<SDOperand,std::vector<unsigned> >::iterator I=Values.begin(),
4557            E = Values.end(); I != E; ++I) {
4558        SDOperand Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, Node->getValueType(0),
4559                                   I->first);
4560        Ops.push_back(Op);
4561      }
4562      Ops.push_back(ShuffleMask);
4563
4564      // Return shuffle(LoValVec, HiValVec, <0,1,0,1>)
4565      return DAG.getNode(ISD::VECTOR_SHUFFLE, Node->getValueType(0),
4566                         &Ops[0], Ops.size());
4567    }
4568  }
4569
4570  // Otherwise, we can't handle this case efficiently.  Allocate a sufficiently
4571  // aligned object on the stack, store each element into it, then load
4572  // the result as a vector.
4573  MVT::ValueType VT = Node->getValueType(0);
4574  // Create the stack frame object.
4575  SDOperand FIPtr = DAG.CreateStackTemporary(VT);
4576
4577  // Emit a store of each element to the stack slot.
4578  SmallVector<SDOperand, 8> Stores;
4579  unsigned TypeByteSize =
4580    MVT::getSizeInBits(Node->getOperand(0).getValueType())/8;
4581  // Store (in the right endianness) the elements to memory.
4582  for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
4583    // Ignore undef elements.
4584    if (Node->getOperand(i).getOpcode() == ISD::UNDEF) continue;
4585
4586    unsigned Offset = TypeByteSize*i;
4587
4588    SDOperand Idx = DAG.getConstant(Offset, FIPtr.getValueType());
4589    Idx = DAG.getNode(ISD::ADD, FIPtr.getValueType(), FIPtr, Idx);
4590
4591    Stores.push_back(DAG.getStore(DAG.getEntryNode(), Node->getOperand(i), Idx,
4592                                  NULL, 0));
4593  }
4594
4595  SDOperand StoreChain;
4596  if (!Stores.empty())    // Not all undef elements?
4597    StoreChain = DAG.getNode(ISD::TokenFactor, MVT::Other,
4598                             &Stores[0], Stores.size());
4599  else
4600    StoreChain = DAG.getEntryNode();
4601
4602  // Result is a load from the stack slot.
4603  return DAG.getLoad(VT, StoreChain, FIPtr, NULL, 0);
4604}
4605
4606void SelectionDAGLegalize::ExpandShiftParts(unsigned NodeOp,
4607                                            SDOperand Op, SDOperand Amt,
4608                                            SDOperand &Lo, SDOperand &Hi) {
4609  // Expand the subcomponents.
4610  SDOperand LHSL, LHSH;
4611  ExpandOp(Op, LHSL, LHSH);
4612
4613  SDOperand Ops[] = { LHSL, LHSH, Amt };
4614  MVT::ValueType VT = LHSL.getValueType();
4615  Lo = DAG.getNode(NodeOp, DAG.getNodeValueTypes(VT, VT), 2, Ops, 3);
4616  Hi = Lo.getValue(1);
4617}
4618
4619
4620/// ExpandShift - Try to find a clever way to expand this shift operation out to
4621/// smaller elements.  If we can't find a way that is more efficient than a
4622/// libcall on this target, return false.  Otherwise, return true with the
4623/// low-parts expanded into Lo and Hi.
4624bool SelectionDAGLegalize::ExpandShift(unsigned Opc, SDOperand Op,SDOperand Amt,
4625                                       SDOperand &Lo, SDOperand &Hi) {
4626  assert((Opc == ISD::SHL || Opc == ISD::SRA || Opc == ISD::SRL) &&
4627         "This is not a shift!");
4628
4629  MVT::ValueType NVT = TLI.getTypeToTransformTo(Op.getValueType());
4630  SDOperand ShAmt = LegalizeOp(Amt);
4631  MVT::ValueType ShTy = ShAmt.getValueType();
4632  unsigned VTBits = MVT::getSizeInBits(Op.getValueType());
4633  unsigned NVTBits = MVT::getSizeInBits(NVT);
4634
4635  // Handle the case when Amt is an immediate.
4636  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Amt.Val)) {
4637    unsigned Cst = CN->getValue();
4638    // Expand the incoming operand to be shifted, so that we have its parts
4639    SDOperand InL, InH;
4640    ExpandOp(Op, InL, InH);
4641    switch(Opc) {
4642    case ISD::SHL:
4643      if (Cst > VTBits) {
4644        Lo = DAG.getConstant(0, NVT);
4645        Hi = DAG.getConstant(0, NVT);
4646      } else if (Cst > NVTBits) {
4647        Lo = DAG.getConstant(0, NVT);
4648        Hi = DAG.getNode(ISD::SHL, NVT, InL, DAG.getConstant(Cst-NVTBits,ShTy));
4649      } else if (Cst == NVTBits) {
4650        Lo = DAG.getConstant(0, NVT);
4651        Hi = InL;
4652      } else {
4653        Lo = DAG.getNode(ISD::SHL, NVT, InL, DAG.getConstant(Cst, ShTy));
4654        Hi = DAG.getNode(ISD::OR, NVT,
4655           DAG.getNode(ISD::SHL, NVT, InH, DAG.getConstant(Cst, ShTy)),
4656           DAG.getNode(ISD::SRL, NVT, InL, DAG.getConstant(NVTBits-Cst, ShTy)));
4657      }
4658      return true;
4659    case ISD::SRL:
4660      if (Cst > VTBits) {
4661        Lo = DAG.getConstant(0, NVT);
4662        Hi = DAG.getConstant(0, NVT);
4663      } else if (Cst > NVTBits) {
4664        Lo = DAG.getNode(ISD::SRL, NVT, InH, DAG.getConstant(Cst-NVTBits,ShTy));
4665        Hi = DAG.getConstant(0, NVT);
4666      } else if (Cst == NVTBits) {
4667        Lo = InH;
4668        Hi = DAG.getConstant(0, NVT);
4669      } else {
4670        Lo = DAG.getNode(ISD::OR, NVT,
4671           DAG.getNode(ISD::SRL, NVT, InL, DAG.getConstant(Cst, ShTy)),
4672           DAG.getNode(ISD::SHL, NVT, InH, DAG.getConstant(NVTBits-Cst, ShTy)));
4673        Hi = DAG.getNode(ISD::SRL, NVT, InH, DAG.getConstant(Cst, ShTy));
4674      }
4675      return true;
4676    case ISD::SRA:
4677      if (Cst > VTBits) {
4678        Hi = Lo = DAG.getNode(ISD::SRA, NVT, InH,
4679                              DAG.getConstant(NVTBits-1, ShTy));
4680      } else if (Cst > NVTBits) {
4681        Lo = DAG.getNode(ISD::SRA, NVT, InH,
4682                           DAG.getConstant(Cst-NVTBits, ShTy));
4683        Hi = DAG.getNode(ISD::SRA, NVT, InH,
4684                              DAG.getConstant(NVTBits-1, ShTy));
4685      } else if (Cst == NVTBits) {
4686        Lo = InH;
4687        Hi = DAG.getNode(ISD::SRA, NVT, InH,
4688                              DAG.getConstant(NVTBits-1, ShTy));
4689      } else {
4690        Lo = DAG.getNode(ISD::OR, NVT,
4691           DAG.getNode(ISD::SRL, NVT, InL, DAG.getConstant(Cst, ShTy)),
4692           DAG.getNode(ISD::SHL, NVT, InH, DAG.getConstant(NVTBits-Cst, ShTy)));
4693        Hi = DAG.getNode(ISD::SRA, NVT, InH, DAG.getConstant(Cst, ShTy));
4694      }
4695      return true;
4696    }
4697  }
4698
4699  // Okay, the shift amount isn't constant.  However, if we can tell that it is
4700  // >= 32 or < 32, we can still simplify it, without knowing the actual value.
4701  uint64_t Mask = NVTBits, KnownZero, KnownOne;
4702  DAG.ComputeMaskedBits(Amt, Mask, KnownZero, KnownOne);
4703
4704  // If we know that the high bit of the shift amount is one, then we can do
4705  // this as a couple of simple shifts.
4706  if (KnownOne & Mask) {
4707    // Mask out the high bit, which we know is set.
4708    Amt = DAG.getNode(ISD::AND, Amt.getValueType(), Amt,
4709                      DAG.getConstant(NVTBits-1, Amt.getValueType()));
4710
4711    // Expand the incoming operand to be shifted, so that we have its parts
4712    SDOperand InL, InH;
4713    ExpandOp(Op, InL, InH);
4714    switch(Opc) {
4715    case ISD::SHL:
4716      Lo = DAG.getConstant(0, NVT);              // Low part is zero.
4717      Hi = DAG.getNode(ISD::SHL, NVT, InL, Amt); // High part from Lo part.
4718      return true;
4719    case ISD::SRL:
4720      Hi = DAG.getConstant(0, NVT);              // Hi part is zero.
4721      Lo = DAG.getNode(ISD::SRL, NVT, InH, Amt); // Lo part from Hi part.
4722      return true;
4723    case ISD::SRA:
4724      Hi = DAG.getNode(ISD::SRA, NVT, InH,       // Sign extend high part.
4725                       DAG.getConstant(NVTBits-1, Amt.getValueType()));
4726      Lo = DAG.getNode(ISD::SRA, NVT, InH, Amt); // Lo part from Hi part.
4727      return true;
4728    }
4729  }
4730
4731  // If we know that the high bit of the shift amount is zero, then we can do
4732  // this as a couple of simple shifts.
4733  if (KnownZero & Mask) {
4734    // Compute 32-amt.
4735    SDOperand Amt2 = DAG.getNode(ISD::SUB, Amt.getValueType(),
4736                                 DAG.getConstant(NVTBits, Amt.getValueType()),
4737                                 Amt);
4738
4739    // Expand the incoming operand to be shifted, so that we have its parts
4740    SDOperand InL, InH;
4741    ExpandOp(Op, InL, InH);
4742    switch(Opc) {
4743    case ISD::SHL:
4744      Lo = DAG.getNode(ISD::SHL, NVT, InL, Amt);
4745      Hi = DAG.getNode(ISD::OR, NVT,
4746                       DAG.getNode(ISD::SHL, NVT, InH, Amt),
4747                       DAG.getNode(ISD::SRL, NVT, InL, Amt2));
4748      return true;
4749    case ISD::SRL:
4750      Hi = DAG.getNode(ISD::SRL, NVT, InH, Amt);
4751      Lo = DAG.getNode(ISD::OR, NVT,
4752                       DAG.getNode(ISD::SRL, NVT, InL, Amt),
4753                       DAG.getNode(ISD::SHL, NVT, InH, Amt2));
4754      return true;
4755    case ISD::SRA:
4756      Hi = DAG.getNode(ISD::SRA, NVT, InH, Amt);
4757      Lo = DAG.getNode(ISD::OR, NVT,
4758                       DAG.getNode(ISD::SRL, NVT, InL, Amt),
4759                       DAG.getNode(ISD::SHL, NVT, InH, Amt2));
4760      return true;
4761    }
4762  }
4763
4764  return false;
4765}
4766
4767
4768// ExpandLibCall - Expand a node into a call to a libcall.  If the result value
4769// does not fit into a register, return the lo part and set the hi part to the
4770// by-reg argument.  If it does fit into a single register, return the result
4771// and leave the Hi part unset.
4772SDOperand SelectionDAGLegalize::ExpandLibCall(const char *Name, SDNode *Node,
4773                                              bool isSigned, SDOperand &Hi) {
4774  assert(!IsLegalizingCall && "Cannot overlap legalization of calls!");
4775  // The input chain to this libcall is the entry node of the function.
4776  // Legalizing the call will automatically add the previous call to the
4777  // dependence.
4778  SDOperand InChain = DAG.getEntryNode();
4779
4780  TargetLowering::ArgListTy Args;
4781  TargetLowering::ArgListEntry Entry;
4782  for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
4783    MVT::ValueType ArgVT = Node->getOperand(i).getValueType();
4784    const Type *ArgTy = MVT::getTypeForValueType(ArgVT);
4785    Entry.Node = Node->getOperand(i); Entry.Ty = ArgTy;
4786    Entry.isSExt = isSigned;
4787    Args.push_back(Entry);
4788  }
4789  SDOperand Callee = DAG.getExternalSymbol(Name, TLI.getPointerTy());
4790
4791  // Splice the libcall in wherever FindInputOutputChains tells us to.
4792  const Type *RetTy = MVT::getTypeForValueType(Node->getValueType(0));
4793  std::pair<SDOperand,SDOperand> CallInfo =
4794    TLI.LowerCallTo(InChain, RetTy, isSigned, false, CallingConv::C, false,
4795                    Callee, Args, DAG);
4796
4797  // Legalize the call sequence, starting with the chain.  This will advance
4798  // the LastCALLSEQ_END to the legalized version of the CALLSEQ_END node that
4799  // was added by LowerCallTo (guaranteeing proper serialization of calls).
4800  LegalizeOp(CallInfo.second);
4801  SDOperand Result;
4802  switch (getTypeAction(CallInfo.first.getValueType())) {
4803  default: assert(0 && "Unknown thing");
4804  case Legal:
4805    Result = CallInfo.first;
4806    break;
4807  case Expand:
4808    ExpandOp(CallInfo.first, Result, Hi);
4809    break;
4810  }
4811  return Result;
4812}
4813
4814
4815/// ExpandIntToFP - Expand a [US]INT_TO_FP operation.
4816///
4817SDOperand SelectionDAGLegalize::
4818ExpandIntToFP(bool isSigned, MVT::ValueType DestTy, SDOperand Source) {
4819  assert(getTypeAction(Source.getValueType()) == Expand &&
4820         "This is not an expansion!");
4821  assert(Source.getValueType() == MVT::i64 && "Only handle expand from i64!");
4822
4823  if (!isSigned) {
4824    assert(Source.getValueType() == MVT::i64 &&
4825           "This only works for 64-bit -> FP");
4826    // The 64-bit value loaded will be incorrectly if the 'sign bit' of the
4827    // incoming integer is set.  To handle this, we dynamically test to see if
4828    // it is set, and, if so, add a fudge factor.
4829    SDOperand Lo, Hi;
4830    ExpandOp(Source, Lo, Hi);
4831
4832    // If this is unsigned, and not supported, first perform the conversion to
4833    // signed, then adjust the result if the sign bit is set.
4834    SDOperand SignedConv = ExpandIntToFP(true, DestTy,
4835                   DAG.getNode(ISD::BUILD_PAIR, Source.getValueType(), Lo, Hi));
4836
4837    SDOperand SignSet = DAG.getSetCC(TLI.getSetCCResultTy(), Hi,
4838                                     DAG.getConstant(0, Hi.getValueType()),
4839                                     ISD::SETLT);
4840    SDOperand Zero = getIntPtrConstant(0), Four = getIntPtrConstant(4);
4841    SDOperand CstOffset = DAG.getNode(ISD::SELECT, Zero.getValueType(),
4842                                      SignSet, Four, Zero);
4843    uint64_t FF = 0x5f800000ULL;
4844    if (TLI.isLittleEndian()) FF <<= 32;
4845    static Constant *FudgeFactor = ConstantInt::get(Type::Int64Ty, FF);
4846
4847    SDOperand CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy());
4848    CPIdx = DAG.getNode(ISD::ADD, TLI.getPointerTy(), CPIdx, CstOffset);
4849    SDOperand FudgeInReg;
4850    if (DestTy == MVT::f32)
4851      FudgeInReg = DAG.getLoad(MVT::f32, DAG.getEntryNode(), CPIdx, NULL, 0);
4852    else if (MVT::getSizeInBits(DestTy) > MVT::getSizeInBits(MVT::f32))
4853      // FIXME: Avoid the extend by construction the right constantpool?
4854      FudgeInReg = DAG.getExtLoad(ISD::EXTLOAD, DestTy, DAG.getEntryNode(),
4855                                  CPIdx, NULL, 0, MVT::f32);
4856    else
4857      assert(0 && "Unexpected conversion");
4858
4859    MVT::ValueType SCVT = SignedConv.getValueType();
4860    if (SCVT != DestTy) {
4861      // Destination type needs to be expanded as well. The FADD now we are
4862      // constructing will be expanded into a libcall.
4863      if (MVT::getSizeInBits(SCVT) != MVT::getSizeInBits(DestTy)) {
4864        assert(SCVT == MVT::i32 && DestTy == MVT::f64);
4865        SignedConv = DAG.getNode(ISD::BUILD_PAIR, MVT::i64,
4866                                 SignedConv, SignedConv.getValue(1));
4867      }
4868      SignedConv = DAG.getNode(ISD::BIT_CONVERT, DestTy, SignedConv);
4869    }
4870    return DAG.getNode(ISD::FADD, DestTy, SignedConv, FudgeInReg);
4871  }
4872
4873  // Check to see if the target has a custom way to lower this.  If so, use it.
4874  switch (TLI.getOperationAction(ISD::SINT_TO_FP, Source.getValueType())) {
4875  default: assert(0 && "This action not implemented for this operation!");
4876  case TargetLowering::Legal:
4877  case TargetLowering::Expand:
4878    break;   // This case is handled below.
4879  case TargetLowering::Custom: {
4880    SDOperand NV = TLI.LowerOperation(DAG.getNode(ISD::SINT_TO_FP, DestTy,
4881                                                  Source), DAG);
4882    if (NV.Val)
4883      return LegalizeOp(NV);
4884    break;   // The target decided this was legal after all
4885  }
4886  }
4887
4888  // Expand the source, then glue it back together for the call.  We must expand
4889  // the source in case it is shared (this pass of legalize must traverse it).
4890  SDOperand SrcLo, SrcHi;
4891  ExpandOp(Source, SrcLo, SrcHi);
4892  Source = DAG.getNode(ISD::BUILD_PAIR, Source.getValueType(), SrcLo, SrcHi);
4893
4894  RTLIB::Libcall LC;
4895  if (DestTy == MVT::f32)
4896    LC = RTLIB::SINTTOFP_I64_F32;
4897  else {
4898    assert(DestTy == MVT::f64 && "Unknown fp value type!");
4899    LC = RTLIB::SINTTOFP_I64_F64;
4900  }
4901
4902  assert(TLI.getLibcallName(LC) && "Don't know how to expand this SINT_TO_FP!");
4903  Source = DAG.getNode(ISD::SINT_TO_FP, DestTy, Source);
4904  SDOperand UnusedHiPart;
4905  return ExpandLibCall(TLI.getLibcallName(LC), Source.Val, isSigned,
4906                       UnusedHiPart);
4907}
4908
4909/// ExpandLegalINT_TO_FP - This function is responsible for legalizing a
4910/// INT_TO_FP operation of the specified operand when the target requests that
4911/// we expand it.  At this point, we know that the result and operand types are
4912/// legal for the target.
4913SDOperand SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned,
4914                                                     SDOperand Op0,
4915                                                     MVT::ValueType DestVT) {
4916  if (Op0.getValueType() == MVT::i32) {
4917    // simple 32-bit [signed|unsigned] integer to float/double expansion
4918
4919    // get the stack frame index of a 8 byte buffer, pessimistically aligned
4920    MachineFunction &MF = DAG.getMachineFunction();
4921    const Type *F64Type = MVT::getTypeForValueType(MVT::f64);
4922    unsigned StackAlign =
4923      (unsigned)TLI.getTargetData()->getPrefTypeAlignment(F64Type);
4924    int SSFI = MF.getFrameInfo()->CreateStackObject(8, StackAlign);
4925    // get address of 8 byte buffer
4926    SDOperand StackSlot = DAG.getFrameIndex(SSFI, TLI.getPointerTy());
4927    // word offset constant for Hi/Lo address computation
4928    SDOperand WordOff = DAG.getConstant(sizeof(int), TLI.getPointerTy());
4929    // set up Hi and Lo (into buffer) address based on endian
4930    SDOperand Hi = StackSlot;
4931    SDOperand Lo = DAG.getNode(ISD::ADD, TLI.getPointerTy(), StackSlot,WordOff);
4932    if (TLI.isLittleEndian())
4933      std::swap(Hi, Lo);
4934
4935    // if signed map to unsigned space
4936    SDOperand Op0Mapped;
4937    if (isSigned) {
4938      // constant used to invert sign bit (signed to unsigned mapping)
4939      SDOperand SignBit = DAG.getConstant(0x80000000u, MVT::i32);
4940      Op0Mapped = DAG.getNode(ISD::XOR, MVT::i32, Op0, SignBit);
4941    } else {
4942      Op0Mapped = Op0;
4943    }
4944    // store the lo of the constructed double - based on integer input
4945    SDOperand Store1 = DAG.getStore(DAG.getEntryNode(),
4946                                    Op0Mapped, Lo, NULL, 0);
4947    // initial hi portion of constructed double
4948    SDOperand InitialHi = DAG.getConstant(0x43300000u, MVT::i32);
4949    // store the hi of the constructed double - biased exponent
4950    SDOperand Store2=DAG.getStore(Store1, InitialHi, Hi, NULL, 0);
4951    // load the constructed double
4952    SDOperand Load = DAG.getLoad(MVT::f64, Store2, StackSlot, NULL, 0);
4953    // FP constant to bias correct the final result
4954    SDOperand Bias = DAG.getConstantFP(isSigned ?
4955                                            BitsToDouble(0x4330000080000000ULL)
4956                                          : BitsToDouble(0x4330000000000000ULL),
4957                                     MVT::f64);
4958    // subtract the bias
4959    SDOperand Sub = DAG.getNode(ISD::FSUB, MVT::f64, Load, Bias);
4960    // final result
4961    SDOperand Result;
4962    // handle final rounding
4963    if (DestVT == MVT::f64) {
4964      // do nothing
4965      Result = Sub;
4966    } else if (MVT::getSizeInBits(DestVT) < MVT::getSizeInBits(MVT::f64)) {
4967      Result = DAG.getNode(ISD::FP_ROUND, DestVT, Sub);
4968    } else if (MVT::getSizeInBits(DestVT) > MVT::getSizeInBits(MVT::f64)) {
4969      Result = DAG.getNode(ISD::FP_EXTEND, DestVT, Sub);
4970    }
4971    return Result;
4972  }
4973  assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet");
4974  SDOperand Tmp1 = DAG.getNode(ISD::SINT_TO_FP, DestVT, Op0);
4975
4976  SDOperand SignSet = DAG.getSetCC(TLI.getSetCCResultTy(), Op0,
4977                                   DAG.getConstant(0, Op0.getValueType()),
4978                                   ISD::SETLT);
4979  SDOperand Zero = getIntPtrConstant(0), Four = getIntPtrConstant(4);
4980  SDOperand CstOffset = DAG.getNode(ISD::SELECT, Zero.getValueType(),
4981                                    SignSet, Four, Zero);
4982
4983  // If the sign bit of the integer is set, the large number will be treated
4984  // as a negative number.  To counteract this, the dynamic code adds an
4985  // offset depending on the data type.
4986  uint64_t FF;
4987  switch (Op0.getValueType()) {
4988  default: assert(0 && "Unsupported integer type!");
4989  case MVT::i8 : FF = 0x43800000ULL; break;  // 2^8  (as a float)
4990  case MVT::i16: FF = 0x47800000ULL; break;  // 2^16 (as a float)
4991  case MVT::i32: FF = 0x4F800000ULL; break;  // 2^32 (as a float)
4992  case MVT::i64: FF = 0x5F800000ULL; break;  // 2^64 (as a float)
4993  }
4994  if (TLI.isLittleEndian()) FF <<= 32;
4995  static Constant *FudgeFactor = ConstantInt::get(Type::Int64Ty, FF);
4996
4997  SDOperand CPIdx = DAG.getConstantPool(FudgeFactor, TLI.getPointerTy());
4998  CPIdx = DAG.getNode(ISD::ADD, TLI.getPointerTy(), CPIdx, CstOffset);
4999  SDOperand FudgeInReg;
5000  if (DestVT == MVT::f32)
5001    FudgeInReg = DAG.getLoad(MVT::f32, DAG.getEntryNode(), CPIdx, NULL, 0);
5002  else {
5003    FudgeInReg = LegalizeOp(DAG.getExtLoad(ISD::EXTLOAD, DestVT,
5004                                           DAG.getEntryNode(), CPIdx,
5005                                           NULL, 0, MVT::f32));
5006  }
5007
5008  return DAG.getNode(ISD::FADD, DestVT, Tmp1, FudgeInReg);
5009}
5010
5011/// PromoteLegalINT_TO_FP - This function is responsible for legalizing a
5012/// *INT_TO_FP operation of the specified operand when the target requests that
5013/// we promote it.  At this point, we know that the result and operand types are
5014/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
5015/// operation that takes a larger input.
5016SDOperand SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDOperand LegalOp,
5017                                                      MVT::ValueType DestVT,
5018                                                      bool isSigned) {
5019  // First step, figure out the appropriate *INT_TO_FP operation to use.
5020  MVT::ValueType NewInTy = LegalOp.getValueType();
5021
5022  unsigned OpToUse = 0;
5023
5024  // Scan for the appropriate larger type to use.
5025  while (1) {
5026    NewInTy = (MVT::ValueType)(NewInTy+1);
5027    assert(MVT::isInteger(NewInTy) && "Ran out of possibilities!");
5028
5029    // If the target supports SINT_TO_FP of this type, use it.
5030    switch (TLI.getOperationAction(ISD::SINT_TO_FP, NewInTy)) {
5031      default: break;
5032      case TargetLowering::Legal:
5033        if (!TLI.isTypeLegal(NewInTy))
5034          break;  // Can't use this datatype.
5035        // FALL THROUGH.
5036      case TargetLowering::Custom:
5037        OpToUse = ISD::SINT_TO_FP;
5038        break;
5039    }
5040    if (OpToUse) break;
5041    if (isSigned) continue;
5042
5043    // If the target supports UINT_TO_FP of this type, use it.
5044    switch (TLI.getOperationAction(ISD::UINT_TO_FP, NewInTy)) {
5045      default: break;
5046      case TargetLowering::Legal:
5047        if (!TLI.isTypeLegal(NewInTy))
5048          break;  // Can't use this datatype.
5049        // FALL THROUGH.
5050      case TargetLowering::Custom:
5051        OpToUse = ISD::UINT_TO_FP;
5052        break;
5053    }
5054    if (OpToUse) break;
5055
5056    // Otherwise, try a larger type.
5057  }
5058
5059  // Okay, we found the operation and type to use.  Zero extend our input to the
5060  // desired type then run the operation on it.
5061  return DAG.getNode(OpToUse, DestVT,
5062                     DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
5063                                 NewInTy, LegalOp));
5064}
5065
5066/// PromoteLegalFP_TO_INT - This function is responsible for legalizing a
5067/// FP_TO_*INT operation of the specified operand when the target requests that
5068/// we promote it.  At this point, we know that the result and operand types are
5069/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
5070/// operation that returns a larger result.
5071SDOperand SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDOperand LegalOp,
5072                                                      MVT::ValueType DestVT,
5073                                                      bool isSigned) {
5074  // First step, figure out the appropriate FP_TO*INT operation to use.
5075  MVT::ValueType NewOutTy = DestVT;
5076
5077  unsigned OpToUse = 0;
5078
5079  // Scan for the appropriate larger type to use.
5080  while (1) {
5081    NewOutTy = (MVT::ValueType)(NewOutTy+1);
5082    assert(MVT::isInteger(NewOutTy) && "Ran out of possibilities!");
5083
5084    // If the target supports FP_TO_SINT returning this type, use it.
5085    switch (TLI.getOperationAction(ISD::FP_TO_SINT, NewOutTy)) {
5086    default: break;
5087    case TargetLowering::Legal:
5088      if (!TLI.isTypeLegal(NewOutTy))
5089        break;  // Can't use this datatype.
5090      // FALL THROUGH.
5091    case TargetLowering::Custom:
5092      OpToUse = ISD::FP_TO_SINT;
5093      break;
5094    }
5095    if (OpToUse) break;
5096
5097    // If the target supports FP_TO_UINT of this type, use it.
5098    switch (TLI.getOperationAction(ISD::FP_TO_UINT, NewOutTy)) {
5099    default: break;
5100    case TargetLowering::Legal:
5101      if (!TLI.isTypeLegal(NewOutTy))
5102        break;  // Can't use this datatype.
5103      // FALL THROUGH.
5104    case TargetLowering::Custom:
5105      OpToUse = ISD::FP_TO_UINT;
5106      break;
5107    }
5108    if (OpToUse) break;
5109
5110    // Otherwise, try a larger type.
5111  }
5112
5113  // Okay, we found the operation and type to use.  Truncate the result of the
5114  // extended FP_TO_*INT operation to the desired size.
5115  return DAG.getNode(ISD::TRUNCATE, DestVT,
5116                     DAG.getNode(OpToUse, NewOutTy, LegalOp));
5117}
5118
5119/// ExpandBSWAP - Open code the operations for BSWAP of the specified operation.
5120///
5121SDOperand SelectionDAGLegalize::ExpandBSWAP(SDOperand Op) {
5122  MVT::ValueType VT = Op.getValueType();
5123  MVT::ValueType SHVT = TLI.getShiftAmountTy();
5124  SDOperand Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
5125  switch (VT) {
5126  default: assert(0 && "Unhandled Expand type in BSWAP!"); abort();
5127  case MVT::i16:
5128    Tmp2 = DAG.getNode(ISD::SHL, VT, Op, DAG.getConstant(8, SHVT));
5129    Tmp1 = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(8, SHVT));
5130    return DAG.getNode(ISD::OR, VT, Tmp1, Tmp2);
5131  case MVT::i32:
5132    Tmp4 = DAG.getNode(ISD::SHL, VT, Op, DAG.getConstant(24, SHVT));
5133    Tmp3 = DAG.getNode(ISD::SHL, VT, Op, DAG.getConstant(8, SHVT));
5134    Tmp2 = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(8, SHVT));
5135    Tmp1 = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(24, SHVT));
5136    Tmp3 = DAG.getNode(ISD::AND, VT, Tmp3, DAG.getConstant(0xFF0000, VT));
5137    Tmp2 = DAG.getNode(ISD::AND, VT, Tmp2, DAG.getConstant(0xFF00, VT));
5138    Tmp4 = DAG.getNode(ISD::OR, VT, Tmp4, Tmp3);
5139    Tmp2 = DAG.getNode(ISD::OR, VT, Tmp2, Tmp1);
5140    return DAG.getNode(ISD::OR, VT, Tmp4, Tmp2);
5141  case MVT::i64:
5142    Tmp8 = DAG.getNode(ISD::SHL, VT, Op, DAG.getConstant(56, SHVT));
5143    Tmp7 = DAG.getNode(ISD::SHL, VT, Op, DAG.getConstant(40, SHVT));
5144    Tmp6 = DAG.getNode(ISD::SHL, VT, Op, DAG.getConstant(24, SHVT));
5145    Tmp5 = DAG.getNode(ISD::SHL, VT, Op, DAG.getConstant(8, SHVT));
5146    Tmp4 = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(8, SHVT));
5147    Tmp3 = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(24, SHVT));
5148    Tmp2 = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(40, SHVT));
5149    Tmp1 = DAG.getNode(ISD::SRL, VT, Op, DAG.getConstant(56, SHVT));
5150    Tmp7 = DAG.getNode(ISD::AND, VT, Tmp7, DAG.getConstant(255ULL<<48, VT));
5151    Tmp6 = DAG.getNode(ISD::AND, VT, Tmp6, DAG.getConstant(255ULL<<40, VT));
5152    Tmp5 = DAG.getNode(ISD::AND, VT, Tmp5, DAG.getConstant(255ULL<<32, VT));
5153    Tmp4 = DAG.getNode(ISD::AND, VT, Tmp4, DAG.getConstant(255ULL<<24, VT));
5154    Tmp3 = DAG.getNode(ISD::AND, VT, Tmp3, DAG.getConstant(255ULL<<16, VT));
5155    Tmp2 = DAG.getNode(ISD::AND, VT, Tmp2, DAG.getConstant(255ULL<<8 , VT));
5156    Tmp8 = DAG.getNode(ISD::OR, VT, Tmp8, Tmp7);
5157    Tmp6 = DAG.getNode(ISD::OR, VT, Tmp6, Tmp5);
5158    Tmp4 = DAG.getNode(ISD::OR, VT, Tmp4, Tmp3);
5159    Tmp2 = DAG.getNode(ISD::OR, VT, Tmp2, Tmp1);
5160    Tmp8 = DAG.getNode(ISD::OR, VT, Tmp8, Tmp6);
5161    Tmp4 = DAG.getNode(ISD::OR, VT, Tmp4, Tmp2);
5162    return DAG.getNode(ISD::OR, VT, Tmp8, Tmp4);
5163  }
5164}
5165
5166/// ExpandBitCount - Expand the specified bitcount instruction into operations.
5167///
5168SDOperand SelectionDAGLegalize::ExpandBitCount(unsigned Opc, SDOperand Op) {
5169  switch (Opc) {
5170  default: assert(0 && "Cannot expand this yet!");
5171  case ISD::CTPOP: {
5172    static const uint64_t mask[6] = {
5173      0x5555555555555555ULL, 0x3333333333333333ULL,
5174      0x0F0F0F0F0F0F0F0FULL, 0x00FF00FF00FF00FFULL,
5175      0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL
5176    };
5177    MVT::ValueType VT = Op.getValueType();
5178    MVT::ValueType ShVT = TLI.getShiftAmountTy();
5179    unsigned len = MVT::getSizeInBits(VT);
5180    for (unsigned i = 0; (1U << i) <= (len / 2); ++i) {
5181      //x = (x & mask[i][len/8]) + (x >> (1 << i) & mask[i][len/8])
5182      SDOperand Tmp2 = DAG.getConstant(mask[i], VT);
5183      SDOperand Tmp3 = DAG.getConstant(1ULL << i, ShVT);
5184      Op = DAG.getNode(ISD::ADD, VT, DAG.getNode(ISD::AND, VT, Op, Tmp2),
5185                       DAG.getNode(ISD::AND, VT,
5186                                   DAG.getNode(ISD::SRL, VT, Op, Tmp3),Tmp2));
5187    }
5188    return Op;
5189  }
5190  case ISD::CTLZ: {
5191    // for now, we do this:
5192    // x = x | (x >> 1);
5193    // x = x | (x >> 2);
5194    // ...
5195    // x = x | (x >>16);
5196    // x = x | (x >>32); // for 64-bit input
5197    // return popcount(~x);
5198    //
5199    // but see also: http://www.hackersdelight.org/HDcode/nlz.cc
5200    MVT::ValueType VT = Op.getValueType();
5201    MVT::ValueType ShVT = TLI.getShiftAmountTy();
5202    unsigned len = MVT::getSizeInBits(VT);
5203    for (unsigned i = 0; (1U << i) <= (len / 2); ++i) {
5204      SDOperand Tmp3 = DAG.getConstant(1ULL << i, ShVT);
5205      Op = DAG.getNode(ISD::OR, VT, Op, DAG.getNode(ISD::SRL, VT, Op, Tmp3));
5206    }
5207    Op = DAG.getNode(ISD::XOR, VT, Op, DAG.getConstant(~0ULL, VT));
5208    return DAG.getNode(ISD::CTPOP, VT, Op);
5209  }
5210  case ISD::CTTZ: {
5211    // for now, we use: { return popcount(~x & (x - 1)); }
5212    // unless the target has ctlz but not ctpop, in which case we use:
5213    // { return 32 - nlz(~x & (x-1)); }
5214    // see also http://www.hackersdelight.org/HDcode/ntz.cc
5215    MVT::ValueType VT = Op.getValueType();
5216    SDOperand Tmp2 = DAG.getConstant(~0ULL, VT);
5217    SDOperand Tmp3 = DAG.getNode(ISD::AND, VT,
5218                       DAG.getNode(ISD::XOR, VT, Op, Tmp2),
5219                       DAG.getNode(ISD::SUB, VT, Op, DAG.getConstant(1, VT)));
5220    // If ISD::CTLZ is legal and CTPOP isn't, then do that instead.
5221    if (!TLI.isOperationLegal(ISD::CTPOP, VT) &&
5222        TLI.isOperationLegal(ISD::CTLZ, VT))
5223      return DAG.getNode(ISD::SUB, VT,
5224                         DAG.getConstant(MVT::getSizeInBits(VT), VT),
5225                         DAG.getNode(ISD::CTLZ, VT, Tmp3));
5226    return DAG.getNode(ISD::CTPOP, VT, Tmp3);
5227  }
5228  }
5229}
5230
5231/// ExpandOp - Expand the specified SDOperand into its two component pieces
5232/// Lo&Hi.  Note that the Op MUST be an expanded type.  As a result of this, the
5233/// LegalizeNodes map is filled in for any results that are not expanded, the
5234/// ExpandedNodes map is filled in for any results that are expanded, and the
5235/// Lo/Hi values are returned.
5236void SelectionDAGLegalize::ExpandOp(SDOperand Op, SDOperand &Lo, SDOperand &Hi){
5237  MVT::ValueType VT = Op.getValueType();
5238  MVT::ValueType NVT = TLI.getTypeToTransformTo(VT);
5239  SDNode *Node = Op.Val;
5240  assert(getTypeAction(VT) == Expand && "Not an expanded type!");
5241  assert(((MVT::isInteger(NVT) && NVT < VT) || MVT::isFloatingPoint(VT) ||
5242         MVT::isVector(VT)) &&
5243         "Cannot expand to FP value or to larger int value!");
5244
5245  // See if we already expanded it.
5246  DenseMap<SDOperand, std::pair<SDOperand, SDOperand> >::iterator I
5247    = ExpandedNodes.find(Op);
5248  if (I != ExpandedNodes.end()) {
5249    Lo = I->second.first;
5250    Hi = I->second.second;
5251    return;
5252  }
5253
5254  switch (Node->getOpcode()) {
5255  case ISD::CopyFromReg:
5256    assert(0 && "CopyFromReg must be legal!");
5257  case ISD::FP_ROUND_INREG:
5258    if (VT == MVT::ppcf128 &&
5259        TLI.getOperationAction(ISD::FP_ROUND_INREG, VT) ==
5260            TargetLowering::Custom) {
5261      SDOperand SrcLo, SrcHi, Src;
5262      ExpandOp(Op.getOperand(0), SrcLo, SrcHi);
5263      Src = DAG.getNode(ISD::BUILD_PAIR, VT, SrcLo, SrcHi);
5264      SDOperand Result = TLI.LowerOperation(
5265        DAG.getNode(ISD::FP_ROUND_INREG, VT, Src, Op.getOperand(1)), DAG);
5266      assert(Result.Val->getOpcode() == ISD::BUILD_PAIR);
5267      Lo = Result.Val->getOperand(0);
5268      Hi = Result.Val->getOperand(1);
5269      break;
5270    }
5271    // fall through
5272  default:
5273#ifndef NDEBUG
5274    cerr << "NODE: "; Node->dump(&DAG); cerr << "\n";
5275#endif
5276    assert(0 && "Do not know how to expand this operator!");
5277    abort();
5278  case ISD::UNDEF:
5279    NVT = TLI.getTypeToExpandTo(VT);
5280    Lo = DAG.getNode(ISD::UNDEF, NVT);
5281    Hi = DAG.getNode(ISD::UNDEF, NVT);
5282    break;
5283  case ISD::Constant: {
5284    uint64_t Cst = cast<ConstantSDNode>(Node)->getValue();
5285    Lo = DAG.getConstant(Cst, NVT);
5286    Hi = DAG.getConstant(Cst >> MVT::getSizeInBits(NVT), NVT);
5287    break;
5288  }
5289  case ISD::ConstantFP: {
5290    ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
5291    if (CFP->getValueType(0) == MVT::ppcf128) {
5292      APInt api = CFP->getValueAPF().convertToAPInt();
5293      Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &api.getRawData()[1])),
5294                             MVT::f64);
5295      Hi = DAG.getConstantFP(APFloat(APInt(64, 1, &api.getRawData()[0])),
5296                             MVT::f64);
5297      break;
5298    }
5299    Lo = ExpandConstantFP(CFP, false, DAG, TLI);
5300    if (getTypeAction(Lo.getValueType()) == Expand)
5301      ExpandOp(Lo, Lo, Hi);
5302    break;
5303  }
5304  case ISD::BUILD_PAIR:
5305    // Return the operands.
5306    Lo = Node->getOperand(0);
5307    Hi = Node->getOperand(1);
5308    break;
5309
5310  case ISD::SIGN_EXTEND_INREG:
5311    ExpandOp(Node->getOperand(0), Lo, Hi);
5312    // sext_inreg the low part if needed.
5313    Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, NVT, Lo, Node->getOperand(1));
5314
5315    // The high part gets the sign extension from the lo-part.  This handles
5316    // things like sextinreg V:i64 from i8.
5317    Hi = DAG.getNode(ISD::SRA, NVT, Lo,
5318                     DAG.getConstant(MVT::getSizeInBits(NVT)-1,
5319                                     TLI.getShiftAmountTy()));
5320    break;
5321
5322  case ISD::BSWAP: {
5323    ExpandOp(Node->getOperand(0), Lo, Hi);
5324    SDOperand TempLo = DAG.getNode(ISD::BSWAP, NVT, Hi);
5325    Hi = DAG.getNode(ISD::BSWAP, NVT, Lo);
5326    Lo = TempLo;
5327    break;
5328  }
5329
5330  case ISD::CTPOP:
5331    ExpandOp(Node->getOperand(0), Lo, Hi);
5332    Lo = DAG.getNode(ISD::ADD, NVT,          // ctpop(HL) -> ctpop(H)+ctpop(L)
5333                     DAG.getNode(ISD::CTPOP, NVT, Lo),
5334                     DAG.getNode(ISD::CTPOP, NVT, Hi));
5335    Hi = DAG.getConstant(0, NVT);
5336    break;
5337
5338  case ISD::CTLZ: {
5339    // ctlz (HL) -> ctlz(H) != 32 ? ctlz(H) : (ctlz(L)+32)
5340    ExpandOp(Node->getOperand(0), Lo, Hi);
5341    SDOperand BitsC = DAG.getConstant(MVT::getSizeInBits(NVT), NVT);
5342    SDOperand HLZ = DAG.getNode(ISD::CTLZ, NVT, Hi);
5343    SDOperand TopNotZero = DAG.getSetCC(TLI.getSetCCResultTy(), HLZ, BitsC,
5344                                        ISD::SETNE);
5345    SDOperand LowPart = DAG.getNode(ISD::CTLZ, NVT, Lo);
5346    LowPart = DAG.getNode(ISD::ADD, NVT, LowPart, BitsC);
5347
5348    Lo = DAG.getNode(ISD::SELECT, NVT, TopNotZero, HLZ, LowPart);
5349    Hi = DAG.getConstant(0, NVT);
5350    break;
5351  }
5352
5353  case ISD::CTTZ: {
5354    // cttz (HL) -> cttz(L) != 32 ? cttz(L) : (cttz(H)+32)
5355    ExpandOp(Node->getOperand(0), Lo, Hi);
5356    SDOperand BitsC = DAG.getConstant(MVT::getSizeInBits(NVT), NVT);
5357    SDOperand LTZ = DAG.getNode(ISD::CTTZ, NVT, Lo);
5358    SDOperand BotNotZero = DAG.getSetCC(TLI.getSetCCResultTy(), LTZ, BitsC,
5359                                        ISD::SETNE);
5360    SDOperand HiPart = DAG.getNode(ISD::CTTZ, NVT, Hi);
5361    HiPart = DAG.getNode(ISD::ADD, NVT, HiPart, BitsC);
5362
5363    Lo = DAG.getNode(ISD::SELECT, NVT, BotNotZero, LTZ, HiPart);
5364    Hi = DAG.getConstant(0, NVT);
5365    break;
5366  }
5367
5368  case ISD::VAARG: {
5369    SDOperand Ch = Node->getOperand(0);   // Legalize the chain.
5370    SDOperand Ptr = Node->getOperand(1);  // Legalize the pointer.
5371    Lo = DAG.getVAArg(NVT, Ch, Ptr, Node->getOperand(2));
5372    Hi = DAG.getVAArg(NVT, Lo.getValue(1), Ptr, Node->getOperand(2));
5373
5374    // Remember that we legalized the chain.
5375    Hi = LegalizeOp(Hi);
5376    AddLegalizedOperand(Op.getValue(1), Hi.getValue(1));
5377    if (!TLI.isLittleEndian())
5378      std::swap(Lo, Hi);
5379    break;
5380  }
5381
5382  case ISD::LOAD: {
5383    LoadSDNode *LD = cast<LoadSDNode>(Node);
5384    SDOperand Ch  = LD->getChain();    // Legalize the chain.
5385    SDOperand Ptr = LD->getBasePtr();  // Legalize the pointer.
5386    ISD::LoadExtType ExtType = LD->getExtensionType();
5387    int SVOffset = LD->getSrcValueOffset();
5388    unsigned Alignment = LD->getAlignment();
5389    bool isVolatile = LD->isVolatile();
5390
5391    if (ExtType == ISD::NON_EXTLOAD) {
5392      Lo = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(), SVOffset,
5393                       isVolatile, Alignment);
5394      if (VT == MVT::f32 || VT == MVT::f64) {
5395        // f32->i32 or f64->i64 one to one expansion.
5396        // Remember that we legalized the chain.
5397        AddLegalizedOperand(SDOperand(Node, 1), LegalizeOp(Lo.getValue(1)));
5398        // Recursively expand the new load.
5399        if (getTypeAction(NVT) == Expand)
5400          ExpandOp(Lo, Lo, Hi);
5401        break;
5402      }
5403
5404      // Increment the pointer to the other half.
5405      unsigned IncrementSize = MVT::getSizeInBits(Lo.getValueType())/8;
5406      Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
5407                        getIntPtrConstant(IncrementSize));
5408      SVOffset += IncrementSize;
5409      if (Alignment > IncrementSize)
5410        Alignment = IncrementSize;
5411      Hi = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(), SVOffset,
5412                       isVolatile, Alignment);
5413
5414      // Build a factor node to remember that this load is independent of the
5415      // other one.
5416      SDOperand TF = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1),
5417                                 Hi.getValue(1));
5418
5419      // Remember that we legalized the chain.
5420      AddLegalizedOperand(Op.getValue(1), LegalizeOp(TF));
5421      if (!TLI.isLittleEndian())
5422        std::swap(Lo, Hi);
5423    } else {
5424      MVT::ValueType EVT = LD->getLoadedVT();
5425
5426      if (VT == MVT::f64 && EVT == MVT::f32) {
5427        // f64 = EXTLOAD f32 should expand to LOAD, FP_EXTEND
5428        SDOperand Load = DAG.getLoad(EVT, Ch, Ptr, LD->getSrcValue(),
5429                                     SVOffset, isVolatile, Alignment);
5430        // Remember that we legalized the chain.
5431        AddLegalizedOperand(SDOperand(Node, 1), LegalizeOp(Load.getValue(1)));
5432        ExpandOp(DAG.getNode(ISD::FP_EXTEND, VT, Load), Lo, Hi);
5433        break;
5434      }
5435
5436      if (EVT == NVT)
5437        Lo = DAG.getLoad(NVT, Ch, Ptr, LD->getSrcValue(),
5438                         SVOffset, isVolatile, Alignment);
5439      else
5440        Lo = DAG.getExtLoad(ExtType, NVT, Ch, Ptr, LD->getSrcValue(),
5441                            SVOffset, EVT, isVolatile,
5442                            Alignment);
5443
5444      // Remember that we legalized the chain.
5445      AddLegalizedOperand(SDOperand(Node, 1), LegalizeOp(Lo.getValue(1)));
5446
5447      if (ExtType == ISD::SEXTLOAD) {
5448        // The high part is obtained by SRA'ing all but one of the bits of the
5449        // lo part.
5450        unsigned LoSize = MVT::getSizeInBits(Lo.getValueType());
5451        Hi = DAG.getNode(ISD::SRA, NVT, Lo,
5452                         DAG.getConstant(LoSize-1, TLI.getShiftAmountTy()));
5453      } else if (ExtType == ISD::ZEXTLOAD) {
5454        // The high part is just a zero.
5455        Hi = DAG.getConstant(0, NVT);
5456      } else /* if (ExtType == ISD::EXTLOAD) */ {
5457        // The high part is undefined.
5458        Hi = DAG.getNode(ISD::UNDEF, NVT);
5459      }
5460    }
5461    break;
5462  }
5463  case ISD::AND:
5464  case ISD::OR:
5465  case ISD::XOR: {   // Simple logical operators -> two trivial pieces.
5466    SDOperand LL, LH, RL, RH;
5467    ExpandOp(Node->getOperand(0), LL, LH);
5468    ExpandOp(Node->getOperand(1), RL, RH);
5469    Lo = DAG.getNode(Node->getOpcode(), NVT, LL, RL);
5470    Hi = DAG.getNode(Node->getOpcode(), NVT, LH, RH);
5471    break;
5472  }
5473  case ISD::SELECT: {
5474    SDOperand LL, LH, RL, RH;
5475    ExpandOp(Node->getOperand(1), LL, LH);
5476    ExpandOp(Node->getOperand(2), RL, RH);
5477    if (getTypeAction(NVT) == Expand)
5478      NVT = TLI.getTypeToExpandTo(NVT);
5479    Lo = DAG.getNode(ISD::SELECT, NVT, Node->getOperand(0), LL, RL);
5480    if (VT != MVT::f32)
5481      Hi = DAG.getNode(ISD::SELECT, NVT, Node->getOperand(0), LH, RH);
5482    break;
5483  }
5484  case ISD::SELECT_CC: {
5485    SDOperand TL, TH, FL, FH;
5486    ExpandOp(Node->getOperand(2), TL, TH);
5487    ExpandOp(Node->getOperand(3), FL, FH);
5488    if (getTypeAction(NVT) == Expand)
5489      NVT = TLI.getTypeToExpandTo(NVT);
5490    Lo = DAG.getNode(ISD::SELECT_CC, NVT, Node->getOperand(0),
5491                     Node->getOperand(1), TL, FL, Node->getOperand(4));
5492    if (VT != MVT::f32)
5493      Hi = DAG.getNode(ISD::SELECT_CC, NVT, Node->getOperand(0),
5494                       Node->getOperand(1), TH, FH, Node->getOperand(4));
5495    break;
5496  }
5497  case ISD::ANY_EXTEND:
5498    // The low part is any extension of the input (which degenerates to a copy).
5499    Lo = DAG.getNode(ISD::ANY_EXTEND, NVT, Node->getOperand(0));
5500    // The high part is undefined.
5501    Hi = DAG.getNode(ISD::UNDEF, NVT);
5502    break;
5503  case ISD::SIGN_EXTEND: {
5504    // The low part is just a sign extension of the input (which degenerates to
5505    // a copy).
5506    Lo = DAG.getNode(ISD::SIGN_EXTEND, NVT, Node->getOperand(0));
5507
5508    // The high part is obtained by SRA'ing all but one of the bits of the lo
5509    // part.
5510    unsigned LoSize = MVT::getSizeInBits(Lo.getValueType());
5511    Hi = DAG.getNode(ISD::SRA, NVT, Lo,
5512                     DAG.getConstant(LoSize-1, TLI.getShiftAmountTy()));
5513    break;
5514  }
5515  case ISD::ZERO_EXTEND:
5516    // The low part is just a zero extension of the input (which degenerates to
5517    // a copy).
5518    Lo = DAG.getNode(ISD::ZERO_EXTEND, NVT, Node->getOperand(0));
5519
5520    // The high part is just a zero.
5521    Hi = DAG.getConstant(0, NVT);
5522    break;
5523
5524  case ISD::TRUNCATE: {
5525    // The input value must be larger than this value.  Expand *it*.
5526    SDOperand NewLo;
5527    ExpandOp(Node->getOperand(0), NewLo, Hi);
5528
5529    // The low part is now either the right size, or it is closer.  If not the
5530    // right size, make an illegal truncate so we recursively expand it.
5531    if (NewLo.getValueType() != Node->getValueType(0))
5532      NewLo = DAG.getNode(ISD::TRUNCATE, Node->getValueType(0), NewLo);
5533    ExpandOp(NewLo, Lo, Hi);
5534    break;
5535  }
5536
5537  case ISD::BIT_CONVERT: {
5538    SDOperand Tmp;
5539    if (TLI.getOperationAction(ISD::BIT_CONVERT, VT) == TargetLowering::Custom){
5540      // If the target wants to, allow it to lower this itself.
5541      switch (getTypeAction(Node->getOperand(0).getValueType())) {
5542      case Expand: assert(0 && "cannot expand FP!");
5543      case Legal:   Tmp = LegalizeOp(Node->getOperand(0)); break;
5544      case Promote: Tmp = PromoteOp (Node->getOperand(0)); break;
5545      }
5546      Tmp = TLI.LowerOperation(DAG.getNode(ISD::BIT_CONVERT, VT, Tmp), DAG);
5547    }
5548
5549    // f32 / f64 must be expanded to i32 / i64.
5550    if (VT == MVT::f32 || VT == MVT::f64) {
5551      Lo = DAG.getNode(ISD::BIT_CONVERT, NVT, Node->getOperand(0));
5552      if (getTypeAction(NVT) == Expand)
5553        ExpandOp(Lo, Lo, Hi);
5554      break;
5555    }
5556
5557    // If source operand will be expanded to the same type as VT, i.e.
5558    // i64 <- f64, i32 <- f32, expand the source operand instead.
5559    MVT::ValueType VT0 = Node->getOperand(0).getValueType();
5560    if (getTypeAction(VT0) == Expand && TLI.getTypeToTransformTo(VT0) == VT) {
5561      ExpandOp(Node->getOperand(0), Lo, Hi);
5562      break;
5563    }
5564
5565    // Turn this into a load/store pair by default.
5566    if (Tmp.Val == 0)
5567      Tmp = ExpandBIT_CONVERT(VT, Node->getOperand(0));
5568
5569    ExpandOp(Tmp, Lo, Hi);
5570    break;
5571  }
5572
5573  case ISD::READCYCLECOUNTER:
5574    assert(TLI.getOperationAction(ISD::READCYCLECOUNTER, VT) ==
5575                 TargetLowering::Custom &&
5576           "Must custom expand ReadCycleCounter");
5577    Lo = TLI.LowerOperation(Op, DAG);
5578    assert(Lo.Val && "Node must be custom expanded!");
5579    Hi = Lo.getValue(1);
5580    AddLegalizedOperand(SDOperand(Node, 1), // Remember we legalized the chain.
5581                        LegalizeOp(Lo.getValue(2)));
5582    break;
5583
5584    // These operators cannot be expanded directly, emit them as calls to
5585    // library functions.
5586  case ISD::FP_TO_SINT: {
5587    if (TLI.getOperationAction(ISD::FP_TO_SINT, VT) == TargetLowering::Custom) {
5588      SDOperand Op;
5589      switch (getTypeAction(Node->getOperand(0).getValueType())) {
5590      case Expand: assert(0 && "cannot expand FP!");
5591      case Legal:   Op = LegalizeOp(Node->getOperand(0)); break;
5592      case Promote: Op = PromoteOp (Node->getOperand(0)); break;
5593      }
5594
5595      Op = TLI.LowerOperation(DAG.getNode(ISD::FP_TO_SINT, VT, Op), DAG);
5596
5597      // Now that the custom expander is done, expand the result, which is still
5598      // VT.
5599      if (Op.Val) {
5600        ExpandOp(Op, Lo, Hi);
5601        break;
5602      }
5603    }
5604
5605    RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5606    if (Node->getOperand(0).getValueType() == MVT::f32)
5607      LC = RTLIB::FPTOSINT_F32_I64;
5608    else if (Node->getOperand(0).getValueType() == MVT::f64)
5609      LC = RTLIB::FPTOSINT_F64_I64;
5610    else if (Node->getOperand(0).getValueType() == MVT::f80)
5611      LC = RTLIB::FPTOSINT_F80_I64;
5612    else if (Node->getOperand(0).getValueType() == MVT::ppcf128)
5613      LC = RTLIB::FPTOSINT_PPCF128_I64;
5614    Lo = ExpandLibCall(TLI.getLibcallName(LC), Node,
5615                       false/*sign irrelevant*/, Hi);
5616    break;
5617  }
5618
5619  case ISD::FP_TO_UINT: {
5620    if (TLI.getOperationAction(ISD::FP_TO_UINT, VT) == TargetLowering::Custom) {
5621      SDOperand Op;
5622      switch (getTypeAction(Node->getOperand(0).getValueType())) {
5623        case Expand: assert(0 && "cannot expand FP!");
5624        case Legal:   Op = LegalizeOp(Node->getOperand(0)); break;
5625        case Promote: Op = PromoteOp (Node->getOperand(0)); break;
5626      }
5627
5628      Op = TLI.LowerOperation(DAG.getNode(ISD::FP_TO_UINT, VT, Op), DAG);
5629
5630      // Now that the custom expander is done, expand the result.
5631      if (Op.Val) {
5632        ExpandOp(Op, Lo, Hi);
5633        break;
5634      }
5635    }
5636
5637    RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5638    if (Node->getOperand(0).getValueType() == MVT::f32)
5639      LC = RTLIB::FPTOUINT_F32_I64;
5640    else if (Node->getOperand(0).getValueType() == MVT::f64)
5641      LC = RTLIB::FPTOUINT_F64_I64;
5642    else if (Node->getOperand(0).getValueType() == MVT::f80)
5643      LC = RTLIB::FPTOUINT_F80_I64;
5644    else if (Node->getOperand(0).getValueType() == MVT::ppcf128)
5645      LC = RTLIB::FPTOUINT_PPCF128_I64;
5646    Lo = ExpandLibCall(TLI.getLibcallName(LC), Node,
5647                       false/*sign irrelevant*/, Hi);
5648    break;
5649  }
5650
5651  case ISD::SHL: {
5652    // If the target wants custom lowering, do so.
5653    SDOperand ShiftAmt = LegalizeOp(Node->getOperand(1));
5654    if (TLI.getOperationAction(ISD::SHL, VT) == TargetLowering::Custom) {
5655      SDOperand Op = DAG.getNode(ISD::SHL, VT, Node->getOperand(0), ShiftAmt);
5656      Op = TLI.LowerOperation(Op, DAG);
5657      if (Op.Val) {
5658        // Now that the custom expander is done, expand the result, which is
5659        // still VT.
5660        ExpandOp(Op, Lo, Hi);
5661        break;
5662      }
5663    }
5664
5665    // If ADDC/ADDE are supported and if the shift amount is a constant 1, emit
5666    // this X << 1 as X+X.
5667    if (ConstantSDNode *ShAmt = dyn_cast<ConstantSDNode>(ShiftAmt)) {
5668      if (ShAmt->getValue() == 1 && TLI.isOperationLegal(ISD::ADDC, NVT) &&
5669          TLI.isOperationLegal(ISD::ADDE, NVT)) {
5670        SDOperand LoOps[2], HiOps[3];
5671        ExpandOp(Node->getOperand(0), LoOps[0], HiOps[0]);
5672        SDVTList VTList = DAG.getVTList(LoOps[0].getValueType(), MVT::Flag);
5673        LoOps[1] = LoOps[0];
5674        Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2);
5675
5676        HiOps[1] = HiOps[0];
5677        HiOps[2] = Lo.getValue(1);
5678        Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3);
5679        break;
5680      }
5681    }
5682
5683    // If we can emit an efficient shift operation, do so now.
5684    if (ExpandShift(ISD::SHL, Node->getOperand(0), ShiftAmt, Lo, Hi))
5685      break;
5686
5687    // If this target supports SHL_PARTS, use it.
5688    TargetLowering::LegalizeAction Action =
5689      TLI.getOperationAction(ISD::SHL_PARTS, NVT);
5690    if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
5691        Action == TargetLowering::Custom) {
5692      ExpandShiftParts(ISD::SHL_PARTS, Node->getOperand(0), ShiftAmt, Lo, Hi);
5693      break;
5694    }
5695
5696    // Otherwise, emit a libcall.
5697    Lo = ExpandLibCall(TLI.getLibcallName(RTLIB::SHL_I64), Node,
5698                       false/*left shift=unsigned*/, Hi);
5699    break;
5700  }
5701
5702  case ISD::SRA: {
5703    // If the target wants custom lowering, do so.
5704    SDOperand ShiftAmt = LegalizeOp(Node->getOperand(1));
5705    if (TLI.getOperationAction(ISD::SRA, VT) == TargetLowering::Custom) {
5706      SDOperand Op = DAG.getNode(ISD::SRA, VT, Node->getOperand(0), ShiftAmt);
5707      Op = TLI.LowerOperation(Op, DAG);
5708      if (Op.Val) {
5709        // Now that the custom expander is done, expand the result, which is
5710        // still VT.
5711        ExpandOp(Op, Lo, Hi);
5712        break;
5713      }
5714    }
5715
5716    // If we can emit an efficient shift operation, do so now.
5717    if (ExpandShift(ISD::SRA, Node->getOperand(0), ShiftAmt, Lo, Hi))
5718      break;
5719
5720    // If this target supports SRA_PARTS, use it.
5721    TargetLowering::LegalizeAction Action =
5722      TLI.getOperationAction(ISD::SRA_PARTS, NVT);
5723    if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
5724        Action == TargetLowering::Custom) {
5725      ExpandShiftParts(ISD::SRA_PARTS, Node->getOperand(0), ShiftAmt, Lo, Hi);
5726      break;
5727    }
5728
5729    // Otherwise, emit a libcall.
5730    Lo = ExpandLibCall(TLI.getLibcallName(RTLIB::SRA_I64), Node,
5731                       true/*ashr is signed*/, Hi);
5732    break;
5733  }
5734
5735  case ISD::SRL: {
5736    // If the target wants custom lowering, do so.
5737    SDOperand ShiftAmt = LegalizeOp(Node->getOperand(1));
5738    if (TLI.getOperationAction(ISD::SRL, VT) == TargetLowering::Custom) {
5739      SDOperand Op = DAG.getNode(ISD::SRL, VT, Node->getOperand(0), ShiftAmt);
5740      Op = TLI.LowerOperation(Op, DAG);
5741      if (Op.Val) {
5742        // Now that the custom expander is done, expand the result, which is
5743        // still VT.
5744        ExpandOp(Op, Lo, Hi);
5745        break;
5746      }
5747    }
5748
5749    // If we can emit an efficient shift operation, do so now.
5750    if (ExpandShift(ISD::SRL, Node->getOperand(0), ShiftAmt, Lo, Hi))
5751      break;
5752
5753    // If this target supports SRL_PARTS, use it.
5754    TargetLowering::LegalizeAction Action =
5755      TLI.getOperationAction(ISD::SRL_PARTS, NVT);
5756    if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
5757        Action == TargetLowering::Custom) {
5758      ExpandShiftParts(ISD::SRL_PARTS, Node->getOperand(0), ShiftAmt, Lo, Hi);
5759      break;
5760    }
5761
5762    // Otherwise, emit a libcall.
5763    Lo = ExpandLibCall(TLI.getLibcallName(RTLIB::SRL_I64), Node,
5764                       false/*lshr is unsigned*/, Hi);
5765    break;
5766  }
5767
5768  case ISD::ADD:
5769  case ISD::SUB: {
5770    // If the target wants to custom expand this, let them.
5771    if (TLI.getOperationAction(Node->getOpcode(), VT) ==
5772            TargetLowering::Custom) {
5773      Op = TLI.LowerOperation(Op, DAG);
5774      if (Op.Val) {
5775        ExpandOp(Op, Lo, Hi);
5776        break;
5777      }
5778    }
5779
5780    // Expand the subcomponents.
5781    SDOperand LHSL, LHSH, RHSL, RHSH;
5782    ExpandOp(Node->getOperand(0), LHSL, LHSH);
5783    ExpandOp(Node->getOperand(1), RHSL, RHSH);
5784    SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag);
5785    SDOperand LoOps[2], HiOps[3];
5786    LoOps[0] = LHSL;
5787    LoOps[1] = RHSL;
5788    HiOps[0] = LHSH;
5789    HiOps[1] = RHSH;
5790    if (Node->getOpcode() == ISD::ADD) {
5791      Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2);
5792      HiOps[2] = Lo.getValue(1);
5793      Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3);
5794    } else {
5795      Lo = DAG.getNode(ISD::SUBC, VTList, LoOps, 2);
5796      HiOps[2] = Lo.getValue(1);
5797      Hi = DAG.getNode(ISD::SUBE, VTList, HiOps, 3);
5798    }
5799    break;
5800  }
5801
5802  case ISD::ADDC:
5803  case ISD::SUBC: {
5804    // Expand the subcomponents.
5805    SDOperand LHSL, LHSH, RHSL, RHSH;
5806    ExpandOp(Node->getOperand(0), LHSL, LHSH);
5807    ExpandOp(Node->getOperand(1), RHSL, RHSH);
5808    SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag);
5809    SDOperand LoOps[2] = { LHSL, RHSL };
5810    SDOperand HiOps[3] = { LHSH, RHSH };
5811
5812    if (Node->getOpcode() == ISD::ADDC) {
5813      Lo = DAG.getNode(ISD::ADDC, VTList, LoOps, 2);
5814      HiOps[2] = Lo.getValue(1);
5815      Hi = DAG.getNode(ISD::ADDE, VTList, HiOps, 3);
5816    } else {
5817      Lo = DAG.getNode(ISD::SUBC, VTList, LoOps, 2);
5818      HiOps[2] = Lo.getValue(1);
5819      Hi = DAG.getNode(ISD::SUBE, VTList, HiOps, 3);
5820    }
5821    // Remember that we legalized the flag.
5822    AddLegalizedOperand(Op.getValue(1), LegalizeOp(Hi.getValue(1)));
5823    break;
5824  }
5825  case ISD::ADDE:
5826  case ISD::SUBE: {
5827    // Expand the subcomponents.
5828    SDOperand LHSL, LHSH, RHSL, RHSH;
5829    ExpandOp(Node->getOperand(0), LHSL, LHSH);
5830    ExpandOp(Node->getOperand(1), RHSL, RHSH);
5831    SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Flag);
5832    SDOperand LoOps[3] = { LHSL, RHSL, Node->getOperand(2) };
5833    SDOperand HiOps[3] = { LHSH, RHSH };
5834
5835    Lo = DAG.getNode(Node->getOpcode(), VTList, LoOps, 3);
5836    HiOps[2] = Lo.getValue(1);
5837    Hi = DAG.getNode(Node->getOpcode(), VTList, HiOps, 3);
5838
5839    // Remember that we legalized the flag.
5840    AddLegalizedOperand(Op.getValue(1), LegalizeOp(Hi.getValue(1)));
5841    break;
5842  }
5843  case ISD::MUL: {
5844    // If the target wants to custom expand this, let them.
5845    if (TLI.getOperationAction(ISD::MUL, VT) == TargetLowering::Custom) {
5846      SDOperand New = TLI.LowerOperation(Op, DAG);
5847      if (New.Val) {
5848        ExpandOp(New, Lo, Hi);
5849        break;
5850      }
5851    }
5852
5853    bool HasMULHS = TLI.isOperationLegal(ISD::MULHS, NVT);
5854    bool HasMULHU = TLI.isOperationLegal(ISD::MULHU, NVT);
5855    bool HasSMUL_LOHI = TLI.isOperationLegal(ISD::SMUL_LOHI, NVT);
5856    bool HasUMUL_LOHI = TLI.isOperationLegal(ISD::UMUL_LOHI, NVT);
5857    if (HasMULHU || HasMULHS || HasUMUL_LOHI || HasSMUL_LOHI) {
5858      SDOperand LL, LH, RL, RH;
5859      ExpandOp(Node->getOperand(0), LL, LH);
5860      ExpandOp(Node->getOperand(1), RL, RH);
5861      unsigned BitSize = MVT::getSizeInBits(RH.getValueType());
5862      unsigned LHSSB = DAG.ComputeNumSignBits(Op.getOperand(0));
5863      unsigned RHSSB = DAG.ComputeNumSignBits(Op.getOperand(1));
5864      // FIXME: generalize this to handle other bit sizes
5865      if (LHSSB == 32 && RHSSB == 32 &&
5866          DAG.MaskedValueIsZero(Op.getOperand(0), 0xFFFFFFFF00000000ULL) &&
5867          DAG.MaskedValueIsZero(Op.getOperand(1), 0xFFFFFFFF00000000ULL)) {
5868        // The inputs are both zero-extended.
5869        if (HasUMUL_LOHI) {
5870          // We can emit a umul_lohi.
5871          Lo = DAG.getNode(ISD::UMUL_LOHI, DAG.getVTList(NVT, NVT), LL, RL);
5872          Hi = SDOperand(Lo.Val, 1);
5873          break;
5874        }
5875        if (HasMULHU) {
5876          // We can emit a mulhu+mul.
5877          Lo = DAG.getNode(ISD::MUL, NVT, LL, RL);
5878          Hi = DAG.getNode(ISD::MULHU, NVT, LL, RL);
5879          break;
5880        }
5881      }
5882      if (LHSSB > BitSize && RHSSB > BitSize) {
5883        // The input values are both sign-extended.
5884        if (HasSMUL_LOHI) {
5885          // We can emit a smul_lohi.
5886          Lo = DAG.getNode(ISD::SMUL_LOHI, DAG.getVTList(NVT, NVT), LL, RL);
5887          Hi = SDOperand(Lo.Val, 1);
5888          break;
5889        }
5890        if (HasMULHS) {
5891          // We can emit a mulhs+mul.
5892          Lo = DAG.getNode(ISD::MUL, NVT, LL, RL);
5893          Hi = DAG.getNode(ISD::MULHS, NVT, LL, RL);
5894          break;
5895        }
5896      }
5897      if (HasUMUL_LOHI) {
5898        // Lo,Hi = umul LHS, RHS.
5899        SDOperand UMulLOHI = DAG.getNode(ISD::UMUL_LOHI,
5900                                         DAG.getVTList(NVT, NVT), LL, RL);
5901        Lo = UMulLOHI;
5902        Hi = UMulLOHI.getValue(1);
5903        RH = DAG.getNode(ISD::MUL, NVT, LL, RH);
5904        LH = DAG.getNode(ISD::MUL, NVT, LH, RL);
5905        Hi = DAG.getNode(ISD::ADD, NVT, Hi, RH);
5906        Hi = DAG.getNode(ISD::ADD, NVT, Hi, LH);
5907        break;
5908      }
5909    }
5910
5911    // If nothing else, we can make a libcall.
5912    Lo = ExpandLibCall(TLI.getLibcallName(RTLIB::MUL_I64), Node,
5913                       false/*sign irrelevant*/, Hi);
5914    break;
5915  }
5916  case ISD::SDIV:
5917    Lo = ExpandLibCall(TLI.getLibcallName(RTLIB::SDIV_I64), Node, true, Hi);
5918    break;
5919  case ISD::UDIV:
5920    Lo = ExpandLibCall(TLI.getLibcallName(RTLIB::UDIV_I64), Node, true, Hi);
5921    break;
5922  case ISD::SREM:
5923    Lo = ExpandLibCall(TLI.getLibcallName(RTLIB::SREM_I64), Node, true, Hi);
5924    break;
5925  case ISD::UREM:
5926    Lo = ExpandLibCall(TLI.getLibcallName(RTLIB::UREM_I64), Node, true, Hi);
5927    break;
5928
5929  case ISD::FADD:
5930    Lo = ExpandLibCall(TLI.getLibcallName(VT == MVT::f32 ? RTLIB::ADD_F32 :
5931                                          VT == MVT::f64 ? RTLIB::ADD_F64 :
5932                                          VT == MVT::ppcf128 ?
5933                                                      RTLIB::ADD_PPCF128 :
5934                                          RTLIB::UNKNOWN_LIBCALL),
5935                       Node, false, Hi);
5936    break;
5937  case ISD::FSUB:
5938    Lo = ExpandLibCall(TLI.getLibcallName(VT == MVT::f32 ? RTLIB::SUB_F32 :
5939                                          VT == MVT::f64 ? RTLIB::SUB_F64 :
5940                                          VT == MVT::ppcf128 ?
5941                                                      RTLIB::SUB_PPCF128 :
5942                                          RTLIB::UNKNOWN_LIBCALL),
5943                       Node, false, Hi);
5944    break;
5945  case ISD::FMUL:
5946    Lo = ExpandLibCall(TLI.getLibcallName(VT == MVT::f32 ? RTLIB::MUL_F32 :
5947                                          VT == MVT::f64 ? RTLIB::MUL_F64 :
5948                                          VT == MVT::ppcf128 ?
5949                                                      RTLIB::MUL_PPCF128 :
5950                                          RTLIB::UNKNOWN_LIBCALL),
5951                       Node, false, Hi);
5952    break;
5953  case ISD::FDIV:
5954    Lo = ExpandLibCall(TLI.getLibcallName(VT == MVT::f32 ? RTLIB::DIV_F32 :
5955                                          VT == MVT::f64 ? RTLIB::DIV_F64 :
5956                                          VT == MVT::ppcf128 ?
5957                                                      RTLIB::DIV_PPCF128 :
5958                                          RTLIB::UNKNOWN_LIBCALL),
5959                       Node, false, Hi);
5960    break;
5961  case ISD::FP_EXTEND:
5962    if (VT == MVT::ppcf128) {
5963      assert(Node->getOperand(0).getValueType()==MVT::f32 ||
5964             Node->getOperand(0).getValueType()==MVT::f64);
5965      const uint64_t zero = 0;
5966      if (Node->getOperand(0).getValueType()==MVT::f32)
5967        Hi = DAG.getNode(ISD::FP_EXTEND, MVT::f64, Node->getOperand(0));
5968      else
5969        Hi = Node->getOperand(0);
5970      Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &zero)), MVT::f64);
5971      break;
5972    }
5973    Lo = ExpandLibCall(TLI.getLibcallName(RTLIB::FPEXT_F32_F64), Node, true,Hi);
5974    break;
5975  case ISD::FP_ROUND:
5976    Lo = ExpandLibCall(TLI.getLibcallName(RTLIB::FPROUND_F64_F32),Node,true,Hi);
5977    break;
5978  case ISD::FPOWI:
5979    Lo = ExpandLibCall(TLI.getLibcallName((VT == MVT::f32) ? RTLIB::POWI_F32 :
5980                                          (VT == MVT::f64) ? RTLIB::POWI_F64 :
5981                                          (VT == MVT::f80) ? RTLIB::POWI_F80 :
5982                                          (VT == MVT::ppcf128) ?
5983                                                         RTLIB::POWI_PPCF128 :
5984                                          RTLIB::UNKNOWN_LIBCALL),
5985                       Node, false, Hi);
5986    break;
5987  case ISD::FSQRT:
5988  case ISD::FSIN:
5989  case ISD::FCOS: {
5990    RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
5991    switch(Node->getOpcode()) {
5992    case ISD::FSQRT:
5993      LC = (VT == MVT::f32) ? RTLIB::SQRT_F32 :
5994           (VT == MVT::f64) ? RTLIB::SQRT_F64 :
5995           (VT == MVT::f80) ? RTLIB::SQRT_F80 :
5996           (VT == MVT::ppcf128) ? RTLIB::SQRT_PPCF128 :
5997           RTLIB::UNKNOWN_LIBCALL;
5998      break;
5999    case ISD::FSIN:
6000      LC = (VT == MVT::f32) ? RTLIB::SIN_F32 : RTLIB::SIN_F64;
6001      break;
6002    case ISD::FCOS:
6003      LC = (VT == MVT::f32) ? RTLIB::COS_F32 : RTLIB::COS_F64;
6004      break;
6005    default: assert(0 && "Unreachable!");
6006    }
6007    Lo = ExpandLibCall(TLI.getLibcallName(LC), Node, false, Hi);
6008    break;
6009  }
6010  case ISD::FABS: {
6011    if (VT == MVT::ppcf128) {
6012      SDOperand Tmp;
6013      ExpandOp(Node->getOperand(0), Lo, Tmp);
6014      Hi = DAG.getNode(ISD::FABS, NVT, Tmp);
6015      // lo = hi==fabs(hi) ? lo : -lo;
6016      Lo = DAG.getNode(ISD::SELECT_CC, NVT, Hi, Tmp,
6017                    Lo, DAG.getNode(ISD::FNEG, NVT, Lo),
6018                    DAG.getCondCode(ISD::SETEQ));
6019      break;
6020    }
6021    SDOperand Mask = (VT == MVT::f64)
6022      ? DAG.getConstantFP(BitsToDouble(~(1ULL << 63)), VT)
6023      : DAG.getConstantFP(BitsToFloat(~(1U << 31)), VT);
6024    Mask = DAG.getNode(ISD::BIT_CONVERT, NVT, Mask);
6025    Lo = DAG.getNode(ISD::BIT_CONVERT, NVT, Node->getOperand(0));
6026    Lo = DAG.getNode(ISD::AND, NVT, Lo, Mask);
6027    if (getTypeAction(NVT) == Expand)
6028      ExpandOp(Lo, Lo, Hi);
6029    break;
6030  }
6031  case ISD::FNEG: {
6032    if (VT == MVT::ppcf128) {
6033      ExpandOp(Node->getOperand(0), Lo, Hi);
6034      Lo = DAG.getNode(ISD::FNEG, MVT::f64, Lo);
6035      Hi = DAG.getNode(ISD::FNEG, MVT::f64, Hi);
6036      break;
6037    }
6038    SDOperand Mask = (VT == MVT::f64)
6039      ? DAG.getConstantFP(BitsToDouble(1ULL << 63), VT)
6040      : DAG.getConstantFP(BitsToFloat(1U << 31), VT);
6041    Mask = DAG.getNode(ISD::BIT_CONVERT, NVT, Mask);
6042    Lo = DAG.getNode(ISD::BIT_CONVERT, NVT, Node->getOperand(0));
6043    Lo = DAG.getNode(ISD::XOR, NVT, Lo, Mask);
6044    if (getTypeAction(NVT) == Expand)
6045      ExpandOp(Lo, Lo, Hi);
6046    break;
6047  }
6048  case ISD::FCOPYSIGN: {
6049    Lo = ExpandFCOPYSIGNToBitwiseOps(Node, NVT, DAG, TLI);
6050    if (getTypeAction(NVT) == Expand)
6051      ExpandOp(Lo, Lo, Hi);
6052    break;
6053  }
6054  case ISD::SINT_TO_FP:
6055  case ISD::UINT_TO_FP: {
6056    bool isSigned = Node->getOpcode() == ISD::SINT_TO_FP;
6057    MVT::ValueType SrcVT = Node->getOperand(0).getValueType();
6058    if (VT == MVT::ppcf128 && SrcVT != MVT::i64) {
6059      static uint64_t zero = 0;
6060      if (isSigned) {
6061        Hi = LegalizeOp(DAG.getNode(ISD::SINT_TO_FP, MVT::f64,
6062                                    Node->getOperand(0)));
6063        Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &zero)), MVT::f64);
6064      } else {
6065        static uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
6066        Hi = LegalizeOp(DAG.getNode(ISD::SINT_TO_FP, MVT::f64,
6067                                    Node->getOperand(0)));
6068        Lo = DAG.getConstantFP(APFloat(APInt(64, 1, &zero)), MVT::f64);
6069        Hi = DAG.getNode(ISD::BUILD_PAIR, VT, Lo, Hi);
6070        // X>=0 ? {(f64)x, 0} : {(f64)x, 0} + 2^32
6071        ExpandOp(DAG.getNode(ISD::SELECT_CC, MVT::ppcf128, Node->getOperand(0),
6072                             DAG.getConstant(0, MVT::i32),
6073                             DAG.getNode(ISD::FADD, MVT::ppcf128, Hi,
6074                                         DAG.getConstantFP(
6075                                            APFloat(APInt(128, 2, TwoE32)),
6076                                            MVT::ppcf128)),
6077                             Hi,
6078                             DAG.getCondCode(ISD::SETLT)),
6079                 Lo, Hi);
6080      }
6081      break;
6082    }
6083    if (VT == MVT::ppcf128 && SrcVT == MVT::i64 && !isSigned) {
6084      // si64->ppcf128 done by libcall, below
6085      static uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
6086      ExpandOp(DAG.getNode(ISD::SINT_TO_FP, MVT::ppcf128, Node->getOperand(0)),
6087               Lo, Hi);
6088      Hi = DAG.getNode(ISD::BUILD_PAIR, VT, Lo, Hi);
6089      // x>=0 ? (ppcf128)(i64)x : (ppcf128)(i64)x + 2^64
6090      ExpandOp(DAG.getNode(ISD::SELECT_CC, MVT::ppcf128, Node->getOperand(0),
6091                           DAG.getConstant(0, MVT::i64),
6092                           DAG.getNode(ISD::FADD, MVT::ppcf128, Hi,
6093                                       DAG.getConstantFP(
6094                                          APFloat(APInt(128, 2, TwoE64)),
6095                                          MVT::ppcf128)),
6096                           Hi,
6097                           DAG.getCondCode(ISD::SETLT)),
6098               Lo, Hi);
6099      break;
6100    }
6101    RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
6102    if (Node->getOperand(0).getValueType() == MVT::i64) {
6103      if (VT == MVT::f32)
6104        LC = isSigned ? RTLIB::SINTTOFP_I64_F32 : RTLIB::UINTTOFP_I64_F32;
6105      else if (VT == MVT::f64)
6106        LC = isSigned ? RTLIB::SINTTOFP_I64_F64 : RTLIB::UINTTOFP_I64_F64;
6107      else if (VT == MVT::f80) {
6108        assert(isSigned);
6109        LC = RTLIB::SINTTOFP_I64_F80;
6110      }
6111      else if (VT == MVT::ppcf128) {
6112        assert(isSigned);
6113        LC = RTLIB::SINTTOFP_I64_PPCF128;
6114      }
6115    } else {
6116      if (VT == MVT::f32)
6117        LC = isSigned ? RTLIB::SINTTOFP_I32_F32 : RTLIB::UINTTOFP_I32_F32;
6118      else
6119        LC = isSigned ? RTLIB::SINTTOFP_I32_F64 : RTLIB::UINTTOFP_I32_F64;
6120    }
6121
6122    // Promote the operand if needed.
6123    if (getTypeAction(SrcVT) == Promote) {
6124      SDOperand Tmp = PromoteOp(Node->getOperand(0));
6125      Tmp = isSigned
6126        ? DAG.getNode(ISD::SIGN_EXTEND_INREG, Tmp.getValueType(), Tmp,
6127                      DAG.getValueType(SrcVT))
6128        : DAG.getZeroExtendInReg(Tmp, SrcVT);
6129      Node = DAG.UpdateNodeOperands(Op, Tmp).Val;
6130    }
6131
6132    const char *LibCall = TLI.getLibcallName(LC);
6133    if (LibCall)
6134      Lo = ExpandLibCall(TLI.getLibcallName(LC), Node, isSigned, Hi);
6135    else  {
6136      Lo = ExpandIntToFP(Node->getOpcode() == ISD::SINT_TO_FP, VT,
6137                         Node->getOperand(0));
6138      if (getTypeAction(Lo.getValueType()) == Expand)
6139        ExpandOp(Lo, Lo, Hi);
6140    }
6141    break;
6142  }
6143  }
6144
6145  // Make sure the resultant values have been legalized themselves, unless this
6146  // is a type that requires multi-step expansion.
6147  if (getTypeAction(NVT) != Expand && NVT != MVT::isVoid) {
6148    Lo = LegalizeOp(Lo);
6149    if (Hi.Val)
6150      // Don't legalize the high part if it is expanded to a single node.
6151      Hi = LegalizeOp(Hi);
6152  }
6153
6154  // Remember in a map if the values will be reused later.
6155  bool isNew = ExpandedNodes.insert(std::make_pair(Op, std::make_pair(Lo, Hi)));
6156  assert(isNew && "Value already expanded?!?");
6157}
6158
6159/// SplitVectorOp - Given an operand of vector type, break it down into
6160/// two smaller values, still of vector type.
6161void SelectionDAGLegalize::SplitVectorOp(SDOperand Op, SDOperand &Lo,
6162                                         SDOperand &Hi) {
6163  assert(MVT::isVector(Op.getValueType()) && "Cannot split non-vector type!");
6164  SDNode *Node = Op.Val;
6165  unsigned NumElements = MVT::getVectorNumElements(Op.getValueType());
6166  assert(NumElements > 1 && "Cannot split a single element vector!");
6167  unsigned NewNumElts = NumElements/2;
6168  MVT::ValueType NewEltVT = MVT::getVectorElementType(Op.getValueType());
6169  MVT::ValueType NewVT = MVT::getVectorType(NewEltVT, NewNumElts);
6170
6171  // See if we already split it.
6172  std::map<SDOperand, std::pair<SDOperand, SDOperand> >::iterator I
6173    = SplitNodes.find(Op);
6174  if (I != SplitNodes.end()) {
6175    Lo = I->second.first;
6176    Hi = I->second.second;
6177    return;
6178  }
6179
6180  switch (Node->getOpcode()) {
6181  default:
6182#ifndef NDEBUG
6183    Node->dump(&DAG);
6184#endif
6185    assert(0 && "Unhandled operation in SplitVectorOp!");
6186  case ISD::BUILD_PAIR:
6187    Lo = Node->getOperand(0);
6188    Hi = Node->getOperand(1);
6189    break;
6190  case ISD::INSERT_VECTOR_ELT: {
6191    SplitVectorOp(Node->getOperand(0), Lo, Hi);
6192    unsigned Index = cast<ConstantSDNode>(Node->getOperand(2))->getValue();
6193    SDOperand ScalarOp = Node->getOperand(1);
6194    if (Index < NewNumElts)
6195      Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, NewVT, Lo, ScalarOp,
6196                       DAG.getConstant(Index, TLI.getPointerTy()));
6197    else
6198      Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, NewVT, Hi, ScalarOp,
6199                       DAG.getConstant(Index - NewNumElts, TLI.getPointerTy()));
6200    break;
6201  }
6202  case ISD::BUILD_VECTOR: {
6203    SmallVector<SDOperand, 8> LoOps(Node->op_begin(),
6204                                    Node->op_begin()+NewNumElts);
6205    Lo = DAG.getNode(ISD::BUILD_VECTOR, NewVT, &LoOps[0], LoOps.size());
6206
6207    SmallVector<SDOperand, 8> HiOps(Node->op_begin()+NewNumElts,
6208                                    Node->op_end());
6209    Hi = DAG.getNode(ISD::BUILD_VECTOR, NewVT, &HiOps[0], HiOps.size());
6210    break;
6211  }
6212  case ISD::CONCAT_VECTORS: {
6213    unsigned NewNumSubvectors = Node->getNumOperands() / 2;
6214    if (NewNumSubvectors == 1) {
6215      Lo = Node->getOperand(0);
6216      Hi = Node->getOperand(1);
6217    } else {
6218      SmallVector<SDOperand, 8> LoOps(Node->op_begin(),
6219                                      Node->op_begin()+NewNumSubvectors);
6220      Lo = DAG.getNode(ISD::CONCAT_VECTORS, NewVT, &LoOps[0], LoOps.size());
6221
6222      SmallVector<SDOperand, 8> HiOps(Node->op_begin()+NewNumSubvectors,
6223                                      Node->op_end());
6224      Hi = DAG.getNode(ISD::CONCAT_VECTORS, NewVT, &HiOps[0], HiOps.size());
6225    }
6226    break;
6227  }
6228  case ISD::ADD:
6229  case ISD::SUB:
6230  case ISD::MUL:
6231  case ISD::FADD:
6232  case ISD::FSUB:
6233  case ISD::FMUL:
6234  case ISD::SDIV:
6235  case ISD::UDIV:
6236  case ISD::FDIV:
6237  case ISD::FPOW:
6238  case ISD::AND:
6239  case ISD::OR:
6240  case ISD::XOR: {
6241    SDOperand LL, LH, RL, RH;
6242    SplitVectorOp(Node->getOperand(0), LL, LH);
6243    SplitVectorOp(Node->getOperand(1), RL, RH);
6244
6245    Lo = DAG.getNode(Node->getOpcode(), NewVT, LL, RL);
6246    Hi = DAG.getNode(Node->getOpcode(), NewVT, LH, RH);
6247    break;
6248  }
6249  case ISD::FPOWI: {
6250    SDOperand L, H;
6251    SplitVectorOp(Node->getOperand(0), L, H);
6252
6253    Lo = DAG.getNode(Node->getOpcode(), NewVT, L, Node->getOperand(1));
6254    Hi = DAG.getNode(Node->getOpcode(), NewVT, H, Node->getOperand(1));
6255    break;
6256  }
6257  case ISD::CTTZ:
6258  case ISD::CTLZ:
6259  case ISD::CTPOP:
6260  case ISD::FNEG:
6261  case ISD::FABS:
6262  case ISD::FSQRT:
6263  case ISD::FSIN:
6264  case ISD::FCOS: {
6265    SDOperand L, H;
6266    SplitVectorOp(Node->getOperand(0), L, H);
6267
6268    Lo = DAG.getNode(Node->getOpcode(), NewVT, L);
6269    Hi = DAG.getNode(Node->getOpcode(), NewVT, H);
6270    break;
6271  }
6272  case ISD::LOAD: {
6273    LoadSDNode *LD = cast<LoadSDNode>(Node);
6274    SDOperand Ch = LD->getChain();
6275    SDOperand Ptr = LD->getBasePtr();
6276    const Value *SV = LD->getSrcValue();
6277    int SVOffset = LD->getSrcValueOffset();
6278    unsigned Alignment = LD->getAlignment();
6279    bool isVolatile = LD->isVolatile();
6280
6281    Lo = DAG.getLoad(NewVT, Ch, Ptr, SV, SVOffset, isVolatile, Alignment);
6282    unsigned IncrementSize = NewNumElts * MVT::getSizeInBits(NewEltVT)/8;
6283    Ptr = DAG.getNode(ISD::ADD, Ptr.getValueType(), Ptr,
6284                      getIntPtrConstant(IncrementSize));
6285    SVOffset += IncrementSize;
6286    if (Alignment > IncrementSize)
6287      Alignment = IncrementSize;
6288    Hi = DAG.getLoad(NewVT, Ch, Ptr, SV, SVOffset, isVolatile, Alignment);
6289
6290    // Build a factor node to remember that this load is independent of the
6291    // other one.
6292    SDOperand TF = DAG.getNode(ISD::TokenFactor, MVT::Other, Lo.getValue(1),
6293                               Hi.getValue(1));
6294
6295    // Remember that we legalized the chain.
6296    AddLegalizedOperand(Op.getValue(1), LegalizeOp(TF));
6297    break;
6298  }
6299  case ISD::BIT_CONVERT: {
6300    // We know the result is a vector.  The input may be either a vector or a
6301    // scalar value.
6302    SDOperand InOp = Node->getOperand(0);
6303    if (!MVT::isVector(InOp.getValueType()) ||
6304        MVT::getVectorNumElements(InOp.getValueType()) == 1) {
6305      // The input is a scalar or single-element vector.
6306      // Lower to a store/load so that it can be split.
6307      // FIXME: this could be improved probably.
6308      SDOperand Ptr = DAG.CreateStackTemporary(InOp.getValueType());
6309
6310      SDOperand St = DAG.getStore(DAG.getEntryNode(),
6311                                  InOp, Ptr, NULL, 0);
6312      InOp = DAG.getLoad(Op.getValueType(), St, Ptr, NULL, 0);
6313    }
6314    // Split the vector and convert each of the pieces now.
6315    SplitVectorOp(InOp, Lo, Hi);
6316    Lo = DAG.getNode(ISD::BIT_CONVERT, NewVT, Lo);
6317    Hi = DAG.getNode(ISD::BIT_CONVERT, NewVT, Hi);
6318    break;
6319  }
6320  }
6321
6322  // Remember in a map if the values will be reused later.
6323  bool isNew =
6324    SplitNodes.insert(std::make_pair(Op, std::make_pair(Lo, Hi))).second;
6325  assert(isNew && "Value already split?!?");
6326}
6327
6328
6329/// ScalarizeVectorOp - Given an operand of single-element vector type
6330/// (e.g. v1f32), convert it into the equivalent operation that returns a
6331/// scalar (e.g. f32) value.
6332SDOperand SelectionDAGLegalize::ScalarizeVectorOp(SDOperand Op) {
6333  assert(MVT::isVector(Op.getValueType()) &&
6334         "Bad ScalarizeVectorOp invocation!");
6335  SDNode *Node = Op.Val;
6336  MVT::ValueType NewVT = MVT::getVectorElementType(Op.getValueType());
6337  assert(MVT::getVectorNumElements(Op.getValueType()) == 1);
6338
6339  // See if we already scalarized it.
6340  std::map<SDOperand, SDOperand>::iterator I = ScalarizedNodes.find(Op);
6341  if (I != ScalarizedNodes.end()) return I->second;
6342
6343  SDOperand Result;
6344  switch (Node->getOpcode()) {
6345  default:
6346#ifndef NDEBUG
6347    Node->dump(&DAG); cerr << "\n";
6348#endif
6349    assert(0 && "Unknown vector operation in ScalarizeVectorOp!");
6350  case ISD::ADD:
6351  case ISD::FADD:
6352  case ISD::SUB:
6353  case ISD::FSUB:
6354  case ISD::MUL:
6355  case ISD::FMUL:
6356  case ISD::SDIV:
6357  case ISD::UDIV:
6358  case ISD::FDIV:
6359  case ISD::SREM:
6360  case ISD::UREM:
6361  case ISD::FREM:
6362  case ISD::FPOW:
6363  case ISD::AND:
6364  case ISD::OR:
6365  case ISD::XOR:
6366    Result = DAG.getNode(Node->getOpcode(),
6367                         NewVT,
6368                         ScalarizeVectorOp(Node->getOperand(0)),
6369                         ScalarizeVectorOp(Node->getOperand(1)));
6370    break;
6371  case ISD::FNEG:
6372  case ISD::FABS:
6373  case ISD::FSQRT:
6374  case ISD::FSIN:
6375  case ISD::FCOS:
6376    Result = DAG.getNode(Node->getOpcode(),
6377                         NewVT,
6378                         ScalarizeVectorOp(Node->getOperand(0)));
6379    break;
6380  case ISD::FPOWI:
6381    Result = DAG.getNode(Node->getOpcode(),
6382                         NewVT,
6383                         ScalarizeVectorOp(Node->getOperand(0)),
6384                         Node->getOperand(1));
6385    break;
6386  case ISD::LOAD: {
6387    LoadSDNode *LD = cast<LoadSDNode>(Node);
6388    SDOperand Ch = LegalizeOp(LD->getChain());     // Legalize the chain.
6389    SDOperand Ptr = LegalizeOp(LD->getBasePtr());  // Legalize the pointer.
6390
6391    const Value *SV = LD->getSrcValue();
6392    int SVOffset = LD->getSrcValueOffset();
6393    Result = DAG.getLoad(NewVT, Ch, Ptr, SV, SVOffset,
6394                         LD->isVolatile(), LD->getAlignment());
6395
6396    // Remember that we legalized the chain.
6397    AddLegalizedOperand(Op.getValue(1), LegalizeOp(Result.getValue(1)));
6398    break;
6399  }
6400  case ISD::BUILD_VECTOR:
6401    Result = Node->getOperand(0);
6402    break;
6403  case ISD::INSERT_VECTOR_ELT:
6404    // Returning the inserted scalar element.
6405    Result = Node->getOperand(1);
6406    break;
6407  case ISD::CONCAT_VECTORS:
6408    assert(Node->getOperand(0).getValueType() == NewVT &&
6409           "Concat of non-legal vectors not yet supported!");
6410    Result = Node->getOperand(0);
6411    break;
6412  case ISD::VECTOR_SHUFFLE: {
6413    // Figure out if the scalar is the LHS or RHS and return it.
6414    SDOperand EltNum = Node->getOperand(2).getOperand(0);
6415    if (cast<ConstantSDNode>(EltNum)->getValue())
6416      Result = ScalarizeVectorOp(Node->getOperand(1));
6417    else
6418      Result = ScalarizeVectorOp(Node->getOperand(0));
6419    break;
6420  }
6421  case ISD::EXTRACT_SUBVECTOR:
6422    Result = Node->getOperand(0);
6423    assert(Result.getValueType() == NewVT);
6424    break;
6425  case ISD::BIT_CONVERT:
6426    Result = DAG.getNode(ISD::BIT_CONVERT, NewVT, Op.getOperand(0));
6427    break;
6428  case ISD::SELECT:
6429    Result = DAG.getNode(ISD::SELECT, NewVT, Op.getOperand(0),
6430                         ScalarizeVectorOp(Op.getOperand(1)),
6431                         ScalarizeVectorOp(Op.getOperand(2)));
6432    break;
6433  }
6434
6435  if (TLI.isTypeLegal(NewVT))
6436    Result = LegalizeOp(Result);
6437  bool isNew = ScalarizedNodes.insert(std::make_pair(Op, Result)).second;
6438  assert(isNew && "Value already scalarized?");
6439  return Result;
6440}
6441
6442
6443// SelectionDAG::Legalize - This is the entry point for the file.
6444//
6445void SelectionDAG::Legalize() {
6446  if (ViewLegalizeDAGs) viewGraph();
6447
6448  /// run - This is the main entry point to this class.
6449  ///
6450  SelectionDAGLegalize(*this).LegalizeDAG();
6451}
6452
6453