1//===-- DAGCombiner.cpp - Implement a DAG node combiner -------------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This pass combines dag nodes to form fewer, simpler DAG nodes.  It can be run
11// both before and after the DAG is legalized.
12//
13// This pass is not a substitute for the LLVM IR instcombine pass. This pass is
14// primarily intended to handle simplification opportunities that are implicit
15// in the LLVM IR and exposed by the various codegen lowering phases.
16//
17//===----------------------------------------------------------------------===//
18
19#include "llvm/CodeGen/SelectionDAG.h"
20#include "llvm/ADT/SmallPtrSet.h"
21#include "llvm/ADT/Statistic.h"
22#include "llvm/Analysis/AliasAnalysis.h"
23#include "llvm/CodeGen/MachineFrameInfo.h"
24#include "llvm/CodeGen/MachineFunction.h"
25#include "llvm/IR/DataLayout.h"
26#include "llvm/IR/DerivedTypes.h"
27#include "llvm/IR/Function.h"
28#include "llvm/IR/LLVMContext.h"
29#include "llvm/Support/CommandLine.h"
30#include "llvm/Support/Debug.h"
31#include "llvm/Support/ErrorHandling.h"
32#include "llvm/Support/MathExtras.h"
33#include "llvm/Support/raw_ostream.h"
34#include "llvm/Target/TargetLowering.h"
35#include "llvm/Target/TargetMachine.h"
36#include "llvm/Target/TargetOptions.h"
37#include "llvm/Target/TargetRegisterInfo.h"
38#include "llvm/Target/TargetSubtargetInfo.h"
39#include <algorithm>
40using namespace llvm;
41
42#define DEBUG_TYPE "dagcombine"
43
44STATISTIC(NodesCombined   , "Number of dag nodes combined");
45STATISTIC(PreIndexedNodes , "Number of pre-indexed nodes created");
46STATISTIC(PostIndexedNodes, "Number of post-indexed nodes created");
47STATISTIC(OpsNarrowed     , "Number of load/op/store narrowed");
48STATISTIC(LdStFP2Int      , "Number of fp load/store pairs transformed to int");
49STATISTIC(SlicedLoads, "Number of load sliced");
50
51namespace {
52  static cl::opt<bool>
53    CombinerAA("combiner-alias-analysis", cl::Hidden,
54               cl::desc("Enable DAG combiner alias-analysis heuristics"));
55
56  static cl::opt<bool>
57    CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden,
58               cl::desc("Enable DAG combiner's use of IR alias analysis"));
59
60  static cl::opt<bool>
61    UseTBAA("combiner-use-tbaa", cl::Hidden, cl::init(true),
62               cl::desc("Enable DAG combiner's use of TBAA"));
63
64#ifndef NDEBUG
65  static cl::opt<std::string>
66    CombinerAAOnlyFunc("combiner-aa-only-func", cl::Hidden,
67               cl::desc("Only use DAG-combiner alias analysis in this"
68                        " function"));
69#endif
70
71  /// Hidden option to stress test load slicing, i.e., when this option
72  /// is enabled, load slicing bypasses most of its profitability guards.
73  static cl::opt<bool>
74  StressLoadSlicing("combiner-stress-load-slicing", cl::Hidden,
75                    cl::desc("Bypass the profitability model of load "
76                             "slicing"),
77                    cl::init(false));
78
79//------------------------------ DAGCombiner ---------------------------------//
80
81  class DAGCombiner {
82    SelectionDAG &DAG;
83    const TargetLowering &TLI;
84    CombineLevel Level;
85    CodeGenOpt::Level OptLevel;
86    bool LegalOperations;
87    bool LegalTypes;
88    bool ForCodeSize;
89
90    // Worklist of all of the nodes that need to be simplified.
91    //
92    // This has the semantics that when adding to the worklist,
93    // the item added must be next to be processed. It should
94    // also only appear once. The naive approach to this takes
95    // linear time.
96    //
97    // To reduce the insert/remove time to logarithmic, we use
98    // a set and a vector to maintain our worklist.
99    //
100    // The set contains the items on the worklist, but does not
101    // maintain the order they should be visited.
102    //
103    // The vector maintains the order nodes should be visited, but may
104    // contain duplicate or removed nodes. When choosing a node to
105    // visit, we pop off the order stack until we find an item that is
106    // also in the contents set. All operations are O(log N).
107    SmallPtrSet<SDNode*, 64> WorkListContents;
108    SmallVector<SDNode*, 64> WorkListOrder;
109
110    // AA - Used for DAG load/store alias analysis.
111    AliasAnalysis &AA;
112
113    /// AddUsersToWorkList - When an instruction is simplified, add all users of
114    /// the instruction to the work lists because they might get more simplified
115    /// now.
116    ///
117    void AddUsersToWorkList(SDNode *N) {
118      for (SDNode *Node : N->uses())
119        AddToWorkList(Node);
120    }
121
122    /// visit - call the node-specific routine that knows how to fold each
123    /// particular type of node.
124    SDValue visit(SDNode *N);
125
126  public:
127    /// AddToWorkList - Add to the work list making sure its instance is at the
128    /// back (next to be processed.)
129    void AddToWorkList(SDNode *N) {
130      WorkListContents.insert(N);
131      WorkListOrder.push_back(N);
132    }
133
134    /// removeFromWorkList - remove all instances of N from the worklist.
135    ///
136    void removeFromWorkList(SDNode *N) {
137      WorkListContents.erase(N);
138    }
139
140    SDValue CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
141                      bool AddTo = true);
142
143    SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true) {
144      return CombineTo(N, &Res, 1, AddTo);
145    }
146
147    SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1,
148                      bool AddTo = true) {
149      SDValue To[] = { Res0, Res1 };
150      return CombineTo(N, To, 2, AddTo);
151    }
152
153    void CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO);
154
155  private:
156
157    /// SimplifyDemandedBits - Check the specified integer node value to see if
158    /// it can be simplified or if things it uses can be simplified by bit
159    /// propagation.  If so, return true.
160    bool SimplifyDemandedBits(SDValue Op) {
161      unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits();
162      APInt Demanded = APInt::getAllOnesValue(BitWidth);
163      return SimplifyDemandedBits(Op, Demanded);
164    }
165
166    bool SimplifyDemandedBits(SDValue Op, const APInt &Demanded);
167
168    bool CombineToPreIndexedLoadStore(SDNode *N);
169    bool CombineToPostIndexedLoadStore(SDNode *N);
170    bool SliceUpLoad(SDNode *N);
171
172    /// \brief Replace an ISD::EXTRACT_VECTOR_ELT of a load with a narrowed
173    ///   load.
174    ///
175    /// \param EVE ISD::EXTRACT_VECTOR_ELT to be replaced.
176    /// \param InVecVT type of the input vector to EVE with bitcasts resolved.
177    /// \param EltNo index of the vector element to load.
178    /// \param OriginalLoad load that EVE came from to be replaced.
179    /// \returns EVE on success SDValue() on failure.
180    SDValue ReplaceExtractVectorEltOfLoadWithNarrowedLoad(
181        SDNode *EVE, EVT InVecVT, SDValue EltNo, LoadSDNode *OriginalLoad);
182    void ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad);
183    SDValue PromoteOperand(SDValue Op, EVT PVT, bool &Replace);
184    SDValue SExtPromoteOperand(SDValue Op, EVT PVT);
185    SDValue ZExtPromoteOperand(SDValue Op, EVT PVT);
186    SDValue PromoteIntBinOp(SDValue Op);
187    SDValue PromoteIntShiftOp(SDValue Op);
188    SDValue PromoteExtend(SDValue Op);
189    bool PromoteLoad(SDValue Op);
190
191    void ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs,
192                         SDValue Trunc, SDValue ExtLoad, SDLoc DL,
193                         ISD::NodeType ExtType);
194
195    /// combine - call the node-specific routine that knows how to fold each
196    /// particular type of node. If that doesn't do anything, try the
197    /// target-specific DAG combines.
198    SDValue combine(SDNode *N);
199
200    // Visitation implementation - Implement dag node combining for different
201    // node types.  The semantics are as follows:
202    // Return Value:
203    //   SDValue.getNode() == 0 - No change was made
204    //   SDValue.getNode() == N - N was replaced, is dead and has been handled.
205    //   otherwise              - N should be replaced by the returned Operand.
206    //
207    SDValue visitTokenFactor(SDNode *N);
208    SDValue visitMERGE_VALUES(SDNode *N);
209    SDValue visitADD(SDNode *N);
210    SDValue visitSUB(SDNode *N);
211    SDValue visitADDC(SDNode *N);
212    SDValue visitSUBC(SDNode *N);
213    SDValue visitADDE(SDNode *N);
214    SDValue visitSUBE(SDNode *N);
215    SDValue visitMUL(SDNode *N);
216    SDValue visitSDIV(SDNode *N);
217    SDValue visitUDIV(SDNode *N);
218    SDValue visitSREM(SDNode *N);
219    SDValue visitUREM(SDNode *N);
220    SDValue visitMULHU(SDNode *N);
221    SDValue visitMULHS(SDNode *N);
222    SDValue visitSMUL_LOHI(SDNode *N);
223    SDValue visitUMUL_LOHI(SDNode *N);
224    SDValue visitSMULO(SDNode *N);
225    SDValue visitUMULO(SDNode *N);
226    SDValue visitSDIVREM(SDNode *N);
227    SDValue visitUDIVREM(SDNode *N);
228    SDValue visitAND(SDNode *N);
229    SDValue visitOR(SDNode *N);
230    SDValue visitXOR(SDNode *N);
231    SDValue SimplifyVBinOp(SDNode *N);
232    SDValue SimplifyVUnaryOp(SDNode *N);
233    SDValue visitSHL(SDNode *N);
234    SDValue visitSRA(SDNode *N);
235    SDValue visitSRL(SDNode *N);
236    SDValue visitRotate(SDNode *N);
237    SDValue visitCTLZ(SDNode *N);
238    SDValue visitCTLZ_ZERO_UNDEF(SDNode *N);
239    SDValue visitCTTZ(SDNode *N);
240    SDValue visitCTTZ_ZERO_UNDEF(SDNode *N);
241    SDValue visitCTPOP(SDNode *N);
242    SDValue visitSELECT(SDNode *N);
243    SDValue visitVSELECT(SDNode *N);
244    SDValue visitSELECT_CC(SDNode *N);
245    SDValue visitSETCC(SDNode *N);
246    SDValue visitSIGN_EXTEND(SDNode *N);
247    SDValue visitZERO_EXTEND(SDNode *N);
248    SDValue visitANY_EXTEND(SDNode *N);
249    SDValue visitSIGN_EXTEND_INREG(SDNode *N);
250    SDValue visitTRUNCATE(SDNode *N);
251    SDValue visitBITCAST(SDNode *N);
252    SDValue visitBUILD_PAIR(SDNode *N);
253    SDValue visitFADD(SDNode *N);
254    SDValue visitFSUB(SDNode *N);
255    SDValue visitFMUL(SDNode *N);
256    SDValue visitFMA(SDNode *N);
257    SDValue visitFDIV(SDNode *N);
258    SDValue visitFREM(SDNode *N);
259    SDValue visitFCOPYSIGN(SDNode *N);
260    SDValue visitSINT_TO_FP(SDNode *N);
261    SDValue visitUINT_TO_FP(SDNode *N);
262    SDValue visitFP_TO_SINT(SDNode *N);
263    SDValue visitFP_TO_UINT(SDNode *N);
264    SDValue visitFP_ROUND(SDNode *N);
265    SDValue visitFP_ROUND_INREG(SDNode *N);
266    SDValue visitFP_EXTEND(SDNode *N);
267    SDValue visitFNEG(SDNode *N);
268    SDValue visitFABS(SDNode *N);
269    SDValue visitFCEIL(SDNode *N);
270    SDValue visitFTRUNC(SDNode *N);
271    SDValue visitFFLOOR(SDNode *N);
272    SDValue visitBRCOND(SDNode *N);
273    SDValue visitBR_CC(SDNode *N);
274    SDValue visitLOAD(SDNode *N);
275    SDValue visitSTORE(SDNode *N);
276    SDValue visitINSERT_VECTOR_ELT(SDNode *N);
277    SDValue visitEXTRACT_VECTOR_ELT(SDNode *N);
278    SDValue visitBUILD_VECTOR(SDNode *N);
279    SDValue visitCONCAT_VECTORS(SDNode *N);
280    SDValue visitEXTRACT_SUBVECTOR(SDNode *N);
281    SDValue visitVECTOR_SHUFFLE(SDNode *N);
282    SDValue visitINSERT_SUBVECTOR(SDNode *N);
283
284    SDValue XformToShuffleWithZero(SDNode *N);
285    SDValue ReassociateOps(unsigned Opc, SDLoc DL, SDValue LHS, SDValue RHS);
286
287    SDValue visitShiftByConstant(SDNode *N, ConstantSDNode *Amt);
288
289    bool SimplifySelectOps(SDNode *SELECT, SDValue LHS, SDValue RHS);
290    SDValue SimplifyBinOpWithSameOpcodeHands(SDNode *N);
291    SDValue SimplifySelect(SDLoc DL, SDValue N0, SDValue N1, SDValue N2);
292    SDValue SimplifySelectCC(SDLoc DL, SDValue N0, SDValue N1, SDValue N2,
293                             SDValue N3, ISD::CondCode CC,
294                             bool NotExtCompare = false);
295    SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
296                          SDLoc DL, bool foldBooleans = true);
297
298    bool isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
299                           SDValue &CC) const;
300    bool isOneUseSetCC(SDValue N) const;
301
302    SDValue SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
303                                         unsigned HiOp);
304    SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
305    SDValue ConstantFoldBITCASTofBUILD_VECTOR(SDNode *, EVT);
306    SDValue BuildSDIV(SDNode *N);
307    SDValue BuildUDIV(SDNode *N);
308    SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
309                               bool DemandHighBits = true);
310    SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1);
311    SDNode *MatchRotatePosNeg(SDValue Shifted, SDValue Pos, SDValue Neg,
312                              SDValue InnerPos, SDValue InnerNeg,
313                              unsigned PosOpcode, unsigned NegOpcode,
314                              SDLoc DL);
315    SDNode *MatchRotate(SDValue LHS, SDValue RHS, SDLoc DL);
316    SDValue ReduceLoadWidth(SDNode *N);
317    SDValue ReduceLoadOpStoreWidth(SDNode *N);
318    SDValue TransformFPLoadStorePair(SDNode *N);
319    SDValue reduceBuildVecExtToExtBuildVec(SDNode *N);
320    SDValue reduceBuildVecConvertToConvertBuildVec(SDNode *N);
321
322    SDValue GetDemandedBits(SDValue V, const APInt &Mask);
323
324    /// GatherAllAliases - Walk up chain skipping non-aliasing memory nodes,
325    /// looking for aliasing nodes and adding them to the Aliases vector.
326    void GatherAllAliases(SDNode *N, SDValue OriginalChain,
327                          SmallVectorImpl<SDValue> &Aliases);
328
329    /// isAlias - Return true if there is any possibility that the two addresses
330    /// overlap.
331    bool isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1) const;
332
333    /// FindBetterChain - Walk up chain skipping non-aliasing memory nodes,
334    /// looking for a better chain (aliasing node.)
335    SDValue FindBetterChain(SDNode *N, SDValue Chain);
336
337    /// Merge consecutive store operations into a wide store.
338    /// This optimization uses wide integers or vectors when possible.
339    /// \return True if some memory operations were changed.
340    bool MergeConsecutiveStores(StoreSDNode *N);
341
342    /// \brief Try to transform a truncation where C is a constant:
343    ///     (trunc (and X, C)) -> (and (trunc X), (trunc C))
344    ///
345    /// \p N needs to be a truncation and its first operand an AND. Other
346    /// requirements are checked by the function (e.g. that trunc is
347    /// single-use) and if missed an empty SDValue is returned.
348    SDValue distributeTruncateThroughAnd(SDNode *N);
349
350  public:
351    DAGCombiner(SelectionDAG &D, AliasAnalysis &A, CodeGenOpt::Level OL)
352        : DAG(D), TLI(D.getTargetLoweringInfo()), Level(BeforeLegalizeTypes),
353          OptLevel(OL), LegalOperations(false), LegalTypes(false), AA(A) {
354      AttributeSet FnAttrs =
355          DAG.getMachineFunction().getFunction()->getAttributes();
356      ForCodeSize =
357          FnAttrs.hasAttribute(AttributeSet::FunctionIndex,
358                               Attribute::OptimizeForSize) ||
359          FnAttrs.hasAttribute(AttributeSet::FunctionIndex, Attribute::MinSize);
360    }
361
362    /// Run - runs the dag combiner on all nodes in the work list
363    void Run(CombineLevel AtLevel);
364
365    SelectionDAG &getDAG() const { return DAG; }
366
367    /// getShiftAmountTy - Returns a type large enough to hold any valid
368    /// shift amount - before type legalization these can be huge.
369    EVT getShiftAmountTy(EVT LHSTy) {
370      assert(LHSTy.isInteger() && "Shift amount is not an integer type!");
371      if (LHSTy.isVector())
372        return LHSTy;
373      return LegalTypes ? TLI.getScalarShiftAmountTy(LHSTy)
374                        : TLI.getPointerTy();
375    }
376
377    /// isTypeLegal - This method returns true if we are running before type
378    /// legalization or if the specified VT is legal.
379    bool isTypeLegal(const EVT &VT) {
380      if (!LegalTypes) return true;
381      return TLI.isTypeLegal(VT);
382    }
383
384    /// getSetCCResultType - Convenience wrapper around
385    /// TargetLowering::getSetCCResultType
386    EVT getSetCCResultType(EVT VT) const {
387      return TLI.getSetCCResultType(*DAG.getContext(), VT);
388    }
389  };
390}
391
392
393namespace {
394/// WorkListRemover - This class is a DAGUpdateListener that removes any deleted
395/// nodes from the worklist.
396class WorkListRemover : public SelectionDAG::DAGUpdateListener {
397  DAGCombiner &DC;
398public:
399  explicit WorkListRemover(DAGCombiner &dc)
400    : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
401
402  void NodeDeleted(SDNode *N, SDNode *E) override {
403    DC.removeFromWorkList(N);
404  }
405};
406}
407
408//===----------------------------------------------------------------------===//
409//  TargetLowering::DAGCombinerInfo implementation
410//===----------------------------------------------------------------------===//
411
412void TargetLowering::DAGCombinerInfo::AddToWorklist(SDNode *N) {
413  ((DAGCombiner*)DC)->AddToWorkList(N);
414}
415
416void TargetLowering::DAGCombinerInfo::RemoveFromWorklist(SDNode *N) {
417  ((DAGCombiner*)DC)->removeFromWorkList(N);
418}
419
420SDValue TargetLowering::DAGCombinerInfo::
421CombineTo(SDNode *N, const std::vector<SDValue> &To, bool AddTo) {
422  return ((DAGCombiner*)DC)->CombineTo(N, &To[0], To.size(), AddTo);
423}
424
425SDValue TargetLowering::DAGCombinerInfo::
426CombineTo(SDNode *N, SDValue Res, bool AddTo) {
427  return ((DAGCombiner*)DC)->CombineTo(N, Res, AddTo);
428}
429
430
431SDValue TargetLowering::DAGCombinerInfo::
432CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo) {
433  return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1, AddTo);
434}
435
436void TargetLowering::DAGCombinerInfo::
437CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
438  return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO);
439}
440
441//===----------------------------------------------------------------------===//
442// Helper Functions
443//===----------------------------------------------------------------------===//
444
445/// isNegatibleForFree - Return 1 if we can compute the negated form of the
446/// specified expression for the same cost as the expression itself, or 2 if we
447/// can compute the negated form more cheaply than the expression itself.
448static char isNegatibleForFree(SDValue Op, bool LegalOperations,
449                               const TargetLowering &TLI,
450                               const TargetOptions *Options,
451                               unsigned Depth = 0) {
452  // fneg is removable even if it has multiple uses.
453  if (Op.getOpcode() == ISD::FNEG) return 2;
454
455  // Don't allow anything with multiple uses.
456  if (!Op.hasOneUse()) return 0;
457
458  // Don't recurse exponentially.
459  if (Depth > 6) return 0;
460
461  switch (Op.getOpcode()) {
462  default: return false;
463  case ISD::ConstantFP:
464    // Don't invert constant FP values after legalize.  The negated constant
465    // isn't necessarily legal.
466    return LegalOperations ? 0 : 1;
467  case ISD::FADD:
468    // FIXME: determine better conditions for this xform.
469    if (!Options->UnsafeFPMath) return 0;
470
471    // After operation legalization, it might not be legal to create new FSUBs.
472    if (LegalOperations &&
473        !TLI.isOperationLegalOrCustom(ISD::FSUB,  Op.getValueType()))
474      return 0;
475
476    // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
477    if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
478                                    Options, Depth + 1))
479      return V;
480    // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
481    return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
482                              Depth + 1);
483  case ISD::FSUB:
484    // We can't turn -(A-B) into B-A when we honor signed zeros.
485    if (!Options->UnsafeFPMath) return 0;
486
487    // fold (fneg (fsub A, B)) -> (fsub B, A)
488    return 1;
489
490  case ISD::FMUL:
491  case ISD::FDIV:
492    if (Options->HonorSignDependentRoundingFPMath()) return 0;
493
494    // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) or (fmul X, (fneg Y))
495    if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
496                                    Options, Depth + 1))
497      return V;
498
499    return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
500                              Depth + 1);
501
502  case ISD::FP_EXTEND:
503  case ISD::FP_ROUND:
504  case ISD::FSIN:
505    return isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI, Options,
506                              Depth + 1);
507  }
508}
509
510/// GetNegatedExpression - If isNegatibleForFree returns true, this function
511/// returns the newly negated expression.
512static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG,
513                                    bool LegalOperations, unsigned Depth = 0) {
514  // fneg is removable even if it has multiple uses.
515  if (Op.getOpcode() == ISD::FNEG) return Op.getOperand(0);
516
517  // Don't allow anything with multiple uses.
518  assert(Op.hasOneUse() && "Unknown reuse!");
519
520  assert(Depth <= 6 && "GetNegatedExpression doesn't match isNegatibleForFree");
521  switch (Op.getOpcode()) {
522  default: llvm_unreachable("Unknown code");
523  case ISD::ConstantFP: {
524    APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
525    V.changeSign();
526    return DAG.getConstantFP(V, Op.getValueType());
527  }
528  case ISD::FADD:
529    // FIXME: determine better conditions for this xform.
530    assert(DAG.getTarget().Options.UnsafeFPMath);
531
532    // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
533    if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
534                           DAG.getTargetLoweringInfo(),
535                           &DAG.getTarget().Options, Depth+1))
536      return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
537                         GetNegatedExpression(Op.getOperand(0), DAG,
538                                              LegalOperations, Depth+1),
539                         Op.getOperand(1));
540    // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
541    return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
542                       GetNegatedExpression(Op.getOperand(1), DAG,
543                                            LegalOperations, Depth+1),
544                       Op.getOperand(0));
545  case ISD::FSUB:
546    // We can't turn -(A-B) into B-A when we honor signed zeros.
547    assert(DAG.getTarget().Options.UnsafeFPMath);
548
549    // fold (fneg (fsub 0, B)) -> B
550    if (ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(Op.getOperand(0)))
551      if (N0CFP->getValueAPF().isZero())
552        return Op.getOperand(1);
553
554    // fold (fneg (fsub A, B)) -> (fsub B, A)
555    return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
556                       Op.getOperand(1), Op.getOperand(0));
557
558  case ISD::FMUL:
559  case ISD::FDIV:
560    assert(!DAG.getTarget().Options.HonorSignDependentRoundingFPMath());
561
562    // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
563    if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
564                           DAG.getTargetLoweringInfo(),
565                           &DAG.getTarget().Options, Depth+1))
566      return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
567                         GetNegatedExpression(Op.getOperand(0), DAG,
568                                              LegalOperations, Depth+1),
569                         Op.getOperand(1));
570
571    // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
572    return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
573                       Op.getOperand(0),
574                       GetNegatedExpression(Op.getOperand(1), DAG,
575                                            LegalOperations, Depth+1));
576
577  case ISD::FP_EXTEND:
578  case ISD::FSIN:
579    return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
580                       GetNegatedExpression(Op.getOperand(0), DAG,
581                                            LegalOperations, Depth+1));
582  case ISD::FP_ROUND:
583      return DAG.getNode(ISD::FP_ROUND, SDLoc(Op), Op.getValueType(),
584                         GetNegatedExpression(Op.getOperand(0), DAG,
585                                              LegalOperations, Depth+1),
586                         Op.getOperand(1));
587  }
588}
589
590// isSetCCEquivalent - Return true if this node is a setcc, or is a select_cc
591// that selects between the target values used for true and false, making it
592// equivalent to a setcc. Also, set the incoming LHS, RHS, and CC references to
593// the appropriate nodes based on the type of node we are checking. This
594// simplifies life a bit for the callers.
595bool DAGCombiner::isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
596                                    SDValue &CC) const {
597  if (N.getOpcode() == ISD::SETCC) {
598    LHS = N.getOperand(0);
599    RHS = N.getOperand(1);
600    CC  = N.getOperand(2);
601    return true;
602  }
603
604  if (N.getOpcode() != ISD::SELECT_CC ||
605      !TLI.isConstTrueVal(N.getOperand(2).getNode()) ||
606      !TLI.isConstFalseVal(N.getOperand(3).getNode()))
607    return false;
608
609  LHS = N.getOperand(0);
610  RHS = N.getOperand(1);
611  CC  = N.getOperand(4);
612  return true;
613}
614
615// isOneUseSetCC - Return true if this is a SetCC-equivalent operation with only
616// one use.  If this is true, it allows the users to invert the operation for
617// free when it is profitable to do so.
618bool DAGCombiner::isOneUseSetCC(SDValue N) const {
619  SDValue N0, N1, N2;
620  if (isSetCCEquivalent(N, N0, N1, N2) && N.getNode()->hasOneUse())
621    return true;
622  return false;
623}
624
625/// isConstantSplatVector - Returns true if N is a BUILD_VECTOR node whose
626/// elements are all the same constant or undefined.
627static bool isConstantSplatVector(SDNode *N, APInt& SplatValue) {
628  BuildVectorSDNode *C = dyn_cast<BuildVectorSDNode>(N);
629  if (!C)
630    return false;
631
632  APInt SplatUndef;
633  unsigned SplatBitSize;
634  bool HasAnyUndefs;
635  EVT EltVT = N->getValueType(0).getVectorElementType();
636  return (C->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
637                             HasAnyUndefs) &&
638          EltVT.getSizeInBits() >= SplatBitSize);
639}
640
641// \brief Returns the SDNode if it is a constant BuildVector or constant.
642static SDNode *isConstantBuildVectorOrConstantInt(SDValue N) {
643  if (isa<ConstantSDNode>(N))
644    return N.getNode();
645  BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N);
646  if(BV && BV->isConstant())
647    return BV;
648  return nullptr;
649}
650
651// \brief Returns the SDNode if it is a constant splat BuildVector or constant
652// int.
653static ConstantSDNode *isConstOrConstSplat(SDValue N) {
654  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N))
655    return CN;
656
657  if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
658    BitVector UndefElements;
659    ConstantSDNode *CN = BV->getConstantSplatNode(&UndefElements);
660
661    // BuildVectors can truncate their operands. Ignore that case here.
662    // FIXME: We blindly ignore splats which include undef which is overly
663    // pessimistic.
664    if (CN && UndefElements.none() &&
665        CN->getValueType(0) == N.getValueType().getScalarType())
666      return CN;
667  }
668
669  return nullptr;
670}
671
672SDValue DAGCombiner::ReassociateOps(unsigned Opc, SDLoc DL,
673                                    SDValue N0, SDValue N1) {
674  EVT VT = N0.getValueType();
675  if (N0.getOpcode() == Opc) {
676    if (SDNode *L = isConstantBuildVectorOrConstantInt(N0.getOperand(1))) {
677      if (SDNode *R = isConstantBuildVectorOrConstantInt(N1)) {
678        // reassoc. (op (op x, c1), c2) -> (op x, (op c1, c2))
679        SDValue OpNode = DAG.FoldConstantArithmetic(Opc, VT, L, R);
680        if (!OpNode.getNode())
681          return SDValue();
682        return DAG.getNode(Opc, DL, VT, N0.getOperand(0), OpNode);
683      }
684      if (N0.hasOneUse()) {
685        // reassoc. (op (op x, c1), y) -> (op (op x, y), c1) iff x+c1 has one
686        // use
687        SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0.getOperand(0), N1);
688        if (!OpNode.getNode())
689          return SDValue();
690        AddToWorkList(OpNode.getNode());
691        return DAG.getNode(Opc, DL, VT, OpNode, N0.getOperand(1));
692      }
693    }
694  }
695
696  if (N1.getOpcode() == Opc) {
697    if (SDNode *R = isConstantBuildVectorOrConstantInt(N1.getOperand(1))) {
698      if (SDNode *L = isConstantBuildVectorOrConstantInt(N0)) {
699        // reassoc. (op c2, (op x, c1)) -> (op x, (op c1, c2))
700        SDValue OpNode = DAG.FoldConstantArithmetic(Opc, VT, R, L);
701        if (!OpNode.getNode())
702          return SDValue();
703        return DAG.getNode(Opc, DL, VT, N1.getOperand(0), OpNode);
704      }
705      if (N1.hasOneUse()) {
706        // reassoc. (op y, (op x, c1)) -> (op (op x, y), c1) iff x+c1 has one
707        // use
708        SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N1.getOperand(0), N0);
709        if (!OpNode.getNode())
710          return SDValue();
711        AddToWorkList(OpNode.getNode());
712        return DAG.getNode(Opc, DL, VT, OpNode, N1.getOperand(1));
713      }
714    }
715  }
716
717  return SDValue();
718}
719
720SDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
721                               bool AddTo) {
722  assert(N->getNumValues() == NumTo && "Broken CombineTo call!");
723  ++NodesCombined;
724  DEBUG(dbgs() << "\nReplacing.1 ";
725        N->dump(&DAG);
726        dbgs() << "\nWith: ";
727        To[0].getNode()->dump(&DAG);
728        dbgs() << " and " << NumTo-1 << " other values\n";
729        for (unsigned i = 0, e = NumTo; i != e; ++i)
730          assert((!To[i].getNode() ||
731                  N->getValueType(i) == To[i].getValueType()) &&
732                 "Cannot combine value to value of different type!"));
733  WorkListRemover DeadNodes(*this);
734  DAG.ReplaceAllUsesWith(N, To);
735  if (AddTo) {
736    // Push the new nodes and any users onto the worklist
737    for (unsigned i = 0, e = NumTo; i != e; ++i) {
738      if (To[i].getNode()) {
739        AddToWorkList(To[i].getNode());
740        AddUsersToWorkList(To[i].getNode());
741      }
742    }
743  }
744
745  // Finally, if the node is now dead, remove it from the graph.  The node
746  // may not be dead if the replacement process recursively simplified to
747  // something else needing this node.
748  if (N->use_empty()) {
749    // Nodes can be reintroduced into the worklist.  Make sure we do not
750    // process a node that has been replaced.
751    removeFromWorkList(N);
752
753    // Finally, since the node is now dead, remove it from the graph.
754    DAG.DeleteNode(N);
755  }
756  return SDValue(N, 0);
757}
758
759void DAGCombiner::
760CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
761  // Replace all uses.  If any nodes become isomorphic to other nodes and
762  // are deleted, make sure to remove them from our worklist.
763  WorkListRemover DeadNodes(*this);
764  DAG.ReplaceAllUsesOfValueWith(TLO.Old, TLO.New);
765
766  // Push the new node and any (possibly new) users onto the worklist.
767  AddToWorkList(TLO.New.getNode());
768  AddUsersToWorkList(TLO.New.getNode());
769
770  // Finally, if the node is now dead, remove it from the graph.  The node
771  // may not be dead if the replacement process recursively simplified to
772  // something else needing this node.
773  if (TLO.Old.getNode()->use_empty()) {
774    removeFromWorkList(TLO.Old.getNode());
775
776    // If the operands of this node are only used by the node, they will now
777    // be dead.  Make sure to visit them first to delete dead nodes early.
778    for (unsigned i = 0, e = TLO.Old.getNode()->getNumOperands(); i != e; ++i)
779      if (TLO.Old.getNode()->getOperand(i).getNode()->hasOneUse())
780        AddToWorkList(TLO.Old.getNode()->getOperand(i).getNode());
781
782    DAG.DeleteNode(TLO.Old.getNode());
783  }
784}
785
786/// SimplifyDemandedBits - Check the specified integer node value to see if
787/// it can be simplified or if things it uses can be simplified by bit
788/// propagation.  If so, return true.
789bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &Demanded) {
790  TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
791  APInt KnownZero, KnownOne;
792  if (!TLI.SimplifyDemandedBits(Op, Demanded, KnownZero, KnownOne, TLO))
793    return false;
794
795  // Revisit the node.
796  AddToWorkList(Op.getNode());
797
798  // Replace the old value with the new one.
799  ++NodesCombined;
800  DEBUG(dbgs() << "\nReplacing.2 ";
801        TLO.Old.getNode()->dump(&DAG);
802        dbgs() << "\nWith: ";
803        TLO.New.getNode()->dump(&DAG);
804        dbgs() << '\n');
805
806  CommitTargetLoweringOpt(TLO);
807  return true;
808}
809
810void DAGCombiner::ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad) {
811  SDLoc dl(Load);
812  EVT VT = Load->getValueType(0);
813  SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, VT, SDValue(ExtLoad, 0));
814
815  DEBUG(dbgs() << "\nReplacing.9 ";
816        Load->dump(&DAG);
817        dbgs() << "\nWith: ";
818        Trunc.getNode()->dump(&DAG);
819        dbgs() << '\n');
820  WorkListRemover DeadNodes(*this);
821  DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), Trunc);
822  DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), SDValue(ExtLoad, 1));
823  removeFromWorkList(Load);
824  DAG.DeleteNode(Load);
825  AddToWorkList(Trunc.getNode());
826}
827
828SDValue DAGCombiner::PromoteOperand(SDValue Op, EVT PVT, bool &Replace) {
829  Replace = false;
830  SDLoc dl(Op);
831  if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
832    EVT MemVT = LD->getMemoryVT();
833    ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD)
834      ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT) ? ISD::ZEXTLOAD
835                                                  : ISD::EXTLOAD)
836      : LD->getExtensionType();
837    Replace = true;
838    return DAG.getExtLoad(ExtType, dl, PVT,
839                          LD->getChain(), LD->getBasePtr(),
840                          MemVT, LD->getMemOperand());
841  }
842
843  unsigned Opc = Op.getOpcode();
844  switch (Opc) {
845  default: break;
846  case ISD::AssertSext:
847    return DAG.getNode(ISD::AssertSext, dl, PVT,
848                       SExtPromoteOperand(Op.getOperand(0), PVT),
849                       Op.getOperand(1));
850  case ISD::AssertZext:
851    return DAG.getNode(ISD::AssertZext, dl, PVT,
852                       ZExtPromoteOperand(Op.getOperand(0), PVT),
853                       Op.getOperand(1));
854  case ISD::Constant: {
855    unsigned ExtOpc =
856      Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
857    return DAG.getNode(ExtOpc, dl, PVT, Op);
858  }
859  }
860
861  if (!TLI.isOperationLegal(ISD::ANY_EXTEND, PVT))
862    return SDValue();
863  return DAG.getNode(ISD::ANY_EXTEND, dl, PVT, Op);
864}
865
866SDValue DAGCombiner::SExtPromoteOperand(SDValue Op, EVT PVT) {
867  if (!TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, PVT))
868    return SDValue();
869  EVT OldVT = Op.getValueType();
870  SDLoc dl(Op);
871  bool Replace = false;
872  SDValue NewOp = PromoteOperand(Op, PVT, Replace);
873  if (!NewOp.getNode())
874    return SDValue();
875  AddToWorkList(NewOp.getNode());
876
877  if (Replace)
878    ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
879  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NewOp.getValueType(), NewOp,
880                     DAG.getValueType(OldVT));
881}
882
883SDValue DAGCombiner::ZExtPromoteOperand(SDValue Op, EVT PVT) {
884  EVT OldVT = Op.getValueType();
885  SDLoc dl(Op);
886  bool Replace = false;
887  SDValue NewOp = PromoteOperand(Op, PVT, Replace);
888  if (!NewOp.getNode())
889    return SDValue();
890  AddToWorkList(NewOp.getNode());
891
892  if (Replace)
893    ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
894  return DAG.getZeroExtendInReg(NewOp, dl, OldVT);
895}
896
897/// PromoteIntBinOp - Promote the specified integer binary operation if the
898/// target indicates it is beneficial. e.g. On x86, it's usually better to
899/// promote i16 operations to i32 since i16 instructions are longer.
900SDValue DAGCombiner::PromoteIntBinOp(SDValue Op) {
901  if (!LegalOperations)
902    return SDValue();
903
904  EVT VT = Op.getValueType();
905  if (VT.isVector() || !VT.isInteger())
906    return SDValue();
907
908  // If operation type is 'undesirable', e.g. i16 on x86, consider
909  // promoting it.
910  unsigned Opc = Op.getOpcode();
911  if (TLI.isTypeDesirableForOp(Opc, VT))
912    return SDValue();
913
914  EVT PVT = VT;
915  // Consult target whether it is a good idea to promote this operation and
916  // what's the right type to promote it to.
917  if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
918    assert(PVT != VT && "Don't know what type to promote to!");
919
920    bool Replace0 = false;
921    SDValue N0 = Op.getOperand(0);
922    SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
923    if (!NN0.getNode())
924      return SDValue();
925
926    bool Replace1 = false;
927    SDValue N1 = Op.getOperand(1);
928    SDValue NN1;
929    if (N0 == N1)
930      NN1 = NN0;
931    else {
932      NN1 = PromoteOperand(N1, PVT, Replace1);
933      if (!NN1.getNode())
934        return SDValue();
935    }
936
937    AddToWorkList(NN0.getNode());
938    if (NN1.getNode())
939      AddToWorkList(NN1.getNode());
940
941    if (Replace0)
942      ReplaceLoadWithPromotedLoad(N0.getNode(), NN0.getNode());
943    if (Replace1)
944      ReplaceLoadWithPromotedLoad(N1.getNode(), NN1.getNode());
945
946    DEBUG(dbgs() << "\nPromoting ";
947          Op.getNode()->dump(&DAG));
948    SDLoc dl(Op);
949    return DAG.getNode(ISD::TRUNCATE, dl, VT,
950                       DAG.getNode(Opc, dl, PVT, NN0, NN1));
951  }
952  return SDValue();
953}
954
955/// PromoteIntShiftOp - Promote the specified integer shift operation if the
956/// target indicates it is beneficial. e.g. On x86, it's usually better to
957/// promote i16 operations to i32 since i16 instructions are longer.
958SDValue DAGCombiner::PromoteIntShiftOp(SDValue Op) {
959  if (!LegalOperations)
960    return SDValue();
961
962  EVT VT = Op.getValueType();
963  if (VT.isVector() || !VT.isInteger())
964    return SDValue();
965
966  // If operation type is 'undesirable', e.g. i16 on x86, consider
967  // promoting it.
968  unsigned Opc = Op.getOpcode();
969  if (TLI.isTypeDesirableForOp(Opc, VT))
970    return SDValue();
971
972  EVT PVT = VT;
973  // Consult target whether it is a good idea to promote this operation and
974  // what's the right type to promote it to.
975  if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
976    assert(PVT != VT && "Don't know what type to promote to!");
977
978    bool Replace = false;
979    SDValue N0 = Op.getOperand(0);
980    if (Opc == ISD::SRA)
981      N0 = SExtPromoteOperand(Op.getOperand(0), PVT);
982    else if (Opc == ISD::SRL)
983      N0 = ZExtPromoteOperand(Op.getOperand(0), PVT);
984    else
985      N0 = PromoteOperand(N0, PVT, Replace);
986    if (!N0.getNode())
987      return SDValue();
988
989    AddToWorkList(N0.getNode());
990    if (Replace)
991      ReplaceLoadWithPromotedLoad(Op.getOperand(0).getNode(), N0.getNode());
992
993    DEBUG(dbgs() << "\nPromoting ";
994          Op.getNode()->dump(&DAG));
995    SDLoc dl(Op);
996    return DAG.getNode(ISD::TRUNCATE, dl, VT,
997                       DAG.getNode(Opc, dl, PVT, N0, Op.getOperand(1)));
998  }
999  return SDValue();
1000}
1001
1002SDValue DAGCombiner::PromoteExtend(SDValue Op) {
1003  if (!LegalOperations)
1004    return SDValue();
1005
1006  EVT VT = Op.getValueType();
1007  if (VT.isVector() || !VT.isInteger())
1008    return SDValue();
1009
1010  // If operation type is 'undesirable', e.g. i16 on x86, consider
1011  // promoting it.
1012  unsigned Opc = Op.getOpcode();
1013  if (TLI.isTypeDesirableForOp(Opc, VT))
1014    return SDValue();
1015
1016  EVT PVT = VT;
1017  // Consult target whether it is a good idea to promote this operation and
1018  // what's the right type to promote it to.
1019  if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1020    assert(PVT != VT && "Don't know what type to promote to!");
1021    // fold (aext (aext x)) -> (aext x)
1022    // fold (aext (zext x)) -> (zext x)
1023    // fold (aext (sext x)) -> (sext x)
1024    DEBUG(dbgs() << "\nPromoting ";
1025          Op.getNode()->dump(&DAG));
1026    return DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, Op.getOperand(0));
1027  }
1028  return SDValue();
1029}
1030
1031bool DAGCombiner::PromoteLoad(SDValue Op) {
1032  if (!LegalOperations)
1033    return false;
1034
1035  EVT VT = Op.getValueType();
1036  if (VT.isVector() || !VT.isInteger())
1037    return false;
1038
1039  // If operation type is 'undesirable', e.g. i16 on x86, consider
1040  // promoting it.
1041  unsigned Opc = Op.getOpcode();
1042  if (TLI.isTypeDesirableForOp(Opc, VT))
1043    return false;
1044
1045  EVT PVT = VT;
1046  // Consult target whether it is a good idea to promote this operation and
1047  // what's the right type to promote it to.
1048  if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
1049    assert(PVT != VT && "Don't know what type to promote to!");
1050
1051    SDLoc dl(Op);
1052    SDNode *N = Op.getNode();
1053    LoadSDNode *LD = cast<LoadSDNode>(N);
1054    EVT MemVT = LD->getMemoryVT();
1055    ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD)
1056      ? (TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT) ? ISD::ZEXTLOAD
1057                                                  : ISD::EXTLOAD)
1058      : LD->getExtensionType();
1059    SDValue NewLD = DAG.getExtLoad(ExtType, dl, PVT,
1060                                   LD->getChain(), LD->getBasePtr(),
1061                                   MemVT, LD->getMemOperand());
1062    SDValue Result = DAG.getNode(ISD::TRUNCATE, dl, VT, NewLD);
1063
1064    DEBUG(dbgs() << "\nPromoting ";
1065          N->dump(&DAG);
1066          dbgs() << "\nTo: ";
1067          Result.getNode()->dump(&DAG);
1068          dbgs() << '\n');
1069    WorkListRemover DeadNodes(*this);
1070    DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
1071    DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), NewLD.getValue(1));
1072    removeFromWorkList(N);
1073    DAG.DeleteNode(N);
1074    AddToWorkList(Result.getNode());
1075    return true;
1076  }
1077  return false;
1078}
1079
1080
1081//===----------------------------------------------------------------------===//
1082//  Main DAG Combiner implementation
1083//===----------------------------------------------------------------------===//
1084
1085void DAGCombiner::Run(CombineLevel AtLevel) {
1086  // set the instance variables, so that the various visit routines may use it.
1087  Level = AtLevel;
1088  LegalOperations = Level >= AfterLegalizeVectorOps;
1089  LegalTypes = Level >= AfterLegalizeTypes;
1090
1091  // Add all the dag nodes to the worklist.
1092  for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
1093       E = DAG.allnodes_end(); I != E; ++I)
1094    AddToWorkList(I);
1095
1096  // Create a dummy node (which is not added to allnodes), that adds a reference
1097  // to the root node, preventing it from being deleted, and tracking any
1098  // changes of the root.
1099  HandleSDNode Dummy(DAG.getRoot());
1100
1101  // The root of the dag may dangle to deleted nodes until the dag combiner is
1102  // done.  Set it to null to avoid confusion.
1103  DAG.setRoot(SDValue());
1104
1105  // while the worklist isn't empty, find a node and
1106  // try and combine it.
1107  while (!WorkListContents.empty()) {
1108    SDNode *N;
1109    // The WorkListOrder holds the SDNodes in order, but it may contain
1110    // duplicates.
1111    // In order to avoid a linear scan, we use a set (O(log N)) to hold what the
1112    // worklist *should* contain, and check the node we want to visit is should
1113    // actually be visited.
1114    do {
1115      N = WorkListOrder.pop_back_val();
1116    } while (!WorkListContents.erase(N));
1117
1118    // If N has no uses, it is dead.  Make sure to revisit all N's operands once
1119    // N is deleted from the DAG, since they too may now be dead or may have a
1120    // reduced number of uses, allowing other xforms.
1121    if (N->use_empty() && N != &Dummy) {
1122      for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
1123        AddToWorkList(N->getOperand(i).getNode());
1124
1125      DAG.DeleteNode(N);
1126      continue;
1127    }
1128
1129    SDValue RV = combine(N);
1130
1131    if (!RV.getNode())
1132      continue;
1133
1134    ++NodesCombined;
1135
1136    // If we get back the same node we passed in, rather than a new node or
1137    // zero, we know that the node must have defined multiple values and
1138    // CombineTo was used.  Since CombineTo takes care of the worklist
1139    // mechanics for us, we have no work to do in this case.
1140    if (RV.getNode() == N)
1141      continue;
1142
1143    assert(N->getOpcode() != ISD::DELETED_NODE &&
1144           RV.getNode()->getOpcode() != ISD::DELETED_NODE &&
1145           "Node was deleted but visit returned new node!");
1146
1147    DEBUG(dbgs() << "\nReplacing.3 ";
1148          N->dump(&DAG);
1149          dbgs() << "\nWith: ";
1150          RV.getNode()->dump(&DAG);
1151          dbgs() << '\n');
1152
1153    // Transfer debug value.
1154    DAG.TransferDbgValues(SDValue(N, 0), RV);
1155    WorkListRemover DeadNodes(*this);
1156    if (N->getNumValues() == RV.getNode()->getNumValues())
1157      DAG.ReplaceAllUsesWith(N, RV.getNode());
1158    else {
1159      assert(N->getValueType(0) == RV.getValueType() &&
1160             N->getNumValues() == 1 && "Type mismatch");
1161      SDValue OpV = RV;
1162      DAG.ReplaceAllUsesWith(N, &OpV);
1163    }
1164
1165    // Push the new node and any users onto the worklist
1166    AddToWorkList(RV.getNode());
1167    AddUsersToWorkList(RV.getNode());
1168
1169    // Add any uses of the old node to the worklist in case this node is the
1170    // last one that uses them.  They may become dead after this node is
1171    // deleted.
1172    for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
1173      AddToWorkList(N->getOperand(i).getNode());
1174
1175    // Finally, if the node is now dead, remove it from the graph.  The node
1176    // may not be dead if the replacement process recursively simplified to
1177    // something else needing this node.
1178    if (N->use_empty()) {
1179      // Nodes can be reintroduced into the worklist.  Make sure we do not
1180      // process a node that has been replaced.
1181      removeFromWorkList(N);
1182
1183      // Finally, since the node is now dead, remove it from the graph.
1184      DAG.DeleteNode(N);
1185    }
1186  }
1187
1188  // If the root changed (e.g. it was a dead load, update the root).
1189  DAG.setRoot(Dummy.getValue());
1190  DAG.RemoveDeadNodes();
1191}
1192
1193SDValue DAGCombiner::visit(SDNode *N) {
1194  switch (N->getOpcode()) {
1195  default: break;
1196  case ISD::TokenFactor:        return visitTokenFactor(N);
1197  case ISD::MERGE_VALUES:       return visitMERGE_VALUES(N);
1198  case ISD::ADD:                return visitADD(N);
1199  case ISD::SUB:                return visitSUB(N);
1200  case ISD::ADDC:               return visitADDC(N);
1201  case ISD::SUBC:               return visitSUBC(N);
1202  case ISD::ADDE:               return visitADDE(N);
1203  case ISD::SUBE:               return visitSUBE(N);
1204  case ISD::MUL:                return visitMUL(N);
1205  case ISD::SDIV:               return visitSDIV(N);
1206  case ISD::UDIV:               return visitUDIV(N);
1207  case ISD::SREM:               return visitSREM(N);
1208  case ISD::UREM:               return visitUREM(N);
1209  case ISD::MULHU:              return visitMULHU(N);
1210  case ISD::MULHS:              return visitMULHS(N);
1211  case ISD::SMUL_LOHI:          return visitSMUL_LOHI(N);
1212  case ISD::UMUL_LOHI:          return visitUMUL_LOHI(N);
1213  case ISD::SMULO:              return visitSMULO(N);
1214  case ISD::UMULO:              return visitUMULO(N);
1215  case ISD::SDIVREM:            return visitSDIVREM(N);
1216  case ISD::UDIVREM:            return visitUDIVREM(N);
1217  case ISD::AND:                return visitAND(N);
1218  case ISD::OR:                 return visitOR(N);
1219  case ISD::XOR:                return visitXOR(N);
1220  case ISD::SHL:                return visitSHL(N);
1221  case ISD::SRA:                return visitSRA(N);
1222  case ISD::SRL:                return visitSRL(N);
1223  case ISD::ROTR:
1224  case ISD::ROTL:               return visitRotate(N);
1225  case ISD::CTLZ:               return visitCTLZ(N);
1226  case ISD::CTLZ_ZERO_UNDEF:    return visitCTLZ_ZERO_UNDEF(N);
1227  case ISD::CTTZ:               return visitCTTZ(N);
1228  case ISD::CTTZ_ZERO_UNDEF:    return visitCTTZ_ZERO_UNDEF(N);
1229  case ISD::CTPOP:              return visitCTPOP(N);
1230  case ISD::SELECT:             return visitSELECT(N);
1231  case ISD::VSELECT:            return visitVSELECT(N);
1232  case ISD::SELECT_CC:          return visitSELECT_CC(N);
1233  case ISD::SETCC:              return visitSETCC(N);
1234  case ISD::SIGN_EXTEND:        return visitSIGN_EXTEND(N);
1235  case ISD::ZERO_EXTEND:        return visitZERO_EXTEND(N);
1236  case ISD::ANY_EXTEND:         return visitANY_EXTEND(N);
1237  case ISD::SIGN_EXTEND_INREG:  return visitSIGN_EXTEND_INREG(N);
1238  case ISD::TRUNCATE:           return visitTRUNCATE(N);
1239  case ISD::BITCAST:            return visitBITCAST(N);
1240  case ISD::BUILD_PAIR:         return visitBUILD_PAIR(N);
1241  case ISD::FADD:               return visitFADD(N);
1242  case ISD::FSUB:               return visitFSUB(N);
1243  case ISD::FMUL:               return visitFMUL(N);
1244  case ISD::FMA:                return visitFMA(N);
1245  case ISD::FDIV:               return visitFDIV(N);
1246  case ISD::FREM:               return visitFREM(N);
1247  case ISD::FCOPYSIGN:          return visitFCOPYSIGN(N);
1248  case ISD::SINT_TO_FP:         return visitSINT_TO_FP(N);
1249  case ISD::UINT_TO_FP:         return visitUINT_TO_FP(N);
1250  case ISD::FP_TO_SINT:         return visitFP_TO_SINT(N);
1251  case ISD::FP_TO_UINT:         return visitFP_TO_UINT(N);
1252  case ISD::FP_ROUND:           return visitFP_ROUND(N);
1253  case ISD::FP_ROUND_INREG:     return visitFP_ROUND_INREG(N);
1254  case ISD::FP_EXTEND:          return visitFP_EXTEND(N);
1255  case ISD::FNEG:               return visitFNEG(N);
1256  case ISD::FABS:               return visitFABS(N);
1257  case ISD::FFLOOR:             return visitFFLOOR(N);
1258  case ISD::FCEIL:              return visitFCEIL(N);
1259  case ISD::FTRUNC:             return visitFTRUNC(N);
1260  case ISD::BRCOND:             return visitBRCOND(N);
1261  case ISD::BR_CC:              return visitBR_CC(N);
1262  case ISD::LOAD:               return visitLOAD(N);
1263  case ISD::STORE:              return visitSTORE(N);
1264  case ISD::INSERT_VECTOR_ELT:  return visitINSERT_VECTOR_ELT(N);
1265  case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N);
1266  case ISD::BUILD_VECTOR:       return visitBUILD_VECTOR(N);
1267  case ISD::CONCAT_VECTORS:     return visitCONCAT_VECTORS(N);
1268  case ISD::EXTRACT_SUBVECTOR:  return visitEXTRACT_SUBVECTOR(N);
1269  case ISD::VECTOR_SHUFFLE:     return visitVECTOR_SHUFFLE(N);
1270  case ISD::INSERT_SUBVECTOR:   return visitINSERT_SUBVECTOR(N);
1271  }
1272  return SDValue();
1273}
1274
1275SDValue DAGCombiner::combine(SDNode *N) {
1276  SDValue RV = visit(N);
1277
1278  // If nothing happened, try a target-specific DAG combine.
1279  if (!RV.getNode()) {
1280    assert(N->getOpcode() != ISD::DELETED_NODE &&
1281           "Node was deleted but visit returned NULL!");
1282
1283    if (N->getOpcode() >= ISD::BUILTIN_OP_END ||
1284        TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode())) {
1285
1286      // Expose the DAG combiner to the target combiner impls.
1287      TargetLowering::DAGCombinerInfo
1288        DagCombineInfo(DAG, Level, false, this);
1289
1290      RV = TLI.PerformDAGCombine(N, DagCombineInfo);
1291    }
1292  }
1293
1294  // If nothing happened still, try promoting the operation.
1295  if (!RV.getNode()) {
1296    switch (N->getOpcode()) {
1297    default: break;
1298    case ISD::ADD:
1299    case ISD::SUB:
1300    case ISD::MUL:
1301    case ISD::AND:
1302    case ISD::OR:
1303    case ISD::XOR:
1304      RV = PromoteIntBinOp(SDValue(N, 0));
1305      break;
1306    case ISD::SHL:
1307    case ISD::SRA:
1308    case ISD::SRL:
1309      RV = PromoteIntShiftOp(SDValue(N, 0));
1310      break;
1311    case ISD::SIGN_EXTEND:
1312    case ISD::ZERO_EXTEND:
1313    case ISD::ANY_EXTEND:
1314      RV = PromoteExtend(SDValue(N, 0));
1315      break;
1316    case ISD::LOAD:
1317      if (PromoteLoad(SDValue(N, 0)))
1318        RV = SDValue(N, 0);
1319      break;
1320    }
1321  }
1322
1323  // If N is a commutative binary node, try commuting it to enable more
1324  // sdisel CSE.
1325  if (!RV.getNode() && SelectionDAG::isCommutativeBinOp(N->getOpcode()) &&
1326      N->getNumValues() == 1) {
1327    SDValue N0 = N->getOperand(0);
1328    SDValue N1 = N->getOperand(1);
1329
1330    // Constant operands are canonicalized to RHS.
1331    if (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1)) {
1332      SDValue Ops[] = {N1, N0};
1333      SDNode *CSENode;
1334      if (const BinaryWithFlagsSDNode *BinNode =
1335              dyn_cast<BinaryWithFlagsSDNode>(N)) {
1336        CSENode = DAG.getNodeIfExists(
1337            N->getOpcode(), N->getVTList(), Ops, BinNode->hasNoUnsignedWrap(),
1338            BinNode->hasNoSignedWrap(), BinNode->isExact());
1339      } else {
1340        CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), Ops);
1341      }
1342      if (CSENode)
1343        return SDValue(CSENode, 0);
1344    }
1345  }
1346
1347  return RV;
1348}
1349
1350/// getInputChainForNode - Given a node, return its input chain if it has one,
1351/// otherwise return a null sd operand.
1352static SDValue getInputChainForNode(SDNode *N) {
1353  if (unsigned NumOps = N->getNumOperands()) {
1354    if (N->getOperand(0).getValueType() == MVT::Other)
1355      return N->getOperand(0);
1356    if (N->getOperand(NumOps-1).getValueType() == MVT::Other)
1357      return N->getOperand(NumOps-1);
1358    for (unsigned i = 1; i < NumOps-1; ++i)
1359      if (N->getOperand(i).getValueType() == MVT::Other)
1360        return N->getOperand(i);
1361  }
1362  return SDValue();
1363}
1364
1365SDValue DAGCombiner::visitTokenFactor(SDNode *N) {
1366  // If N has two operands, where one has an input chain equal to the other,
1367  // the 'other' chain is redundant.
1368  if (N->getNumOperands() == 2) {
1369    if (getInputChainForNode(N->getOperand(0).getNode()) == N->getOperand(1))
1370      return N->getOperand(0);
1371    if (getInputChainForNode(N->getOperand(1).getNode()) == N->getOperand(0))
1372      return N->getOperand(1);
1373  }
1374
1375  SmallVector<SDNode *, 8> TFs;     // List of token factors to visit.
1376  SmallVector<SDValue, 8> Ops;    // Ops for replacing token factor.
1377  SmallPtrSet<SDNode*, 16> SeenOps;
1378  bool Changed = false;             // If we should replace this token factor.
1379
1380  // Start out with this token factor.
1381  TFs.push_back(N);
1382
1383  // Iterate through token factors.  The TFs grows when new token factors are
1384  // encountered.
1385  for (unsigned i = 0; i < TFs.size(); ++i) {
1386    SDNode *TF = TFs[i];
1387
1388    // Check each of the operands.
1389    for (unsigned i = 0, ie = TF->getNumOperands(); i != ie; ++i) {
1390      SDValue Op = TF->getOperand(i);
1391
1392      switch (Op.getOpcode()) {
1393      case ISD::EntryToken:
1394        // Entry tokens don't need to be added to the list. They are
1395        // rededundant.
1396        Changed = true;
1397        break;
1398
1399      case ISD::TokenFactor:
1400        if (Op.hasOneUse() &&
1401            std::find(TFs.begin(), TFs.end(), Op.getNode()) == TFs.end()) {
1402          // Queue up for processing.
1403          TFs.push_back(Op.getNode());
1404          // Clean up in case the token factor is removed.
1405          AddToWorkList(Op.getNode());
1406          Changed = true;
1407          break;
1408        }
1409        // Fall thru
1410
1411      default:
1412        // Only add if it isn't already in the list.
1413        if (SeenOps.insert(Op.getNode()))
1414          Ops.push_back(Op);
1415        else
1416          Changed = true;
1417        break;
1418      }
1419    }
1420  }
1421
1422  SDValue Result;
1423
1424  // If we've change things around then replace token factor.
1425  if (Changed) {
1426    if (Ops.empty()) {
1427      // The entry token is the only possible outcome.
1428      Result = DAG.getEntryNode();
1429    } else {
1430      // New and improved token factor.
1431      Result = DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, Ops);
1432    }
1433
1434    // Don't add users to work list.
1435    return CombineTo(N, Result, false);
1436  }
1437
1438  return Result;
1439}
1440
1441/// MERGE_VALUES can always be eliminated.
1442SDValue DAGCombiner::visitMERGE_VALUES(SDNode *N) {
1443  WorkListRemover DeadNodes(*this);
1444  // Replacing results may cause a different MERGE_VALUES to suddenly
1445  // be CSE'd with N, and carry its uses with it. Iterate until no
1446  // uses remain, to ensure that the node can be safely deleted.
1447  // First add the users of this node to the work list so that they
1448  // can be tried again once they have new operands.
1449  AddUsersToWorkList(N);
1450  do {
1451    for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
1452      DAG.ReplaceAllUsesOfValueWith(SDValue(N, i), N->getOperand(i));
1453  } while (!N->use_empty());
1454  removeFromWorkList(N);
1455  DAG.DeleteNode(N);
1456  return SDValue(N, 0);   // Return N so it doesn't get rechecked!
1457}
1458
1459static
1460SDValue combineShlAddConstant(SDLoc DL, SDValue N0, SDValue N1,
1461                              SelectionDAG &DAG) {
1462  EVT VT = N0.getValueType();
1463  SDValue N00 = N0.getOperand(0);
1464  SDValue N01 = N0.getOperand(1);
1465  ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N01);
1466
1467  if (N01C && N00.getOpcode() == ISD::ADD && N00.getNode()->hasOneUse() &&
1468      isa<ConstantSDNode>(N00.getOperand(1))) {
1469    // fold (add (shl (add x, c1), c2), ) -> (add (add (shl x, c2), c1<<c2), )
1470    N0 = DAG.getNode(ISD::ADD, SDLoc(N0), VT,
1471                     DAG.getNode(ISD::SHL, SDLoc(N00), VT,
1472                                 N00.getOperand(0), N01),
1473                     DAG.getNode(ISD::SHL, SDLoc(N01), VT,
1474                                 N00.getOperand(1), N01));
1475    return DAG.getNode(ISD::ADD, DL, VT, N0, N1);
1476  }
1477
1478  return SDValue();
1479}
1480
1481SDValue DAGCombiner::visitADD(SDNode *N) {
1482  SDValue N0 = N->getOperand(0);
1483  SDValue N1 = N->getOperand(1);
1484  ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
1485  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
1486  EVT VT = N0.getValueType();
1487
1488  // fold vector ops
1489  if (VT.isVector()) {
1490    SDValue FoldedVOp = SimplifyVBinOp(N);
1491    if (FoldedVOp.getNode()) return FoldedVOp;
1492
1493    // fold (add x, 0) -> x, vector edition
1494    if (ISD::isBuildVectorAllZeros(N1.getNode()))
1495      return N0;
1496    if (ISD::isBuildVectorAllZeros(N0.getNode()))
1497      return N1;
1498  }
1499
1500  // fold (add x, undef) -> undef
1501  if (N0.getOpcode() == ISD::UNDEF)
1502    return N0;
1503  if (N1.getOpcode() == ISD::UNDEF)
1504    return N1;
1505  // fold (add c1, c2) -> c1+c2
1506  if (N0C && N1C)
1507    return DAG.FoldConstantArithmetic(ISD::ADD, VT, N0C, N1C);
1508  // canonicalize constant to RHS
1509  if (N0C && !N1C)
1510    return DAG.getNode(ISD::ADD, SDLoc(N), VT, N1, N0);
1511  // fold (add x, 0) -> x
1512  if (N1C && N1C->isNullValue())
1513    return N0;
1514  // fold (add Sym, c) -> Sym+c
1515  if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N0))
1516    if (!LegalOperations && TLI.isOffsetFoldingLegal(GA) && N1C &&
1517        GA->getOpcode() == ISD::GlobalAddress)
1518      return DAG.getGlobalAddress(GA->getGlobal(), SDLoc(N1C), VT,
1519                                  GA->getOffset() +
1520                                    (uint64_t)N1C->getSExtValue());
1521  // fold ((c1-A)+c2) -> (c1+c2)-A
1522  if (N1C && N0.getOpcode() == ISD::SUB)
1523    if (ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0.getOperand(0)))
1524      return DAG.getNode(ISD::SUB, SDLoc(N), VT,
1525                         DAG.getConstant(N1C->getAPIntValue()+
1526                                         N0C->getAPIntValue(), VT),
1527                         N0.getOperand(1));
1528  // reassociate add
1529  SDValue RADD = ReassociateOps(ISD::ADD, SDLoc(N), N0, N1);
1530  if (RADD.getNode())
1531    return RADD;
1532  // fold ((0-A) + B) -> B-A
1533  if (N0.getOpcode() == ISD::SUB && isa<ConstantSDNode>(N0.getOperand(0)) &&
1534      cast<ConstantSDNode>(N0.getOperand(0))->isNullValue())
1535    return DAG.getNode(ISD::SUB, SDLoc(N), VT, N1, N0.getOperand(1));
1536  // fold (A + (0-B)) -> A-B
1537  if (N1.getOpcode() == ISD::SUB && isa<ConstantSDNode>(N1.getOperand(0)) &&
1538      cast<ConstantSDNode>(N1.getOperand(0))->isNullValue())
1539    return DAG.getNode(ISD::SUB, SDLoc(N), VT, N0, N1.getOperand(1));
1540  // fold (A+(B-A)) -> B
1541  if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(1))
1542    return N1.getOperand(0);
1543  // fold ((B-A)+A) -> B
1544  if (N0.getOpcode() == ISD::SUB && N1 == N0.getOperand(1))
1545    return N0.getOperand(0);
1546  // fold (A+(B-(A+C))) to (B-C)
1547  if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
1548      N0 == N1.getOperand(1).getOperand(0))
1549    return DAG.getNode(ISD::SUB, SDLoc(N), VT, N1.getOperand(0),
1550                       N1.getOperand(1).getOperand(1));
1551  // fold (A+(B-(C+A))) to (B-C)
1552  if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
1553      N0 == N1.getOperand(1).getOperand(1))
1554    return DAG.getNode(ISD::SUB, SDLoc(N), VT, N1.getOperand(0),
1555                       N1.getOperand(1).getOperand(0));
1556  // fold (A+((B-A)+or-C)) to (B+or-C)
1557  if ((N1.getOpcode() == ISD::SUB || N1.getOpcode() == ISD::ADD) &&
1558      N1.getOperand(0).getOpcode() == ISD::SUB &&
1559      N0 == N1.getOperand(0).getOperand(1))
1560    return DAG.getNode(N1.getOpcode(), SDLoc(N), VT,
1561                       N1.getOperand(0).getOperand(0), N1.getOperand(1));
1562
1563  // fold (A-B)+(C-D) to (A+C)-(B+D) when A or C is constant
1564  if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB) {
1565    SDValue N00 = N0.getOperand(0);
1566    SDValue N01 = N0.getOperand(1);
1567    SDValue N10 = N1.getOperand(0);
1568    SDValue N11 = N1.getOperand(1);
1569
1570    if (isa<ConstantSDNode>(N00) || isa<ConstantSDNode>(N10))
1571      return DAG.getNode(ISD::SUB, SDLoc(N), VT,
1572                         DAG.getNode(ISD::ADD, SDLoc(N0), VT, N00, N10),
1573                         DAG.getNode(ISD::ADD, SDLoc(N1), VT, N01, N11));
1574  }
1575
1576  if (!VT.isVector() && SimplifyDemandedBits(SDValue(N, 0)))
1577    return SDValue(N, 0);
1578
1579  // fold (a+b) -> (a|b) iff a and b share no bits.
1580  if (VT.isInteger() && !VT.isVector()) {
1581    APInt LHSZero, LHSOne;
1582    APInt RHSZero, RHSOne;
1583    DAG.computeKnownBits(N0, LHSZero, LHSOne);
1584
1585    if (LHSZero.getBoolValue()) {
1586      DAG.computeKnownBits(N1, RHSZero, RHSOne);
1587
1588      // If all possibly-set bits on the LHS are clear on the RHS, return an OR.
1589      // If all possibly-set bits on the RHS are clear on the LHS, return an OR.
1590      if ((RHSZero & ~LHSZero) == ~LHSZero || (LHSZero & ~RHSZero) == ~RHSZero){
1591        if (!LegalOperations || TLI.isOperationLegal(ISD::OR, VT))
1592          return DAG.getNode(ISD::OR, SDLoc(N), VT, N0, N1);
1593      }
1594    }
1595  }
1596
1597  // fold (add (shl (add x, c1), c2), ) -> (add (add (shl x, c2), c1<<c2), )
1598  if (N0.getOpcode() == ISD::SHL && N0.getNode()->hasOneUse()) {
1599    SDValue Result = combineShlAddConstant(SDLoc(N), N0, N1, DAG);
1600    if (Result.getNode()) return Result;
1601  }
1602  if (N1.getOpcode() == ISD::SHL && N1.getNode()->hasOneUse()) {
1603    SDValue Result = combineShlAddConstant(SDLoc(N), N1, N0, DAG);
1604    if (Result.getNode()) return Result;
1605  }
1606
1607  // fold (add x, shl(0 - y, n)) -> sub(x, shl(y, n))
1608  if (N1.getOpcode() == ISD::SHL &&
1609      N1.getOperand(0).getOpcode() == ISD::SUB)
1610    if (ConstantSDNode *C =
1611          dyn_cast<ConstantSDNode>(N1.getOperand(0).getOperand(0)))
1612      if (C->getAPIntValue() == 0)
1613        return DAG.getNode(ISD::SUB, SDLoc(N), VT, N0,
1614                           DAG.getNode(ISD::SHL, SDLoc(N), VT,
1615                                       N1.getOperand(0).getOperand(1),
1616                                       N1.getOperand(1)));
1617  if (N0.getOpcode() == ISD::SHL &&
1618      N0.getOperand(0).getOpcode() == ISD::SUB)
1619    if (ConstantSDNode *C =
1620          dyn_cast<ConstantSDNode>(N0.getOperand(0).getOperand(0)))
1621      if (C->getAPIntValue() == 0)
1622        return DAG.getNode(ISD::SUB, SDLoc(N), VT, N1,
1623                           DAG.getNode(ISD::SHL, SDLoc(N), VT,
1624                                       N0.getOperand(0).getOperand(1),
1625                                       N0.getOperand(1)));
1626
1627  if (N1.getOpcode() == ISD::AND) {
1628    SDValue AndOp0 = N1.getOperand(0);
1629    ConstantSDNode *AndOp1 = dyn_cast<ConstantSDNode>(N1->getOperand(1));
1630    unsigned NumSignBits = DAG.ComputeNumSignBits(AndOp0);
1631    unsigned DestBits = VT.getScalarType().getSizeInBits();
1632
1633    // (add z, (and (sbbl x, x), 1)) -> (sub z, (sbbl x, x))
1634    // and similar xforms where the inner op is either ~0 or 0.
1635    if (NumSignBits == DestBits && AndOp1 && AndOp1->isOne()) {
1636      SDLoc DL(N);
1637      return DAG.getNode(ISD::SUB, DL, VT, N->getOperand(0), AndOp0);
1638    }
1639  }
1640
1641  // add (sext i1), X -> sub X, (zext i1)
1642  if (N0.getOpcode() == ISD::SIGN_EXTEND &&
1643      N0.getOperand(0).getValueType() == MVT::i1 &&
1644      !TLI.isOperationLegal(ISD::SIGN_EXTEND, MVT::i1)) {
1645    SDLoc DL(N);
1646    SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
1647    return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt);
1648  }
1649
1650  return SDValue();
1651}
1652
1653SDValue DAGCombiner::visitADDC(SDNode *N) {
1654  SDValue N0 = N->getOperand(0);
1655  SDValue N1 = N->getOperand(1);
1656  ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
1657  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
1658  EVT VT = N0.getValueType();
1659
1660  // If the flag result is dead, turn this into an ADD.
1661  if (!N->hasAnyUseOfValue(1))
1662    return CombineTo(N, DAG.getNode(ISD::ADD, SDLoc(N), VT, N0, N1),
1663                     DAG.getNode(ISD::CARRY_FALSE,
1664                                 SDLoc(N), MVT::Glue));
1665
1666  // canonicalize constant to RHS.
1667  if (N0C && !N1C)
1668    return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N1, N0);
1669
1670  // fold (addc x, 0) -> x + no carry out
1671  if (N1C && N1C->isNullValue())
1672    return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE,
1673                                        SDLoc(N), MVT::Glue));
1674
1675  // fold (addc a, b) -> (or a, b), CARRY_FALSE iff a and b share no bits.
1676  APInt LHSZero, LHSOne;
1677  APInt RHSZero, RHSOne;
1678  DAG.computeKnownBits(N0, LHSZero, LHSOne);
1679
1680  if (LHSZero.getBoolValue()) {
1681    DAG.computeKnownBits(N1, RHSZero, RHSOne);
1682
1683    // If all possibly-set bits on the LHS are clear on the RHS, return an OR.
1684    // If all possibly-set bits on the RHS are clear on the LHS, return an OR.
1685    if ((RHSZero & ~LHSZero) == ~LHSZero || (LHSZero & ~RHSZero) == ~RHSZero)
1686      return CombineTo(N, DAG.getNode(ISD::OR, SDLoc(N), VT, N0, N1),
1687                       DAG.getNode(ISD::CARRY_FALSE,
1688                                   SDLoc(N), MVT::Glue));
1689  }
1690
1691  return SDValue();
1692}
1693
1694SDValue DAGCombiner::visitADDE(SDNode *N) {
1695  SDValue N0 = N->getOperand(0);
1696  SDValue N1 = N->getOperand(1);
1697  SDValue CarryIn = N->getOperand(2);
1698  ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
1699  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
1700
1701  // canonicalize constant to RHS
1702  if (N0C && !N1C)
1703    return DAG.getNode(ISD::ADDE, SDLoc(N), N->getVTList(),
1704                       N1, N0, CarryIn);
1705
1706  // fold (adde x, y, false) -> (addc x, y)
1707  if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
1708    return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N0, N1);
1709
1710  return SDValue();
1711}
1712
1713// Since it may not be valid to emit a fold to zero for vector initializers
1714// check if we can before folding.
1715static SDValue tryFoldToZero(SDLoc DL, const TargetLowering &TLI, EVT VT,
1716                             SelectionDAG &DAG,
1717                             bool LegalOperations, bool LegalTypes) {
1718  if (!VT.isVector())
1719    return DAG.getConstant(0, VT);
1720  if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))
1721    return DAG.getConstant(0, VT);
1722  return SDValue();
1723}
1724
1725SDValue DAGCombiner::visitSUB(SDNode *N) {
1726  SDValue N0 = N->getOperand(0);
1727  SDValue N1 = N->getOperand(1);
1728  ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0.getNode());
1729  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode());
1730  ConstantSDNode *N1C1 = N1.getOpcode() != ISD::ADD ? nullptr :
1731    dyn_cast<ConstantSDNode>(N1.getOperand(1).getNode());
1732  EVT VT = N0.getValueType();
1733
1734  // fold vector ops
1735  if (VT.isVector()) {
1736    SDValue FoldedVOp = SimplifyVBinOp(N);
1737    if (FoldedVOp.getNode()) return FoldedVOp;
1738
1739    // fold (sub x, 0) -> x, vector edition
1740    if (ISD::isBuildVectorAllZeros(N1.getNode()))
1741      return N0;
1742  }
1743
1744  // fold (sub x, x) -> 0
1745  // FIXME: Refactor this and xor and other similar operations together.
1746  if (N0 == N1)
1747    return tryFoldToZero(SDLoc(N), TLI, VT, DAG, LegalOperations, LegalTypes);
1748  // fold (sub c1, c2) -> c1-c2
1749  if (N0C && N1C)
1750    return DAG.FoldConstantArithmetic(ISD::SUB, VT, N0C, N1C);
1751  // fold (sub x, c) -> (add x, -c)
1752  if (N1C)
1753    return DAG.getNode(ISD::ADD, SDLoc(N), VT, N0,
1754                       DAG.getConstant(-N1C->getAPIntValue(), VT));
1755  // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1)
1756  if (N0C && N0C->isAllOnesValue())
1757    return DAG.getNode(ISD::XOR, SDLoc(N), VT, N1, N0);
1758  // fold A-(A-B) -> B
1759  if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(0))
1760    return N1.getOperand(1);
1761  // fold (A+B)-A -> B
1762  if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1)
1763    return N0.getOperand(1);
1764  // fold (A+B)-B -> A
1765  if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1)
1766    return N0.getOperand(0);
1767  // fold C2-(A+C1) -> (C2-C1)-A
1768  if (N1.getOpcode() == ISD::ADD && N0C && N1C1) {
1769    SDValue NewC = DAG.getConstant(N0C->getAPIntValue() - N1C1->getAPIntValue(),
1770                                   VT);
1771    return DAG.getNode(ISD::SUB, SDLoc(N), VT, NewC,
1772                       N1.getOperand(0));
1773  }
1774  // fold ((A+(B+or-C))-B) -> A+or-C
1775  if (N0.getOpcode() == ISD::ADD &&
1776      (N0.getOperand(1).getOpcode() == ISD::SUB ||
1777       N0.getOperand(1).getOpcode() == ISD::ADD) &&
1778      N0.getOperand(1).getOperand(0) == N1)
1779    return DAG.getNode(N0.getOperand(1).getOpcode(), SDLoc(N), VT,
1780                       N0.getOperand(0), N0.getOperand(1).getOperand(1));
1781  // fold ((A+(C+B))-B) -> A+C
1782  if (N0.getOpcode() == ISD::ADD &&
1783      N0.getOperand(1).getOpcode() == ISD::ADD &&
1784      N0.getOperand(1).getOperand(1) == N1)
1785    return DAG.getNode(ISD::ADD, SDLoc(N), VT,
1786                       N0.getOperand(0), N0.getOperand(1).getOperand(0));
1787  // fold ((A-(B-C))-C) -> A-B
1788  if (N0.getOpcode() == ISD::SUB &&
1789      N0.getOperand(1).getOpcode() == ISD::SUB &&
1790      N0.getOperand(1).getOperand(1) == N1)
1791    return DAG.getNode(ISD::SUB, SDLoc(N), VT,
1792                       N0.getOperand(0), N0.getOperand(1).getOperand(0));
1793
1794  // If either operand of a sub is undef, the result is undef
1795  if (N0.getOpcode() == ISD::UNDEF)
1796    return N0;
1797  if (N1.getOpcode() == ISD::UNDEF)
1798    return N1;
1799
1800  // If the relocation model supports it, consider symbol offsets.
1801  if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N0))
1802    if (!LegalOperations && TLI.isOffsetFoldingLegal(GA)) {
1803      // fold (sub Sym, c) -> Sym-c
1804      if (N1C && GA->getOpcode() == ISD::GlobalAddress)
1805        return DAG.getGlobalAddress(GA->getGlobal(), SDLoc(N1C), VT,
1806                                    GA->getOffset() -
1807                                      (uint64_t)N1C->getSExtValue());
1808      // fold (sub Sym+c1, Sym+c2) -> c1-c2
1809      if (GlobalAddressSDNode *GB = dyn_cast<GlobalAddressSDNode>(N1))
1810        if (GA->getGlobal() == GB->getGlobal())
1811          return DAG.getConstant((uint64_t)GA->getOffset() - GB->getOffset(),
1812                                 VT);
1813    }
1814
1815  return SDValue();
1816}
1817
1818SDValue DAGCombiner::visitSUBC(SDNode *N) {
1819  SDValue N0 = N->getOperand(0);
1820  SDValue N1 = N->getOperand(1);
1821  ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
1822  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
1823  EVT VT = N0.getValueType();
1824
1825  // If the flag result is dead, turn this into an SUB.
1826  if (!N->hasAnyUseOfValue(1))
1827    return CombineTo(N, DAG.getNode(ISD::SUB, SDLoc(N), VT, N0, N1),
1828                     DAG.getNode(ISD::CARRY_FALSE, SDLoc(N),
1829                                 MVT::Glue));
1830
1831  // fold (subc x, x) -> 0 + no borrow
1832  if (N0 == N1)
1833    return CombineTo(N, DAG.getConstant(0, VT),
1834                     DAG.getNode(ISD::CARRY_FALSE, SDLoc(N),
1835                                 MVT::Glue));
1836
1837  // fold (subc x, 0) -> x + no borrow
1838  if (N1C && N1C->isNullValue())
1839    return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, SDLoc(N),
1840                                        MVT::Glue));
1841
1842  // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1) + no borrow
1843  if (N0C && N0C->isAllOnesValue())
1844    return CombineTo(N, DAG.getNode(ISD::XOR, SDLoc(N), VT, N1, N0),
1845                     DAG.getNode(ISD::CARRY_FALSE, SDLoc(N),
1846                                 MVT::Glue));
1847
1848  return SDValue();
1849}
1850
1851SDValue DAGCombiner::visitSUBE(SDNode *N) {
1852  SDValue N0 = N->getOperand(0);
1853  SDValue N1 = N->getOperand(1);
1854  SDValue CarryIn = N->getOperand(2);
1855
1856  // fold (sube x, y, false) -> (subc x, y)
1857  if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
1858    return DAG.getNode(ISD::SUBC, SDLoc(N), N->getVTList(), N0, N1);
1859
1860  return SDValue();
1861}
1862
1863SDValue DAGCombiner::visitMUL(SDNode *N) {
1864  SDValue N0 = N->getOperand(0);
1865  SDValue N1 = N->getOperand(1);
1866  EVT VT = N0.getValueType();
1867
1868  // fold (mul x, undef) -> 0
1869  if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
1870    return DAG.getConstant(0, VT);
1871
1872  bool N0IsConst = false;
1873  bool N1IsConst = false;
1874  APInt ConstValue0, ConstValue1;
1875  // fold vector ops
1876  if (VT.isVector()) {
1877    SDValue FoldedVOp = SimplifyVBinOp(N);
1878    if (FoldedVOp.getNode()) return FoldedVOp;
1879
1880    N0IsConst = isConstantSplatVector(N0.getNode(), ConstValue0);
1881    N1IsConst = isConstantSplatVector(N1.getNode(), ConstValue1);
1882  } else {
1883    N0IsConst = dyn_cast<ConstantSDNode>(N0) != nullptr;
1884    ConstValue0 = N0IsConst ? (dyn_cast<ConstantSDNode>(N0))->getAPIntValue()
1885                            : APInt();
1886    N1IsConst = dyn_cast<ConstantSDNode>(N1) != nullptr;
1887    ConstValue1 = N1IsConst ? (dyn_cast<ConstantSDNode>(N1))->getAPIntValue()
1888                            : APInt();
1889  }
1890
1891  // fold (mul c1, c2) -> c1*c2
1892  if (N0IsConst && N1IsConst)
1893    return DAG.FoldConstantArithmetic(ISD::MUL, VT, N0.getNode(), N1.getNode());
1894
1895  // canonicalize constant to RHS
1896  if (N0IsConst && !N1IsConst)
1897    return DAG.getNode(ISD::MUL, SDLoc(N), VT, N1, N0);
1898  // fold (mul x, 0) -> 0
1899  if (N1IsConst && ConstValue1 == 0)
1900    return N1;
1901  // We require a splat of the entire scalar bit width for non-contiguous
1902  // bit patterns.
1903  bool IsFullSplat =
1904    ConstValue1.getBitWidth() == VT.getScalarType().getSizeInBits();
1905  // fold (mul x, 1) -> x
1906  if (N1IsConst && ConstValue1 == 1 && IsFullSplat)
1907    return N0;
1908  // fold (mul x, -1) -> 0-x
1909  if (N1IsConst && ConstValue1.isAllOnesValue())
1910    return DAG.getNode(ISD::SUB, SDLoc(N), VT,
1911                       DAG.getConstant(0, VT), N0);
1912  // fold (mul x, (1 << c)) -> x << c
1913  if (N1IsConst && ConstValue1.isPowerOf2() && IsFullSplat)
1914    return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0,
1915                       DAG.getConstant(ConstValue1.logBase2(),
1916                                       getShiftAmountTy(N0.getValueType())));
1917  // fold (mul x, -(1 << c)) -> -(x << c) or (-x) << c
1918  if (N1IsConst && (-ConstValue1).isPowerOf2() && IsFullSplat) {
1919    unsigned Log2Val = (-ConstValue1).logBase2();
1920    // FIXME: If the input is something that is easily negated (e.g. a
1921    // single-use add), we should put the negate there.
1922    return DAG.getNode(ISD::SUB, SDLoc(N), VT,
1923                       DAG.getConstant(0, VT),
1924                       DAG.getNode(ISD::SHL, SDLoc(N), VT, N0,
1925                            DAG.getConstant(Log2Val,
1926                                      getShiftAmountTy(N0.getValueType()))));
1927  }
1928
1929  APInt Val;
1930  // (mul (shl X, c1), c2) -> (mul X, c2 << c1)
1931  if (N1IsConst && N0.getOpcode() == ISD::SHL &&
1932      (isConstantSplatVector(N0.getOperand(1).getNode(), Val) ||
1933                     isa<ConstantSDNode>(N0.getOperand(1)))) {
1934    SDValue C3 = DAG.getNode(ISD::SHL, SDLoc(N), VT,
1935                             N1, N0.getOperand(1));
1936    AddToWorkList(C3.getNode());
1937    return DAG.getNode(ISD::MUL, SDLoc(N), VT,
1938                       N0.getOperand(0), C3);
1939  }
1940
1941  // Change (mul (shl X, C), Y) -> (shl (mul X, Y), C) when the shift has one
1942  // use.
1943  {
1944    SDValue Sh(nullptr,0), Y(nullptr,0);
1945    // Check for both (mul (shl X, C), Y)  and  (mul Y, (shl X, C)).
1946    if (N0.getOpcode() == ISD::SHL &&
1947        (isConstantSplatVector(N0.getOperand(1).getNode(), Val) ||
1948                       isa<ConstantSDNode>(N0.getOperand(1))) &&
1949        N0.getNode()->hasOneUse()) {
1950      Sh = N0; Y = N1;
1951    } else if (N1.getOpcode() == ISD::SHL &&
1952               isa<ConstantSDNode>(N1.getOperand(1)) &&
1953               N1.getNode()->hasOneUse()) {
1954      Sh = N1; Y = N0;
1955    }
1956
1957    if (Sh.getNode()) {
1958      SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N), VT,
1959                                Sh.getOperand(0), Y);
1960      return DAG.getNode(ISD::SHL, SDLoc(N), VT,
1961                         Mul, Sh.getOperand(1));
1962    }
1963  }
1964
1965  // fold (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2)
1966  if (N1IsConst && N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse() &&
1967      (isConstantSplatVector(N0.getOperand(1).getNode(), Val) ||
1968                     isa<ConstantSDNode>(N0.getOperand(1))))
1969    return DAG.getNode(ISD::ADD, SDLoc(N), VT,
1970                       DAG.getNode(ISD::MUL, SDLoc(N0), VT,
1971                                   N0.getOperand(0), N1),
1972                       DAG.getNode(ISD::MUL, SDLoc(N1), VT,
1973                                   N0.getOperand(1), N1));
1974
1975  // reassociate mul
1976  SDValue RMUL = ReassociateOps(ISD::MUL, SDLoc(N), N0, N1);
1977  if (RMUL.getNode())
1978    return RMUL;
1979
1980  return SDValue();
1981}
1982
1983SDValue DAGCombiner::visitSDIV(SDNode *N) {
1984  SDValue N0 = N->getOperand(0);
1985  SDValue N1 = N->getOperand(1);
1986  ConstantSDNode *N0C = isConstOrConstSplat(N0);
1987  ConstantSDNode *N1C = isConstOrConstSplat(N1);
1988  EVT VT = N->getValueType(0);
1989
1990  // fold vector ops
1991  if (VT.isVector()) {
1992    SDValue FoldedVOp = SimplifyVBinOp(N);
1993    if (FoldedVOp.getNode()) return FoldedVOp;
1994  }
1995
1996  // fold (sdiv c1, c2) -> c1/c2
1997  if (N0C && N1C && !N1C->isNullValue())
1998    return DAG.FoldConstantArithmetic(ISD::SDIV, VT, N0C, N1C);
1999  // fold (sdiv X, 1) -> X
2000  if (N1C && N1C->getAPIntValue() == 1LL)
2001    return N0;
2002  // fold (sdiv X, -1) -> 0-X
2003  if (N1C && N1C->isAllOnesValue())
2004    return DAG.getNode(ISD::SUB, SDLoc(N), VT,
2005                       DAG.getConstant(0, VT), N0);
2006  // If we know the sign bits of both operands are zero, strength reduce to a
2007  // udiv instead.  Handles (X&15) /s 4 -> X&15 >> 2
2008  if (!VT.isVector()) {
2009    if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
2010      return DAG.getNode(ISD::UDIV, SDLoc(N), N1.getValueType(),
2011                         N0, N1);
2012  }
2013
2014  // fold (sdiv X, pow2) -> simple ops after legalize
2015  if (N1C && !N1C->isNullValue() && (N1C->getAPIntValue().isPowerOf2() ||
2016                                     (-N1C->getAPIntValue()).isPowerOf2())) {
2017    // If dividing by powers of two is cheap, then don't perform the following
2018    // fold.
2019    if (TLI.isPow2DivCheap())
2020      return SDValue();
2021
2022    unsigned lg2 = N1C->getAPIntValue().countTrailingZeros();
2023
2024    // Splat the sign bit into the register
2025    SDValue SGN =
2026        DAG.getNode(ISD::SRA, SDLoc(N), VT, N0,
2027                    DAG.getConstant(VT.getScalarSizeInBits() - 1,
2028                                    getShiftAmountTy(N0.getValueType())));
2029    AddToWorkList(SGN.getNode());
2030
2031    // Add (N0 < 0) ? abs2 - 1 : 0;
2032    SDValue SRL =
2033        DAG.getNode(ISD::SRL, SDLoc(N), VT, SGN,
2034                    DAG.getConstant(VT.getScalarSizeInBits() - lg2,
2035                                    getShiftAmountTy(SGN.getValueType())));
2036    SDValue ADD = DAG.getNode(ISD::ADD, SDLoc(N), VT, N0, SRL);
2037    AddToWorkList(SRL.getNode());
2038    AddToWorkList(ADD.getNode());    // Divide by pow2
2039    SDValue SRA = DAG.getNode(ISD::SRA, SDLoc(N), VT, ADD,
2040                  DAG.getConstant(lg2, getShiftAmountTy(ADD.getValueType())));
2041
2042    // If we're dividing by a positive value, we're done.  Otherwise, we must
2043    // negate the result.
2044    if (N1C->getAPIntValue().isNonNegative())
2045      return SRA;
2046
2047    AddToWorkList(SRA.getNode());
2048    return DAG.getNode(ISD::SUB, SDLoc(N), VT, DAG.getConstant(0, VT), SRA);
2049  }
2050
2051  // if integer divide is expensive and we satisfy the requirements, emit an
2052  // alternate sequence.
2053  if (N1C && !TLI.isIntDivCheap()) {
2054    SDValue Op = BuildSDIV(N);
2055    if (Op.getNode()) return Op;
2056  }
2057
2058  // undef / X -> 0
2059  if (N0.getOpcode() == ISD::UNDEF)
2060    return DAG.getConstant(0, VT);
2061  // X / undef -> undef
2062  if (N1.getOpcode() == ISD::UNDEF)
2063    return N1;
2064
2065  return SDValue();
2066}
2067
2068SDValue DAGCombiner::visitUDIV(SDNode *N) {
2069  SDValue N0 = N->getOperand(0);
2070  SDValue N1 = N->getOperand(1);
2071  ConstantSDNode *N0C = isConstOrConstSplat(N0);
2072  ConstantSDNode *N1C = isConstOrConstSplat(N1);
2073  EVT VT = N->getValueType(0);
2074
2075  // fold vector ops
2076  if (VT.isVector()) {
2077    SDValue FoldedVOp = SimplifyVBinOp(N);
2078    if (FoldedVOp.getNode()) return FoldedVOp;
2079  }
2080
2081  // fold (udiv c1, c2) -> c1/c2
2082  if (N0C && N1C && !N1C->isNullValue())
2083    return DAG.FoldConstantArithmetic(ISD::UDIV, VT, N0C, N1C);
2084  // fold (udiv x, (1 << c)) -> x >>u c
2085  if (N1C && N1C->getAPIntValue().isPowerOf2())
2086    return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0,
2087                       DAG.getConstant(N1C->getAPIntValue().logBase2(),
2088                                       getShiftAmountTy(N0.getValueType())));
2089  // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2
2090  if (N1.getOpcode() == ISD::SHL) {
2091    if (ConstantSDNode *SHC = dyn_cast<ConstantSDNode>(N1.getOperand(0))) {
2092      if (SHC->getAPIntValue().isPowerOf2()) {
2093        EVT ADDVT = N1.getOperand(1).getValueType();
2094        SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N), ADDVT,
2095                                  N1.getOperand(1),
2096                                  DAG.getConstant(SHC->getAPIntValue()
2097                                                                  .logBase2(),
2098                                                  ADDVT));
2099        AddToWorkList(Add.getNode());
2100        return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, Add);
2101      }
2102    }
2103  }
2104  // fold (udiv x, c) -> alternate
2105  if (N1C && !TLI.isIntDivCheap()) {
2106    SDValue Op = BuildUDIV(N);
2107    if (Op.getNode()) return Op;
2108  }
2109
2110  // undef / X -> 0
2111  if (N0.getOpcode() == ISD::UNDEF)
2112    return DAG.getConstant(0, VT);
2113  // X / undef -> undef
2114  if (N1.getOpcode() == ISD::UNDEF)
2115    return N1;
2116
2117  return SDValue();
2118}
2119
2120SDValue DAGCombiner::visitSREM(SDNode *N) {
2121  SDValue N0 = N->getOperand(0);
2122  SDValue N1 = N->getOperand(1);
2123  ConstantSDNode *N0C = isConstOrConstSplat(N0);
2124  ConstantSDNode *N1C = isConstOrConstSplat(N1);
2125  EVT VT = N->getValueType(0);
2126
2127  // fold (srem c1, c2) -> c1%c2
2128  if (N0C && N1C && !N1C->isNullValue())
2129    return DAG.FoldConstantArithmetic(ISD::SREM, VT, N0C, N1C);
2130  // If we know the sign bits of both operands are zero, strength reduce to a
2131  // urem instead.  Handles (X & 0x0FFFFFFF) %s 16 -> X&15
2132  if (!VT.isVector()) {
2133    if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
2134      return DAG.getNode(ISD::UREM, SDLoc(N), VT, N0, N1);
2135  }
2136
2137  // If X/C can be simplified by the division-by-constant logic, lower
2138  // X%C to the equivalent of X-X/C*C.
2139  if (N1C && !N1C->isNullValue()) {
2140    SDValue Div = DAG.getNode(ISD::SDIV, SDLoc(N), VT, N0, N1);
2141    AddToWorkList(Div.getNode());
2142    SDValue OptimizedDiv = combine(Div.getNode());
2143    if (OptimizedDiv.getNode() && OptimizedDiv.getNode() != Div.getNode()) {
2144      SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N), VT,
2145                                OptimizedDiv, N1);
2146      SDValue Sub = DAG.getNode(ISD::SUB, SDLoc(N), VT, N0, Mul);
2147      AddToWorkList(Mul.getNode());
2148      return Sub;
2149    }
2150  }
2151
2152  // undef % X -> 0
2153  if (N0.getOpcode() == ISD::UNDEF)
2154    return DAG.getConstant(0, VT);
2155  // X % undef -> undef
2156  if (N1.getOpcode() == ISD::UNDEF)
2157    return N1;
2158
2159  return SDValue();
2160}
2161
2162SDValue DAGCombiner::visitUREM(SDNode *N) {
2163  SDValue N0 = N->getOperand(0);
2164  SDValue N1 = N->getOperand(1);
2165  ConstantSDNode *N0C = isConstOrConstSplat(N0);
2166  ConstantSDNode *N1C = isConstOrConstSplat(N1);
2167  EVT VT = N->getValueType(0);
2168
2169  // fold (urem c1, c2) -> c1%c2
2170  if (N0C && N1C && !N1C->isNullValue())
2171    return DAG.FoldConstantArithmetic(ISD::UREM, VT, N0C, N1C);
2172  // fold (urem x, pow2) -> (and x, pow2-1)
2173  if (N1C && !N1C->isNullValue() && N1C->getAPIntValue().isPowerOf2())
2174    return DAG.getNode(ISD::AND, SDLoc(N), VT, N0,
2175                       DAG.getConstant(N1C->getAPIntValue()-1,VT));
2176  // fold (urem x, (shl pow2, y)) -> (and x, (add (shl pow2, y), -1))
2177  if (N1.getOpcode() == ISD::SHL) {
2178    if (ConstantSDNode *SHC = dyn_cast<ConstantSDNode>(N1.getOperand(0))) {
2179      if (SHC->getAPIntValue().isPowerOf2()) {
2180        SDValue Add =
2181          DAG.getNode(ISD::ADD, SDLoc(N), VT, N1,
2182                 DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()),
2183                                 VT));
2184        AddToWorkList(Add.getNode());
2185        return DAG.getNode(ISD::AND, SDLoc(N), VT, N0, Add);
2186      }
2187    }
2188  }
2189
2190  // If X/C can be simplified by the division-by-constant logic, lower
2191  // X%C to the equivalent of X-X/C*C.
2192  if (N1C && !N1C->isNullValue()) {
2193    SDValue Div = DAG.getNode(ISD::UDIV, SDLoc(N), VT, N0, N1);
2194    AddToWorkList(Div.getNode());
2195    SDValue OptimizedDiv = combine(Div.getNode());
2196    if (OptimizedDiv.getNode() && OptimizedDiv.getNode() != Div.getNode()) {
2197      SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N), VT,
2198                                OptimizedDiv, N1);
2199      SDValue Sub = DAG.getNode(ISD::SUB, SDLoc(N), VT, N0, Mul);
2200      AddToWorkList(Mul.getNode());
2201      return Sub;
2202    }
2203  }
2204
2205  // undef % X -> 0
2206  if (N0.getOpcode() == ISD::UNDEF)
2207    return DAG.getConstant(0, VT);
2208  // X % undef -> undef
2209  if (N1.getOpcode() == ISD::UNDEF)
2210    return N1;
2211
2212  return SDValue();
2213}
2214
2215SDValue DAGCombiner::visitMULHS(SDNode *N) {
2216  SDValue N0 = N->getOperand(0);
2217  SDValue N1 = N->getOperand(1);
2218  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2219  EVT VT = N->getValueType(0);
2220  SDLoc DL(N);
2221
2222  // fold (mulhs x, 0) -> 0
2223  if (N1C && N1C->isNullValue())
2224    return N1;
2225  // fold (mulhs x, 1) -> (sra x, size(x)-1)
2226  if (N1C && N1C->getAPIntValue() == 1)
2227    return DAG.getNode(ISD::SRA, SDLoc(N), N0.getValueType(), N0,
2228                       DAG.getConstant(N0.getValueType().getSizeInBits() - 1,
2229                                       getShiftAmountTy(N0.getValueType())));
2230  // fold (mulhs x, undef) -> 0
2231  if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
2232    return DAG.getConstant(0, VT);
2233
2234  // If the type twice as wide is legal, transform the mulhs to a wider multiply
2235  // plus a shift.
2236  if (VT.isSimple() && !VT.isVector()) {
2237    MVT Simple = VT.getSimpleVT();
2238    unsigned SimpleSize = Simple.getSizeInBits();
2239    EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
2240    if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
2241      N0 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0);
2242      N1 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1);
2243      N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
2244      N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
2245            DAG.getConstant(SimpleSize, getShiftAmountTy(N1.getValueType())));
2246      return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
2247    }
2248  }
2249
2250  return SDValue();
2251}
2252
2253SDValue DAGCombiner::visitMULHU(SDNode *N) {
2254  SDValue N0 = N->getOperand(0);
2255  SDValue N1 = N->getOperand(1);
2256  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2257  EVT VT = N->getValueType(0);
2258  SDLoc DL(N);
2259
2260  // fold (mulhu x, 0) -> 0
2261  if (N1C && N1C->isNullValue())
2262    return N1;
2263  // fold (mulhu x, 1) -> 0
2264  if (N1C && N1C->getAPIntValue() == 1)
2265    return DAG.getConstant(0, N0.getValueType());
2266  // fold (mulhu x, undef) -> 0
2267  if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
2268    return DAG.getConstant(0, VT);
2269
2270  // If the type twice as wide is legal, transform the mulhu to a wider multiply
2271  // plus a shift.
2272  if (VT.isSimple() && !VT.isVector()) {
2273    MVT Simple = VT.getSimpleVT();
2274    unsigned SimpleSize = Simple.getSizeInBits();
2275    EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
2276    if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
2277      N0 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0);
2278      N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1);
2279      N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
2280      N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
2281            DAG.getConstant(SimpleSize, getShiftAmountTy(N1.getValueType())));
2282      return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
2283    }
2284  }
2285
2286  return SDValue();
2287}
2288
2289/// SimplifyNodeWithTwoResults - Perform optimizations common to nodes that
2290/// compute two values. LoOp and HiOp give the opcodes for the two computations
2291/// that are being performed. Return true if a simplification was made.
2292///
2293SDValue DAGCombiner::SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
2294                                                unsigned HiOp) {
2295  // If the high half is not needed, just compute the low half.
2296  bool HiExists = N->hasAnyUseOfValue(1);
2297  if (!HiExists &&
2298      (!LegalOperations ||
2299       TLI.isOperationLegalOrCustom(LoOp, N->getValueType(0)))) {
2300    SDValue Res = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0),
2301                              ArrayRef<SDUse>(N->op_begin(), N->op_end()));
2302    return CombineTo(N, Res, Res);
2303  }
2304
2305  // If the low half is not needed, just compute the high half.
2306  bool LoExists = N->hasAnyUseOfValue(0);
2307  if (!LoExists &&
2308      (!LegalOperations ||
2309       TLI.isOperationLegal(HiOp, N->getValueType(1)))) {
2310    SDValue Res = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1),
2311                              ArrayRef<SDUse>(N->op_begin(), N->op_end()));
2312    return CombineTo(N, Res, Res);
2313  }
2314
2315  // If both halves are used, return as it is.
2316  if (LoExists && HiExists)
2317    return SDValue();
2318
2319  // If the two computed results can be simplified separately, separate them.
2320  if (LoExists) {
2321    SDValue Lo = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0),
2322                             ArrayRef<SDUse>(N->op_begin(), N->op_end()));
2323    AddToWorkList(Lo.getNode());
2324    SDValue LoOpt = combine(Lo.getNode());
2325    if (LoOpt.getNode() && LoOpt.getNode() != Lo.getNode() &&
2326        (!LegalOperations ||
2327         TLI.isOperationLegal(LoOpt.getOpcode(), LoOpt.getValueType())))
2328      return CombineTo(N, LoOpt, LoOpt);
2329  }
2330
2331  if (HiExists) {
2332    SDValue Hi = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1),
2333                             ArrayRef<SDUse>(N->op_begin(), N->op_end()));
2334    AddToWorkList(Hi.getNode());
2335    SDValue HiOpt = combine(Hi.getNode());
2336    if (HiOpt.getNode() && HiOpt != Hi &&
2337        (!LegalOperations ||
2338         TLI.isOperationLegal(HiOpt.getOpcode(), HiOpt.getValueType())))
2339      return CombineTo(N, HiOpt, HiOpt);
2340  }
2341
2342  return SDValue();
2343}
2344
2345SDValue DAGCombiner::visitSMUL_LOHI(SDNode *N) {
2346  SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS);
2347  if (Res.getNode()) return Res;
2348
2349  EVT VT = N->getValueType(0);
2350  SDLoc DL(N);
2351
2352  // If the type twice as wide is legal, transform the mulhu to a wider multiply
2353  // plus a shift.
2354  if (VT.isSimple() && !VT.isVector()) {
2355    MVT Simple = VT.getSimpleVT();
2356    unsigned SimpleSize = Simple.getSizeInBits();
2357    EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
2358    if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
2359      SDValue Lo = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(0));
2360      SDValue Hi = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N->getOperand(1));
2361      Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
2362      // Compute the high part as N1.
2363      Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
2364            DAG.getConstant(SimpleSize, getShiftAmountTy(Lo.getValueType())));
2365      Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
2366      // Compute the low part as N0.
2367      Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
2368      return CombineTo(N, Lo, Hi);
2369    }
2370  }
2371
2372  return SDValue();
2373}
2374
2375SDValue DAGCombiner::visitUMUL_LOHI(SDNode *N) {
2376  SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU);
2377  if (Res.getNode()) return Res;
2378
2379  EVT VT = N->getValueType(0);
2380  SDLoc DL(N);
2381
2382  // If the type twice as wide is legal, transform the mulhu to a wider multiply
2383  // plus a shift.
2384  if (VT.isSimple() && !VT.isVector()) {
2385    MVT Simple = VT.getSimpleVT();
2386    unsigned SimpleSize = Simple.getSizeInBits();
2387    EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
2388    if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
2389      SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(0));
2390      SDValue Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N->getOperand(1));
2391      Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
2392      // Compute the high part as N1.
2393      Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
2394            DAG.getConstant(SimpleSize, getShiftAmountTy(Lo.getValueType())));
2395      Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
2396      // Compute the low part as N0.
2397      Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
2398      return CombineTo(N, Lo, Hi);
2399    }
2400  }
2401
2402  return SDValue();
2403}
2404
2405SDValue DAGCombiner::visitSMULO(SDNode *N) {
2406  // (smulo x, 2) -> (saddo x, x)
2407  if (ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1)))
2408    if (C2->getAPIntValue() == 2)
2409      return DAG.getNode(ISD::SADDO, SDLoc(N), N->getVTList(),
2410                         N->getOperand(0), N->getOperand(0));
2411
2412  return SDValue();
2413}
2414
2415SDValue DAGCombiner::visitUMULO(SDNode *N) {
2416  // (umulo x, 2) -> (uaddo x, x)
2417  if (ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1)))
2418    if (C2->getAPIntValue() == 2)
2419      return DAG.getNode(ISD::UADDO, SDLoc(N), N->getVTList(),
2420                         N->getOperand(0), N->getOperand(0));
2421
2422  return SDValue();
2423}
2424
2425SDValue DAGCombiner::visitSDIVREM(SDNode *N) {
2426  SDValue Res = SimplifyNodeWithTwoResults(N, ISD::SDIV, ISD::SREM);
2427  if (Res.getNode()) return Res;
2428
2429  return SDValue();
2430}
2431
2432SDValue DAGCombiner::visitUDIVREM(SDNode *N) {
2433  SDValue Res = SimplifyNodeWithTwoResults(N, ISD::UDIV, ISD::UREM);
2434  if (Res.getNode()) return Res;
2435
2436  return SDValue();
2437}
2438
2439/// SimplifyBinOpWithSameOpcodeHands - If this is a binary operator with
2440/// two operands of the same opcode, try to simplify it.
2441SDValue DAGCombiner::SimplifyBinOpWithSameOpcodeHands(SDNode *N) {
2442  SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
2443  EVT VT = N0.getValueType();
2444  assert(N0.getOpcode() == N1.getOpcode() && "Bad input!");
2445
2446  // Bail early if none of these transforms apply.
2447  if (N0.getNode()->getNumOperands() == 0) return SDValue();
2448
2449  // For each of OP in AND/OR/XOR:
2450  // fold (OP (zext x), (zext y)) -> (zext (OP x, y))
2451  // fold (OP (sext x), (sext y)) -> (sext (OP x, y))
2452  // fold (OP (aext x), (aext y)) -> (aext (OP x, y))
2453  // fold (OP (trunc x), (trunc y)) -> (trunc (OP x, y)) (if trunc isn't free)
2454  //
2455  // do not sink logical op inside of a vector extend, since it may combine
2456  // into a vsetcc.
2457  EVT Op0VT = N0.getOperand(0).getValueType();
2458  if ((N0.getOpcode() == ISD::ZERO_EXTEND ||
2459       N0.getOpcode() == ISD::SIGN_EXTEND ||
2460       // Avoid infinite looping with PromoteIntBinOp.
2461       (N0.getOpcode() == ISD::ANY_EXTEND &&
2462        (!LegalTypes || TLI.isTypeDesirableForOp(N->getOpcode(), Op0VT))) ||
2463       (N0.getOpcode() == ISD::TRUNCATE &&
2464        (!TLI.isZExtFree(VT, Op0VT) ||
2465         !TLI.isTruncateFree(Op0VT, VT)) &&
2466        TLI.isTypeLegal(Op0VT))) &&
2467      !VT.isVector() &&
2468      Op0VT == N1.getOperand(0).getValueType() &&
2469      (!LegalOperations || TLI.isOperationLegal(N->getOpcode(), Op0VT))) {
2470    SDValue ORNode = DAG.getNode(N->getOpcode(), SDLoc(N0),
2471                                 N0.getOperand(0).getValueType(),
2472                                 N0.getOperand(0), N1.getOperand(0));
2473    AddToWorkList(ORNode.getNode());
2474    return DAG.getNode(N0.getOpcode(), SDLoc(N), VT, ORNode);
2475  }
2476
2477  // For each of OP in SHL/SRL/SRA/AND...
2478  //   fold (and (OP x, z), (OP y, z)) -> (OP (and x, y), z)
2479  //   fold (or  (OP x, z), (OP y, z)) -> (OP (or  x, y), z)
2480  //   fold (xor (OP x, z), (OP y, z)) -> (OP (xor x, y), z)
2481  if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL ||
2482       N0.getOpcode() == ISD::SRA || N0.getOpcode() == ISD::AND) &&
2483      N0.getOperand(1) == N1.getOperand(1)) {
2484    SDValue ORNode = DAG.getNode(N->getOpcode(), SDLoc(N0),
2485                                 N0.getOperand(0).getValueType(),
2486                                 N0.getOperand(0), N1.getOperand(0));
2487    AddToWorkList(ORNode.getNode());
2488    return DAG.getNode(N0.getOpcode(), SDLoc(N), VT,
2489                       ORNode, N0.getOperand(1));
2490  }
2491
2492  // Simplify xor/and/or (bitcast(A), bitcast(B)) -> bitcast(op (A,B))
2493  // Only perform this optimization after type legalization and before
2494  // LegalizeVectorOprs. LegalizeVectorOprs promotes vector operations by
2495  // adding bitcasts. For example (xor v4i32) is promoted to (v2i64), and
2496  // we don't want to undo this promotion.
2497  // We also handle SCALAR_TO_VECTOR because xor/or/and operations are cheaper
2498  // on scalars.
2499  if ((N0.getOpcode() == ISD::BITCAST ||
2500       N0.getOpcode() == ISD::SCALAR_TO_VECTOR) &&
2501      Level == AfterLegalizeTypes) {
2502    SDValue In0 = N0.getOperand(0);
2503    SDValue In1 = N1.getOperand(0);
2504    EVT In0Ty = In0.getValueType();
2505    EVT In1Ty = In1.getValueType();
2506    SDLoc DL(N);
2507    // If both incoming values are integers, and the original types are the
2508    // same.
2509    if (In0Ty.isInteger() && In1Ty.isInteger() && In0Ty == In1Ty) {
2510      SDValue Op = DAG.getNode(N->getOpcode(), DL, In0Ty, In0, In1);
2511      SDValue BC = DAG.getNode(N0.getOpcode(), DL, VT, Op);
2512      AddToWorkList(Op.getNode());
2513      return BC;
2514    }
2515  }
2516
2517  // Xor/and/or are indifferent to the swizzle operation (shuffle of one value).
2518  // Simplify xor/and/or (shuff(A), shuff(B)) -> shuff(op (A,B))
2519  // If both shuffles use the same mask, and both shuffle within a single
2520  // vector, then it is worthwhile to move the swizzle after the operation.
2521  // The type-legalizer generates this pattern when loading illegal
2522  // vector types from memory. In many cases this allows additional shuffle
2523  // optimizations.
2524  // There are other cases where moving the shuffle after the xor/and/or
2525  // is profitable even if shuffles don't perform a swizzle.
2526  // If both shuffles use the same mask, and both shuffles have the same first
2527  // or second operand, then it might still be profitable to move the shuffle
2528  // after the xor/and/or operation.
2529  if (N0.getOpcode() == ISD::VECTOR_SHUFFLE && Level < AfterLegalizeDAG) {
2530    ShuffleVectorSDNode *SVN0 = cast<ShuffleVectorSDNode>(N0);
2531    ShuffleVectorSDNode *SVN1 = cast<ShuffleVectorSDNode>(N1);
2532
2533    assert(N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType() &&
2534           "Inputs to shuffles are not the same type");
2535
2536    // Check that both shuffles use the same mask. The masks are known to be of
2537    // the same length because the result vector type is the same.
2538    // Check also that shuffles have only one use to avoid introducing extra
2539    // instructions.
2540    if (SVN0->hasOneUse() && SVN1->hasOneUse() &&
2541        SVN0->getMask().equals(SVN1->getMask())) {
2542      SDValue ShOp = N0->getOperand(1);
2543
2544      // Don't try to fold this node if it requires introducing a
2545      // build vector of all zeros that might be illegal at this stage.
2546      if (N->getOpcode() == ISD::XOR && ShOp.getOpcode() != ISD::UNDEF) {
2547        if (!LegalTypes)
2548          ShOp = DAG.getConstant(0, VT);
2549        else
2550          ShOp = SDValue();
2551      }
2552
2553      // (AND (shuf (A, C), shuf (B, C)) -> shuf (AND (A, B), C)
2554      // (OR  (shuf (A, C), shuf (B, C)) -> shuf (OR  (A, B), C)
2555      // (XOR (shuf (A, C), shuf (B, C)) -> shuf (XOR (A, B), V_0)
2556      if (N0.getOperand(1) == N1.getOperand(1) && ShOp.getNode()) {
2557        SDValue NewNode = DAG.getNode(N->getOpcode(), SDLoc(N), VT,
2558                                      N0->getOperand(0), N1->getOperand(0));
2559        AddToWorkList(NewNode.getNode());
2560        return DAG.getVectorShuffle(VT, SDLoc(N), NewNode, ShOp,
2561                                    &SVN0->getMask()[0]);
2562      }
2563
2564      // Don't try to fold this node if it requires introducing a
2565      // build vector of all zeros that might be illegal at this stage.
2566      ShOp = N0->getOperand(0);
2567      if (N->getOpcode() == ISD::XOR && ShOp.getOpcode() != ISD::UNDEF) {
2568        if (!LegalTypes)
2569          ShOp = DAG.getConstant(0, VT);
2570        else
2571          ShOp = SDValue();
2572      }
2573
2574      // (AND (shuf (C, A), shuf (C, B)) -> shuf (C, AND (A, B))
2575      // (OR  (shuf (C, A), shuf (C, B)) -> shuf (C, OR  (A, B))
2576      // (XOR (shuf (C, A), shuf (C, B)) -> shuf (V_0, XOR (A, B))
2577      if (N0->getOperand(0) == N1->getOperand(0) && ShOp.getNode()) {
2578        SDValue NewNode = DAG.getNode(N->getOpcode(), SDLoc(N), VT,
2579                                      N0->getOperand(1), N1->getOperand(1));
2580        AddToWorkList(NewNode.getNode());
2581        return DAG.getVectorShuffle(VT, SDLoc(N), ShOp, NewNode,
2582                                    &SVN0->getMask()[0]);
2583      }
2584    }
2585  }
2586
2587  return SDValue();
2588}
2589
2590SDValue DAGCombiner::visitAND(SDNode *N) {
2591  SDValue N0 = N->getOperand(0);
2592  SDValue N1 = N->getOperand(1);
2593  SDValue LL, LR, RL, RR, CC0, CC1;
2594  ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
2595  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2596  EVT VT = N1.getValueType();
2597  unsigned BitWidth = VT.getScalarType().getSizeInBits();
2598
2599  // fold vector ops
2600  if (VT.isVector()) {
2601    SDValue FoldedVOp = SimplifyVBinOp(N);
2602    if (FoldedVOp.getNode()) return FoldedVOp;
2603
2604    // fold (and x, 0) -> 0, vector edition
2605    if (ISD::isBuildVectorAllZeros(N0.getNode()))
2606      return N0;
2607    if (ISD::isBuildVectorAllZeros(N1.getNode()))
2608      return N1;
2609
2610    // fold (and x, -1) -> x, vector edition
2611    if (ISD::isBuildVectorAllOnes(N0.getNode()))
2612      return N1;
2613    if (ISD::isBuildVectorAllOnes(N1.getNode()))
2614      return N0;
2615  }
2616
2617  // fold (and x, undef) -> 0
2618  if (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)
2619    return DAG.getConstant(0, VT);
2620  // fold (and c1, c2) -> c1&c2
2621  if (N0C && N1C)
2622    return DAG.FoldConstantArithmetic(ISD::AND, VT, N0C, N1C);
2623  // canonicalize constant to RHS
2624  if (N0C && !N1C)
2625    return DAG.getNode(ISD::AND, SDLoc(N), VT, N1, N0);
2626  // fold (and x, -1) -> x
2627  if (N1C && N1C->isAllOnesValue())
2628    return N0;
2629  // if (and x, c) is known to be zero, return 0
2630  if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
2631                                   APInt::getAllOnesValue(BitWidth)))
2632    return DAG.getConstant(0, VT);
2633  // reassociate and
2634  SDValue RAND = ReassociateOps(ISD::AND, SDLoc(N), N0, N1);
2635  if (RAND.getNode())
2636    return RAND;
2637  // fold (and (or x, C), D) -> D if (C & D) == D
2638  if (N1C && N0.getOpcode() == ISD::OR)
2639    if (ConstantSDNode *ORI = dyn_cast<ConstantSDNode>(N0.getOperand(1)))
2640      if ((ORI->getAPIntValue() & N1C->getAPIntValue()) == N1C->getAPIntValue())
2641        return N1;
2642  // fold (and (any_ext V), c) -> (zero_ext V) if 'and' only clears top bits.
2643  if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
2644    SDValue N0Op0 = N0.getOperand(0);
2645    APInt Mask = ~N1C->getAPIntValue();
2646    Mask = Mask.trunc(N0Op0.getValueSizeInBits());
2647    if (DAG.MaskedValueIsZero(N0Op0, Mask)) {
2648      SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N),
2649                                 N0.getValueType(), N0Op0);
2650
2651      // Replace uses of the AND with uses of the Zero extend node.
2652      CombineTo(N, Zext);
2653
2654      // We actually want to replace all uses of the any_extend with the
2655      // zero_extend, to avoid duplicating things.  This will later cause this
2656      // AND to be folded.
2657      CombineTo(N0.getNode(), Zext);
2658      return SDValue(N, 0);   // Return N so it doesn't get rechecked!
2659    }
2660  }
2661  // similarly fold (and (X (load ([non_ext|any_ext|zero_ext] V))), c) ->
2662  // (X (load ([non_ext|zero_ext] V))) if 'and' only clears top bits which must
2663  // already be zero by virtue of the width of the base type of the load.
2664  //
2665  // the 'X' node here can either be nothing or an extract_vector_elt to catch
2666  // more cases.
2667  if ((N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
2668       N0.getOperand(0).getOpcode() == ISD::LOAD) ||
2669      N0.getOpcode() == ISD::LOAD) {
2670    LoadSDNode *Load = cast<LoadSDNode>( (N0.getOpcode() == ISD::LOAD) ?
2671                                         N0 : N0.getOperand(0) );
2672
2673    // Get the constant (if applicable) the zero'th operand is being ANDed with.
2674    // This can be a pure constant or a vector splat, in which case we treat the
2675    // vector as a scalar and use the splat value.
2676    APInt Constant = APInt::getNullValue(1);
2677    if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
2678      Constant = C->getAPIntValue();
2679    } else if (BuildVectorSDNode *Vector = dyn_cast<BuildVectorSDNode>(N1)) {
2680      APInt SplatValue, SplatUndef;
2681      unsigned SplatBitSize;
2682      bool HasAnyUndefs;
2683      bool IsSplat = Vector->isConstantSplat(SplatValue, SplatUndef,
2684                                             SplatBitSize, HasAnyUndefs);
2685      if (IsSplat) {
2686        // Undef bits can contribute to a possible optimisation if set, so
2687        // set them.
2688        SplatValue |= SplatUndef;
2689
2690        // The splat value may be something like "0x00FFFFFF", which means 0 for
2691        // the first vector value and FF for the rest, repeating. We need a mask
2692        // that will apply equally to all members of the vector, so AND all the
2693        // lanes of the constant together.
2694        EVT VT = Vector->getValueType(0);
2695        unsigned BitWidth = VT.getVectorElementType().getSizeInBits();
2696
2697        // If the splat value has been compressed to a bitlength lower
2698        // than the size of the vector lane, we need to re-expand it to
2699        // the lane size.
2700        if (BitWidth > SplatBitSize)
2701          for (SplatValue = SplatValue.zextOrTrunc(BitWidth);
2702               SplatBitSize < BitWidth;
2703               SplatBitSize = SplatBitSize * 2)
2704            SplatValue |= SplatValue.shl(SplatBitSize);
2705
2706        Constant = APInt::getAllOnesValue(BitWidth);
2707        for (unsigned i = 0, n = SplatBitSize/BitWidth; i < n; ++i)
2708          Constant &= SplatValue.lshr(i*BitWidth).zextOrTrunc(BitWidth);
2709      }
2710    }
2711
2712    // If we want to change an EXTLOAD to a ZEXTLOAD, ensure a ZEXTLOAD is
2713    // actually legal and isn't going to get expanded, else this is a false
2714    // optimisation.
2715    bool CanZextLoadProfitably = TLI.isLoadExtLegal(ISD::ZEXTLOAD,
2716                                                    Load->getMemoryVT());
2717
2718    // Resize the constant to the same size as the original memory access before
2719    // extension. If it is still the AllOnesValue then this AND is completely
2720    // unneeded.
2721    Constant =
2722      Constant.zextOrTrunc(Load->getMemoryVT().getScalarType().getSizeInBits());
2723
2724    bool B;
2725    switch (Load->getExtensionType()) {
2726    default: B = false; break;
2727    case ISD::EXTLOAD: B = CanZextLoadProfitably; break;
2728    case ISD::ZEXTLOAD:
2729    case ISD::NON_EXTLOAD: B = true; break;
2730    }
2731
2732    if (B && Constant.isAllOnesValue()) {
2733      // If the load type was an EXTLOAD, convert to ZEXTLOAD in order to
2734      // preserve semantics once we get rid of the AND.
2735      SDValue NewLoad(Load, 0);
2736      if (Load->getExtensionType() == ISD::EXTLOAD) {
2737        NewLoad = DAG.getLoad(Load->getAddressingMode(), ISD::ZEXTLOAD,
2738                              Load->getValueType(0), SDLoc(Load),
2739                              Load->getChain(), Load->getBasePtr(),
2740                              Load->getOffset(), Load->getMemoryVT(),
2741                              Load->getMemOperand());
2742        // Replace uses of the EXTLOAD with the new ZEXTLOAD.
2743        if (Load->getNumValues() == 3) {
2744          // PRE/POST_INC loads have 3 values.
2745          SDValue To[] = { NewLoad.getValue(0), NewLoad.getValue(1),
2746                           NewLoad.getValue(2) };
2747          CombineTo(Load, To, 3, true);
2748        } else {
2749          CombineTo(Load, NewLoad.getValue(0), NewLoad.getValue(1));
2750        }
2751      }
2752
2753      // Fold the AND away, taking care not to fold to the old load node if we
2754      // replaced it.
2755      CombineTo(N, (N0.getNode() == Load) ? NewLoad : N0);
2756
2757      return SDValue(N, 0); // Return N so it doesn't get rechecked!
2758    }
2759  }
2760  // fold (and (setcc x), (setcc y)) -> (setcc (and x, y))
2761  if (isSetCCEquivalent(N0, LL, LR, CC0) && isSetCCEquivalent(N1, RL, RR, CC1)){
2762    ISD::CondCode Op0 = cast<CondCodeSDNode>(CC0)->get();
2763    ISD::CondCode Op1 = cast<CondCodeSDNode>(CC1)->get();
2764
2765    if (LR == RR && isa<ConstantSDNode>(LR) && Op0 == Op1 &&
2766        LL.getValueType().isInteger()) {
2767      // fold (and (seteq X, 0), (seteq Y, 0)) -> (seteq (or X, Y), 0)
2768      if (cast<ConstantSDNode>(LR)->isNullValue() && Op1 == ISD::SETEQ) {
2769        SDValue ORNode = DAG.getNode(ISD::OR, SDLoc(N0),
2770                                     LR.getValueType(), LL, RL);
2771        AddToWorkList(ORNode.getNode());
2772        return DAG.getSetCC(SDLoc(N), VT, ORNode, LR, Op1);
2773      }
2774      // fold (and (seteq X, -1), (seteq Y, -1)) -> (seteq (and X, Y), -1)
2775      if (cast<ConstantSDNode>(LR)->isAllOnesValue() && Op1 == ISD::SETEQ) {
2776        SDValue ANDNode = DAG.getNode(ISD::AND, SDLoc(N0),
2777                                      LR.getValueType(), LL, RL);
2778        AddToWorkList(ANDNode.getNode());
2779        return DAG.getSetCC(SDLoc(N), VT, ANDNode, LR, Op1);
2780      }
2781      // fold (and (setgt X,  -1), (setgt Y,  -1)) -> (setgt (or X, Y), -1)
2782      if (cast<ConstantSDNode>(LR)->isAllOnesValue() && Op1 == ISD::SETGT) {
2783        SDValue ORNode = DAG.getNode(ISD::OR, SDLoc(N0),
2784                                     LR.getValueType(), LL, RL);
2785        AddToWorkList(ORNode.getNode());
2786        return DAG.getSetCC(SDLoc(N), VT, ORNode, LR, Op1);
2787      }
2788    }
2789    // Simplify (and (setne X, 0), (setne X, -1)) -> (setuge (add X, 1), 2)
2790    if (LL == RL && isa<ConstantSDNode>(LR) && isa<ConstantSDNode>(RR) &&
2791        Op0 == Op1 && LL.getValueType().isInteger() &&
2792      Op0 == ISD::SETNE && ((cast<ConstantSDNode>(LR)->isNullValue() &&
2793                                 cast<ConstantSDNode>(RR)->isAllOnesValue()) ||
2794                                (cast<ConstantSDNode>(LR)->isAllOnesValue() &&
2795                                 cast<ConstantSDNode>(RR)->isNullValue()))) {
2796      SDValue ADDNode = DAG.getNode(ISD::ADD, SDLoc(N0), LL.getValueType(),
2797                                    LL, DAG.getConstant(1, LL.getValueType()));
2798      AddToWorkList(ADDNode.getNode());
2799      return DAG.getSetCC(SDLoc(N), VT, ADDNode,
2800                          DAG.getConstant(2, LL.getValueType()), ISD::SETUGE);
2801    }
2802    // canonicalize equivalent to ll == rl
2803    if (LL == RR && LR == RL) {
2804      Op1 = ISD::getSetCCSwappedOperands(Op1);
2805      std::swap(RL, RR);
2806    }
2807    if (LL == RL && LR == RR) {
2808      bool isInteger = LL.getValueType().isInteger();
2809      ISD::CondCode Result = ISD::getSetCCAndOperation(Op0, Op1, isInteger);
2810      if (Result != ISD::SETCC_INVALID &&
2811          (!LegalOperations ||
2812           (TLI.isCondCodeLegal(Result, LL.getSimpleValueType()) &&
2813            TLI.isOperationLegal(ISD::SETCC,
2814                            getSetCCResultType(N0.getSimpleValueType())))))
2815        return DAG.getSetCC(SDLoc(N), N0.getValueType(),
2816                            LL, LR, Result);
2817    }
2818  }
2819
2820  // Simplify: (and (op x...), (op y...))  -> (op (and x, y))
2821  if (N0.getOpcode() == N1.getOpcode()) {
2822    SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N);
2823    if (Tmp.getNode()) return Tmp;
2824  }
2825
2826  // fold (and (sign_extend_inreg x, i16 to i32), 1) -> (and x, 1)
2827  // fold (and (sra)) -> (and (srl)) when possible.
2828  if (!VT.isVector() &&
2829      SimplifyDemandedBits(SDValue(N, 0)))
2830    return SDValue(N, 0);
2831
2832  // fold (zext_inreg (extload x)) -> (zextload x)
2833  if (ISD::isEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode())) {
2834    LoadSDNode *LN0 = cast<LoadSDNode>(N0);
2835    EVT MemVT = LN0->getMemoryVT();
2836    // If we zero all the possible extended bits, then we can turn this into
2837    // a zextload if we are running before legalize or the operation is legal.
2838    unsigned BitWidth = N1.getValueType().getScalarType().getSizeInBits();
2839    if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth,
2840                           BitWidth - MemVT.getScalarType().getSizeInBits())) &&
2841        ((!LegalOperations && !LN0->isVolatile()) ||
2842         TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT))) {
2843      SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT,
2844                                       LN0->getChain(), LN0->getBasePtr(),
2845                                       MemVT, LN0->getMemOperand());
2846      AddToWorkList(N);
2847      CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
2848      return SDValue(N, 0);   // Return N so it doesn't get rechecked!
2849    }
2850  }
2851  // fold (zext_inreg (sextload x)) -> (zextload x) iff load has one use
2852  if (ISD::isSEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) &&
2853      N0.hasOneUse()) {
2854    LoadSDNode *LN0 = cast<LoadSDNode>(N0);
2855    EVT MemVT = LN0->getMemoryVT();
2856    // If we zero all the possible extended bits, then we can turn this into
2857    // a zextload if we are running before legalize or the operation is legal.
2858    unsigned BitWidth = N1.getValueType().getScalarType().getSizeInBits();
2859    if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth,
2860                           BitWidth - MemVT.getScalarType().getSizeInBits())) &&
2861        ((!LegalOperations && !LN0->isVolatile()) ||
2862         TLI.isLoadExtLegal(ISD::ZEXTLOAD, MemVT))) {
2863      SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT,
2864                                       LN0->getChain(), LN0->getBasePtr(),
2865                                       MemVT, LN0->getMemOperand());
2866      AddToWorkList(N);
2867      CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
2868      return SDValue(N, 0);   // Return N so it doesn't get rechecked!
2869    }
2870  }
2871
2872  // fold (and (load x), 255) -> (zextload x, i8)
2873  // fold (and (extload x, i16), 255) -> (zextload x, i8)
2874  // fold (and (any_ext (extload x, i16)), 255) -> (zextload x, i8)
2875  if (N1C && (N0.getOpcode() == ISD::LOAD ||
2876              (N0.getOpcode() == ISD::ANY_EXTEND &&
2877               N0.getOperand(0).getOpcode() == ISD::LOAD))) {
2878    bool HasAnyExt = N0.getOpcode() == ISD::ANY_EXTEND;
2879    LoadSDNode *LN0 = HasAnyExt
2880      ? cast<LoadSDNode>(N0.getOperand(0))
2881      : cast<LoadSDNode>(N0);
2882    if (LN0->getExtensionType() != ISD::SEXTLOAD &&
2883        LN0->isUnindexed() && N0.hasOneUse() && SDValue(LN0, 0).hasOneUse()) {
2884      uint32_t ActiveBits = N1C->getAPIntValue().getActiveBits();
2885      if (ActiveBits > 0 && APIntOps::isMask(ActiveBits, N1C->getAPIntValue())){
2886        EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
2887        EVT LoadedVT = LN0->getMemoryVT();
2888
2889        if (ExtVT == LoadedVT &&
2890            (!LegalOperations || TLI.isLoadExtLegal(ISD::ZEXTLOAD, ExtVT))) {
2891          EVT LoadResultTy = HasAnyExt ? LN0->getValueType(0) : VT;
2892
2893          SDValue NewLoad =
2894            DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(LN0), LoadResultTy,
2895                           LN0->getChain(), LN0->getBasePtr(), ExtVT,
2896                           LN0->getMemOperand());
2897          AddToWorkList(N);
2898          CombineTo(LN0, NewLoad, NewLoad.getValue(1));
2899          return SDValue(N, 0);   // Return N so it doesn't get rechecked!
2900        }
2901
2902        // Do not change the width of a volatile load.
2903        // Do not generate loads of non-round integer types since these can
2904        // be expensive (and would be wrong if the type is not byte sized).
2905        if (!LN0->isVolatile() && LoadedVT.bitsGT(ExtVT) && ExtVT.isRound() &&
2906            (!LegalOperations || TLI.isLoadExtLegal(ISD::ZEXTLOAD, ExtVT))) {
2907          EVT PtrType = LN0->getOperand(1).getValueType();
2908
2909          unsigned Alignment = LN0->getAlignment();
2910          SDValue NewPtr = LN0->getBasePtr();
2911
2912          // For big endian targets, we need to add an offset to the pointer
2913          // to load the correct bytes.  For little endian systems, we merely
2914          // need to read fewer bytes from the same pointer.
2915          if (TLI.isBigEndian()) {
2916            unsigned LVTStoreBytes = LoadedVT.getStoreSize();
2917            unsigned EVTStoreBytes = ExtVT.getStoreSize();
2918            unsigned PtrOff = LVTStoreBytes - EVTStoreBytes;
2919            NewPtr = DAG.getNode(ISD::ADD, SDLoc(LN0), PtrType,
2920                                 NewPtr, DAG.getConstant(PtrOff, PtrType));
2921            Alignment = MinAlign(Alignment, PtrOff);
2922          }
2923
2924          AddToWorkList(NewPtr.getNode());
2925
2926          EVT LoadResultTy = HasAnyExt ? LN0->getValueType(0) : VT;
2927          SDValue Load =
2928            DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(LN0), LoadResultTy,
2929                           LN0->getChain(), NewPtr,
2930                           LN0->getPointerInfo(),
2931                           ExtVT, LN0->isVolatile(), LN0->isNonTemporal(),
2932                           Alignment, LN0->getTBAAInfo());
2933          AddToWorkList(N);
2934          CombineTo(LN0, Load, Load.getValue(1));
2935          return SDValue(N, 0);   // Return N so it doesn't get rechecked!
2936        }
2937      }
2938    }
2939  }
2940
2941  if (N0.getOpcode() == ISD::ADD && N1.getOpcode() == ISD::SRL &&
2942      VT.getSizeInBits() <= 64) {
2943    if (ConstantSDNode *ADDI = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
2944      APInt ADDC = ADDI->getAPIntValue();
2945      if (!TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
2946        // Look for (and (add x, c1), (lshr y, c2)). If C1 wasn't a legal
2947        // immediate for an add, but it is legal if its top c2 bits are set,
2948        // transform the ADD so the immediate doesn't need to be materialized
2949        // in a register.
2950        if (ConstantSDNode *SRLI = dyn_cast<ConstantSDNode>(N1.getOperand(1))) {
2951          APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(),
2952                                             SRLI->getZExtValue());
2953          if (DAG.MaskedValueIsZero(N0.getOperand(1), Mask)) {
2954            ADDC |= Mask;
2955            if (TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
2956              SDValue NewAdd =
2957                DAG.getNode(ISD::ADD, SDLoc(N0), VT,
2958                            N0.getOperand(0), DAG.getConstant(ADDC, VT));
2959              CombineTo(N0.getNode(), NewAdd);
2960              return SDValue(N, 0); // Return N so it doesn't get rechecked!
2961            }
2962          }
2963        }
2964      }
2965    }
2966  }
2967
2968  // fold (and (or (srl N, 8), (shl N, 8)), 0xffff) -> (srl (bswap N), const)
2969  if (N1C && N1C->getAPIntValue() == 0xffff && N0.getOpcode() == ISD::OR) {
2970    SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0),
2971                                       N0.getOperand(1), false);
2972    if (BSwap.getNode())
2973      return BSwap;
2974  }
2975
2976  return SDValue();
2977}
2978
2979/// MatchBSwapHWord - Match (a >> 8) | (a << 8) as (bswap a) >> 16
2980///
2981SDValue DAGCombiner::MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
2982                                        bool DemandHighBits) {
2983  if (!LegalOperations)
2984    return SDValue();
2985
2986  EVT VT = N->getValueType(0);
2987  if (VT != MVT::i64 && VT != MVT::i32 && VT != MVT::i16)
2988    return SDValue();
2989  if (!TLI.isOperationLegal(ISD::BSWAP, VT))
2990    return SDValue();
2991
2992  // Recognize (and (shl a, 8), 0xff), (and (srl a, 8), 0xff00)
2993  bool LookPassAnd0 = false;
2994  bool LookPassAnd1 = false;
2995  if (N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::SRL)
2996      std::swap(N0, N1);
2997  if (N1.getOpcode() == ISD::AND && N1.getOperand(0).getOpcode() == ISD::SHL)
2998      std::swap(N0, N1);
2999  if (N0.getOpcode() == ISD::AND) {
3000    if (!N0.getNode()->hasOneUse())
3001      return SDValue();
3002    ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
3003    if (!N01C || N01C->getZExtValue() != 0xFF00)
3004      return SDValue();
3005    N0 = N0.getOperand(0);
3006    LookPassAnd0 = true;
3007  }
3008
3009  if (N1.getOpcode() == ISD::AND) {
3010    if (!N1.getNode()->hasOneUse())
3011      return SDValue();
3012    ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
3013    if (!N11C || N11C->getZExtValue() != 0xFF)
3014      return SDValue();
3015    N1 = N1.getOperand(0);
3016    LookPassAnd1 = true;
3017  }
3018
3019  if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL)
3020    std::swap(N0, N1);
3021  if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL)
3022    return SDValue();
3023  if (!N0.getNode()->hasOneUse() ||
3024      !N1.getNode()->hasOneUse())
3025    return SDValue();
3026
3027  ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
3028  ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
3029  if (!N01C || !N11C)
3030    return SDValue();
3031  if (N01C->getZExtValue() != 8 || N11C->getZExtValue() != 8)
3032    return SDValue();
3033
3034  // Look for (shl (and a, 0xff), 8), (srl (and a, 0xff00), 8)
3035  SDValue N00 = N0->getOperand(0);
3036  if (!LookPassAnd0 && N00.getOpcode() == ISD::AND) {
3037    if (!N00.getNode()->hasOneUse())
3038      return SDValue();
3039    ConstantSDNode *N001C = dyn_cast<ConstantSDNode>(N00.getOperand(1));
3040    if (!N001C || N001C->getZExtValue() != 0xFF)
3041      return SDValue();
3042    N00 = N00.getOperand(0);
3043    LookPassAnd0 = true;
3044  }
3045
3046  SDValue N10 = N1->getOperand(0);
3047  if (!LookPassAnd1 && N10.getOpcode() == ISD::AND) {
3048    if (!N10.getNode()->hasOneUse())
3049      return SDValue();
3050    ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N10.getOperand(1));
3051    if (!N101C || N101C->getZExtValue() != 0xFF00)
3052      return SDValue();
3053    N10 = N10.getOperand(0);
3054    LookPassAnd1 = true;
3055  }
3056
3057  if (N00 != N10)
3058    return SDValue();
3059
3060  // Make sure everything beyond the low halfword gets set to zero since the SRL
3061  // 16 will clear the top bits.
3062  unsigned OpSizeInBits = VT.getSizeInBits();
3063  if (DemandHighBits && OpSizeInBits > 16) {
3064    // If the left-shift isn't masked out then the only way this is a bswap is
3065    // if all bits beyond the low 8 are 0. In that case the entire pattern
3066    // reduces to a left shift anyway: leave it for other parts of the combiner.
3067    if (!LookPassAnd0)
3068      return SDValue();
3069
3070    // However, if the right shift isn't masked out then it might be because
3071    // it's not needed. See if we can spot that too.
3072    if (!LookPassAnd1 &&
3073        !DAG.MaskedValueIsZero(
3074            N10, APInt::getHighBitsSet(OpSizeInBits, OpSizeInBits - 16)))
3075      return SDValue();
3076  }
3077
3078  SDValue Res = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N00);
3079  if (OpSizeInBits > 16)
3080    Res = DAG.getNode(ISD::SRL, SDLoc(N), VT, Res,
3081                      DAG.getConstant(OpSizeInBits-16, getShiftAmountTy(VT)));
3082  return Res;
3083}
3084
3085/// isBSwapHWordElement - Return true if the specified node is an element
3086/// that makes up a 32-bit packed halfword byteswap. i.e.
3087/// ((x&0xff)<<8)|((x&0xff00)>>8)|((x&0x00ff0000)<<8)|((x&0xff000000)>>8)
3088static bool isBSwapHWordElement(SDValue N, SmallVectorImpl<SDNode *> &Parts) {
3089  if (!N.getNode()->hasOneUse())
3090    return false;
3091
3092  unsigned Opc = N.getOpcode();
3093  if (Opc != ISD::AND && Opc != ISD::SHL && Opc != ISD::SRL)
3094    return false;
3095
3096  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N.getOperand(1));
3097  if (!N1C)
3098    return false;
3099
3100  unsigned Num;
3101  switch (N1C->getZExtValue()) {
3102  default:
3103    return false;
3104  case 0xFF:       Num = 0; break;
3105  case 0xFF00:     Num = 1; break;
3106  case 0xFF0000:   Num = 2; break;
3107  case 0xFF000000: Num = 3; break;
3108  }
3109
3110  // Look for (x & 0xff) << 8 as well as ((x << 8) & 0xff00).
3111  SDValue N0 = N.getOperand(0);
3112  if (Opc == ISD::AND) {
3113    if (Num == 0 || Num == 2) {
3114      // (x >> 8) & 0xff
3115      // (x >> 8) & 0xff0000
3116      if (N0.getOpcode() != ISD::SRL)
3117        return false;
3118      ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
3119      if (!C || C->getZExtValue() != 8)
3120        return false;
3121    } else {
3122      // (x << 8) & 0xff00
3123      // (x << 8) & 0xff000000
3124      if (N0.getOpcode() != ISD::SHL)
3125        return false;
3126      ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
3127      if (!C || C->getZExtValue() != 8)
3128        return false;
3129    }
3130  } else if (Opc == ISD::SHL) {
3131    // (x & 0xff) << 8
3132    // (x & 0xff0000) << 8
3133    if (Num != 0 && Num != 2)
3134      return false;
3135    ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
3136    if (!C || C->getZExtValue() != 8)
3137      return false;
3138  } else { // Opc == ISD::SRL
3139    // (x & 0xff00) >> 8
3140    // (x & 0xff000000) >> 8
3141    if (Num != 1 && Num != 3)
3142      return false;
3143    ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
3144    if (!C || C->getZExtValue() != 8)
3145      return false;
3146  }
3147
3148  if (Parts[Num])
3149    return false;
3150
3151  Parts[Num] = N0.getOperand(0).getNode();
3152  return true;
3153}
3154
3155/// MatchBSwapHWord - Match a 32-bit packed halfword bswap. That is
3156/// ((x&0xff)<<8)|((x&0xff00)>>8)|((x&0x00ff0000)<<8)|((x&0xff000000)>>8)
3157/// => (rotl (bswap x), 16)
3158SDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) {
3159  if (!LegalOperations)
3160    return SDValue();
3161
3162  EVT VT = N->getValueType(0);
3163  if (VT != MVT::i32)
3164    return SDValue();
3165  if (!TLI.isOperationLegal(ISD::BSWAP, VT))
3166    return SDValue();
3167
3168  SmallVector<SDNode*,4> Parts(4, (SDNode*)nullptr);
3169  // Look for either
3170  // (or (or (and), (and)), (or (and), (and)))
3171  // (or (or (or (and), (and)), (and)), (and))
3172  if (N0.getOpcode() != ISD::OR)
3173    return SDValue();
3174  SDValue N00 = N0.getOperand(0);
3175  SDValue N01 = N0.getOperand(1);
3176
3177  if (N1.getOpcode() == ISD::OR &&
3178      N00.getNumOperands() == 2 && N01.getNumOperands() == 2) {
3179    // (or (or (and), (and)), (or (and), (and)))
3180    SDValue N000 = N00.getOperand(0);
3181    if (!isBSwapHWordElement(N000, Parts))
3182      return SDValue();
3183
3184    SDValue N001 = N00.getOperand(1);
3185    if (!isBSwapHWordElement(N001, Parts))
3186      return SDValue();
3187    SDValue N010 = N01.getOperand(0);
3188    if (!isBSwapHWordElement(N010, Parts))
3189      return SDValue();
3190    SDValue N011 = N01.getOperand(1);
3191    if (!isBSwapHWordElement(N011, Parts))
3192      return SDValue();
3193  } else {
3194    // (or (or (or (and), (and)), (and)), (and))
3195    if (!isBSwapHWordElement(N1, Parts))
3196      return SDValue();
3197    if (!isBSwapHWordElement(N01, Parts))
3198      return SDValue();
3199    if (N00.getOpcode() != ISD::OR)
3200      return SDValue();
3201    SDValue N000 = N00.getOperand(0);
3202    if (!isBSwapHWordElement(N000, Parts))
3203      return SDValue();
3204    SDValue N001 = N00.getOperand(1);
3205    if (!isBSwapHWordElement(N001, Parts))
3206      return SDValue();
3207  }
3208
3209  // Make sure the parts are all coming from the same node.
3210  if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3])
3211    return SDValue();
3212
3213  SDValue BSwap = DAG.getNode(ISD::BSWAP, SDLoc(N), VT,
3214                              SDValue(Parts[0],0));
3215
3216  // Result of the bswap should be rotated by 16. If it's not legal, then
3217  // do  (x << 16) | (x >> 16).
3218  SDValue ShAmt = DAG.getConstant(16, getShiftAmountTy(VT));
3219  if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT))
3220    return DAG.getNode(ISD::ROTL, SDLoc(N), VT, BSwap, ShAmt);
3221  if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT))
3222    return DAG.getNode(ISD::ROTR, SDLoc(N), VT, BSwap, ShAmt);
3223  return DAG.getNode(ISD::OR, SDLoc(N), VT,
3224                     DAG.getNode(ISD::SHL, SDLoc(N), VT, BSwap, ShAmt),
3225                     DAG.getNode(ISD::SRL, SDLoc(N), VT, BSwap, ShAmt));
3226}
3227
3228SDValue DAGCombiner::visitOR(SDNode *N) {
3229  SDValue N0 = N->getOperand(0);
3230  SDValue N1 = N->getOperand(1);
3231  SDValue LL, LR, RL, RR, CC0, CC1;
3232  ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
3233  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
3234  EVT VT = N1.getValueType();
3235
3236  // fold vector ops
3237  if (VT.isVector()) {
3238    SDValue FoldedVOp = SimplifyVBinOp(N);
3239    if (FoldedVOp.getNode()) return FoldedVOp;
3240
3241    // fold (or x, 0) -> x, vector edition
3242    if (ISD::isBuildVectorAllZeros(N0.getNode()))
3243      return N1;
3244    if (ISD::isBuildVectorAllZeros(N1.getNode()))
3245      return N0;
3246
3247    // fold (or x, -1) -> -1, vector edition
3248    if (ISD::isBuildVectorAllOnes(N0.getNode()))
3249      return N0;
3250    if (ISD::isBuildVectorAllOnes(N1.getNode()))
3251      return N1;
3252
3253    // fold (or (shuf A, V_0, MA), (shuf B, V_0, MB)) -> (shuf A, B, Mask1)
3254    // fold (or (shuf A, V_0, MA), (shuf B, V_0, MB)) -> (shuf B, A, Mask2)
3255    // Do this only if the resulting shuffle is legal.
3256    if (isa<ShuffleVectorSDNode>(N0) &&
3257        isa<ShuffleVectorSDNode>(N1) &&
3258        N0->getOperand(1) == N1->getOperand(1) &&
3259        ISD::isBuildVectorAllZeros(N0.getOperand(1).getNode())) {
3260      bool CanFold = true;
3261      unsigned NumElts = VT.getVectorNumElements();
3262      const ShuffleVectorSDNode *SV0 = cast<ShuffleVectorSDNode>(N0);
3263      const ShuffleVectorSDNode *SV1 = cast<ShuffleVectorSDNode>(N1);
3264      // We construct two shuffle masks:
3265      // - Mask1 is a shuffle mask for a shuffle with N0 as the first operand
3266      // and N1 as the second operand.
3267      // - Mask2 is a shuffle mask for a shuffle with N1 as the first operand
3268      // and N0 as the second operand.
3269      // We do this because OR is commutable and therefore there might be
3270      // two ways to fold this node into a shuffle.
3271      SmallVector<int,4> Mask1;
3272      SmallVector<int,4> Mask2;
3273
3274      for (unsigned i = 0; i != NumElts && CanFold; ++i) {
3275        int M0 = SV0->getMaskElt(i);
3276        int M1 = SV1->getMaskElt(i);
3277
3278        // Both shuffle indexes are undef. Propagate Undef.
3279        if (M0 < 0 && M1 < 0) {
3280          Mask1.push_back(M0);
3281          Mask2.push_back(M0);
3282          continue;
3283        }
3284
3285        if (M0 < 0 || M1 < 0 ||
3286            (M0 < (int)NumElts && M1 < (int)NumElts) ||
3287            (M0 >= (int)NumElts && M1 >= (int)NumElts)) {
3288          CanFold = false;
3289          break;
3290        }
3291
3292        Mask1.push_back(M0 < (int)NumElts ? M0 : M1 + NumElts);
3293        Mask2.push_back(M1 < (int)NumElts ? M1 : M0 + NumElts);
3294      }
3295
3296      if (CanFold) {
3297        // Fold this sequence only if the resulting shuffle is 'legal'.
3298        if (TLI.isShuffleMaskLegal(Mask1, VT))
3299          return DAG.getVectorShuffle(VT, SDLoc(N), N0->getOperand(0),
3300                                      N1->getOperand(0), &Mask1[0]);
3301        if (TLI.isShuffleMaskLegal(Mask2, VT))
3302          return DAG.getVectorShuffle(VT, SDLoc(N), N1->getOperand(0),
3303                                      N0->getOperand(0), &Mask2[0]);
3304      }
3305    }
3306  }
3307
3308  // fold (or x, undef) -> -1
3309  if (!LegalOperations &&
3310      (N0.getOpcode() == ISD::UNDEF || N1.getOpcode() == ISD::UNDEF)) {
3311    EVT EltVT = VT.isVector() ? VT.getVectorElementType() : VT;
3312    return DAG.getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT);
3313  }
3314  // fold (or c1, c2) -> c1|c2
3315  if (N0C && N1C)
3316    return DAG.FoldConstantArithmetic(ISD::OR, VT, N0C, N1C);
3317  // canonicalize constant to RHS
3318  if (N0C && !N1C)
3319    return DAG.getNode(ISD::OR, SDLoc(N), VT, N1, N0);
3320  // fold (or x, 0) -> x
3321  if (N1C && N1C->isNullValue())
3322    return N0;
3323  // fold (or x, -1) -> -1
3324  if (N1C && N1C->isAllOnesValue())
3325    return N1;
3326  // fold (or x, c) -> c iff (x & ~c) == 0
3327  if (N1C && DAG.MaskedValueIsZero(N0, ~N1C->getAPIntValue()))
3328    return N1;
3329
3330  // Recognize halfword bswaps as (bswap + rotl 16) or (bswap + shl 16)
3331  SDValue BSwap = MatchBSwapHWord(N, N0, N1);
3332  if (BSwap.getNode())
3333    return BSwap;
3334  BSwap = MatchBSwapHWordLow(N, N0, N1);
3335  if (BSwap.getNode())
3336    return BSwap;
3337
3338  // reassociate or
3339  SDValue ROR = ReassociateOps(ISD::OR, SDLoc(N), N0, N1);
3340  if (ROR.getNode())
3341    return ROR;
3342  // Canonicalize (or (and X, c1), c2) -> (and (or X, c2), c1|c2)
3343  // iff (c1 & c2) == 0.
3344  if (N1C && N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() &&
3345             isa<ConstantSDNode>(N0.getOperand(1))) {
3346    ConstantSDNode *C1 = cast<ConstantSDNode>(N0.getOperand(1));
3347    if ((C1->getAPIntValue() & N1C->getAPIntValue()) != 0) {
3348      SDValue COR = DAG.FoldConstantArithmetic(ISD::OR, VT, N1C, C1);
3349      if (!COR.getNode())
3350        return SDValue();
3351      return DAG.getNode(ISD::AND, SDLoc(N), VT,
3352                         DAG.getNode(ISD::OR, SDLoc(N0), VT,
3353                                     N0.getOperand(0), N1), COR);
3354    }
3355  }
3356  // fold (or (setcc x), (setcc y)) -> (setcc (or x, y))
3357  if (isSetCCEquivalent(N0, LL, LR, CC0) && isSetCCEquivalent(N1, RL, RR, CC1)){
3358    ISD::CondCode Op0 = cast<CondCodeSDNode>(CC0)->get();
3359    ISD::CondCode Op1 = cast<CondCodeSDNode>(CC1)->get();
3360
3361    if (LR == RR && isa<ConstantSDNode>(LR) && Op0 == Op1 &&
3362        LL.getValueType().isInteger()) {
3363      // fold (or (setne X, 0), (setne Y, 0)) -> (setne (or X, Y), 0)
3364      // fold (or (setlt X, 0), (setlt Y, 0)) -> (setne (or X, Y), 0)
3365      if (cast<ConstantSDNode>(LR)->isNullValue() &&
3366          (Op1 == ISD::SETNE || Op1 == ISD::SETLT)) {
3367        SDValue ORNode = DAG.getNode(ISD::OR, SDLoc(LR),
3368                                     LR.getValueType(), LL, RL);
3369        AddToWorkList(ORNode.getNode());
3370        return DAG.getSetCC(SDLoc(N), VT, ORNode, LR, Op1);
3371      }
3372      // fold (or (setne X, -1), (setne Y, -1)) -> (setne (and X, Y), -1)
3373      // fold (or (setgt X, -1), (setgt Y  -1)) -> (setgt (and X, Y), -1)
3374      if (cast<ConstantSDNode>(LR)->isAllOnesValue() &&
3375          (Op1 == ISD::SETNE || Op1 == ISD::SETGT)) {
3376        SDValue ANDNode = DAG.getNode(ISD::AND, SDLoc(LR),
3377                                      LR.getValueType(), LL, RL);
3378        AddToWorkList(ANDNode.getNode());
3379        return DAG.getSetCC(SDLoc(N), VT, ANDNode, LR, Op1);
3380      }
3381    }
3382    // canonicalize equivalent to ll == rl
3383    if (LL == RR && LR == RL) {
3384      Op1 = ISD::getSetCCSwappedOperands(Op1);
3385      std::swap(RL, RR);
3386    }
3387    if (LL == RL && LR == RR) {
3388      bool isInteger = LL.getValueType().isInteger();
3389      ISD::CondCode Result = ISD::getSetCCOrOperation(Op0, Op1, isInteger);
3390      if (Result != ISD::SETCC_INVALID &&
3391          (!LegalOperations ||
3392           (TLI.isCondCodeLegal(Result, LL.getSimpleValueType()) &&
3393            TLI.isOperationLegal(ISD::SETCC,
3394              getSetCCResultType(N0.getValueType())))))
3395        return DAG.getSetCC(SDLoc(N), N0.getValueType(),
3396                            LL, LR, Result);
3397    }
3398  }
3399
3400  // Simplify: (or (op x...), (op y...))  -> (op (or x, y))
3401  if (N0.getOpcode() == N1.getOpcode()) {
3402    SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N);
3403    if (Tmp.getNode()) return Tmp;
3404  }
3405
3406  // (or (and X, C1), (and Y, C2))  -> (and (or X, Y), C3) if possible.
3407  if (N0.getOpcode() == ISD::AND &&
3408      N1.getOpcode() == ISD::AND &&
3409      N0.getOperand(1).getOpcode() == ISD::Constant &&
3410      N1.getOperand(1).getOpcode() == ISD::Constant &&
3411      // Don't increase # computations.
3412      (N0.getNode()->hasOneUse() || N1.getNode()->hasOneUse())) {
3413    // We can only do this xform if we know that bits from X that are set in C2
3414    // but not in C1 are already zero.  Likewise for Y.
3415    const APInt &LHSMask =
3416      cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
3417    const APInt &RHSMask =
3418      cast<ConstantSDNode>(N1.getOperand(1))->getAPIntValue();
3419
3420    if (DAG.MaskedValueIsZero(N0.getOperand(0), RHSMask&~LHSMask) &&
3421        DAG.MaskedValueIsZero(N1.getOperand(0), LHSMask&~RHSMask)) {
3422      SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
3423                              N0.getOperand(0), N1.getOperand(0));
3424      return DAG.getNode(ISD::AND, SDLoc(N), VT, X,
3425                         DAG.getConstant(LHSMask | RHSMask, VT));
3426    }
3427  }
3428
3429  // See if this is some rotate idiom.
3430  if (SDNode *Rot = MatchRotate(N0, N1, SDLoc(N)))
3431    return SDValue(Rot, 0);
3432
3433  // Simplify the operands using demanded-bits information.
3434  if (!VT.isVector() &&
3435      SimplifyDemandedBits(SDValue(N, 0)))
3436    return SDValue(N, 0);
3437
3438  return SDValue();
3439}
3440
3441/// MatchRotateHalf - Match "(X shl/srl V1) & V2" where V2 may not be present.
3442static bool MatchRotateHalf(SDValue Op, SDValue &Shift, SDValue &Mask) {
3443  if (Op.getOpcode() == ISD::AND) {
3444    if (isa<ConstantSDNode>(Op.getOperand(1))) {
3445      Mask = Op.getOperand(1);
3446      Op = Op.getOperand(0);
3447    } else {
3448      return false;
3449    }
3450  }
3451
3452  if (Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL) {
3453    Shift = Op;
3454    return true;
3455  }
3456
3457  return false;
3458}
3459
3460// Return true if we can prove that, whenever Neg and Pos are both in the
3461// range [0, OpSize), Neg == (Pos == 0 ? 0 : OpSize - Pos).  This means that
3462// for two opposing shifts shift1 and shift2 and a value X with OpBits bits:
3463//
3464//     (or (shift1 X, Neg), (shift2 X, Pos))
3465//
3466// reduces to a rotate in direction shift2 by Pos or (equivalently) a rotate
3467// in direction shift1 by Neg.  The range [0, OpSize) means that we only need
3468// to consider shift amounts with defined behavior.
3469static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned OpSize) {
3470  // If OpSize is a power of 2 then:
3471  //
3472  //  (a) (Pos == 0 ? 0 : OpSize - Pos) == (OpSize - Pos) & (OpSize - 1)
3473  //  (b) Neg == Neg & (OpSize - 1) whenever Neg is in [0, OpSize).
3474  //
3475  // So if OpSize is a power of 2 and Neg is (and Neg', OpSize-1), we check
3476  // for the stronger condition:
3477  //
3478  //     Neg & (OpSize - 1) == (OpSize - Pos) & (OpSize - 1)    [A]
3479  //
3480  // for all Neg and Pos.  Since Neg & (OpSize - 1) == Neg' & (OpSize - 1)
3481  // we can just replace Neg with Neg' for the rest of the function.
3482  //
3483  // In other cases we check for the even stronger condition:
3484  //
3485  //     Neg == OpSize - Pos                                    [B]
3486  //
3487  // for all Neg and Pos.  Note that the (or ...) then invokes undefined
3488  // behavior if Pos == 0 (and consequently Neg == OpSize).
3489  //
3490  // We could actually use [A] whenever OpSize is a power of 2, but the
3491  // only extra cases that it would match are those uninteresting ones
3492  // where Neg and Pos are never in range at the same time.  E.g. for
3493  // OpSize == 32, using [A] would allow a Neg of the form (sub 64, Pos)
3494  // as well as (sub 32, Pos), but:
3495  //
3496  //     (or (shift1 X, (sub 64, Pos)), (shift2 X, Pos))
3497  //
3498  // always invokes undefined behavior for 32-bit X.
3499  //
3500  // Below, Mask == OpSize - 1 when using [A] and is all-ones otherwise.
3501  unsigned MaskLoBits = 0;
3502  if (Neg.getOpcode() == ISD::AND &&
3503      isPowerOf2_64(OpSize) &&
3504      Neg.getOperand(1).getOpcode() == ISD::Constant &&
3505      cast<ConstantSDNode>(Neg.getOperand(1))->getAPIntValue() == OpSize - 1) {
3506    Neg = Neg.getOperand(0);
3507    MaskLoBits = Log2_64(OpSize);
3508  }
3509
3510  // Check whether Neg has the form (sub NegC, NegOp1) for some NegC and NegOp1.
3511  if (Neg.getOpcode() != ISD::SUB)
3512    return 0;
3513  ConstantSDNode *NegC = dyn_cast<ConstantSDNode>(Neg.getOperand(0));
3514  if (!NegC)
3515    return 0;
3516  SDValue NegOp1 = Neg.getOperand(1);
3517
3518  // On the RHS of [A], if Pos is Pos' & (OpSize - 1), just replace Pos with
3519  // Pos'.  The truncation is redundant for the purpose of the equality.
3520  if (MaskLoBits &&
3521      Pos.getOpcode() == ISD::AND &&
3522      Pos.getOperand(1).getOpcode() == ISD::Constant &&
3523      cast<ConstantSDNode>(Pos.getOperand(1))->getAPIntValue() == OpSize - 1)
3524    Pos = Pos.getOperand(0);
3525
3526  // The condition we need is now:
3527  //
3528  //     (NegC - NegOp1) & Mask == (OpSize - Pos) & Mask
3529  //
3530  // If NegOp1 == Pos then we need:
3531  //
3532  //              OpSize & Mask == NegC & Mask
3533  //
3534  // (because "x & Mask" is a truncation and distributes through subtraction).
3535  APInt Width;
3536  if (Pos == NegOp1)
3537    Width = NegC->getAPIntValue();
3538  // Check for cases where Pos has the form (add NegOp1, PosC) for some PosC.
3539  // Then the condition we want to prove becomes:
3540  //
3541  //     (NegC - NegOp1) & Mask == (OpSize - (NegOp1 + PosC)) & Mask
3542  //
3543  // which, again because "x & Mask" is a truncation, becomes:
3544  //
3545  //                NegC & Mask == (OpSize - PosC) & Mask
3546  //              OpSize & Mask == (NegC + PosC) & Mask
3547  else if (Pos.getOpcode() == ISD::ADD &&
3548           Pos.getOperand(0) == NegOp1 &&
3549           Pos.getOperand(1).getOpcode() == ISD::Constant)
3550    Width = (cast<ConstantSDNode>(Pos.getOperand(1))->getAPIntValue() +
3551             NegC->getAPIntValue());
3552  else
3553    return false;
3554
3555  // Now we just need to check that OpSize & Mask == Width & Mask.
3556  if (MaskLoBits)
3557    // Opsize & Mask is 0 since Mask is Opsize - 1.
3558    return Width.getLoBits(MaskLoBits) == 0;
3559  return Width == OpSize;
3560}
3561
3562// A subroutine of MatchRotate used once we have found an OR of two opposite
3563// shifts of Shifted.  If Neg == <operand size> - Pos then the OR reduces
3564// to both (PosOpcode Shifted, Pos) and (NegOpcode Shifted, Neg), with the
3565// former being preferred if supported.  InnerPos and InnerNeg are Pos and
3566// Neg with outer conversions stripped away.
3567SDNode *DAGCombiner::MatchRotatePosNeg(SDValue Shifted, SDValue Pos,
3568                                       SDValue Neg, SDValue InnerPos,
3569                                       SDValue InnerNeg, unsigned PosOpcode,
3570                                       unsigned NegOpcode, SDLoc DL) {
3571  // fold (or (shl x, (*ext y)),
3572  //          (srl x, (*ext (sub 32, y)))) ->
3573  //   (rotl x, y) or (rotr x, (sub 32, y))
3574  //
3575  // fold (or (shl x, (*ext (sub 32, y))),
3576  //          (srl x, (*ext y))) ->
3577  //   (rotr x, y) or (rotl x, (sub 32, y))
3578  EVT VT = Shifted.getValueType();
3579  if (matchRotateSub(InnerPos, InnerNeg, VT.getSizeInBits())) {
3580    bool HasPos = TLI.isOperationLegalOrCustom(PosOpcode, VT);
3581    return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, Shifted,
3582                       HasPos ? Pos : Neg).getNode();
3583  }
3584
3585  return nullptr;
3586}
3587
3588// MatchRotate - Handle an 'or' of two operands.  If this is one of the many
3589// idioms for rotate, and if the target supports rotation instructions, generate
3590// a rot[lr].
3591SDNode *DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, SDLoc DL) {
3592  // Must be a legal type.  Expanded 'n promoted things won't work with rotates.
3593  EVT VT = LHS.getValueType();
3594  if (!TLI.isTypeLegal(VT)) return nullptr;
3595
3596  // The target must have at least one rotate flavor.
3597  bool HasROTL = TLI.isOperationLegalOrCustom(ISD::ROTL, VT);
3598  bool HasROTR = TLI.isOperationLegalOrCustom(ISD::ROTR, VT);
3599  if (!HasROTL && !HasROTR) return nullptr;
3600
3601  // Match "(X shl/srl V1) & V2" where V2 may not be present.
3602  SDValue LHSShift;   // The shift.
3603  SDValue LHSMask;    // AND value if any.
3604  if (!MatchRotateHalf(LHS, LHSShift, LHSMask))
3605    return nullptr; // Not part of a rotate.
3606
3607  SDValue RHSShift;   // The shift.
3608  SDValue RHSMask;    // AND value if any.
3609  if (!MatchRotateHalf(RHS, RHSShift, RHSMask))
3610    return nullptr; // Not part of a rotate.
3611
3612  if (LHSShift.getOperand(0) != RHSShift.getOperand(0))
3613    return nullptr;   // Not shifting the same value.
3614
3615  if (LHSShift.getOpcode() == RHSShift.getOpcode())
3616    return nullptr;   // Shifts must disagree.
3617
3618  // Canonicalize shl to left side in a shl/srl pair.
3619  if (RHSShift.getOpcode() == ISD::SHL) {
3620    std::swap(LHS, RHS);
3621    std::swap(LHSShift, RHSShift);
3622    std::swap(LHSMask , RHSMask );
3623  }
3624
3625  unsigned OpSizeInBits = VT.getSizeInBits();
3626  SDValue LHSShiftArg = LHSShift.getOperand(0);
3627  SDValue LHSShiftAmt = LHSShift.getOperand(1);
3628  SDValue RHSShiftArg = RHSShift.getOperand(0);
3629  SDValue RHSShiftAmt = RHSShift.getOperand(1);
3630
3631  // fold (or (shl x, C1), (srl x, C2)) -> (rotl x, C1)
3632  // fold (or (shl x, C1), (srl x, C2)) -> (rotr x, C2)
3633  if (LHSShiftAmt.getOpcode() == ISD::Constant &&
3634      RHSShiftAmt.getOpcode() == ISD::Constant) {
3635    uint64_t LShVal = cast<ConstantSDNode>(LHSShiftAmt)->getZExtValue();
3636    uint64_t RShVal = cast<ConstantSDNode>(RHSShiftAmt)->getZExtValue();
3637    if ((LShVal + RShVal) != OpSizeInBits)
3638      return nullptr;
3639
3640    SDValue Rot = DAG.getNode(HasROTL ? ISD::ROTL : ISD::ROTR, DL, VT,
3641                              LHSShiftArg, HasROTL ? LHSShiftAmt : RHSShiftAmt);
3642
3643    // If there is an AND of either shifted operand, apply it to the result.
3644    if (LHSMask.getNode() || RHSMask.getNode()) {
3645      APInt Mask = APInt::getAllOnesValue(OpSizeInBits);
3646
3647      if (LHSMask.getNode()) {
3648        APInt RHSBits = APInt::getLowBitsSet(OpSizeInBits, LShVal);
3649        Mask &= cast<ConstantSDNode>(LHSMask)->getAPIntValue() | RHSBits;
3650      }
3651      if (RHSMask.getNode()) {
3652        APInt LHSBits = APInt::getHighBitsSet(OpSizeInBits, RShVal);
3653        Mask &= cast<ConstantSDNode>(RHSMask)->getAPIntValue() | LHSBits;
3654      }
3655
3656      Rot = DAG.getNode(ISD::AND, DL, VT, Rot, DAG.getConstant(Mask, VT));
3657    }
3658
3659    return Rot.getNode();
3660  }
3661
3662  // If there is a mask here, and we have a variable shift, we can't be sure
3663  // that we're masking out the right stuff.
3664  if (LHSMask.getNode() || RHSMask.getNode())
3665    return nullptr;
3666
3667  // If the shift amount is sign/zext/any-extended just peel it off.
3668  SDValue LExtOp0 = LHSShiftAmt;
3669  SDValue RExtOp0 = RHSShiftAmt;
3670  if ((LHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND ||
3671       LHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND ||
3672       LHSShiftAmt.getOpcode() == ISD::ANY_EXTEND ||
3673       LHSShiftAmt.getOpcode() == ISD::TRUNCATE) &&
3674      (RHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND ||
3675       RHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND ||
3676       RHSShiftAmt.getOpcode() == ISD::ANY_EXTEND ||
3677       RHSShiftAmt.getOpcode() == ISD::TRUNCATE)) {
3678    LExtOp0 = LHSShiftAmt.getOperand(0);
3679    RExtOp0 = RHSShiftAmt.getOperand(0);
3680  }
3681
3682  SDNode *TryL = MatchRotatePosNeg(LHSShiftArg, LHSShiftAmt, RHSShiftAmt,
3683                                   LExtOp0, RExtOp0, ISD::ROTL, ISD::ROTR, DL);
3684  if (TryL)
3685    return TryL;
3686
3687  SDNode *TryR = MatchRotatePosNeg(RHSShiftArg, RHSShiftAmt, LHSShiftAmt,
3688                                   RExtOp0, LExtOp0, ISD::ROTR, ISD::ROTL, DL);
3689  if (TryR)
3690    return TryR;
3691
3692  return nullptr;
3693}
3694
3695SDValue DAGCombiner::visitXOR(SDNode *N) {
3696  SDValue N0 = N->getOperand(0);
3697  SDValue N1 = N->getOperand(1);
3698  SDValue LHS, RHS, CC;
3699  ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
3700  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
3701  EVT VT = N0.getValueType();
3702
3703  // fold vector ops
3704  if (VT.isVector()) {
3705    SDValue FoldedVOp = SimplifyVBinOp(N);
3706    if (FoldedVOp.getNode()) return FoldedVOp;
3707
3708    // fold (xor x, 0) -> x, vector edition
3709    if (ISD::isBuildVectorAllZeros(N0.getNode()))
3710      return N1;
3711    if (ISD::isBuildVectorAllZeros(N1.getNode()))
3712      return N0;
3713  }
3714
3715  // fold (xor undef, undef) -> 0. This is a common idiom (misuse).
3716  if (N0.getOpcode() == ISD::UNDEF && N1.getOpcode() == ISD::UNDEF)
3717    return DAG.getConstant(0, VT);
3718  // fold (xor x, undef) -> undef
3719  if (N0.getOpcode() == ISD::UNDEF)
3720    return N0;
3721  if (N1.getOpcode() == ISD::UNDEF)
3722    return N1;
3723  // fold (xor c1, c2) -> c1^c2
3724  if (N0C && N1C)
3725    return DAG.FoldConstantArithmetic(ISD::XOR, VT, N0C, N1C);
3726  // canonicalize constant to RHS
3727  if (N0C && !N1C)
3728    return DAG.getNode(ISD::XOR, SDLoc(N), VT, N1, N0);
3729  // fold (xor x, 0) -> x
3730  if (N1C && N1C->isNullValue())
3731    return N0;
3732  // reassociate xor
3733  SDValue RXOR = ReassociateOps(ISD::XOR, SDLoc(N), N0, N1);
3734  if (RXOR.getNode())
3735    return RXOR;
3736
3737  // fold !(x cc y) -> (x !cc y)
3738  if (N1C && N1C->getAPIntValue() == 1 && isSetCCEquivalent(N0, LHS, RHS, CC)) {
3739    bool isInt = LHS.getValueType().isInteger();
3740    ISD::CondCode NotCC = ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(),
3741                                               isInt);
3742
3743    if (!LegalOperations ||
3744        TLI.isCondCodeLegal(NotCC, LHS.getSimpleValueType())) {
3745      switch (N0.getOpcode()) {
3746      default:
3747        llvm_unreachable("Unhandled SetCC Equivalent!");
3748      case ISD::SETCC:
3749        return DAG.getSetCC(SDLoc(N), VT, LHS, RHS, NotCC);
3750      case ISD::SELECT_CC:
3751        return DAG.getSelectCC(SDLoc(N), LHS, RHS, N0.getOperand(2),
3752                               N0.getOperand(3), NotCC);
3753      }
3754    }
3755  }
3756
3757  // fold (not (zext (setcc x, y))) -> (zext (not (setcc x, y)))
3758  if (N1C && N1C->getAPIntValue() == 1 && N0.getOpcode() == ISD::ZERO_EXTEND &&
3759      N0.getNode()->hasOneUse() &&
3760      isSetCCEquivalent(N0.getOperand(0), LHS, RHS, CC)){
3761    SDValue V = N0.getOperand(0);
3762    V = DAG.getNode(ISD::XOR, SDLoc(N0), V.getValueType(), V,
3763                    DAG.getConstant(1, V.getValueType()));
3764    AddToWorkList(V.getNode());
3765    return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, V);
3766  }
3767
3768  // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are setcc
3769  if (N1C && N1C->getAPIntValue() == 1 && VT == MVT::i1 &&
3770      (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
3771    SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
3772    if (isOneUseSetCC(RHS) || isOneUseSetCC(LHS)) {
3773      unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND;
3774      LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), VT, LHS, N1); // LHS = ~LHS
3775      RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), VT, RHS, N1); // RHS = ~RHS
3776      AddToWorkList(LHS.getNode()); AddToWorkList(RHS.getNode());
3777      return DAG.getNode(NewOpcode, SDLoc(N), VT, LHS, RHS);
3778    }
3779  }
3780  // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are constants
3781  if (N1C && N1C->isAllOnesValue() &&
3782      (N0.getOpcode() == ISD::OR || N0.getOpcode() == ISD::AND)) {
3783    SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
3784    if (isa<ConstantSDNode>(RHS) || isa<ConstantSDNode>(LHS)) {
3785      unsigned NewOpcode = N0.getOpcode() == ISD::AND ? ISD::OR : ISD::AND;
3786      LHS = DAG.getNode(ISD::XOR, SDLoc(LHS), VT, LHS, N1); // LHS = ~LHS
3787      RHS = DAG.getNode(ISD::XOR, SDLoc(RHS), VT, RHS, N1); // RHS = ~RHS
3788      AddToWorkList(LHS.getNode()); AddToWorkList(RHS.getNode());
3789      return DAG.getNode(NewOpcode, SDLoc(N), VT, LHS, RHS);
3790    }
3791  }
3792  // fold (xor (and x, y), y) -> (and (not x), y)
3793  if (N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() &&
3794      N0->getOperand(1) == N1) {
3795    SDValue X = N0->getOperand(0);
3796    SDValue NotX = DAG.getNOT(SDLoc(X), X, VT);
3797    AddToWorkList(NotX.getNode());
3798    return DAG.getNode(ISD::AND, SDLoc(N), VT, NotX, N1);
3799  }
3800  // fold (xor (xor x, c1), c2) -> (xor x, (xor c1, c2))
3801  if (N1C && N0.getOpcode() == ISD::XOR) {
3802    ConstantSDNode *N00C = dyn_cast<ConstantSDNode>(N0.getOperand(0));
3803    ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
3804    if (N00C)
3805      return DAG.getNode(ISD::XOR, SDLoc(N), VT, N0.getOperand(1),
3806                         DAG.getConstant(N1C->getAPIntValue() ^
3807                                         N00C->getAPIntValue(), VT));
3808    if (N01C)
3809      return DAG.getNode(ISD::XOR, SDLoc(N), VT, N0.getOperand(0),
3810                         DAG.getConstant(N1C->getAPIntValue() ^
3811                                         N01C->getAPIntValue(), VT));
3812  }
3813  // fold (xor x, x) -> 0
3814  if (N0 == N1)
3815    return tryFoldToZero(SDLoc(N), TLI, VT, DAG, LegalOperations, LegalTypes);
3816
3817  // Simplify: xor (op x...), (op y...)  -> (op (xor x, y))
3818  if (N0.getOpcode() == N1.getOpcode()) {
3819    SDValue Tmp = SimplifyBinOpWithSameOpcodeHands(N);
3820    if (Tmp.getNode()) return Tmp;
3821  }
3822
3823  // Simplify the expression using non-local knowledge.
3824  if (!VT.isVector() &&
3825      SimplifyDemandedBits(SDValue(N, 0)))
3826    return SDValue(N, 0);
3827
3828  return SDValue();
3829}
3830
3831/// visitShiftByConstant - Handle transforms common to the three shifts, when
3832/// the shift amount is a constant.
3833SDValue DAGCombiner::visitShiftByConstant(SDNode *N, ConstantSDNode *Amt) {
3834  // We can't and shouldn't fold opaque constants.
3835  if (Amt->isOpaque())
3836    return SDValue();
3837
3838  SDNode *LHS = N->getOperand(0).getNode();
3839  if (!LHS->hasOneUse()) return SDValue();
3840
3841  // We want to pull some binops through shifts, so that we have (and (shift))
3842  // instead of (shift (and)), likewise for add, or, xor, etc.  This sort of
3843  // thing happens with address calculations, so it's important to canonicalize
3844  // it.
3845  bool HighBitSet = false;  // Can we transform this if the high bit is set?
3846
3847  switch (LHS->getOpcode()) {
3848  default: return SDValue();
3849  case ISD::OR:
3850  case ISD::XOR:
3851    HighBitSet = false; // We can only transform sra if the high bit is clear.
3852    break;
3853  case ISD::AND:
3854    HighBitSet = true;  // We can only transform sra if the high bit is set.
3855    break;
3856  case ISD::ADD:
3857    if (N->getOpcode() != ISD::SHL)
3858      return SDValue(); // only shl(add) not sr[al](add).
3859    HighBitSet = false; // We can only transform sra if the high bit is clear.
3860    break;
3861  }
3862
3863  // We require the RHS of the binop to be a constant and not opaque as well.
3864  ConstantSDNode *BinOpCst = dyn_cast<ConstantSDNode>(LHS->getOperand(1));
3865  if (!BinOpCst || BinOpCst->isOpaque()) return SDValue();
3866
3867  // FIXME: disable this unless the input to the binop is a shift by a constant.
3868  // If it is not a shift, it pessimizes some common cases like:
3869  //
3870  //    void foo(int *X, int i) { X[i & 1235] = 1; }
3871  //    int bar(int *X, int i) { return X[i & 255]; }
3872  SDNode *BinOpLHSVal = LHS->getOperand(0).getNode();
3873  if ((BinOpLHSVal->getOpcode() != ISD::SHL &&
3874       BinOpLHSVal->getOpcode() != ISD::SRA &&
3875       BinOpLHSVal->getOpcode() != ISD::SRL) ||
3876      !isa<ConstantSDNode>(BinOpLHSVal->getOperand(1)))
3877    return SDValue();
3878
3879  EVT VT = N->getValueType(0);
3880
3881  // If this is a signed shift right, and the high bit is modified by the
3882  // logical operation, do not perform the transformation. The highBitSet
3883  // boolean indicates the value of the high bit of the constant which would
3884  // cause it to be modified for this operation.
3885  if (N->getOpcode() == ISD::SRA) {
3886    bool BinOpRHSSignSet = BinOpCst->getAPIntValue().isNegative();
3887    if (BinOpRHSSignSet != HighBitSet)
3888      return SDValue();
3889  }
3890
3891  if (!TLI.isDesirableToCommuteWithShift(LHS))
3892    return SDValue();
3893
3894  // Fold the constants, shifting the binop RHS by the shift amount.
3895  SDValue NewRHS = DAG.getNode(N->getOpcode(), SDLoc(LHS->getOperand(1)),
3896                               N->getValueType(0),
3897                               LHS->getOperand(1), N->getOperand(1));
3898  assert(isa<ConstantSDNode>(NewRHS) && "Folding was not successful!");
3899
3900  // Create the new shift.
3901  SDValue NewShift = DAG.getNode(N->getOpcode(),
3902                                 SDLoc(LHS->getOperand(0)),
3903                                 VT, LHS->getOperand(0), N->getOperand(1));
3904
3905  // Create the new binop.
3906  return DAG.getNode(LHS->getOpcode(), SDLoc(N), VT, NewShift, NewRHS);
3907}
3908
3909SDValue DAGCombiner::distributeTruncateThroughAnd(SDNode *N) {
3910  assert(N->getOpcode() == ISD::TRUNCATE);
3911  assert(N->getOperand(0).getOpcode() == ISD::AND);
3912
3913  // (truncate:TruncVT (and N00, N01C)) -> (and (truncate:TruncVT N00), TruncC)
3914  if (N->hasOneUse() && N->getOperand(0).hasOneUse()) {
3915    SDValue N01 = N->getOperand(0).getOperand(1);
3916
3917    if (ConstantSDNode *N01C = isConstOrConstSplat(N01)) {
3918      EVT TruncVT = N->getValueType(0);
3919      SDValue N00 = N->getOperand(0).getOperand(0);
3920      APInt TruncC = N01C->getAPIntValue();
3921      TruncC = TruncC.trunc(TruncVT.getScalarSizeInBits());
3922
3923      return DAG.getNode(ISD::AND, SDLoc(N), TruncVT,
3924                         DAG.getNode(ISD::TRUNCATE, SDLoc(N), TruncVT, N00),
3925                         DAG.getConstant(TruncC, TruncVT));
3926    }
3927  }
3928
3929  return SDValue();
3930}
3931
3932SDValue DAGCombiner::visitRotate(SDNode *N) {
3933  // fold (rot* x, (trunc (and y, c))) -> (rot* x, (and (trunc y), (trunc c))).
3934  if (N->getOperand(1).getOpcode() == ISD::TRUNCATE &&
3935      N->getOperand(1).getOperand(0).getOpcode() == ISD::AND) {
3936    SDValue NewOp1 = distributeTruncateThroughAnd(N->getOperand(1).getNode());
3937    if (NewOp1.getNode())
3938      return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
3939                         N->getOperand(0), NewOp1);
3940  }
3941  return SDValue();
3942}
3943
3944SDValue DAGCombiner::visitSHL(SDNode *N) {
3945  SDValue N0 = N->getOperand(0);
3946  SDValue N1 = N->getOperand(1);
3947  ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
3948  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
3949  EVT VT = N0.getValueType();
3950  unsigned OpSizeInBits = VT.getScalarSizeInBits();
3951
3952  // fold vector ops
3953  if (VT.isVector()) {
3954    SDValue FoldedVOp = SimplifyVBinOp(N);
3955    if (FoldedVOp.getNode()) return FoldedVOp;
3956
3957    BuildVectorSDNode *N1CV = dyn_cast<BuildVectorSDNode>(N1);
3958    // If setcc produces all-one true value then:
3959    // (shl (and (setcc) N01CV) N1CV) -> (and (setcc) N01CV<<N1CV)
3960    if (N1CV && N1CV->isConstant()) {
3961      if (N0.getOpcode() == ISD::AND) {
3962        SDValue N00 = N0->getOperand(0);
3963        SDValue N01 = N0->getOperand(1);
3964        BuildVectorSDNode *N01CV = dyn_cast<BuildVectorSDNode>(N01);
3965
3966        if (N01CV && N01CV->isConstant() && N00.getOpcode() == ISD::SETCC &&
3967            TLI.getBooleanContents(N00.getOperand(0).getValueType()) ==
3968                TargetLowering::ZeroOrNegativeOneBooleanContent) {
3969          SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, VT, N01CV, N1CV);
3970          if (C.getNode())
3971            return DAG.getNode(ISD::AND, SDLoc(N), VT, N00, C);
3972        }
3973      } else {
3974        N1C = isConstOrConstSplat(N1);
3975      }
3976    }
3977  }
3978
3979  // fold (shl c1, c2) -> c1<<c2
3980  if (N0C && N1C)
3981    return DAG.FoldConstantArithmetic(ISD::SHL, VT, N0C, N1C);
3982  // fold (shl 0, x) -> 0
3983  if (N0C && N0C->isNullValue())
3984    return N0;
3985  // fold (shl x, c >= size(x)) -> undef
3986  if (N1C && N1C->getZExtValue() >= OpSizeInBits)
3987    return DAG.getUNDEF(VT);
3988  // fold (shl x, 0) -> x
3989  if (N1C && N1C->isNullValue())
3990    return N0;
3991  // fold (shl undef, x) -> 0
3992  if (N0.getOpcode() == ISD::UNDEF)
3993    return DAG.getConstant(0, VT);
3994  // if (shl x, c) is known to be zero, return 0
3995  if (DAG.MaskedValueIsZero(SDValue(N, 0),
3996                            APInt::getAllOnesValue(OpSizeInBits)))
3997    return DAG.getConstant(0, VT);
3998  // fold (shl x, (trunc (and y, c))) -> (shl x, (and (trunc y), (trunc c))).
3999  if (N1.getOpcode() == ISD::TRUNCATE &&
4000      N1.getOperand(0).getOpcode() == ISD::AND) {
4001    SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode());
4002    if (NewOp1.getNode())
4003      return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, NewOp1);
4004  }
4005
4006  if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
4007    return SDValue(N, 0);
4008
4009  // fold (shl (shl x, c1), c2) -> 0 or (shl x, (add c1, c2))
4010  if (N1C && N0.getOpcode() == ISD::SHL) {
4011    if (ConstantSDNode *N0C1 = isConstOrConstSplat(N0.getOperand(1))) {
4012      uint64_t c1 = N0C1->getZExtValue();
4013      uint64_t c2 = N1C->getZExtValue();
4014      if (c1 + c2 >= OpSizeInBits)
4015        return DAG.getConstant(0, VT);
4016      return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0.getOperand(0),
4017                         DAG.getConstant(c1 + c2, N1.getValueType()));
4018    }
4019  }
4020
4021  // fold (shl (ext (shl x, c1)), c2) -> (ext (shl x, (add c1, c2)))
4022  // For this to be valid, the second form must not preserve any of the bits
4023  // that are shifted out by the inner shift in the first form.  This means
4024  // the outer shift size must be >= the number of bits added by the ext.
4025  // As a corollary, we don't care what kind of ext it is.
4026  if (N1C && (N0.getOpcode() == ISD::ZERO_EXTEND ||
4027              N0.getOpcode() == ISD::ANY_EXTEND ||
4028              N0.getOpcode() == ISD::SIGN_EXTEND) &&
4029      N0.getOperand(0).getOpcode() == ISD::SHL) {
4030    SDValue N0Op0 = N0.getOperand(0);
4031    if (ConstantSDNode *N0Op0C1 = isConstOrConstSplat(N0Op0.getOperand(1))) {
4032      uint64_t c1 = N0Op0C1->getZExtValue();
4033      uint64_t c2 = N1C->getZExtValue();
4034      EVT InnerShiftVT = N0Op0.getValueType();
4035      uint64_t InnerShiftSize = InnerShiftVT.getScalarSizeInBits();
4036      if (c2 >= OpSizeInBits - InnerShiftSize) {
4037        if (c1 + c2 >= OpSizeInBits)
4038          return DAG.getConstant(0, VT);
4039        return DAG.getNode(ISD::SHL, SDLoc(N0), VT,
4040                           DAG.getNode(N0.getOpcode(), SDLoc(N0), VT,
4041                                       N0Op0->getOperand(0)),
4042                           DAG.getConstant(c1 + c2, N1.getValueType()));
4043      }
4044    }
4045  }
4046
4047  // fold (shl (zext (srl x, C)), C) -> (zext (shl (srl x, C), C))
4048  // Only fold this if the inner zext has no other uses to avoid increasing
4049  // the total number of instructions.
4050  if (N1C && N0.getOpcode() == ISD::ZERO_EXTEND && N0.hasOneUse() &&
4051      N0.getOperand(0).getOpcode() == ISD::SRL) {
4052    SDValue N0Op0 = N0.getOperand(0);
4053    if (ConstantSDNode *N0Op0C1 = isConstOrConstSplat(N0Op0.getOperand(1))) {
4054      uint64_t c1 = N0Op0C1->getZExtValue();
4055      if (c1 < VT.getScalarSizeInBits()) {
4056        uint64_t c2 = N1C->getZExtValue();
4057        if (c1 == c2) {
4058          SDValue NewOp0 = N0.getOperand(0);
4059          EVT CountVT = NewOp0.getOperand(1).getValueType();
4060          SDValue NewSHL = DAG.getNode(ISD::SHL, SDLoc(N), NewOp0.getValueType(),
4061                                       NewOp0, DAG.getConstant(c2, CountVT));
4062          AddToWorkList(NewSHL.getNode());
4063          return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N0), VT, NewSHL);
4064        }
4065      }
4066    }
4067  }
4068
4069  // fold (shl (srl x, c1), c2) -> (and (shl x, (sub c2, c1), MASK) or
4070  //                               (and (srl x, (sub c1, c2), MASK)
4071  // Only fold this if the inner shift has no other uses -- if it does, folding
4072  // this will increase the total number of instructions.
4073  if (N1C && N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
4074    if (ConstantSDNode *N0C1 = isConstOrConstSplat(N0.getOperand(1))) {
4075      uint64_t c1 = N0C1->getZExtValue();
4076      if (c1 < OpSizeInBits) {
4077        uint64_t c2 = N1C->getZExtValue();
4078        APInt Mask = APInt::getHighBitsSet(OpSizeInBits, OpSizeInBits - c1);
4079        SDValue Shift;
4080        if (c2 > c1) {
4081          Mask = Mask.shl(c2 - c1);
4082          Shift = DAG.getNode(ISD::SHL, SDLoc(N), VT, N0.getOperand(0),
4083                              DAG.getConstant(c2 - c1, N1.getValueType()));
4084        } else {
4085          Mask = Mask.lshr(c1 - c2);
4086          Shift = DAG.getNode(ISD::SRL, SDLoc(N), VT, N0.getOperand(0),
4087                              DAG.getConstant(c1 - c2, N1.getValueType()));
4088        }
4089        return DAG.getNode(ISD::AND, SDLoc(N0), VT, Shift,
4090                           DAG.getConstant(Mask, VT));
4091      }
4092    }
4093  }
4094  // fold (shl (sra x, c1), c1) -> (and x, (shl -1, c1))
4095  if (N1C && N0.getOpcode() == ISD::SRA && N1 == N0.getOperand(1)) {
4096    unsigned BitSize = VT.getScalarSizeInBits();
4097    SDValue HiBitsMask =
4098      DAG.getConstant(APInt::getHighBitsSet(BitSize,
4099                                            BitSize - N1C->getZExtValue()), VT);
4100    return DAG.getNode(ISD::AND, SDLoc(N), VT, N0.getOperand(0),
4101                       HiBitsMask);
4102  }
4103
4104  if (N1C) {
4105    SDValue NewSHL = visitShiftByConstant(N, N1C);
4106    if (NewSHL.getNode())
4107      return NewSHL;
4108  }
4109
4110  return SDValue();
4111}
4112
4113SDValue DAGCombiner::visitSRA(SDNode *N) {
4114  SDValue N0 = N->getOperand(0);
4115  SDValue N1 = N->getOperand(1);
4116  ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
4117  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
4118  EVT VT = N0.getValueType();
4119  unsigned OpSizeInBits = VT.getScalarType().getSizeInBits();
4120
4121  // fold vector ops
4122  if (VT.isVector()) {
4123    SDValue FoldedVOp = SimplifyVBinOp(N);
4124    if (FoldedVOp.getNode()) return FoldedVOp;
4125
4126    N1C = isConstOrConstSplat(N1);
4127  }
4128
4129  // fold (sra c1, c2) -> (sra c1, c2)
4130  if (N0C && N1C)
4131    return DAG.FoldConstantArithmetic(ISD::SRA, VT, N0C, N1C);
4132  // fold (sra 0, x) -> 0
4133  if (N0C && N0C->isNullValue())
4134    return N0;
4135  // fold (sra -1, x) -> -1
4136  if (N0C && N0C->isAllOnesValue())
4137    return N0;
4138  // fold (sra x, (setge c, size(x))) -> undef
4139  if (N1C && N1C->getZExtValue() >= OpSizeInBits)
4140    return DAG.getUNDEF(VT);
4141  // fold (sra x, 0) -> x
4142  if (N1C && N1C->isNullValue())
4143    return N0;
4144  // fold (sra (shl x, c1), c1) -> sext_inreg for some c1 and target supports
4145  // sext_inreg.
4146  if (N1C && N0.getOpcode() == ISD::SHL && N1 == N0.getOperand(1)) {
4147    unsigned LowBits = OpSizeInBits - (unsigned)N1C->getZExtValue();
4148    EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), LowBits);
4149    if (VT.isVector())
4150      ExtVT = EVT::getVectorVT(*DAG.getContext(),
4151                               ExtVT, VT.getVectorNumElements());
4152    if ((!LegalOperations ||
4153         TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, ExtVT)))
4154      return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), VT,
4155                         N0.getOperand(0), DAG.getValueType(ExtVT));
4156  }
4157
4158  // fold (sra (sra x, c1), c2) -> (sra x, (add c1, c2))
4159  if (N1C && N0.getOpcode() == ISD::SRA) {
4160    if (ConstantSDNode *C1 = isConstOrConstSplat(N0.getOperand(1))) {
4161      unsigned Sum = N1C->getZExtValue() + C1->getZExtValue();
4162      if (Sum >= OpSizeInBits)
4163        Sum = OpSizeInBits - 1;
4164      return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0.getOperand(0),
4165                         DAG.getConstant(Sum, N1.getValueType()));
4166    }
4167  }
4168
4169  // fold (sra (shl X, m), (sub result_size, n))
4170  // -> (sign_extend (trunc (shl X, (sub (sub result_size, n), m)))) for
4171  // result_size - n != m.
4172  // If truncate is free for the target sext(shl) is likely to result in better
4173  // code.
4174  if (N0.getOpcode() == ISD::SHL && N1C) {
4175    // Get the two constanst of the shifts, CN0 = m, CN = n.
4176    const ConstantSDNode *N01C = isConstOrConstSplat(N0.getOperand(1));
4177    if (N01C) {
4178      LLVMContext &Ctx = *DAG.getContext();
4179      // Determine what the truncate's result bitsize and type would be.
4180      EVT TruncVT = EVT::getIntegerVT(Ctx, OpSizeInBits - N1C->getZExtValue());
4181
4182      if (VT.isVector())
4183        TruncVT = EVT::getVectorVT(Ctx, TruncVT, VT.getVectorNumElements());
4184
4185      // Determine the residual right-shift amount.
4186      signed ShiftAmt = N1C->getZExtValue() - N01C->getZExtValue();
4187
4188      // If the shift is not a no-op (in which case this should be just a sign
4189      // extend already), the truncated to type is legal, sign_extend is legal
4190      // on that type, and the truncate to that type is both legal and free,
4191      // perform the transform.
4192      if ((ShiftAmt > 0) &&
4193          TLI.isOperationLegalOrCustom(ISD::SIGN_EXTEND, TruncVT) &&
4194          TLI.isOperationLegalOrCustom(ISD::TRUNCATE, VT) &&
4195          TLI.isTruncateFree(VT, TruncVT)) {
4196
4197          SDValue Amt = DAG.getConstant(ShiftAmt,
4198              getShiftAmountTy(N0.getOperand(0).getValueType()));
4199          SDValue Shift = DAG.getNode(ISD::SRL, SDLoc(N0), VT,
4200                                      N0.getOperand(0), Amt);
4201          SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), TruncVT,
4202                                      Shift);
4203          return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N),
4204                             N->getValueType(0), Trunc);
4205      }
4206    }
4207  }
4208
4209  // fold (sra x, (trunc (and y, c))) -> (sra x, (and (trunc y), (trunc c))).
4210  if (N1.getOpcode() == ISD::TRUNCATE &&
4211      N1.getOperand(0).getOpcode() == ISD::AND) {
4212    SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode());
4213    if (NewOp1.getNode())
4214      return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0, NewOp1);
4215  }
4216
4217  // fold (sra (trunc (srl x, c1)), c2) -> (trunc (sra x, c1 + c2))
4218  //      if c1 is equal to the number of bits the trunc removes
4219  if (N0.getOpcode() == ISD::TRUNCATE &&
4220      (N0.getOperand(0).getOpcode() == ISD::SRL ||
4221       N0.getOperand(0).getOpcode() == ISD::SRA) &&
4222      N0.getOperand(0).hasOneUse() &&
4223      N0.getOperand(0).getOperand(1).hasOneUse() &&
4224      N1C) {
4225    SDValue N0Op0 = N0.getOperand(0);
4226    if (ConstantSDNode *LargeShift = isConstOrConstSplat(N0Op0.getOperand(1))) {
4227      unsigned LargeShiftVal = LargeShift->getZExtValue();
4228      EVT LargeVT = N0Op0.getValueType();
4229
4230      if (LargeVT.getScalarSizeInBits() - OpSizeInBits == LargeShiftVal) {
4231        SDValue Amt =
4232          DAG.getConstant(LargeShiftVal + N1C->getZExtValue(),
4233                          getShiftAmountTy(N0Op0.getOperand(0).getValueType()));
4234        SDValue SRA = DAG.getNode(ISD::SRA, SDLoc(N), LargeVT,
4235                                  N0Op0.getOperand(0), Amt);
4236        return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, SRA);
4237      }
4238    }
4239  }
4240
4241  // Simplify, based on bits shifted out of the LHS.
4242  if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
4243    return SDValue(N, 0);
4244
4245
4246  // If the sign bit is known to be zero, switch this to a SRL.
4247  if (DAG.SignBitIsZero(N0))
4248    return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, N1);
4249
4250  if (N1C) {
4251    SDValue NewSRA = visitShiftByConstant(N, N1C);
4252    if (NewSRA.getNode())
4253      return NewSRA;
4254  }
4255
4256  return SDValue();
4257}
4258
4259SDValue DAGCombiner::visitSRL(SDNode *N) {
4260  SDValue N0 = N->getOperand(0);
4261  SDValue N1 = N->getOperand(1);
4262  ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
4263  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
4264  EVT VT = N0.getValueType();
4265  unsigned OpSizeInBits = VT.getScalarType().getSizeInBits();
4266
4267  // fold vector ops
4268  if (VT.isVector()) {
4269    SDValue FoldedVOp = SimplifyVBinOp(N);
4270    if (FoldedVOp.getNode()) return FoldedVOp;
4271
4272    N1C = isConstOrConstSplat(N1);
4273  }
4274
4275  // fold (srl c1, c2) -> c1 >>u c2
4276  if (N0C && N1C)
4277    return DAG.FoldConstantArithmetic(ISD::SRL, VT, N0C, N1C);
4278  // fold (srl 0, x) -> 0
4279  if (N0C && N0C->isNullValue())
4280    return N0;
4281  // fold (srl x, c >= size(x)) -> undef
4282  if (N1C && N1C->getZExtValue() >= OpSizeInBits)
4283    return DAG.getUNDEF(VT);
4284  // fold (srl x, 0) -> x
4285  if (N1C && N1C->isNullValue())
4286    return N0;
4287  // if (srl x, c) is known to be zero, return 0
4288  if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0),
4289                                   APInt::getAllOnesValue(OpSizeInBits)))
4290    return DAG.getConstant(0, VT);
4291
4292  // fold (srl (srl x, c1), c2) -> 0 or (srl x, (add c1, c2))
4293  if (N1C && N0.getOpcode() == ISD::SRL) {
4294    if (ConstantSDNode *N01C = isConstOrConstSplat(N0.getOperand(1))) {
4295      uint64_t c1 = N01C->getZExtValue();
4296      uint64_t c2 = N1C->getZExtValue();
4297      if (c1 + c2 >= OpSizeInBits)
4298        return DAG.getConstant(0, VT);
4299      return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0.getOperand(0),
4300                         DAG.getConstant(c1 + c2, N1.getValueType()));
4301    }
4302  }
4303
4304  // fold (srl (trunc (srl x, c1)), c2) -> 0 or (trunc (srl x, (add c1, c2)))
4305  if (N1C && N0.getOpcode() == ISD::TRUNCATE &&
4306      N0.getOperand(0).getOpcode() == ISD::SRL &&
4307      isa<ConstantSDNode>(N0.getOperand(0)->getOperand(1))) {
4308    uint64_t c1 =
4309      cast<ConstantSDNode>(N0.getOperand(0)->getOperand(1))->getZExtValue();
4310    uint64_t c2 = N1C->getZExtValue();
4311    EVT InnerShiftVT = N0.getOperand(0).getValueType();
4312    EVT ShiftCountVT = N0.getOperand(0)->getOperand(1).getValueType();
4313    uint64_t InnerShiftSize = InnerShiftVT.getScalarType().getSizeInBits();
4314    // This is only valid if the OpSizeInBits + c1 = size of inner shift.
4315    if (c1 + OpSizeInBits == InnerShiftSize) {
4316      if (c1 + c2 >= InnerShiftSize)
4317        return DAG.getConstant(0, VT);
4318      return DAG.getNode(ISD::TRUNCATE, SDLoc(N0), VT,
4319                         DAG.getNode(ISD::SRL, SDLoc(N0), InnerShiftVT,
4320                                     N0.getOperand(0)->getOperand(0),
4321                                     DAG.getConstant(c1 + c2, ShiftCountVT)));
4322    }
4323  }
4324
4325  // fold (srl (shl x, c), c) -> (and x, cst2)
4326  if (N1C && N0.getOpcode() == ISD::SHL && N0.getOperand(1) == N1) {
4327    unsigned BitSize = N0.getScalarValueSizeInBits();
4328    if (BitSize <= 64) {
4329      uint64_t ShAmt = N1C->getZExtValue() + 64 - BitSize;
4330      return DAG.getNode(ISD::AND, SDLoc(N), VT, N0.getOperand(0),
4331                         DAG.getConstant(~0ULL >> ShAmt, VT));
4332    }
4333  }
4334
4335  // fold (srl (anyextend x), c) -> (and (anyextend (srl x, c)), mask)
4336  if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
4337    // Shifting in all undef bits?
4338    EVT SmallVT = N0.getOperand(0).getValueType();
4339    unsigned BitSize = SmallVT.getScalarSizeInBits();
4340    if (N1C->getZExtValue() >= BitSize)
4341      return DAG.getUNDEF(VT);
4342
4343    if (!LegalTypes || TLI.isTypeDesirableForOp(ISD::SRL, SmallVT)) {
4344      uint64_t ShiftAmt = N1C->getZExtValue();
4345      SDValue SmallShift = DAG.getNode(ISD::SRL, SDLoc(N0), SmallVT,
4346                                       N0.getOperand(0),
4347                          DAG.getConstant(ShiftAmt, getShiftAmountTy(SmallVT)));
4348      AddToWorkList(SmallShift.getNode());
4349      APInt Mask = APInt::getAllOnesValue(OpSizeInBits).lshr(ShiftAmt);
4350      return DAG.getNode(ISD::AND, SDLoc(N), VT,
4351                         DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, SmallShift),
4352                         DAG.getConstant(Mask, VT));
4353    }
4354  }
4355
4356  // fold (srl (sra X, Y), 31) -> (srl X, 31).  This srl only looks at the sign
4357  // bit, which is unmodified by sra.
4358  if (N1C && N1C->getZExtValue() + 1 == OpSizeInBits) {
4359    if (N0.getOpcode() == ISD::SRA)
4360      return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0.getOperand(0), N1);
4361  }
4362
4363  // fold (srl (ctlz x), "5") -> x  iff x has one bit set (the low bit).
4364  if (N1C && N0.getOpcode() == ISD::CTLZ &&
4365      N1C->getAPIntValue() == Log2_32(OpSizeInBits)) {
4366    APInt KnownZero, KnownOne;
4367    DAG.computeKnownBits(N0.getOperand(0), KnownZero, KnownOne);
4368
4369    // If any of the input bits are KnownOne, then the input couldn't be all
4370    // zeros, thus the result of the srl will always be zero.
4371    if (KnownOne.getBoolValue()) return DAG.getConstant(0, VT);
4372
4373    // If all of the bits input the to ctlz node are known to be zero, then
4374    // the result of the ctlz is "32" and the result of the shift is one.
4375    APInt UnknownBits = ~KnownZero;
4376    if (UnknownBits == 0) return DAG.getConstant(1, VT);
4377
4378    // Otherwise, check to see if there is exactly one bit input to the ctlz.
4379    if ((UnknownBits & (UnknownBits - 1)) == 0) {
4380      // Okay, we know that only that the single bit specified by UnknownBits
4381      // could be set on input to the CTLZ node. If this bit is set, the SRL
4382      // will return 0, if it is clear, it returns 1. Change the CTLZ/SRL pair
4383      // to an SRL/XOR pair, which is likely to simplify more.
4384      unsigned ShAmt = UnknownBits.countTrailingZeros();
4385      SDValue Op = N0.getOperand(0);
4386
4387      if (ShAmt) {
4388        Op = DAG.getNode(ISD::SRL, SDLoc(N0), VT, Op,
4389                  DAG.getConstant(ShAmt, getShiftAmountTy(Op.getValueType())));
4390        AddToWorkList(Op.getNode());
4391      }
4392
4393      return DAG.getNode(ISD::XOR, SDLoc(N), VT,
4394                         Op, DAG.getConstant(1, VT));
4395    }
4396  }
4397
4398  // fold (srl x, (trunc (and y, c))) -> (srl x, (and (trunc y), (trunc c))).
4399  if (N1.getOpcode() == ISD::TRUNCATE &&
4400      N1.getOperand(0).getOpcode() == ISD::AND) {
4401    SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode());
4402    if (NewOp1.getNode())
4403      return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0, NewOp1);
4404  }
4405
4406  // fold operands of srl based on knowledge that the low bits are not
4407  // demanded.
4408  if (N1C && SimplifyDemandedBits(SDValue(N, 0)))
4409    return SDValue(N, 0);
4410
4411  if (N1C) {
4412    SDValue NewSRL = visitShiftByConstant(N, N1C);
4413    if (NewSRL.getNode())
4414      return NewSRL;
4415  }
4416
4417  // Attempt to convert a srl of a load into a narrower zero-extending load.
4418  SDValue NarrowLoad = ReduceLoadWidth(N);
4419  if (NarrowLoad.getNode())
4420    return NarrowLoad;
4421
4422  // Here is a common situation. We want to optimize:
4423  //
4424  //   %a = ...
4425  //   %b = and i32 %a, 2
4426  //   %c = srl i32 %b, 1
4427  //   brcond i32 %c ...
4428  //
4429  // into
4430  //
4431  //   %a = ...
4432  //   %b = and %a, 2
4433  //   %c = setcc eq %b, 0
4434  //   brcond %c ...
4435  //
4436  // However when after the source operand of SRL is optimized into AND, the SRL
4437  // itself may not be optimized further. Look for it and add the BRCOND into
4438  // the worklist.
4439  if (N->hasOneUse()) {
4440    SDNode *Use = *N->use_begin();
4441    if (Use->getOpcode() == ISD::BRCOND)
4442      AddToWorkList(Use);
4443    else if (Use->getOpcode() == ISD::TRUNCATE && Use->hasOneUse()) {
4444      // Also look pass the truncate.
4445      Use = *Use->use_begin();
4446      if (Use->getOpcode() == ISD::BRCOND)
4447        AddToWorkList(Use);
4448    }
4449  }
4450
4451  return SDValue();
4452}
4453
4454SDValue DAGCombiner::visitCTLZ(SDNode *N) {
4455  SDValue N0 = N->getOperand(0);
4456  EVT VT = N->getValueType(0);
4457
4458  // fold (ctlz c1) -> c2
4459  if (isa<ConstantSDNode>(N0))
4460    return DAG.getNode(ISD::CTLZ, SDLoc(N), VT, N0);
4461  return SDValue();
4462}
4463
4464SDValue DAGCombiner::visitCTLZ_ZERO_UNDEF(SDNode *N) {
4465  SDValue N0 = N->getOperand(0);
4466  EVT VT = N->getValueType(0);
4467
4468  // fold (ctlz_zero_undef c1) -> c2
4469  if (isa<ConstantSDNode>(N0))
4470    return DAG.getNode(ISD::CTLZ_ZERO_UNDEF, SDLoc(N), VT, N0);
4471  return SDValue();
4472}
4473
4474SDValue DAGCombiner::visitCTTZ(SDNode *N) {
4475  SDValue N0 = N->getOperand(0);
4476  EVT VT = N->getValueType(0);
4477
4478  // fold (cttz c1) -> c2
4479  if (isa<ConstantSDNode>(N0))
4480    return DAG.getNode(ISD::CTTZ, SDLoc(N), VT, N0);
4481  return SDValue();
4482}
4483
4484SDValue DAGCombiner::visitCTTZ_ZERO_UNDEF(SDNode *N) {
4485  SDValue N0 = N->getOperand(0);
4486  EVT VT = N->getValueType(0);
4487
4488  // fold (cttz_zero_undef c1) -> c2
4489  if (isa<ConstantSDNode>(N0))
4490    return DAG.getNode(ISD::CTTZ_ZERO_UNDEF, SDLoc(N), VT, N0);
4491  return SDValue();
4492}
4493
4494SDValue DAGCombiner::visitCTPOP(SDNode *N) {
4495  SDValue N0 = N->getOperand(0);
4496  EVT VT = N->getValueType(0);
4497
4498  // fold (ctpop c1) -> c2
4499  if (isa<ConstantSDNode>(N0))
4500    return DAG.getNode(ISD::CTPOP, SDLoc(N), VT, N0);
4501  return SDValue();
4502}
4503
4504SDValue DAGCombiner::visitSELECT(SDNode *N) {
4505  SDValue N0 = N->getOperand(0);
4506  SDValue N1 = N->getOperand(1);
4507  SDValue N2 = N->getOperand(2);
4508  ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
4509  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
4510  ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
4511  EVT VT = N->getValueType(0);
4512  EVT VT0 = N0.getValueType();
4513
4514  // fold (select C, X, X) -> X