X86ISelDAGToDAG.cpp revision f522068412218cd14b2c2df74a3437717d255381
1800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany//===- X86ISelDAGToDAG.cpp - A DAG pattern matching inst selector for X86 -===// 2800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany// 3800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany// The LLVM Compiler Infrastructure 4800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany// 5800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany// This file is distributed under the University of Illinois Open Source 6800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany// License. See LICENSE.TXT for details. 7800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany// 8800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany//===----------------------------------------------------------------------===// 9800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany// 10800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany// This file defines a DAG pattern matching instruction selector for X86, 11800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany// converting from a legalized dag to a X86 dag. 12800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany// 13800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany//===----------------------------------------------------------------------===// 14800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 15800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany#define DEBUG_TYPE "x86-isel" 16800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany#include "X86.h" 17800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany#include "X86InstrBuilder.h" 18d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth#include "X86ISelLowering.h" 19800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany#include "X86MachineFunctionInfo.h" 201c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov#include "X86RegisterInfo.h" 2159cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov#include "X86Subtarget.h" 22800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany#include "X86TargetMachine.h" 23800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany#include "llvm/GlobalValue.h" 24800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany#include "llvm/Instructions.h" 25800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany#include "llvm/Intrinsics.h" 26800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany#include "llvm/Support/CFG.h" 2706fdbaa9145f01a291d4981ca5120b7bdcad44c6Evgeniy Stepanov#include "llvm/Type.h" 281afbb517965e29b07cb42e2335d5eadd87de6535Alexey Samsonov#include "llvm/CodeGen/MachineConstantPool.h" 290b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/CodeGen/MachineFunction.h" 300b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/CodeGen/MachineFrameInfo.h" 310b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/CodeGen/MachineInstrBuilder.h" 320b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/CodeGen/MachineRegisterInfo.h" 330b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/CodeGen/SelectionDAGISel.h" 340b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/Target/TargetMachine.h" 350b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/Target/TargetOptions.h" 360b8c9a80f20772c3793201ab5b251d3520b9cea3Chandler Carruth#include "llvm/Support/Compiler.h" 3759cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov#include "llvm/Support/Debug.h" 38800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany#include "llvm/Support/MathExtras.h" 39800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany#include "llvm/Support/Streams.h" 40800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany#include "llvm/ADT/SmallPtrSet.h" 41800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany#include "llvm/ADT/Statistic.h" 42800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryanyusing namespace llvm; 43800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 44800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya SerebryanySTATISTIC(NumFPKill , "Number of FP_REG_KILL instructions added"); 4590230c84668269fbd53d163e398cd16486d5d414Chandler CarruthSTATISTIC(NumLoadMoved, "Number of loads moved below TokenFactor"); 461afbb517965e29b07cb42e2335d5eadd87de6535Alexey Samsonov 47800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany//===----------------------------------------------------------------------===// 48800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany// Pattern Matcher Implementation 49d04a8d4b33ff316ca4cf961e06c9e312eff8e64fChandler Carruth//===----------------------------------------------------------------------===// 50800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 51800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryanynamespace { 52800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany /// X86ISelAddressMode - This corresponds to X86AddressMode, but uses 53800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany /// SDValue's instead of register numbers for the leaves of the matched 54800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany /// tree. 55800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany struct X86ISelAddressMode { 5648a615fee78c3c262c60147a65dc6fff5fd6bb3bKostya Serebryany enum { 57800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany RegBase, 58800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany FrameIndexBase 59800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany } BaseType; 60800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 61800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany struct { // This is really a union, discriminated by BaseType! 62800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDValue Reg; 637bcfc9950bac0f411f9671e8d6ce483bd219727eKostya Serebryany int FrameIndex; 647bcfc9950bac0f411f9671e8d6ce483bd219727eKostya Serebryany } Base; 65800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 66800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany bool isRIPRel; // RIP as base? 677bcfc9950bac0f411f9671e8d6ce483bd219727eKostya Serebryany unsigned Scale; 689b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany SDValue IndexReg; 699b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany unsigned Disp; 70800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany GlobalValue *GV; 7195e3cf44a9d4672655dee3bd558bfeefa631dd55Kostya Serebryany Constant *CP; 72800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany const char *ES; 73800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany int JT; 74800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany unsigned Align; // CP alignment. 75800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 7651c7c65e32f76ec5a50cdecfe2b4c287c57da127Kostya Serebryany X86ISelAddressMode() 77f985f44b13681071e585acb7a5703a2c1c23b6ceAlexey Samsonov : BaseType(RegBase), isRIPRel(false), Scale(1), IndexReg(), Disp(0), 78f985f44b13681071e585acb7a5703a2c1c23b6ceAlexey Samsonov GV(0), CP(0), ES(0), JT(-1), Align(0) { 79f985f44b13681071e585acb7a5703a2c1c23b6ceAlexey Samsonov } 80800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany void dump() { 81800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany cerr << "X86ISelAddressMode " << this << "\n"; 82800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany cerr << "Base.Reg "; 83800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (Base.Reg.getNode() != 0) Base.Reg.getNode()->dump(); 84800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany else cerr << "nul"; 85800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany cerr << " Base.FrameIndex " << Base.FrameIndex << "\n"; 86c0ed3e548c6f688e22685de04e210c7b59ac3433Kostya Serebryany cerr << "isRIPRel " << isRIPRel << " Scale" << Scale << "\n"; 87c0ed3e548c6f688e22685de04e210c7b59ac3433Kostya Serebryany cerr << "IndexReg "; 88c0ed3e548c6f688e22685de04e210c7b59ac3433Kostya Serebryany if (IndexReg.getNode() != 0) IndexReg.getNode()->dump(); 89800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany else cerr << "nul"; 90800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany cerr << " Disp " << Disp << "\n"; 91800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany cerr << "GV "; if (GV) GV->dump(); 92800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany else cerr << "nul"; 93800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany cerr << " CP "; if (CP) CP->dump(); 94800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany else cerr << "nul"; 95800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany cerr << "\n"; 96e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany cerr << "ES "; if (ES) cerr << ES; else cerr << "nul"; 97e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany cerr << " JT" << JT << " Align" << Align << "\n"; 98e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany } 996e2d506dc962873a0e05092bbb034f9a615d1084Kostya Serebryany }; 1006e2d506dc962873a0e05092bbb034f9a615d1084Kostya Serebryany} 1016e2d506dc962873a0e05092bbb034f9a615d1084Kostya Serebryany 102c0ed3e548c6f688e22685de04e210c7b59ac3433Kostya Serebryanynamespace { 103324cbb89f2759fb9ad347bd2af4631e50c39c8f3Kostya Serebryany //===--------------------------------------------------------------------===// 104324cbb89f2759fb9ad347bd2af4631e50c39c8f3Kostya Serebryany /// ISel - X86 specific code to select X86 machine instructions for 105324cbb89f2759fb9ad347bd2af4631e50c39c8f3Kostya Serebryany /// SelectionDAG operations. 106324cbb89f2759fb9ad347bd2af4631e50c39c8f3Kostya Serebryany /// 107324cbb89f2759fb9ad347bd2af4631e50c39c8f3Kostya Serebryany class VISIBILITY_HIDDEN X86DAGToDAGISel : public SelectionDAGISel { 108324cbb89f2759fb9ad347bd2af4631e50c39c8f3Kostya Serebryany /// TM - Keep a reference to X86TargetMachine. 109324cbb89f2759fb9ad347bd2af4631e50c39c8f3Kostya Serebryany /// 110800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany X86TargetMachine &TM; 111800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 112800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany /// X86Lowering - This object fully describes how to lower LLVM code to an 113800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany /// X86-specific SelectionDAG. 114800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany X86TargetLowering &X86Lowering; 115800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 116800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany /// Subtarget - Keep a pointer to the X86Subtarget around so that we can 117800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany /// make the right decision when generating code for different targets. 118800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany const X86Subtarget *Subtarget; 1199b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany 1209b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany /// CurBB - Current BB being isel'd. 121800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany /// 122800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany MachineBasicBlock *CurBB; 1236c55412ea4b39103a3a5764d49ddfdf50e066d56Kostya Serebryany 1246c55412ea4b39103a3a5764d49ddfdf50e066d56Kostya Serebryany /// OptForSize - If true, selector should try to optimize for code size 125b0dcf61252e58715a3bea79f4c112572df361c30Alexey Samsonov /// instead of performance. 126b0dcf61252e58715a3bea79f4c112572df361c30Alexey Samsonov bool OptForSize; 127800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 128800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany public: 129800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany X86DAGToDAGISel(X86TargetMachine &tm, bool fast) 130800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany : SelectionDAGISel(*tm.getTargetLowering(), fast), 131800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany TM(tm), X86Lowering(*TM.getTargetLowering()), 132800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany Subtarget(&TM.getSubtarget<X86Subtarget>()), 133800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany OptForSize(false) {} 134800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 135800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany virtual const char *getPassName() const { 136800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany return "X86 DAG->DAG Instruction Selection"; 137800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany } 138800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 139800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany /// InstructionSelect - This callback is invoked by 140800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany /// SelectionDAGISel when it has created a SelectionDAG for us to codegen. 141800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany virtual void InstructionSelect(); 142800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 143800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany /// InstructionSelectPostProcessing - Post processing of selected and 144800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany /// scheduled basic blocks. 145800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany virtual void InstructionSelectPostProcessing(); 146800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 147ee548275c63a1eeffda9d3edd2bea04e1dadcc67Alexey Samsonov virtual void EmitFunctionEntryCode(Function &Fn, MachineFunction &MF); 148ee548275c63a1eeffda9d3edd2bea04e1dadcc67Alexey Samsonov 149ee548275c63a1eeffda9d3edd2bea04e1dadcc67Alexey Samsonov virtual bool CanBeFoldedBy(SDNode *N, SDNode *U, SDNode *Root) const; 150ee548275c63a1eeffda9d3edd2bea04e1dadcc67Alexey Samsonov 151800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany// Include the pieces autogenerated from the target description. 152800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany#include "X86GenDAGISel.inc" 153800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 154800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany private: 155800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDNode *Select(SDValue N); 156800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDNode *SelectAtomic64(SDNode *Node, unsigned Opc); 157800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 158800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany bool MatchAddress(SDValue N, X86ISelAddressMode &AM, 159800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany bool isRoot = true, unsigned Depth = 0); 160800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany bool MatchAddressBase(SDValue N, X86ISelAddressMode &AM, 161800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany bool isRoot, unsigned Depth); 162800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany bool SelectAddr(SDValue Op, SDValue N, SDValue &Base, 163800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDValue &Scale, SDValue &Index, SDValue &Disp); 164ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryany bool SelectLEAAddr(SDValue Op, SDValue N, SDValue &Base, 165ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryany SDValue &Scale, SDValue &Index, SDValue &Disp); 166ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryany bool SelectScalarSSELoad(SDValue Op, SDValue Pred, 167ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryany SDValue N, SDValue &Base, SDValue &Scale, 168ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryany SDValue &Index, SDValue &Disp, 169ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryany SDValue &InChain, SDValue &OutChain); 170ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryany bool TryFoldLoad(SDValue P, SDValue N, 171ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryany SDValue &Base, SDValue &Scale, 172ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryany SDValue &Index, SDValue &Disp); 173ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryany void PreprocessForRMW(); 174ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryany void PreprocessForFPConvert(); 175ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryany 176ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryany /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for 177ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryany /// inline asm expressions. 178ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryany virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, 179ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryany char ConstraintCode, 180ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryany std::vector<SDValue> &OutOps); 181ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryany 182ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryany void EmitSpecialCodeForMain(MachineBasicBlock *BB, MachineFrameInfo *MFI); 183ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryany 184ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryany inline void getAddressOperands(X86ISelAddressMode &AM, SDValue &Base, 185ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryany SDValue &Scale, SDValue &Index, 186ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryany SDValue &Disp) { 187ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryany Base = (AM.BaseType == X86ISelAddressMode::FrameIndexBase) ? 188ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryany CurDAG->getTargetFrameIndex(AM.Base.FrameIndex, TLI.getPointerTy()) : 18919cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov AM.Base.Reg; 19048a615fee78c3c262c60147a65dc6fff5fd6bb3bKostya Serebryany Scale = getI8Imm(AM.Scale); 19119cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov Index = AM.IndexReg; 19219cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov // These are 32-bit even in 64-bit mode since RIP relative offset 19319cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov // is 32-bit. 19448a615fee78c3c262c60147a65dc6fff5fd6bb3bKostya Serebryany if (AM.GV) 19519cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov Disp = CurDAG->getTargetGlobalAddress(AM.GV, MVT::i32, AM.Disp); 19619cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov else if (AM.CP) 19711af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i32, 19811af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov AM.Align, AM.Disp); 19911af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov else if (AM.ES) 20011af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov Disp = CurDAG->getTargetExternalSymbol(AM.ES, MVT::i32); 20148a615fee78c3c262c60147a65dc6fff5fd6bb3bKostya Serebryany else if (AM.JT != -1) 20219cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i32); 20319cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov else 20419cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov Disp = getI32Imm(AM.Disp); 20548a615fee78c3c262c60147a65dc6fff5fd6bb3bKostya Serebryany } 20648a615fee78c3c262c60147a65dc6fff5fd6bb3bKostya Serebryany 20748a615fee78c3c262c60147a65dc6fff5fd6bb3bKostya Serebryany /// getI8Imm - Return a target constant with the specified value, of type 20848a615fee78c3c262c60147a65dc6fff5fd6bb3bKostya Serebryany /// i8. 20948a615fee78c3c262c60147a65dc6fff5fd6bb3bKostya Serebryany inline SDValue getI8Imm(unsigned Imm) { 21011af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov return CurDAG->getTargetConstant(Imm, MVT::i8); 21148a615fee78c3c262c60147a65dc6fff5fd6bb3bKostya Serebryany } 21248a615fee78c3c262c60147a65dc6fff5fd6bb3bKostya Serebryany 21319cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov /// getI16Imm - Return a target constant with the specified value, of type 21419cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov /// i16. 21519cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov inline SDValue getI16Imm(unsigned Imm) { 21619cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov return CurDAG->getTargetConstant(Imm, MVT::i16); 21719cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov } 21819cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov 21919cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov /// getI32Imm - Return a target constant with the specified value, of type 22019cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov /// i32. 22119cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov inline SDValue getI32Imm(unsigned Imm) { 22219cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov return CurDAG->getTargetConstant(Imm, MVT::i32); 22319cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov } 224b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany 225b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany /// getGlobalBaseReg - Return an SDNode that returns the value of 22619cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov /// the global base register. Output instructions required to 227b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany /// initialize the global base register, if necessary. 228b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany /// 22919cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov SDNode *getGlobalBaseReg(); 230b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany 231ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryany /// getTruncateTo8Bit - return an SDNode that implements a subreg based 232800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany /// truncate of the specified operand to i8. This can be done with tablegen, 233ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryany /// except that this code uses MVT::Flag in a tricky way that happens to 234ee548275c63a1eeffda9d3edd2bea04e1dadcc67Alexey Samsonov /// improve scheduling in some cases. 235ee548275c63a1eeffda9d3edd2bea04e1dadcc67Alexey Samsonov SDNode *getTruncateTo8Bit(SDValue N0); 236b0dcf61252e58715a3bea79f4c112572df361c30Alexey Samsonov 23711af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov#ifndef NDEBUG 23811af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov unsigned Indent; 239ee548275c63a1eeffda9d3edd2bea04e1dadcc67Alexey Samsonov#endif 240ee548275c63a1eeffda9d3edd2bea04e1dadcc67Alexey Samsonov }; 241ee548275c63a1eeffda9d3edd2bea04e1dadcc67Alexey Samsonov} 242b0dcf61252e58715a3bea79f4c112572df361c30Alexey Samsonov 243b0dcf61252e58715a3bea79f4c112572df361c30Alexey Samsonov/// findFlagUse - Return use of MVT::Flag value produced by the specified 24411af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov/// SDNode. 24511af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov/// 2461416edc30adbd53b2be863f7f3de56de4a4c9d0aKostya Serebryanystatic SDNode *findFlagUse(SDNode *N) { 2471416edc30adbd53b2be863f7f3de56de4a4c9d0aKostya Serebryany unsigned FlagResNo = N->getNumValues()-1; 2481416edc30adbd53b2be863f7f3de56de4a4c9d0aKostya Serebryany for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) { 249ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryany SDNode *User = *I; 250ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryany for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) { 251800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDValue Op = User->getOperand(i); 252c0ed3e548c6f688e22685de04e210c7b59ac3433Kostya Serebryany if (Op.getNode() == N && Op.getResNo() == FlagResNo) 253c0ed3e548c6f688e22685de04e210c7b59ac3433Kostya Serebryany return User; 254ebd6454929f2ba3b92f61c151eccde0b8b0a8dedKostya Serebryany } 2552735cf4aa52e31b8d2de90f836c3ad991215e04eKostya Serebryany } 256ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryany return NULL; 257ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryany} 258c0ed3e548c6f688e22685de04e210c7b59ac3433Kostya Serebryany 259800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany/// findNonImmUse - Return true by reference in "found" if "Use" is an 260800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany/// non-immediate use of "Def". This function recursively traversing 261ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryany/// up the operand chain ignoring certain nodes. 2629b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryanystatic void findNonImmUse(SDNode *Use, SDNode* Def, SDNode *ImmedUse, 2639b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany SDNode *Root, bool &found, 264a1a8a323f43a95620fc4c6b5aad8d5ff5e5eb590Kostya Serebryany SmallPtrSet<SDNode*, 16> &Visited) { 26519cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov if (found || 266ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryany Use->getNodeId() < Def->getNodeId() || 267800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany !Visited.insert(Use)) 268800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany return; 269800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 2708b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany for (unsigned i = 0, e = Use->getNumOperands(); !found && i != e; ++i) { 271800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDNode *N = Use->getOperand(i).getNode(); 2729b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany if (N == Def) { 2735a3a9c937198084498a196dae856ac5a5a005bccKostya Serebryany if (Use == ImmedUse || Use == Root) 2749b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany continue; // We are not looking for immediate use. 275800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany assert(N != Root); 276ee548275c63a1eeffda9d3edd2bea04e1dadcc67Alexey Samsonov found = true; 277ee548275c63a1eeffda9d3edd2bea04e1dadcc67Alexey Samsonov break; 278ee548275c63a1eeffda9d3edd2bea04e1dadcc67Alexey Samsonov } 27911af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov 28011af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov // Traverse up the operand chain. 28111af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov findNonImmUse(N, Def, ImmedUse, Root, found, Visited); 282800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany } 2833574eca1b02600bac4e625297f4ecf745f4c4f32Micah Villmow} 284800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 285800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany/// isNonImmUse - Start searching from Root up the DAG to check is Def can 28619cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov/// be reached. Return true if that's the case. However, ignore direct uses 287800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany/// by ImmedUse (which would be U in the example illustrated in 288800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany/// CanBeFoldedBy) and by Root (which can happen in the store case). 289ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryany/// FIXME: to be really generic, we should allow direct use by any node 290b5b86d263a651566cb25c0f406f75ceffb771029Kostya Serebryany/// that is being folded. But realisticly since we only fold loads which 2919db5b5ffa9fccd5c7f1f39a3e9aa66cc4a5eedc1Kostya Serebryany/// have one non-chain use, we only need to watch out for load/op/store 2929db5b5ffa9fccd5c7f1f39a3e9aa66cc4a5eedc1Kostya Serebryany/// and load/op/cmp case where the root (store / cmp) may reach the load via 293f7b08226eb44458f6f38cbeaca527028803c725aKostya Serebryany/// its chain operand. 294ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryanystatic inline bool isNonImmUse(SDNode *Root, SDNode *Def, SDNode *ImmedUse) { 29559cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov SmallPtrSet<SDNode*, 16> Visited; 29659cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov bool found = false; 297800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany findNonImmUse(Root, Def, ImmedUse, Root, found, Visited); 298c0ed3e548c6f688e22685de04e210c7b59ac3433Kostya Serebryany return found; 2991416edc30adbd53b2be863f7f3de56de4a4c9d0aKostya Serebryany} 300b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany 301b0dcf61252e58715a3bea79f4c112572df361c30Alexey Samsonov 30211af9a873f9e1409a422ab31e22729368805afafAlexey Samsonovbool X86DAGToDAGISel::CanBeFoldedBy(SDNode *N, SDNode *U, SDNode *Root) const { 30311af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov if (Fast) return false; 304ee548275c63a1eeffda9d3edd2bea04e1dadcc67Alexey Samsonov 305b0dcf61252e58715a3bea79f4c112572df361c30Alexey Samsonov // If Root use can somehow reach N through a path that that doesn't contain 306b0dcf61252e58715a3bea79f4c112572df361c30Alexey Samsonov // U then folding N would create a cycle. e.g. In the following 30711af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov // diagram, Root can reach N through X. If N is folded into into Root, then 30811af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov // X is both a predecessor and a successor of U. 3091416edc30adbd53b2be863f7f3de56de4a4c9d0aKostya Serebryany // 3101416edc30adbd53b2be863f7f3de56de4a4c9d0aKostya Serebryany // [N*] // 3111416edc30adbd53b2be863f7f3de56de4a4c9d0aKostya Serebryany // ^ ^ // 3121416edc30adbd53b2be863f7f3de56de4a4c9d0aKostya Serebryany // / \ // 3131416edc30adbd53b2be863f7f3de56de4a4c9d0aKostya Serebryany // [U*] [X]? // 314f985f44b13681071e585acb7a5703a2c1c23b6ceAlexey Samsonov // ^ ^ // 315b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany // \ / // 3164684858624d7ffe82379783e9b678227d5e0b515Alexey Samsonov // \ / // 3174684858624d7ffe82379783e9b678227d5e0b515Alexey Samsonov // [Root*] // 318b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany // 319b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany // * indicates nodes to be folded together. 320b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany // 32119cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov // If Root produces a flag, then it gets (even more) interesting. Since it 32219cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov // will be "glued" together with its flag use in the scheduler, we need to 32319cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov // check if it might reach N. 324b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany // 325ee548275c63a1eeffda9d3edd2bea04e1dadcc67Alexey Samsonov // [N*] // 326b0dcf61252e58715a3bea79f4c112572df361c30Alexey Samsonov // ^ ^ // 32711af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov // / \ // 32811af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov // [U*] [X]? // 329b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany // ^ ^ // 330b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany // \ \ // 331b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany // \ | // 332b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany // [Root*] | // 3331416edc30adbd53b2be863f7f3de56de4a4c9d0aKostya Serebryany // ^ | // 33419cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov // f | // 3354684858624d7ffe82379783e9b678227d5e0b515Alexey Samsonov // | / // 3364684858624d7ffe82379783e9b678227d5e0b515Alexey Samsonov // [Y] / // 3374684858624d7ffe82379783e9b678227d5e0b515Alexey Samsonov // ^ / // 3384684858624d7ffe82379783e9b678227d5e0b515Alexey Samsonov // f / // 339b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany // | / // 340b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany // [FU] // 34159cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov // 34259cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov // If FU (flag use) indirectly reaches N (the load), and Root folds N 34359cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov // (call it Fold), then X is a predecessor of FU and a successor of 34459cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov // Fold. But since Fold and FU are flagged together, this will create 34559cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov // a cycle in the scheduling graph. 34659cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov 34759cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov MVT VT = Root->getValueType(Root->getNumValues()-1); 34859cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov while (VT == MVT::Flag) { 34959cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov SDNode *FU = findFlagUse(Root); 35059cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov if (FU == NULL) 35159cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov break; 35259cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov Root = FU; 35359cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov VT = Root->getValueType(Root->getNumValues()-1); 35459cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov } 35559cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov 35659cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov return !isNonImmUse(Root, N, U); 35719cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov} 35859cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov 35959cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// MoveBelowTokenFactor - Replace TokenFactor operand with load's chain operand 36059cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// and move load below the TokenFactor. Replace store's chain operand with 36159cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// load's chain result. 36259cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonovstatic void MoveBelowTokenFactor(SelectionDAG *CurDAG, SDValue Load, 36359cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov SDValue Store, SDValue TF) { 36459cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov SmallVector<SDValue, 4> Ops; 36559cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov for (unsigned i = 0, e = TF.getNode()->getNumOperands(); i != e; ++i) 36659cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov if (Load.getNode() == TF.getOperand(i).getNode()) 3671c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov Ops.push_back(Load.getOperand(0)); 3681c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov else 3691c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov Ops.push_back(TF.getOperand(i)); 3701c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov CurDAG->UpdateNodeOperands(TF, &Ops[0], Ops.size()); 3711c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov CurDAG->UpdateNodeOperands(Load, TF, Load.getOperand(1), Load.getOperand(2)); 3721c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov CurDAG->UpdateNodeOperands(Store, Load.getValue(1), Store.getOperand(1), 3731c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov Store.getOperand(2), Store.getOperand(3)); 3741c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov} 3751c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov 3761c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov/// isRMWLoad - Return true if N is a load that's part of RMW sub-DAG. 3771c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov/// 3781c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonovstatic bool isRMWLoad(SDValue N, SDValue Chain, SDValue Address, 37959cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov SDValue &Load) { 38059cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov if (N.getOpcode() == ISD::BIT_CONVERT) 38159cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov N = N.getOperand(0); 38219cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov 38319cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov LoadSDNode *LD = dyn_cast<LoadSDNode>(N); 38459cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov if (!LD || LD->isVolatile()) 38559cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov return false; 38659cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov if (LD->getAddressingMode() != ISD::UNINDEXED) 38759cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov return false; 38859cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov 38959cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov ISD::LoadExtType ExtType = LD->getExtensionType(); 39059cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov if (ExtType != ISD::NON_EXTLOAD && ExtType != ISD::EXTLOAD) 39159cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov return false; 39259cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov 39359cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov if (N.hasOneUse() && 39459cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov N.getOperand(1) == Address && 39559cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov N.getNode()->isOperandOf(Chain.getNode())) { 39659cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov Load = N; 39759cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov return true; 39859cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov } 39959cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov return false; 40059cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov} 40159cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov 40259cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// MoveBelowCallSeqStart - Replace CALLSEQ_START operand with load's chain 40359cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// operand and move load below the call's chain operand. 40459cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonovstatic void MoveBelowCallSeqStart(SelectionDAG *CurDAG, SDValue Load, 40559cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov SDValue Call, SDValue Chain) { 40659cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov SmallVector<SDValue, 8> Ops; 40759cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov for (unsigned i = 0, e = Chain.getNode()->getNumOperands(); i != e; ++i) 40859cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov if (Load.getNode() == Chain.getOperand(i).getNode()) 40959cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov Ops.push_back(Load.getOperand(0)); 41059cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov else 41159cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov Ops.push_back(Chain.getOperand(i)); 41259cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov CurDAG->UpdateNodeOperands(Chain, &Ops[0], Ops.size()); 41359cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov CurDAG->UpdateNodeOperands(Load, Call.getOperand(0), 41459cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov Load.getOperand(1), Load.getOperand(2)); 41559cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov Ops.clear(); 41659cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov Ops.push_back(SDValue(Load.getNode(), 1)); 41759cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov for (unsigned i = 1, e = Call.getNode()->getNumOperands(); i != e; ++i) 4181c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov Ops.push_back(Call.getOperand(i)); 41959cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov CurDAG->UpdateNodeOperands(Call, &Ops[0], Ops.size()); 42059cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov} 42159cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov 42259cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// isCalleeLoad - Return true if call address is a load and it can be 42359cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// moved below CALLSEQ_START and the chains leading up to the call. 42459cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// Return the CALLSEQ_START by reference as a second output. 42559cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonovstatic bool isCalleeLoad(SDValue Callee, SDValue &Chain) { 4261c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov if (Callee.getNode() == Chain.getNode() || !Callee.hasOneUse()) 4271c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov return false; 4281c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov LoadSDNode *LD = dyn_cast<LoadSDNode>(Callee.getNode()); 4291c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov if (!LD || 4301c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov LD->isVolatile() || 4311c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov LD->getAddressingMode() != ISD::UNINDEXED || 4321c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov LD->getExtensionType() != ISD::NON_EXTLOAD) 4331c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov return false; 4341c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov 4351c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov // Now let's find the callseq_start. 4361c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov while (Chain.getOpcode() != ISD::CALLSEQ_START) { 4371c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov if (!Chain.hasOneUse()) 4381c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov return false; 4391c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov Chain = Chain.getOperand(0); 4401c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov } 4411c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov return Chain.getOperand(0).getNode() == Callee.getNode(); 4421c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov} 4431c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov 4441c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov 4451c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov/// PreprocessForRMW - Preprocess the DAG to make instruction selection better. 4461c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov/// This is only run if not in -fast mode (aka -O0). 4471c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov/// This allows the instruction selector to pick more read-modify-write 4481c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov/// instructions. This is a common case: 4491c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov/// 4501c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov/// [Load chain] 4511c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov/// ^ 45259cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// | 45359cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// [Load] 45459cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// ^ ^ 4551c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov/// | | 4561c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov/// / \- 4571c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov/// / | 4581c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov/// [TokenFactor] [Op] 4591c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov/// ^ ^ 4601c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov/// | | 4611c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov/// \ / 46219cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov/// \ / 46319cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov/// [Store] 46419cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov/// 46559cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// The fact the store's chain operand != load's chain will prevent the 46659cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// (store (op (load))) instruction from being selected. We can transform it to: 46759cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// 46859cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// [Load chain] 46959cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// ^ 47059cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// | 47159cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// [TokenFactor] 47259cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// ^ 47359cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// | 47459cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// [Load] 47559cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// ^ ^ 47659cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// | | 47759cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// | \- 4781c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov/// | | 4791c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov/// | [Op] 48059cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// | ^ 48159cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// | | 48259cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// \ / 48359cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// \ / 48459cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// [Store] 485800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryanyvoid X86DAGToDAGISel::PreprocessForRMW() { 486800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(), 487800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany E = CurDAG->allnodes_end(); I != E; ++I) { 488800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (I->getOpcode() == X86ISD::CALL) { 489800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany /// Also try moving call address load from outside callseq_start to just 490800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany /// before the call to allow it to be folded. 491ee548275c63a1eeffda9d3edd2bea04e1dadcc67Alexey Samsonov /// 492b0dcf61252e58715a3bea79f4c112572df361c30Alexey Samsonov /// [Load chain] 49311af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov /// ^ 494ee548275c63a1eeffda9d3edd2bea04e1dadcc67Alexey Samsonov /// | 49511af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov /// [Load] 496800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany /// ^ ^ 497800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany /// | | 4981416edc30adbd53b2be863f7f3de56de4a4c9d0aKostya Serebryany /// / \-- 4991416edc30adbd53b2be863f7f3de56de4a4c9d0aKostya Serebryany /// / | 5001416edc30adbd53b2be863f7f3de56de4a4c9d0aKostya Serebryany ///[CALLSEQ_START] | 5011416edc30adbd53b2be863f7f3de56de4a4c9d0aKostya Serebryany /// ^ | 502b0dcf61252e58715a3bea79f4c112572df361c30Alexey Samsonov /// | | 50311af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov /// [LOAD/C2Reg] | 50411af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov /// | | 50511af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov /// \ / 50625878042030e85c244b41bfcdfad27c32360e2ecAlexander Potapenko /// \ / 50725878042030e85c244b41bfcdfad27c32360e2ecAlexander Potapenko /// [CALL] 5082735cf4aa52e31b8d2de90f836c3ad991215e04eKostya Serebryany SDValue Chain = I->getOperand(0); 5092735cf4aa52e31b8d2de90f836c3ad991215e04eKostya Serebryany SDValue Load = I->getOperand(1); 5102735cf4aa52e31b8d2de90f836c3ad991215e04eKostya Serebryany if (!isCalleeLoad(Load, Chain)) 5112735cf4aa52e31b8d2de90f836c3ad991215e04eKostya Serebryany continue; 5122735cf4aa52e31b8d2de90f836c3ad991215e04eKostya Serebryany MoveBelowCallSeqStart(CurDAG, Load, SDValue(I, 0), Chain); 5132735cf4aa52e31b8d2de90f836c3ad991215e04eKostya Serebryany ++NumLoadMoved; 514800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany continue; 515800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany } 51618c7f80b3e83ab584bd8572695a3cde8bafd9d3cChris Lattner 517800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (!ISD::isNON_TRUNCStore(I)) 51851c7c65e32f76ec5a50cdecfe2b4c287c57da127Kostya Serebryany continue; 51951c7c65e32f76ec5a50cdecfe2b4c287c57da127Kostya Serebryany SDValue Chain = I->getOperand(0); 52051c7c65e32f76ec5a50cdecfe2b4c287c57da127Kostya Serebryany 52151c7c65e32f76ec5a50cdecfe2b4c287c57da127Kostya Serebryany if (Chain.getNode()->getOpcode() != ISD::TokenFactor) 52251c7c65e32f76ec5a50cdecfe2b4c287c57da127Kostya Serebryany continue; 52351c7c65e32f76ec5a50cdecfe2b4c287c57da127Kostya Serebryany 524800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDValue N1 = I->getOperand(1); 525800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDValue N2 = I->getOperand(2); 526800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if ((N1.getValueType().isFloatingPoint() && 527800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany !N1.getValueType().isVector()) || 52819cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov !N1.hasOneUse()) 52919cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov continue; 530800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 531800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany bool RModW = false; 53248a615fee78c3c262c60147a65dc6fff5fd6bb3bKostya Serebryany SDValue Load; 53348a615fee78c3c262c60147a65dc6fff5fd6bb3bKostya Serebryany unsigned Opcode = N1.getNode()->getOpcode(); 53448a615fee78c3c262c60147a65dc6fff5fd6bb3bKostya Serebryany switch (Opcode) { 53548a615fee78c3c262c60147a65dc6fff5fd6bb3bKostya Serebryany case ISD::ADD: 536800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany case ISD::MUL: 537800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany case ISD::AND: 538c0ed3e548c6f688e22685de04e210c7b59ac3433Kostya Serebryany case ISD::OR: 539ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryany case ISD::XOR: 540800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany case ISD::ADDC: 541800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany case ISD::ADDE: 542800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany case ISD::VECTOR_SHUFFLE: { 543800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDValue N10 = N1.getOperand(0); 544ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryany SDValue N11 = N1.getOperand(1); 545800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany RModW = isRMWLoad(N10, Chain, N2, Load); 546800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (!RModW) 547800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany RModW = isRMWLoad(N11, Chain, N2, Load); 548800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany break; 549800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany } 550800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany case ISD::SUB: 551800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany case ISD::SHL: 552800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany case ISD::SRA: 553800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany case ISD::SRL: 554ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryany case ISD::ROTL: 555800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany case ISD::ROTR: 556800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany case ISD::SUBC: 557800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany case ISD::SUBE: 558800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany case X86ISD::SHLD: 559ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryany case X86ISD::SHRD: { 560800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDValue N10 = N1.getOperand(0); 561800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany RModW = isRMWLoad(N10, Chain, N2, Load); 5622735cf4aa52e31b8d2de90f836c3ad991215e04eKostya Serebryany break; 563800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany } 564800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany } 565800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 566800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (RModW) { 567800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany MoveBelowTokenFactor(CurDAG, Load, SDValue(I, 0), Chain); 568800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany ++NumLoadMoved; 569800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany } 570800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany } 571800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany} 572800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 573800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 57456139bc493790612ee6281630678e293be6b2eb2Kostya Serebryany/// PreprocessForFPConvert - Walk over the dag lowering fpround and fpextend 5754a2dec05cef5882b745dd248d79e42a42cdbc87bEvgeniy Stepanov/// nodes that target the FP stack to be store and load to the stack. This is a 576800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany/// gross hack. We would like to simply mark these as being illegal, but when 577800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany/// we do that, legalize produces these when it expands calls, then expands 578ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryany/// these in the same legalize pass. We would like dag combine to be able to 579800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany/// hack on these between the call expansion and the node legalization. As such 580ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryany/// this pass basically does "really late" legalization of these inline with the 581800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany/// X86 isel pass. 582800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryanyvoid X86DAGToDAGISel::PreprocessForFPConvert() { 583800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(), 584e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany E = CurDAG->allnodes_end(); I != E; ) { 585e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany SDNode *N = I++; // Preincrement iterator to avoid invalidation issues. 586e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany if (N->getOpcode() != ISD::FP_ROUND && N->getOpcode() != ISD::FP_EXTEND) 587800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany continue; 588e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany 589e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany // If the source and destination are SSE registers, then this is a legal 590800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany // conversion that should not be lowered. 591800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany MVT SrcVT = N->getOperand(0).getValueType(); 592e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany MVT DstVT = N->getValueType(0); 593e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany bool SrcIsSSE = X86Lowering.isScalarFPTypeInSSEReg(SrcVT); 594e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany bool DstIsSSE = X86Lowering.isScalarFPTypeInSSEReg(DstVT); 595e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany if (SrcIsSSE && DstIsSSE) 596e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany continue; 597e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany 598e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany if (!SrcIsSSE && !DstIsSSE) { 599e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany // If this is an FPStack extension, it is a noop. 600e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany if (N->getOpcode() == ISD::FP_EXTEND) 601e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany continue; 602e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany // If this is a value-preserving FPStack truncation, it is a noop. 603e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany if (N->getConstantOperandVal(1)) 604e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany continue; 605e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany } 606e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany 607e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany // Here we could have an FP stack truncation or an FPStack <-> SSE convert. 608800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany // FPStack has extload and truncstore. SSE can fold direct loads into other 609800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany // operations. Based on this, decide what we want to do. 610ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryany MVT MemVT; 6113780ad8b998d93d7db406919c06137cdb786ef05Axel Naumann if (N->getOpcode() == ISD::FP_ROUND) 612e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany MemVT = DstVT; // FP_ROUND must use DstVT, we can't do a 'trunc load'. 613e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany else 6149b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany MemVT = SrcIsSSE ? SrcVT : DstVT; 6159b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany 6169b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT); 6179b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany 618ee548275c63a1eeffda9d3edd2bea04e1dadcc67Alexey Samsonov // FIXME: optimize the case where the src/dest is a load or store? 6199b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany SDValue Store = CurDAG->getTruncStore(CurDAG->getEntryNode(), 6209b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany N->getOperand(0), 621407790604b8f71f7172bbdfb76c27e1799d241c2Kostya Serebryany MemTmp, NULL, 0, MemVT); 622407790604b8f71f7172bbdfb76c27e1799d241c2Kostya Serebryany SDValue Result = CurDAG->getExtLoad(ISD::EXTLOAD, DstVT, Store, MemTmp, 623407790604b8f71f7172bbdfb76c27e1799d241c2Kostya Serebryany NULL, 0, MemVT); 6249b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany 6259b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany // We're about to replace all uses of the FP_ROUND/FP_EXTEND with the 626800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany // extload we created. This will cause general havok on the dag because 6279b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany // anything below the conversion could be folded into other existing nodes. 628800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany // To avoid invalidating 'I', back it up to the convert node. 629800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany --I; 630800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Result); 631800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 632800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany // Now that we did that, the node is dead. Increment the iterator to the 633800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany // next node to process, then delete N. 634800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany ++I; 635800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany CurDAG->DeleteNode(N); 636800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany } 637800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany} 638800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 639800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany/// InstructionSelectBasicBlock - This callback is invoked by SelectionDAGISel 640800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany/// when it has created a SelectionDAG for us to codegen. 641ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryanyvoid X86DAGToDAGISel::InstructionSelect() { 642800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany CurBB = BB; // BB can change as result of isel. 643800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany const Function *F = CurDAG->getMachineFunction().getFunction(); 64455cabae685991ac6bc6d96b14e2139f64a0c9967Alexander Potapenko OptForSize = F->hasFnAttr(Attribute::OptimizeForSize); 64555cabae685991ac6bc6d96b14e2139f64a0c9967Alexander Potapenko 64655cabae685991ac6bc6d96b14e2139f64a0c9967Alexander Potapenko DEBUG(BB->dump()); 64755cabae685991ac6bc6d96b14e2139f64a0c9967Alexander Potapenko if (!Fast) 648b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany PreprocessForRMW(); 64955cabae685991ac6bc6d96b14e2139f64a0c9967Alexander Potapenko 65055cabae685991ac6bc6d96b14e2139f64a0c9967Alexander Potapenko // FIXME: This should only happen when not -fast. 65155cabae685991ac6bc6d96b14e2139f64a0c9967Alexander Potapenko PreprocessForFPConvert(); 65255cabae685991ac6bc6d96b14e2139f64a0c9967Alexander Potapenko 65355cabae685991ac6bc6d96b14e2139f64a0c9967Alexander Potapenko // Codegen the basic block. 65455cabae685991ac6bc6d96b14e2139f64a0c9967Alexander Potapenko#ifndef NDEBUG 655800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany DOUT << "===== Instruction selection begins:\n"; 656ebd6454929f2ba3b92f61c151eccde0b8b0a8dedKostya Serebryany Indent = 0; 6574f0c69623c10a3a49f6926fd53694ee532e06a85Kostya Serebryany#endif 658ebd6454929f2ba3b92f61c151eccde0b8b0a8dedKostya Serebryany SelectRoot(); 659ebd6454929f2ba3b92f61c151eccde0b8b0a8dedKostya Serebryany#ifndef NDEBUG 660ebd6454929f2ba3b92f61c151eccde0b8b0a8dedKostya Serebryany DOUT << "===== Instruction selection ends:\n"; 661f7b08226eb44458f6f38cbeaca527028803c725aKostya Serebryany#endif 662f7b08226eb44458f6f38cbeaca527028803c725aKostya Serebryany 663f7b08226eb44458f6f38cbeaca527028803c725aKostya Serebryany CurDAG->RemoveDeadNodes(); 664f7b08226eb44458f6f38cbeaca527028803c725aKostya Serebryany} 6653c7faae346f548c55cad86d82a2e242443001f23Kostya Serebryany 666800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryanyvoid X86DAGToDAGISel::InstructionSelectPostProcessing() { 667800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany // If we are emitting FP stack code, scan the basic block to determine if this 6682735cf4aa52e31b8d2de90f836c3ad991215e04eKostya Serebryany // block defines any FP values. If so, put an FP_REG_KILL instruction before 669c0ed3e548c6f688e22685de04e210c7b59ac3433Kostya Serebryany // the terminator of the block. 670c0ed3e548c6f688e22685de04e210c7b59ac3433Kostya Serebryany 67119cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov // Note that FP stack instructions are used in all modes for long double, 672c0ed3e548c6f688e22685de04e210c7b59ac3433Kostya Serebryany // so we always need to do this check. 673c0ed3e548c6f688e22685de04e210c7b59ac3433Kostya Serebryany // Also note that it's possible for an FP stack register to be live across 674c0ed3e548c6f688e22685de04e210c7b59ac3433Kostya Serebryany // an instruction that produces multiple basic blocks (SSE CMOV) so we 675c0ed3e548c6f688e22685de04e210c7b59ac3433Kostya Serebryany // must check all the generated basic blocks. 676c0ed3e548c6f688e22685de04e210c7b59ac3433Kostya Serebryany 677c0ed3e548c6f688e22685de04e210c7b59ac3433Kostya Serebryany // Scan all of the machine instructions in these MBBs, checking for FP 678c0ed3e548c6f688e22685de04e210c7b59ac3433Kostya Serebryany // stores. (RFP32 and RFP64 will not exist in SSE mode, but RFP80 might.) 679c0ed3e548c6f688e22685de04e210c7b59ac3433Kostya Serebryany MachineFunction::iterator MBBI = CurBB; 680c0ed3e548c6f688e22685de04e210c7b59ac3433Kostya Serebryany MachineFunction::iterator EndMBB = BB; ++EndMBB; 6816e2d506dc962873a0e05092bbb034f9a615d1084Kostya Serebryany for (; MBBI != EndMBB; ++MBBI) { 682c0ed3e548c6f688e22685de04e210c7b59ac3433Kostya Serebryany MachineBasicBlock *MBB = MBBI; 683c0ed3e548c6f688e22685de04e210c7b59ac3433Kostya Serebryany 684c0ed3e548c6f688e22685de04e210c7b59ac3433Kostya Serebryany // If this block returns, ignore it. We don't want to insert an FP_REG_KILL 685c0ed3e548c6f688e22685de04e210c7b59ac3433Kostya Serebryany // before the return. 686ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryany if (!MBB->empty()) { 687800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany MachineBasicBlock::iterator EndI = MBB->end(); 688800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany --EndI; 689800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (EndI->getDesc().isReturn()) 690800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany continue; 691800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany } 69219cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov 693800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany bool ContainsFPCode = false; 694800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end(); 695800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany !ContainsFPCode && I != E; ++I) { 696800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (I->getNumOperands() != 0 && I->getOperand(0).isReg()) { 697800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany const TargetRegisterClass *clas; 698800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany for (unsigned op = 0, e = I->getNumOperands(); op != e; ++op) { 699800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (I->getOperand(op).isReg() && I->getOperand(op).isDef() && 70011c2a47af825a0f89d75aaa97ad873ed2acef266Kostya Serebryany TargetRegisterInfo::isVirtualRegister(I->getOperand(op).getReg()) && 70119cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov ((clas = RegInfo->getRegClass(I->getOperand(0).getReg())) == 702ebd6454929f2ba3b92f61c151eccde0b8b0a8dedKostya Serebryany X86::RFP32RegisterClass || 703ebd6454929f2ba3b92f61c151eccde0b8b0a8dedKostya Serebryany clas == X86::RFP64RegisterClass || 7046e2d506dc962873a0e05092bbb034f9a615d1084Kostya Serebryany clas == X86::RFP80RegisterClass)) { 7054a2dec05cef5882b745dd248d79e42a42cdbc87bEvgeniy Stepanov ContainsFPCode = true; 7064a2dec05cef5882b745dd248d79e42a42cdbc87bEvgeniy Stepanov break; 707ebd6454929f2ba3b92f61c151eccde0b8b0a8dedKostya Serebryany } 708f7b08226eb44458f6f38cbeaca527028803c725aKostya Serebryany } 709c0ed3e548c6f688e22685de04e210c7b59ac3433Kostya Serebryany } 710c0ed3e548c6f688e22685de04e210c7b59ac3433Kostya Serebryany } 711ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryany // Check PHI nodes in successor blocks. These PHI's will be lowered to have 712ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryany // a copy of the input value in this block. In SSE mode, we only care about 713ebd6454929f2ba3b92f61c151eccde0b8b0a8dedKostya Serebryany // 80-bit values. 714f7b08226eb44458f6f38cbeaca527028803c725aKostya Serebryany if (!ContainsFPCode) { 715f7b08226eb44458f6f38cbeaca527028803c725aKostya Serebryany // Final check, check LLVM BB's that are successors to the LLVM BB 716c0ed3e548c6f688e22685de04e210c7b59ac3433Kostya Serebryany // corresponding to BB for FP PHI nodes. 7174a2dec05cef5882b745dd248d79e42a42cdbc87bEvgeniy Stepanov const BasicBlock *LLVMBB = BB->getBasicBlock(); 718800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany const PHINode *PN; 719ebd6454929f2ba3b92f61c151eccde0b8b0a8dedKostya Serebryany for (succ_const_iterator SI = succ_begin(LLVMBB), E = succ_end(LLVMBB); 720ebd6454929f2ba3b92f61c151eccde0b8b0a8dedKostya Serebryany !ContainsFPCode && SI != E; ++SI) { 721ebd6454929f2ba3b92f61c151eccde0b8b0a8dedKostya Serebryany for (BasicBlock::const_iterator II = SI->begin(); 722ebd6454929f2ba3b92f61c151eccde0b8b0a8dedKostya Serebryany (PN = dyn_cast<PHINode>(II)); ++II) { 723800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (PN->getType()==Type::X86_FP80Ty || 724800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany (!Subtarget->hasSSE1() && PN->getType()->isFloatingPoint()) || 7251416edc30adbd53b2be863f7f3de56de4a4c9d0aKostya Serebryany (!Subtarget->hasSSE2() && PN->getType()==Type::DoubleTy)) { 726b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany ContainsFPCode = true; 7279b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany break; 7289b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany } 7299b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany } 7309b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany } 7319b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany } 7329b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany // Finally, if we found any FP code, emit the FP_REG_KILL instruction. 7339b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany if (ContainsFPCode) { 7349b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany BuildMI(*MBB, MBBI->getFirstTerminator(), 7359b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany TM.getInstrInfo()->get(X86::FP_REG_KILL)); 7369b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany ++NumFPKill; 7379b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany } 7389b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany } 7399b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany} 7409b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany 7419b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany/// EmitSpecialCodeForMain - Emit any code that needs to be executed only in 7429b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany/// the main function. 7439b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryanyvoid X86DAGToDAGISel::EmitSpecialCodeForMain(MachineBasicBlock *BB, 7449b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany MachineFrameInfo *MFI) { 7459b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany const TargetInstrInfo *TII = TM.getInstrInfo(); 7469b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany if (Subtarget->isTargetCygMing()) 7479b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany BuildMI(BB, TII->get(X86::CALLpcrel32)).addExternalSymbol("__main"); 7489b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany} 7491416edc30adbd53b2be863f7f3de56de4a4c9d0aKostya Serebryany 7509b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryanyvoid X86DAGToDAGISel::EmitFunctionEntryCode(Function &Fn, MachineFunction &MF) { 751324d96b9e265b0fd8bf63a28340910def64e2164Kostya Serebryany // If this is main, emit special code for main. 7529b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany MachineBasicBlock *BB = MF.begin(); 75359a4a47a7bea7cc17877c6a3954ad9f8309ff1cbKostya Serebryany if (Fn.hasExternalLinkage() && Fn.getName() == "main") 7549b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany EmitSpecialCodeForMain(BB, MF.getFrameInfo()); 7559b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany} 75651c7c65e32f76ec5a50cdecfe2b4c287c57da127Kostya Serebryany 7579b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany/// MatchAddress - Add the specified node to the specified addressing mode, 7589b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany/// returning true if it cannot be done. This just pattern matches for the 7599b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany/// addressing mode. 7609b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryanybool X86DAGToDAGISel::MatchAddress(SDValue N, X86ISelAddressMode &AM, 7619b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany bool isRoot, unsigned Depth) { 7629b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany DOUT << "MatchAddress: "; DEBUG(AM.dump()); 7639b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany // Limit recursion. 7649b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany if (Depth > 5) 7659b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany return MatchAddressBase(N, AM, isRoot, Depth); 7669b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany 7679b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany // RIP relative addressing: %rip + 32-bit displacement! 7689b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany if (AM.isRIPRel) { 769b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany if (!AM.ES && AM.JT != -1 && N.getOpcode() == ISD::Constant) { 7709b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany int64_t Val = cast<ConstantSDNode>(N)->getSExtValue(); 7719b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany if (isInt32(AM.Disp + Val)) { 7729b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany AM.Disp += Val; 7739b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany return false; 7749b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany } 7759b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany } 7769b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany return true; 7779b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany } 7789b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany 7799b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany int id = N.getNode()->getNodeId(); 7809b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany bool AlreadySelected = isSelected(id); // Already selected, not yet replaced. 7819b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany 7829b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany switch (N.getOpcode()) { 7839b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany default: break; 7849b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany case ISD::Constant: { 7859b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany int64_t Val = cast<ConstantSDNode>(N)->getSExtValue(); 7869b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany if (isInt32(AM.Disp + Val)) { 7879b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany AM.Disp += Val; 7889b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany return false; 7899b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany } 7909b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany break; 7919b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany } 7929b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany 7939b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany case X86ISD::Wrapper: { 7949b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya SerebryanyDOUT << "Wrapper: 64bit " << Subtarget->is64Bit(); 7959b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya SerebryanyDOUT << " AM "; DEBUG(AM.dump()); DOUT << "\n"; 7969b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya SerebryanyDOUT << "AlreadySelected " << AlreadySelected << "\n"; 7979b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany bool is64Bit = Subtarget->is64Bit(); 7989b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany // Under X86-64 non-small code model, GV (and friends) are 64-bits. 7999b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany // Also, base and index reg must be 0 in order to use rip as base. 8009b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany if (is64Bit && (TM.getCodeModel() != CodeModel::Small || 8019b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany AM.Base.Reg.getNode() || AM.IndexReg.getNode())) 8029b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany break; 8039b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany if (AM.GV != 0 || AM.CP != 0 || AM.ES != 0 || AM.JT != -1) 8049b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany break; 8059b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany // If value is available in a register both base and index components have 8069b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany // been picked, we can't fit the result available in the register in the 8079b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany // addressing mode. Duplicate GlobalAddress or ConstantPool as displacement. 8084684858624d7ffe82379783e9b678227d5e0b515Alexey Samsonov if (!AlreadySelected || (AM.Base.Reg.getNode() && AM.IndexReg.getNode())) { 8094684858624d7ffe82379783e9b678227d5e0b515Alexey Samsonov SDValue N0 = N.getOperand(0); 8104684858624d7ffe82379783e9b678227d5e0b515Alexey Samsonov if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(N0)) { 8114684858624d7ffe82379783e9b678227d5e0b515Alexey Samsonov GlobalValue *GV = G->getGlobal(); 8124684858624d7ffe82379783e9b678227d5e0b515Alexey Samsonov AM.GV = GV; 8134684858624d7ffe82379783e9b678227d5e0b515Alexey Samsonov AM.Disp += G->getOffset(); 8144684858624d7ffe82379783e9b678227d5e0b515Alexey Samsonov AM.isRIPRel = TM.symbolicAddressesAreRIPRel(); 8154684858624d7ffe82379783e9b678227d5e0b515Alexey Samsonov return false; 8164684858624d7ffe82379783e9b678227d5e0b515Alexey Samsonov } else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(N0)) { 8174684858624d7ffe82379783e9b678227d5e0b515Alexey Samsonov AM.CP = CP->getConstVal(); 8184684858624d7ffe82379783e9b678227d5e0b515Alexey Samsonov AM.Align = CP->getAlignment(); 8194684858624d7ffe82379783e9b678227d5e0b515Alexey Samsonov AM.Disp += CP->getOffset(); 8204684858624d7ffe82379783e9b678227d5e0b515Alexey Samsonov AM.isRIPRel = TM.symbolicAddressesAreRIPRel(); 8214684858624d7ffe82379783e9b678227d5e0b515Alexey Samsonov return false; 8224684858624d7ffe82379783e9b678227d5e0b515Alexey Samsonov } else if (ExternalSymbolSDNode *S =dyn_cast<ExternalSymbolSDNode>(N0)) { 8234684858624d7ffe82379783e9b678227d5e0b515Alexey Samsonov AM.ES = S->getSymbol(); 8244684858624d7ffe82379783e9b678227d5e0b515Alexey Samsonov AM.isRIPRel = TM.symbolicAddressesAreRIPRel(); 8254684858624d7ffe82379783e9b678227d5e0b515Alexey Samsonov return false; 8264684858624d7ffe82379783e9b678227d5e0b515Alexey Samsonov } else if (JumpTableSDNode *J = dyn_cast<JumpTableSDNode>(N0)) { 8274684858624d7ffe82379783e9b678227d5e0b515Alexey Samsonov AM.JT = J->getIndex(); 828800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany AM.isRIPRel = TM.symbolicAddressesAreRIPRel(); 829800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany return false; 830800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany } 8311416edc30adbd53b2be863f7f3de56de4a4c9d0aKostya Serebryany } 8321416edc30adbd53b2be863f7f3de56de4a4c9d0aKostya Serebryany break; 8331416edc30adbd53b2be863f7f3de56de4a4c9d0aKostya Serebryany } 8341416edc30adbd53b2be863f7f3de56de4a4c9d0aKostya Serebryany 8351416edc30adbd53b2be863f7f3de56de4a4c9d0aKostya Serebryany case ISD::FrameIndex: 836b0dcf61252e58715a3bea79f4c112572df361c30Alexey Samsonov if (AM.BaseType == X86ISelAddressMode::RegBase 837d6f62c8da5aa4f3388cec1542309ffa623cac601Alexey Samsonov && AM.Base.Reg.getNode() == 0) { 838b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany AM.BaseType = X86ISelAddressMode::FrameIndexBase; 83919cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov AM.Base.FrameIndex = cast<FrameIndexSDNode>(N)->getIndex(); 84019cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov return false; 84111af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov } 8424684858624d7ffe82379783e9b678227d5e0b515Alexey Samsonov break; 8434684858624d7ffe82379783e9b678227d5e0b515Alexey Samsonov 844b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany case ISD::SHL: 845800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (AlreadySelected || AM.IndexReg.getNode() != 0 846800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany || AM.Scale != 1 || AM.isRIPRel) 8479b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany break; 8489b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany 8499b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany if (ConstantSDNode 8509b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany *CN = dyn_cast<ConstantSDNode>(N.getNode()->getOperand(1))) { 851800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany unsigned Val = CN->getZExtValue(); 852800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (Val == 1 || Val == 2 || Val == 3) { 853800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany AM.Scale = 1 << Val; 854800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDValue ShVal = N.getNode()->getOperand(0); 855800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 856800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany // Okay, we know that we have a scale by now. However, if the scaled 857800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany // value is an add of something and a constant, we can fold the 858800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany // constant into the disp field here. 859800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (ShVal.getNode()->getOpcode() == ISD::ADD && ShVal.hasOneUse() && 860800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany isa<ConstantSDNode>(ShVal.getNode()->getOperand(1))) { 8619b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany AM.IndexReg = ShVal.getNode()->getOperand(0); 862800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany ConstantSDNode *AddVal = 863800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany cast<ConstantSDNode>(ShVal.getNode()->getOperand(1)); 8649b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany uint64_t Disp = AM.Disp + (AddVal->getZExtValue() << Val); 8659b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany if (isInt32(Disp)) 8669b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany AM.Disp = Disp; 867800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany else 868b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany AM.IndexReg = ShVal; 869b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany } else { 870b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany AM.IndexReg = ShVal; 871b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany } 872800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany return false; 8739b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany } 8749b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany break; 8759b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany } 8769b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany 877800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany case ISD::SMUL_LOHI: 878800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany case ISD::UMUL_LOHI: 879800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany // A mul_lohi where we need the low part can be folded as a plain multiply. 880800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (N.getResNo() != 0) break; 881208a4ff2b56f453910bb817540f34b8169f7702aKostya Serebryany // FALL THROUGH 882b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany case ISD::MUL: 883b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany // X*[3,5,9] -> X+X*[2,4,8] 884800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (!AlreadySelected && 8859b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany AM.BaseType == X86ISelAddressMode::RegBase && 886ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryany AM.Base.Reg.getNode() == 0 && 887ca23d43084c45a7d6f4371d62f45be044650ce38Kostya Serebryany AM.IndexReg.getNode() == 0 && 88859a4a47a7bea7cc17877c6a3954ad9f8309ff1cbKostya Serebryany !AM.isRIPRel) { 8897dadac65d375ff21b7a36bfb7642578d2f467525Kostya Serebryany if (ConstantSDNode 890800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany *CN = dyn_cast<ConstantSDNode>(N.getNode()->getOperand(1))) 891800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (CN->getZExtValue() == 3 || CN->getZExtValue() == 5 || 892800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany CN->getZExtValue() == 9) { 893800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany AM.Scale = unsigned(CN->getZExtValue())-1; 894800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 895800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDValue MulVal = N.getNode()->getOperand(0); 896a4b2b1d8fbb84b0b6fac5abce21a2a5c5e907282Kostya Serebryany SDValue Reg; 897a4b2b1d8fbb84b0b6fac5abce21a2a5c5e907282Kostya Serebryany 898a4b2b1d8fbb84b0b6fac5abce21a2a5c5e907282Kostya Serebryany // Okay, we know that we have a scale by now. However, if the scaled 899a4b2b1d8fbb84b0b6fac5abce21a2a5c5e907282Kostya Serebryany // value is an add of something and a constant, we can fold the 900a4b2b1d8fbb84b0b6fac5abce21a2a5c5e907282Kostya Serebryany // constant into the disp field here. 901800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (MulVal.getNode()->getOpcode() == ISD::ADD && MulVal.hasOneUse() && 902800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany isa<ConstantSDNode>(MulVal.getNode()->getOperand(1))) { 903800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany Reg = MulVal.getNode()->getOperand(0); 904800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany ConstantSDNode *AddVal = 905ce718ff9f42c7da092eaa01dd0242e8d5ba84713Hans Wennborg cast<ConstantSDNode>(MulVal.getNode()->getOperand(1)); 906800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany uint64_t Disp = AM.Disp + AddVal->getZExtValue() * 907b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany CN->getZExtValue(); 908800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (isInt32(Disp)) 909800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany AM.Disp = Disp; 910800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany else 911800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany Reg = N.getNode()->getOperand(0); 912800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany } else { 913800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany Reg = N.getNode()->getOperand(0); 914f1639abf1aaba1448f719f595156cd0f4cd560ccKostya Serebryany } 915800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 916800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany AM.IndexReg = AM.Base.Reg = Reg; 917800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany return false; 918800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany } 919800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany } 920800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany break; 921800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 922800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany case ISD::ADD: 923800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (!AlreadySelected) { 9249b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany X86ISelAddressMode Backup = AM; 925800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (!MatchAddress(N.getNode()->getOperand(0), AM, false, Depth+1) && 9269b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany !MatchAddress(N.getNode()->getOperand(1), AM, false, Depth+1)) 9279b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany return false; 928ee548275c63a1eeffda9d3edd2bea04e1dadcc67Alexey Samsonov AM = Backup; 9299b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany if (!MatchAddress(N.getNode()->getOperand(1), AM, false, Depth+1) && 9309b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany !MatchAddress(N.getNode()->getOperand(0), AM, false, Depth+1)) 9319b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany return false; 9329b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany AM = Backup; 9339b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany } 934324d96b9e265b0fd8bf63a28340910def64e2164Kostya Serebryany break; 935800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 936800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany case ISD::OR: 937800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany // Handle "X | C" as "X + C" iff X is known to have C bits clear. 938800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (AlreadySelected) break; 939800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 940800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N.getOperand(1))) { 941800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany X86ISelAddressMode Backup = AM; 9429b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany // Start with the LHS as an addr mode. 943ee548275c63a1eeffda9d3edd2bea04e1dadcc67Alexey Samsonov if (!MatchAddress(N.getOperand(0), AM, false) && 9449b9f87a87ac1b373c4c6a70904315bebbd01c50cKostya Serebryany // Address could not have picked a GV address for the displacement. 945800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany AM.GV == NULL && 946800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany // On x86-64, the resultant disp must fit in 32-bits. 947800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany isInt32(AM.Disp + CN->getSExtValue()) && 948800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany // Check to see if the LHS & C is zero. 9497bcfc9950bac0f411f9671e8d6ce483bd219727eKostya Serebryany CurDAG->MaskedValueIsZero(N.getOperand(0), CN->getAPIntValue())) { 9507bcfc9950bac0f411f9671e8d6ce483bd219727eKostya Serebryany AM.Disp += CN->getZExtValue(); 9517bcfc9950bac0f411f9671e8d6ce483bd219727eKostya Serebryany return false; 9527bcfc9950bac0f411f9671e8d6ce483bd219727eKostya Serebryany } 9537bcfc9950bac0f411f9671e8d6ce483bd219727eKostya Serebryany AM = Backup; 9547bcfc9950bac0f411f9671e8d6ce483bd219727eKostya Serebryany } 9557bcfc9950bac0f411f9671e8d6ce483bd219727eKostya Serebryany break; 9567bcfc9950bac0f411f9671e8d6ce483bd219727eKostya Serebryany 9577bcfc9950bac0f411f9671e8d6ce483bd219727eKostya Serebryany case ISD::AND: { 9587bcfc9950bac0f411f9671e8d6ce483bd219727eKostya Serebryany // Handle "(x << C1) & C2" as "(X & (C2>>C1)) << C1" if safe and if this 9597bcfc9950bac0f411f9671e8d6ce483bd219727eKostya Serebryany // allows us to fold the shift into this addressing mode. 9607bcfc9950bac0f411f9671e8d6ce483bd219727eKostya Serebryany if (AlreadySelected) break; 961800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDValue Shift = N.getOperand(0); 962800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (Shift.getOpcode() != ISD::SHL) break; 963800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 964800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany // Scale must not be used already. 9658b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany if (AM.IndexReg.getNode() != 0 || AM.Scale != 1) break; 9668b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany 9679db5b5ffa9fccd5c7f1f39a3e9aa66cc4a5eedc1Kostya Serebryany // Not when RIP is used as the base. 9689db5b5ffa9fccd5c7f1f39a3e9aa66cc4a5eedc1Kostya Serebryany if (AM.isRIPRel) break; 9699db5b5ffa9fccd5c7f1f39a3e9aa66cc4a5eedc1Kostya Serebryany 9709db5b5ffa9fccd5c7f1f39a3e9aa66cc4a5eedc1Kostya Serebryany ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N.getOperand(1)); 9719db5b5ffa9fccd5c7f1f39a3e9aa66cc4a5eedc1Kostya Serebryany ConstantSDNode *C1 = dyn_cast<ConstantSDNode>(Shift.getOperand(1)); 9729db5b5ffa9fccd5c7f1f39a3e9aa66cc4a5eedc1Kostya Serebryany if (!C1 || !C2) break; 9739db5b5ffa9fccd5c7f1f39a3e9aa66cc4a5eedc1Kostya Serebryany 9744f0c69623c10a3a49f6926fd53694ee532e06a85Kostya Serebryany // Not likely to be profitable if either the AND or SHIFT node has more 9757846c1c851a53a8280f9d8ed57cd98d82c742551Kostya Serebryany // than one use (unless all uses are for address computation). Besides, 9767846c1c851a53a8280f9d8ed57cd98d82c742551Kostya Serebryany // isel mechanism requires their node ids to be reused. 9777846c1c851a53a8280f9d8ed57cd98d82c742551Kostya Serebryany if (!N.hasOneUse() || !Shift.hasOneUse()) 9789db5b5ffa9fccd5c7f1f39a3e9aa66cc4a5eedc1Kostya Serebryany break; 9799db5b5ffa9fccd5c7f1f39a3e9aa66cc4a5eedc1Kostya Serebryany 980ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryany // Verify that the shift amount is something we can fold. 981ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryany unsigned ShiftCst = C1->getZExtValue(); 982ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryany if (ShiftCst != 1 && ShiftCst != 2 && ShiftCst != 3) 983f7b08226eb44458f6f38cbeaca527028803c725aKostya Serebryany break; 984f7b08226eb44458f6f38cbeaca527028803c725aKostya Serebryany 985f7b08226eb44458f6f38cbeaca527028803c725aKostya Serebryany // Get the new AND mask, this folds to a constant. 986f7b08226eb44458f6f38cbeaca527028803c725aKostya Serebryany SDValue NewANDMask = CurDAG->getNode(ISD::SRL, N.getValueType(), 9878b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany SDValue(C2, 0), SDValue(C1, 0)); 9888b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany SDValue NewAND = CurDAG->getNode(ISD::AND, N.getValueType(), 98919cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov Shift.getOperand(0), NewANDMask); 99011af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov SDValue NewSHIFT = CurDAG->getNode(ISD::SHL, N.getValueType(), 99111af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov NewAND, SDValue(C1, 0)); 99211af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov NewANDMask.getNode()->setNodeId(Shift.getNode()->getNodeId()); 99311af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov NewAND.getNode()->setNodeId(N.getNode()->getNodeId()); 99411af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov CurDAG->ReplaceAllUsesWith(N, NewSHIFT); 99511af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov 99611af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov AM.Scale = 1 << ShiftCst; 99711af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov AM.IndexReg = NewAND; 99811af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov return false; 99911af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov } 100011af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov } 100111af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov 100211af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov return MatchAddressBase(N, AM, isRoot, Depth); 100311af9a873f9e1409a422ab31e22729368805afafAlexey Samsonov} 100419cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov 100519cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov/// MatchAddressBase - Helper for MatchAddress. Add the specified node to the 10068b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany/// specified addressing mode without any further recursion. 10078b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryanybool X86DAGToDAGISel::MatchAddressBase(SDValue N, X86ISelAddressMode &AM, 10088b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany bool isRoot, unsigned Depth) { 10098b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany // Is the base register already occupied? 10108b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany if (AM.BaseType != X86ISelAddressMode::RegBase || AM.Base.Reg.getNode()) { 10118b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany // If so, check to see if the scale index register is set. 10128b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany if (AM.IndexReg.getNode() == 0 && !AM.isRIPRel) { 1013b0dcf61252e58715a3bea79f4c112572df361c30Alexey Samsonov AM.IndexReg = N; 10148b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany AM.Scale = 1; 10158b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany return false; 10168b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany } 10178b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany 10188b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany // Otherwise, we cannot select it. 10198b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany return true; 10208b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany } 10218b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany 10228b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany // Default, generate it as a register. 10238b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany AM.BaseType = X86ISelAddressMode::RegBase; 10248b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany AM.Base.Reg = N; 10258b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany return false; 10268b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany} 10278b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany 10288b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany/// SelectAddr - returns true if it is able pattern match an addressing mode. 10298b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany/// It returns the operands which make up the maximal addressing mode it can 10309db5b5ffa9fccd5c7f1f39a3e9aa66cc4a5eedc1Kostya Serebryany/// match by reference. 103111af9a873f9e1409a422ab31e22729368805afafAlexey Samsonovbool X86DAGToDAGISel::SelectAddr(SDValue Op, SDValue N, SDValue &Base, 103219cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov SDValue &Scale, SDValue &Index, 1033800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDValue &Disp) { 10347bcfc9950bac0f411f9671e8d6ce483bd219727eKostya Serebryany X86ISelAddressMode AM; 1035ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryany if (MatchAddress(N, AM)) 1036ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryany return false; 1037ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryany 1038a1a8a323f43a95620fc4c6b5aad8d5ff5e5eb590Kostya Serebryany MVT VT = N.getValueType(); 1039a1a8a323f43a95620fc4c6b5aad8d5ff5e5eb590Kostya Serebryany if (AM.BaseType == X86ISelAddressMode::RegBase) { 1040a1a8a323f43a95620fc4c6b5aad8d5ff5e5eb590Kostya Serebryany if (!AM.Base.Reg.getNode()) 1041a1a8a323f43a95620fc4c6b5aad8d5ff5e5eb590Kostya Serebryany AM.Base.Reg = CurDAG->getRegister(0, VT); 1042a1a8a323f43a95620fc4c6b5aad8d5ff5e5eb590Kostya Serebryany } 1043a1a8a323f43a95620fc4c6b5aad8d5ff5e5eb590Kostya Serebryany 1044a1a8a323f43a95620fc4c6b5aad8d5ff5e5eb590Kostya Serebryany if (!AM.IndexReg.getNode()) 1045a1a8a323f43a95620fc4c6b5aad8d5ff5e5eb590Kostya Serebryany AM.IndexReg = CurDAG->getRegister(0, VT); 1046a1a8a323f43a95620fc4c6b5aad8d5ff5e5eb590Kostya Serebryany 1047a1a8a323f43a95620fc4c6b5aad8d5ff5e5eb590Kostya Serebryany getAddressOperands(AM, Base, Scale, Index, Disp); 1048a1a8a323f43a95620fc4c6b5aad8d5ff5e5eb590Kostya Serebryany return true; 1049a1a8a323f43a95620fc4c6b5aad8d5ff5e5eb590Kostya Serebryany} 1050a1a8a323f43a95620fc4c6b5aad8d5ff5e5eb590Kostya Serebryany 1051a1a8a323f43a95620fc4c6b5aad8d5ff5e5eb590Kostya Serebryany/// SelectScalarSSELoad - Match a scalar SSE load. In particular, we want to 1052a1a8a323f43a95620fc4c6b5aad8d5ff5e5eb590Kostya Serebryany/// match a load whose top elements are either undef or zeros. The load flavor 1053a1a8a323f43a95620fc4c6b5aad8d5ff5e5eb590Kostya Serebryany/// is derived from the type of N, which is either v4f32 or v2f64. 1054ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryanybool X86DAGToDAGISel::SelectScalarSSELoad(SDValue Op, SDValue Pred, 1055800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDValue N, SDValue &Base, 1056800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDValue &Scale, SDValue &Index, 1057324d96b9e265b0fd8bf63a28340910def64e2164Kostya Serebryany SDValue &Disp, SDValue &InChain, 10588b390ffbfdd52a23a45a21de99aa1c31f3ce623fKostya Serebryany SDValue &OutChain) { 1059a1a8a323f43a95620fc4c6b5aad8d5ff5e5eb590Kostya Serebryany if (N.getOpcode() == ISD::SCALAR_TO_VECTOR) { 1060a1a8a323f43a95620fc4c6b5aad8d5ff5e5eb590Kostya Serebryany InChain = N.getOperand(0).getValue(1); 1061a1a8a323f43a95620fc4c6b5aad8d5ff5e5eb590Kostya Serebryany if (ISD::isNON_EXTLoad(InChain.getNode()) && 1062a1a8a323f43a95620fc4c6b5aad8d5ff5e5eb590Kostya Serebryany InChain.getValue(0).hasOneUse() && 1063831737d329a727f53a1fb0572f7b7a8127208881Bill Wendling N.hasOneUse() && 1064831737d329a727f53a1fb0572f7b7a8127208881Bill Wendling CanBeFoldedBy(N.getNode(), Pred.getNode(), Op.getNode())) { 10656765834754cbb3cb0f15b4b15e98c5e73fa50066Bill Wendling LoadSDNode *LD = cast<LoadSDNode>(InChain); 1066800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (!SelectAddr(Op, LD->getBasePtr(), Base, Scale, Index, Disp)) 1067800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany return false; 1068800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany OutChain = LD->getChain(); 10696765834754cbb3cb0f15b4b15e98c5e73fa50066Bill Wendling return true; 10706765834754cbb3cb0f15b4b15e98c5e73fa50066Bill Wendling } 10716765834754cbb3cb0f15b4b15e98c5e73fa50066Bill Wendling } 1072800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 1073800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany // Also handle the case where we explicitly require zeros in the top 107495e3cf44a9d4672655dee3bd558bfeefa631dd55Kostya Serebryany // elements. This is a vector shuffle from the zero vector. 1075e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany if (N.getOpcode() == X86ISD::VZEXT_MOVL && N.getNode()->hasOneUse() && 1076800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany // Check to see if the top elements are all zeros (or bitcast of zeros). 1077800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany N.getOperand(0).getOpcode() == ISD::SCALAR_TO_VECTOR && 1078800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany N.getOperand(0).getNode()->hasOneUse() && 1079800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany ISD::isNON_EXTLoad(N.getOperand(0).getOperand(0).getNode()) && 1080800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany N.getOperand(0).getOperand(0).hasOneUse()) { 1081324cbb89f2759fb9ad347bd2af4631e50c39c8f3Kostya Serebryany // Okay, this is a zero extending load. Fold it. 1082800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany LoadSDNode *LD = cast<LoadSDNode>(N.getOperand(0).getOperand(0)); 1083800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (!SelectAddr(Op, LD->getBasePtr(), Base, Scale, Index, Disp)) 1084bcb55ce3862bbbedac4e09d7099c9e0efc434e4bKostya Serebryany return false; 1085e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany OutChain = LD->getChain(); 1086800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany InChain = SDValue(LD, 1); 1087800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany return true; 1088800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany } 1089800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany return false; 1090800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany} 1091800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 1092800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 109395e3cf44a9d4672655dee3bd558bfeefa631dd55Kostya Serebryany/// SelectLEAAddr - it calls SelectAddr and determines if the maximal addressing 1094800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany/// mode it matches can be cost effectively emitted as an LEA instruction. 1095800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryanybool X86DAGToDAGISel::SelectLEAAddr(SDValue Op, SDValue N, 1096a17babb021aa9dc3441ecce1ac8a62d2b27edecbKostya Serebryany SDValue &Base, SDValue &Scale, 109795e3cf44a9d4672655dee3bd558bfeefa631dd55Kostya Serebryany SDValue &Index, SDValue &Disp) { 109895e3cf44a9d4672655dee3bd558bfeefa631dd55Kostya Serebryany X86ISelAddressMode AM; 1099800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (MatchAddress(N, AM)) 1100800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany return false; 1101800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 1102800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany MVT VT = N.getValueType(); 1103324cbb89f2759fb9ad347bd2af4631e50c39c8f3Kostya Serebryany unsigned Complexity = 0; 1104324cbb89f2759fb9ad347bd2af4631e50c39c8f3Kostya Serebryany if (AM.BaseType == X86ISelAddressMode::RegBase) 1105324cbb89f2759fb9ad347bd2af4631e50c39c8f3Kostya Serebryany if (AM.Base.Reg.getNode()) 1106800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany Complexity = 1; 1107800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany else 1108800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany AM.Base.Reg = CurDAG->getRegister(0, VT); 1109800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany else if (AM.BaseType == X86ISelAddressMode::FrameIndexBase) 1110800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany Complexity = 4; 1111800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 1112800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (AM.IndexReg.getNode()) 1113800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany Complexity++; 1114800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany else 1115e6cf2e0bd09544eeb69665deb908d264e62a71c2Kostya Serebryany AM.IndexReg = CurDAG->getRegister(0, VT); 1116ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryany 1117800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany // Don't match just leal(,%reg,2). It's cheaper to do addl %reg, %reg, or with 1118ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryany // a simple shift. 1119800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (AM.Scale > 1) 1120800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany Complexity++; 1121800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 1122800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany // FIXME: We are artificially lowering the criteria to turn ADD %reg, $GA 112359cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov // to a LEA. This is determined with some expermentation but is by no means 112459cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov // optimal (especially for code size consideration). LEA is nice because of 112595e3cf44a9d4672655dee3bd558bfeefa631dd55Kostya Serebryany // its three-address nature. Tweak the cost function again when we can run 112695e3cf44a9d4672655dee3bd558bfeefa631dd55Kostya Serebryany // convertToThreeAddress() at register allocation time. 112795e3cf44a9d4672655dee3bd558bfeefa631dd55Kostya Serebryany if (AM.GV || AM.CP || AM.ES || AM.JT != -1) { 112895e3cf44a9d4672655dee3bd558bfeefa631dd55Kostya Serebryany // For X86-64, we should always use lea to materialize RIP relative 112995e3cf44a9d4672655dee3bd558bfeefa631dd55Kostya Serebryany // addresses. 113095e3cf44a9d4672655dee3bd558bfeefa631dd55Kostya Serebryany if (Subtarget->is64Bit()) 1131ee4edeccabe1854ee895f52d4ac5588bd5f40c80Kostya Serebryany Complexity = 4; 113295e3cf44a9d4672655dee3bd558bfeefa631dd55Kostya Serebryany else 1133324d96b9e265b0fd8bf63a28340910def64e2164Kostya Serebryany Complexity += 2; 113495e3cf44a9d4672655dee3bd558bfeefa631dd55Kostya Serebryany } 113595e3cf44a9d4672655dee3bd558bfeefa631dd55Kostya Serebryany 1136800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (AM.Disp && (AM.Base.Reg.getNode() || AM.IndexReg.getNode())) 1137800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany Complexity++; 1138800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 1139800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (Complexity > 2) { 1140800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany getAddressOperands(AM, Base, Scale, Index, Disp); 1141800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany return true; 1142800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany } 1143800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany return false; 1144858143816d43e58b17bfd11cb1b57afbd7f0f893Craig Topper} 1145800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 1146800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryanybool X86DAGToDAGISel::TryFoldLoad(SDValue P, SDValue N, 1147800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDValue &Base, SDValue &Scale, 1148800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDValue &Index, SDValue &Disp) { 1149b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany if (ISD::isNON_EXTLoad(N.getNode()) && 1150800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany N.hasOneUse() && 1151800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany CanBeFoldedBy(N.getNode(), P.getNode(), P.getNode())) 1152b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany return SelectAddr(P, N.getOperand(1), Base, Scale, Index, Disp); 1153800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany return false; 1154800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany} 1155800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 1156800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany/// getGlobalBaseReg - Return an SDNode that returns the value of 1157800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany/// the global base register. Output instructions required to 1158800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany/// initialize the global base register, if necessary. 1159800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany/// 1160800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya SerebryanySDNode *X86DAGToDAGISel::getGlobalBaseReg() { 1161800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany MachineFunction *MF = CurBB->getParent(); 1162800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany unsigned GlobalBaseReg = TM.getInstrInfo()->getGlobalBaseReg(MF); 1163800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany return CurDAG->getRegister(GlobalBaseReg, TLI.getPointerTy()).getNode(); 116459cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov} 116559cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov 116659cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonovstatic SDNode *FindCallStartFromCall(SDNode *Node) { 116759cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov if (Node->getOpcode() == ISD::CALLSEQ_START) return Node; 116859cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov assert(Node->getOperand(0).getValueType() == MVT::Other && 116959cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov "Node doesn't have a token chain argument!"); 117059cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov return FindCallStartFromCall(Node->getOperand(0).getNode()); 117159cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov} 117259cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov 117359cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// getTruncateTo8Bit - return an SDNode that implements a subreg based 117459cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// truncate of the specified operand to i8. This can be done with tablegen, 117559cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// except that this code uses MVT::Flag in a tricky way that happens to 117659cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov/// improve scheduling in some cases. 117759cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey SamsonovSDNode *X86DAGToDAGISel::getTruncateTo8Bit(SDValue N0) { 117859cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov assert(!Subtarget->is64Bit() && 117959cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov "getTruncateTo8Bit is only needed on x86-32!"); 118059cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov SDValue SRIdx = CurDAG->getTargetConstant(1, MVT::i32); // SubRegSet 1 118159cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov 118259cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov // Ensure that the source register has an 8-bit subreg on 32-bit targets 118359cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov unsigned Opc; 118459cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov MVT N0VT = N0.getValueType(); 118559cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov switch (N0VT.getSimpleVT()) { 118659cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov default: assert(0 && "Unknown truncate!"); 118759cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov case MVT::i16: 118859cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov Opc = X86::MOV16to16_; 118959cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov break; 119059cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov case MVT::i32: 119159cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov Opc = X86::MOV32to32_; 119219cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov break; 1193800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany } 1194800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 1195800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany // The use of MVT::Flag here is not strictly accurate, but it helps 1196800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany // scheduling in some cases. 1197800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany N0 = SDValue(CurDAG->getTargetNode(Opc, N0VT, MVT::Flag, N0), 0); 1198800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany return CurDAG->getTargetNode(X86::EXTRACT_SUBREG, 1199800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany MVT::i8, N0, SRIdx, N0.getValue(1)); 1200800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany} 1201800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 1202800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya SerebryanySDNode *X86DAGToDAGISel::SelectAtomic64(SDNode *Node, unsigned Opc) { 1203800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDValue Chain = Node->getOperand(0); 1204800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDValue In1 = Node->getOperand(1); 1205800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDValue In2L = Node->getOperand(2); 1206800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDValue In2H = Node->getOperand(3); 1207800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDValue Tmp0, Tmp1, Tmp2, Tmp3; 1208b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany if (!SelectAddr(In1, In1, Tmp0, Tmp1, Tmp2, Tmp3)) 1209800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany return NULL; 1210800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDValue LSI = Node->getOperand(4); // MemOperand 1211800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany AddToISelQueue(Tmp0); 1212800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany AddToISelQueue(Tmp1); 1213b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany AddToISelQueue(Tmp2); 1214800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany AddToISelQueue(Tmp3); 1215800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany AddToISelQueue(In2L); 1216800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany AddToISelQueue(In2H); 1217800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany // For now, don't select the MemOperand object, we don't know how. 1218800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany AddToISelQueue(Chain); 1219800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany const SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, In2L, In2H, LSI, Chain }; 1220800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany return CurDAG->getTargetNode(Opc, MVT::i32, MVT::i32, MVT::Other, Ops, 8); 1221800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany} 1222800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 122319cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey SamsonovSDNode *X86DAGToDAGISel::Select(SDValue N) { 1224b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany SDNode *Node = N.getNode(); 1225800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany MVT NVT = Node->getValueType(0); 1226800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany unsigned Opc, MOpc; 1227800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany unsigned Opcode = Node->getOpcode(); 1228b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany 122919cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov#ifndef NDEBUG 1230800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany DOUT << std::string(Indent, ' ') << "Selecting: "; 1231800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany DEBUG(Node->dump(CurDAG)); 1232800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany DOUT << "\n"; 1233800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany Indent += 2; 1234800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany#endif 1235800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 1236800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (Node->isMachineOpcode()) { 1237800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany#ifndef NDEBUG 123819cd7e9ce28ed7f3326ebcd386eec215ab3763f9Alexey Samsonov DOUT << std::string(Indent-2, ' ') << "== "; 12391c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov DEBUG(Node->dump(CurDAG)); 12401c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov DOUT << "\n"; 1241800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany Indent -= 2; 1242800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany#endif 1243b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany return NULL; // Already selected. 1244800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany } 1245800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 1246800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany switch (Opcode) { 124759cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov default: break; 124859cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov case X86ISD::GlobalBaseReg: 124959cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov return getGlobalBaseReg(); 1250800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 125159cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov case ISD::ADD: { 1252800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany // Turn ADD X, c to MOV32ri X+c. This cannot be done with tblgen'd 1253800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany // code and is matched first so to prevent it from being turned into 12541c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov // LEA32r X+c. 1255800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany // In 64-bit small code size mode, use LEA to take advantage of 1256800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany // RIP-relative addressing. 1257800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (TM.getCodeModel() != CodeModel::Small) 1258800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany break; 1259800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany MVT PtrVT = TLI.getPointerTy(); 1260800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDValue N0 = N.getOperand(0); 1261800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDValue N1 = N.getOperand(1); 126259cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov if (N.getNode()->getValueType(0) == PtrVT && 126359cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov N0.getOpcode() == X86ISD::Wrapper && 126459cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov N1.getOpcode() == ISD::Constant) { 1265800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany unsigned Offset = (unsigned)cast<ConstantSDNode>(N1)->getZExtValue(); 1266800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDValue C(0, 0); 1267800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany // TODO: handle ExternalSymbolSDNode. 1268800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany if (GlobalAddressSDNode *G = 1269800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany dyn_cast<GlobalAddressSDNode>(N0.getOperand(0))) { 1270800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany C = CurDAG->getTargetGlobalAddress(G->getGlobal(), PtrVT, 1271800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany G->getOffset() + Offset); 1272800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany } else if (ConstantPoolSDNode *CP = 1273800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany dyn_cast<ConstantPoolSDNode>(N0.getOperand(0))) { 1274800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany C = CurDAG->getTargetConstantPool(CP->getConstVal(), PtrVT, 1275800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany CP->getAlignment(), 1276800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany CP->getOffset()+Offset); 1277800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany } 1278800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 12791c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov if (C.getNode()) { 12801c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov if (Subtarget->is64Bit()) { 12811c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov SDValue Ops[] = { CurDAG->getRegister(0, PtrVT), getI8Imm(1), 12821c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov CurDAG->getRegister(0, PtrVT), C }; 12831c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov return CurDAG->SelectNodeTo(N.getNode(), X86::LEA64r, 12841c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov MVT::i64, Ops, 4); 12851c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov } else 12861c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov return CurDAG->SelectNodeTo(N.getNode(), X86::MOV32ri, PtrVT, C); 12871c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov } 12881c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov } 12891c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov 12901c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov // Other cases are handled by auto-generated code. 1291b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany break; 1292800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany } 1293800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 1294800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany case X86ISD::ATOMOR64_DAG: 1295800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany return SelectAtomic64(Node, X86::ATOMOR6432); 1296800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany case X86ISD::ATOMXOR64_DAG: 1297800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany return SelectAtomic64(Node, X86::ATOMXOR6432); 1298800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany case X86ISD::ATOMADD64_DAG: 1299800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany return SelectAtomic64(Node, X86::ATOMADD6432); 1300b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany case X86ISD::ATOMSUB64_DAG: 1301f985f44b13681071e585acb7a5703a2c1c23b6ceAlexey Samsonov return SelectAtomic64(Node, X86::ATOMSUB6432); 1302800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany case X86ISD::ATOMNAND64_DAG: 1303f985f44b13681071e585acb7a5703a2c1c23b6ceAlexey Samsonov return SelectAtomic64(Node, X86::ATOMNAND6432); 13041afbb517965e29b07cb42e2335d5eadd87de6535Alexey Samsonov case X86ISD::ATOMAND64_DAG: 1305f985f44b13681071e585acb7a5703a2c1c23b6ceAlexey Samsonov return SelectAtomic64(Node, X86::ATOMAND6432); 1306b9a12ea0fd92bfdb4c6eb5af333648a618f68686Kostya Serebryany case X86ISD::ATOMSWAP64_DAG: 1307800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany return SelectAtomic64(Node, X86::ATOMSWAP6432); 1308800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 1309800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany case ISD::SMUL_LOHI: 1310800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany case ISD::UMUL_LOHI: { 1311800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDValue N0 = Node->getOperand(0); 1312800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDValue N1 = Node->getOperand(1); 1313800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 1314800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany bool isSigned = Opcode == ISD::SMUL_LOHI; 131559cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov if (!isSigned) 131659cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov switch (NVT.getSimpleVT()) { 1317800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany default: assert(0 && "Unsupported VT!"); 13189ce84c1c95c0153a2f33e188ce0db00770425f9eAlexey Samsonov case MVT::i8: Opc = X86::MUL8r; MOpc = X86::MUL8m; break; 1319a5f54f14435d881b10d8eb65e19fa42af95757e9Kostya Serebryany case MVT::i16: Opc = X86::MUL16r; MOpc = X86::MUL16m; break; 132059cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov case MVT::i32: Opc = X86::MUL32r; MOpc = X86::MUL32m; break; 132159cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov case MVT::i64: Opc = X86::MUL64r; MOpc = X86::MUL64m; break; 1322800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany } 1323800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany else 1324800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany switch (NVT.getSimpleVT()) { 132559cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov default: assert(0 && "Unsupported VT!"); 132659cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov case MVT::i8: Opc = X86::IMUL8r; MOpc = X86::IMUL8m; break; 1327800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany case MVT::i16: Opc = X86::IMUL16r; MOpc = X86::IMUL16m; break; 1328800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany case MVT::i32: Opc = X86::IMUL32r; MOpc = X86::IMUL32m; break; 1329800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany case MVT::i64: Opc = X86::IMUL64r; MOpc = X86::IMUL64m; break; 1330800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany } 1331800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 1332800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany unsigned LoReg, HiReg; 1333800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany switch (NVT.getSimpleVT()) { 1334800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany default: assert(0 && "Unsupported VT!"); 1335800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany case MVT::i8: LoReg = X86::AL; HiReg = X86::AH; break; 133659cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov case MVT::i16: LoReg = X86::AX; HiReg = X86::DX; break; 1337800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany case MVT::i32: LoReg = X86::EAX; HiReg = X86::EDX; break; 1338f985f44b13681071e585acb7a5703a2c1c23b6ceAlexey Samsonov case MVT::i64: LoReg = X86::RAX; HiReg = X86::RDX; break; 1339800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany } 1340800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 1341800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany SDValue Tmp0, Tmp1, Tmp2, Tmp3; 1342f985f44b13681071e585acb7a5703a2c1c23b6ceAlexey Samsonov bool foldedLoad = TryFoldLoad(N, N1, Tmp0, Tmp1, Tmp2, Tmp3); 1343f985f44b13681071e585acb7a5703a2c1c23b6ceAlexey Samsonov // multiplty is commmutative 1344f985f44b13681071e585acb7a5703a2c1c23b6ceAlexey Samsonov if (!foldedLoad) { 1345f985f44b13681071e585acb7a5703a2c1c23b6ceAlexey Samsonov foldedLoad = TryFoldLoad(N, N0, Tmp0, Tmp1, Tmp2, Tmp3); 1346f985f44b13681071e585acb7a5703a2c1c23b6ceAlexey Samsonov if (foldedLoad) 1347800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany std::swap(N0, N1); 1348800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany } 1349800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 1350bd0052a0f26f04b8fcf59e8f645e5e33751e1f6eKostya Serebryany AddToISelQueue(N0); 1351bd0052a0f26f04b8fcf59e8f645e5e33751e1f6eKostya Serebryany SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), LoReg, 1352bd0052a0f26f04b8fcf59e8f645e5e33751e1f6eKostya Serebryany N0, SDValue()).getValue(1); 1353800e03f59896ef4b26d988f1878370bb5aeec0d8Kostya Serebryany 1354f985f44b13681071e585acb7a5703a2c1c23b6ceAlexey Samsonov if (foldedLoad) { 135559cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov AddToISelQueue(N1.getOperand(0)); 135659cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov AddToISelQueue(Tmp0); 1357f985f44b13681071e585acb7a5703a2c1c23b6ceAlexey Samsonov AddToISelQueue(Tmp1); 1358f985f44b13681071e585acb7a5703a2c1c23b6ceAlexey Samsonov AddToISelQueue(Tmp2); 1359f985f44b13681071e585acb7a5703a2c1c23b6ceAlexey Samsonov AddToISelQueue(Tmp3); 1360f985f44b13681071e585acb7a5703a2c1c23b6ceAlexey Samsonov SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, N1.getOperand(0), InFlag }; 1361f985f44b13681071e585acb7a5703a2c1c23b6ceAlexey Samsonov SDNode *CNode = 1362f985f44b13681071e585acb7a5703a2c1c23b6ceAlexey Samsonov CurDAG->getTargetNode(MOpc, MVT::Other, MVT::Flag, Ops, 6); 1363f985f44b13681071e585acb7a5703a2c1c23b6ceAlexey Samsonov InFlag = SDValue(CNode, 1); 136459cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov // Update the chain. 136559cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov ReplaceUses(N1.getValue(1), SDValue(CNode, 0)); 136659cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov } else { 136759cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov AddToISelQueue(N1); 136859cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov InFlag = 136959cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov SDValue(CurDAG->getTargetNode(Opc, MVT::Flag, N1, InFlag), 0); 137059cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov } 137159cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov 137259cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov // Copy the low half of the result, if it is needed. 137359cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov if (!N.getValue(0).use_empty()) { 13741c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), 13751c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov LoReg, NVT, InFlag); 13761c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov InFlag = Result.getValue(2); 13771c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov ReplaceUses(N.getValue(0), Result); 13781c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov#ifndef NDEBUG 13791c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov DOUT << std::string(Indent-2, ' ') << "=> "; 13801c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov DEBUG(Result.getNode()->dump(CurDAG)); 13811c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov DOUT << "\n"; 13821c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov#endif 13831c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov } 13841c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov // Copy the high half of the result, if it is needed. 13851c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov if (!N.getValue(1).use_empty()) { 13861c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov SDValue Result; 13871c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov if (HiReg == X86::AH && Subtarget->is64Bit()) { 13881c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov // Prevent use of AH in a REX instruction by referencing AX instead. 13891c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov // Shift it down 8 bits. 13901c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), 13911c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov X86::AX, MVT::i16, InFlag); 13921c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov InFlag = Result.getValue(2); 13931c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov Result = SDValue(CurDAG->getTargetNode(X86::SHR16ri, MVT::i16, Result, 13941c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov CurDAG->getTargetConstant(8, MVT::i8)), 0); 13951c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov // Then truncate it down to i8. 13961c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov SDValue SRIdx = CurDAG->getTargetConstant(1, MVT::i32); // SubRegSet 1 13971c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov Result = SDValue(CurDAG->getTargetNode(X86::EXTRACT_SUBREG, 13981c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov MVT::i8, Result, SRIdx), 0); 139959cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov } else { 140059cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), 14011c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov HiReg, NVT, InFlag); 14021c8b825c4396ad9cd38f713d9e9a51adae1d4c4eAlexey Samsonov InFlag = Result.getValue(2); 140359cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov } 140459cca13a80ed78ccbcdaa9bf20381bd48243f2f1Alexey Samsonov ReplaceUses(N.getValue(1), Result); 1405#ifndef NDEBUG 1406 DOUT << std::string(Indent-2, ' ') << "=> "; 1407 DEBUG(Result.getNode()->dump(CurDAG)); 1408 DOUT << "\n"; 1409#endif 1410 } 1411 1412#ifndef NDEBUG 1413 Indent -= 2; 1414#endif 1415 1416 return NULL; 1417 } 1418 1419 case ISD::SDIVREM: 1420 case ISD::UDIVREM: { 1421 SDValue N0 = Node->getOperand(0); 1422 SDValue N1 = Node->getOperand(1); 1423 1424 bool isSigned = Opcode == ISD::SDIVREM; 1425 if (!isSigned) 1426 switch (NVT.getSimpleVT()) { 1427 default: assert(0 && "Unsupported VT!"); 1428 case MVT::i8: Opc = X86::DIV8r; MOpc = X86::DIV8m; break; 1429 case MVT::i16: Opc = X86::DIV16r; MOpc = X86::DIV16m; break; 1430 case MVT::i32: Opc = X86::DIV32r; MOpc = X86::DIV32m; break; 1431 case MVT::i64: Opc = X86::DIV64r; MOpc = X86::DIV64m; break; 1432 } 1433 else 1434 switch (NVT.getSimpleVT()) { 1435 default: assert(0 && "Unsupported VT!"); 1436 case MVT::i8: Opc = X86::IDIV8r; MOpc = X86::IDIV8m; break; 1437 case MVT::i16: Opc = X86::IDIV16r; MOpc = X86::IDIV16m; break; 1438 case MVT::i32: Opc = X86::IDIV32r; MOpc = X86::IDIV32m; break; 1439 case MVT::i64: Opc = X86::IDIV64r; MOpc = X86::IDIV64m; break; 1440 } 1441 1442 unsigned LoReg, HiReg; 1443 unsigned ClrOpcode, SExtOpcode; 1444 switch (NVT.getSimpleVT()) { 1445 default: assert(0 && "Unsupported VT!"); 1446 case MVT::i8: 1447 LoReg = X86::AL; HiReg = X86::AH; 1448 ClrOpcode = 0; 1449 SExtOpcode = X86::CBW; 1450 break; 1451 case MVT::i16: 1452 LoReg = X86::AX; HiReg = X86::DX; 1453 ClrOpcode = X86::MOV16r0; 1454 SExtOpcode = X86::CWD; 1455 break; 1456 case MVT::i32: 1457 LoReg = X86::EAX; HiReg = X86::EDX; 1458 ClrOpcode = X86::MOV32r0; 1459 SExtOpcode = X86::CDQ; 1460 break; 1461 case MVT::i64: 1462 LoReg = X86::RAX; HiReg = X86::RDX; 1463 ClrOpcode = X86::MOV64r0; 1464 SExtOpcode = X86::CQO; 1465 break; 1466 } 1467 1468 SDValue Tmp0, Tmp1, Tmp2, Tmp3; 1469 bool foldedLoad = TryFoldLoad(N, N1, Tmp0, Tmp1, Tmp2, Tmp3); 1470 1471 SDValue InFlag; 1472 if (NVT == MVT::i8 && !isSigned) { 1473 // Special case for div8, just use a move with zero extension to AX to 1474 // clear the upper 8 bits (AH). 1475 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Move, Chain; 1476 if (TryFoldLoad(N, N0, Tmp0, Tmp1, Tmp2, Tmp3)) { 1477 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, N0.getOperand(0) }; 1478 AddToISelQueue(N0.getOperand(0)); 1479 AddToISelQueue(Tmp0); 1480 AddToISelQueue(Tmp1); 1481 AddToISelQueue(Tmp2); 1482 AddToISelQueue(Tmp3); 1483 Move = 1484 SDValue(CurDAG->getTargetNode(X86::MOVZX16rm8, MVT::i16, MVT::Other, 1485 Ops, 5), 0); 1486 Chain = Move.getValue(1); 1487 ReplaceUses(N0.getValue(1), Chain); 1488 } else { 1489 AddToISelQueue(N0); 1490 Move = 1491 SDValue(CurDAG->getTargetNode(X86::MOVZX16rr8, MVT::i16, N0), 0); 1492 Chain = CurDAG->getEntryNode(); 1493 } 1494 Chain = CurDAG->getCopyToReg(Chain, X86::AX, Move, SDValue()); 1495 InFlag = Chain.getValue(1); 1496 } else { 1497 AddToISelQueue(N0); 1498 InFlag = 1499 CurDAG->getCopyToReg(CurDAG->getEntryNode(), 1500 LoReg, N0, SDValue()).getValue(1); 1501 if (isSigned) { 1502 // Sign extend the low part into the high part. 1503 InFlag = 1504 SDValue(CurDAG->getTargetNode(SExtOpcode, MVT::Flag, InFlag), 0); 1505 } else { 1506 // Zero out the high part, effectively zero extending the input. 1507 SDValue ClrNode = SDValue(CurDAG->getTargetNode(ClrOpcode, NVT), 0); 1508 InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), HiReg, 1509 ClrNode, InFlag).getValue(1); 1510 } 1511 } 1512 1513 if (foldedLoad) { 1514 AddToISelQueue(N1.getOperand(0)); 1515 AddToISelQueue(Tmp0); 1516 AddToISelQueue(Tmp1); 1517 AddToISelQueue(Tmp2); 1518 AddToISelQueue(Tmp3); 1519 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, N1.getOperand(0), InFlag }; 1520 SDNode *CNode = 1521 CurDAG->getTargetNode(MOpc, MVT::Other, MVT::Flag, Ops, 6); 1522 InFlag = SDValue(CNode, 1); 1523 // Update the chain. 1524 ReplaceUses(N1.getValue(1), SDValue(CNode, 0)); 1525 } else { 1526 AddToISelQueue(N1); 1527 InFlag = 1528 SDValue(CurDAG->getTargetNode(Opc, MVT::Flag, N1, InFlag), 0); 1529 } 1530 1531 // Copy the division (low) result, if it is needed. 1532 if (!N.getValue(0).use_empty()) { 1533 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), 1534 LoReg, NVT, InFlag); 1535 InFlag = Result.getValue(2); 1536 ReplaceUses(N.getValue(0), Result); 1537#ifndef NDEBUG 1538 DOUT << std::string(Indent-2, ' ') << "=> "; 1539 DEBUG(Result.getNode()->dump(CurDAG)); 1540 DOUT << "\n"; 1541#endif 1542 } 1543 // Copy the remainder (high) result, if it is needed. 1544 if (!N.getValue(1).use_empty()) { 1545 SDValue Result; 1546 if (HiReg == X86::AH && Subtarget->is64Bit()) { 1547 // Prevent use of AH in a REX instruction by referencing AX instead. 1548 // Shift it down 8 bits. 1549 Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), 1550 X86::AX, MVT::i16, InFlag); 1551 InFlag = Result.getValue(2); 1552 Result = SDValue(CurDAG->getTargetNode(X86::SHR16ri, MVT::i16, Result, 1553 CurDAG->getTargetConstant(8, MVT::i8)), 0); 1554 // Then truncate it down to i8. 1555 SDValue SRIdx = CurDAG->getTargetConstant(1, MVT::i32); // SubRegSet 1 1556 Result = SDValue(CurDAG->getTargetNode(X86::EXTRACT_SUBREG, 1557 MVT::i8, Result, SRIdx), 0); 1558 } else { 1559 Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), 1560 HiReg, NVT, InFlag); 1561 InFlag = Result.getValue(2); 1562 } 1563 ReplaceUses(N.getValue(1), Result); 1564#ifndef NDEBUG 1565 DOUT << std::string(Indent-2, ' ') << "=> "; 1566 DEBUG(Result.getNode()->dump(CurDAG)); 1567 DOUT << "\n"; 1568#endif 1569 } 1570 1571#ifndef NDEBUG 1572 Indent -= 2; 1573#endif 1574 1575 return NULL; 1576 } 1577 1578 case ISD::SIGN_EXTEND_INREG: { 1579 MVT SVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); 1580 if (SVT == MVT::i8 && !Subtarget->is64Bit()) { 1581 SDValue N0 = Node->getOperand(0); 1582 AddToISelQueue(N0); 1583 1584 SDValue TruncOp = SDValue(getTruncateTo8Bit(N0), 0); 1585 unsigned Opc = 0; 1586 switch (NVT.getSimpleVT()) { 1587 default: assert(0 && "Unknown sign_extend_inreg!"); 1588 case MVT::i16: 1589 Opc = X86::MOVSX16rr8; 1590 break; 1591 case MVT::i32: 1592 Opc = X86::MOVSX32rr8; 1593 break; 1594 } 1595 1596 SDNode *ResNode = CurDAG->getTargetNode(Opc, NVT, TruncOp); 1597 1598#ifndef NDEBUG 1599 DOUT << std::string(Indent-2, ' ') << "=> "; 1600 DEBUG(TruncOp.getNode()->dump(CurDAG)); 1601 DOUT << "\n"; 1602 DOUT << std::string(Indent-2, ' ') << "=> "; 1603 DEBUG(ResNode->dump(CurDAG)); 1604 DOUT << "\n"; 1605 Indent -= 2; 1606#endif 1607 return ResNode; 1608 } 1609 break; 1610 } 1611 1612 case ISD::TRUNCATE: { 1613 if (NVT == MVT::i8 && !Subtarget->is64Bit()) { 1614 SDValue Input = Node->getOperand(0); 1615 AddToISelQueue(Node->getOperand(0)); 1616 SDNode *ResNode = getTruncateTo8Bit(Input); 1617 1618#ifndef NDEBUG 1619 DOUT << std::string(Indent-2, ' ') << "=> "; 1620 DEBUG(ResNode->dump(CurDAG)); 1621 DOUT << "\n"; 1622 Indent -= 2; 1623#endif 1624 return ResNode; 1625 } 1626 break; 1627 } 1628 1629 case ISD::DECLARE: { 1630 // Handle DECLARE nodes here because the second operand may have been 1631 // wrapped in X86ISD::Wrapper. 1632 SDValue Chain = Node->getOperand(0); 1633 SDValue N1 = Node->getOperand(1); 1634 SDValue N2 = Node->getOperand(2); 1635 if (!isa<FrameIndexSDNode>(N1)) 1636 break; 1637 int FI = cast<FrameIndexSDNode>(N1)->getIndex(); 1638 if (N2.getOpcode() == ISD::ADD && 1639 N2.getOperand(0).getOpcode() == X86ISD::GlobalBaseReg) 1640 N2 = N2.getOperand(1); 1641 if (N2.getOpcode() == X86ISD::Wrapper && 1642 isa<GlobalAddressSDNode>(N2.getOperand(0))) { 1643 GlobalValue *GV = 1644 cast<GlobalAddressSDNode>(N2.getOperand(0))->getGlobal(); 1645 SDValue Tmp1 = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy()); 1646 SDValue Tmp2 = CurDAG->getTargetGlobalAddress(GV, TLI.getPointerTy()); 1647 AddToISelQueue(Chain); 1648 SDValue Ops[] = { Tmp1, Tmp2, Chain }; 1649 return CurDAG->getTargetNode(TargetInstrInfo::DECLARE, 1650 MVT::Other, Ops, 3); 1651 } 1652 break; 1653 } 1654 } 1655 1656 SDNode *ResNode = SelectCode(N); 1657 1658#ifndef NDEBUG 1659 DOUT << std::string(Indent-2, ' ') << "=> "; 1660 if (ResNode == NULL || ResNode == N.getNode()) 1661 DEBUG(N.getNode()->dump(CurDAG)); 1662 else 1663 DEBUG(ResNode->dump(CurDAG)); 1664 DOUT << "\n"; 1665 Indent -= 2; 1666#endif 1667 1668 return ResNode; 1669} 1670 1671bool X86DAGToDAGISel:: 1672SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode, 1673 std::vector<SDValue> &OutOps) { 1674 SDValue Op0, Op1, Op2, Op3; 1675 switch (ConstraintCode) { 1676 case 'o': // offsetable ?? 1677 case 'v': // not offsetable ?? 1678 default: return true; 1679 case 'm': // memory 1680 if (!SelectAddr(Op, Op, Op0, Op1, Op2, Op3)) 1681 return true; 1682 break; 1683 } 1684 1685 OutOps.push_back(Op0); 1686 OutOps.push_back(Op1); 1687 OutOps.push_back(Op2); 1688 OutOps.push_back(Op3); 1689 AddToISelQueue(Op0); 1690 AddToISelQueue(Op1); 1691 AddToISelQueue(Op2); 1692 AddToISelQueue(Op3); 1693 return false; 1694} 1695 1696/// createX86ISelDag - This pass converts a legalized DAG into a 1697/// X86-specific DAG, ready for instruction scheduling. 1698/// 1699FunctionPass *llvm::createX86ISelDag(X86TargetMachine &TM, bool Fast) { 1700 return new X86DAGToDAGISel(TM, Fast); 1701} 1702