1894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman//===- X86ISelDAGToDAG.cpp - A DAG pattern matching inst selector for X86 -===// 2894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman// 3894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman// The LLVM Compiler Infrastructure 4894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman// 5894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman// This file is distributed under the University of Illinois Open Source 6894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman// License. See LICENSE.TXT for details. 7894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman// 8894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman//===----------------------------------------------------------------------===// 9894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman// 10894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman// This file defines a DAG pattern matching instruction selector for X86, 11894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman// converting from a legalized dag to a X86 dag. 12894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman// 13894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman//===----------------------------------------------------------------------===// 14894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 15894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman#define DEBUG_TYPE "x86-isel" 16894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman#include "X86.h" 17894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman#include "X86InstrBuilder.h" 18894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman#include "X86MachineFunctionInfo.h" 19894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman#include "X86RegisterInfo.h" 20894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman#include "X86Subtarget.h" 21894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman#include "X86TargetMachine.h" 22894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman#include "llvm/Instructions.h" 23894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman#include "llvm/Intrinsics.h" 24894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman#include "llvm/Support/CFG.h" 25894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman#include "llvm/Type.h" 2619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman#include "llvm/CodeGen/FunctionLoweringInfo.h" 27894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman#include "llvm/CodeGen/MachineConstantPool.h" 28894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman#include "llvm/CodeGen/MachineFunction.h" 29894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman#include "llvm/CodeGen/MachineFrameInfo.h" 30894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman#include "llvm/CodeGen/MachineInstrBuilder.h" 31894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman#include "llvm/CodeGen/MachineRegisterInfo.h" 32894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman#include "llvm/CodeGen/SelectionDAGISel.h" 33894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman#include "llvm/Target/TargetMachine.h" 34894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman#include "llvm/Target/TargetOptions.h" 35894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman#include "llvm/Support/Debug.h" 36894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman#include "llvm/Support/ErrorHandling.h" 37894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman#include "llvm/Support/MathExtras.h" 38894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman#include "llvm/Support/raw_ostream.h" 39894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman#include "llvm/ADT/SmallPtrSet.h" 40894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman#include "llvm/ADT/Statistic.h" 41894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Baumanusing namespace llvm; 42894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 43894018228b0e0bdbd7aa7e8f47d4a9458789ca82John BaumanSTATISTIC(NumLoadMoved, "Number of loads moved below TokenFactor"); 44894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 45894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman//===----------------------------------------------------------------------===// 46894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman// Pattern Matcher Implementation 47894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman//===----------------------------------------------------------------------===// 48894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 49894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Baumannamespace { 50894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// X86ISelAddressMode - This corresponds to X86AddressMode, but uses 51894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// SDValue's instead of register numbers for the leaves of the matched 52894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// tree. 53894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman struct X86ISelAddressMode { 54894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman enum { 55894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman RegBase, 56894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman FrameIndexBase 57894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } BaseType; 58894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 59894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // This is really a union, discriminated by BaseType! 60894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Base_Reg; 61894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman int Base_FrameIndex; 62894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 63894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman unsigned Scale; 64894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue IndexReg; 65894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman int32_t Disp; 66894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Segment; 67894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman const GlobalValue *GV; 68894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman const Constant *CP; 69894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman const BlockAddress *BlockAddr; 70894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman const char *ES; 71894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman int JT; 72894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman unsigned Align; // CP alignment. 73894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman unsigned char SymbolFlags; // X86II::MO_* 74894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 75894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman X86ISelAddressMode() 76894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman : BaseType(RegBase), Base_FrameIndex(0), Scale(1), IndexReg(), Disp(0), 77894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Segment(), GV(0), CP(0), BlockAddr(0), ES(0), JT(-1), Align(0), 78894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SymbolFlags(X86II::MO_NO_FLAG) { 79894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 80894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 81894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman bool hasSymbolicDisplacement() const { 82894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return GV != 0 || CP != 0 || ES != 0 || JT != -1 || BlockAddr != 0; 83894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 84894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 85894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman bool hasBaseOrIndexReg() const { 86894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return IndexReg.getNode() != 0 || Base_Reg.getNode() != 0; 87894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 88894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 89894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// isRIPRelative - Return true if this addressing mode is already RIP 90894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// relative. 91894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman bool isRIPRelative() const { 92894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (BaseType != RegBase) return false; 93894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (RegisterSDNode *RegNode = 94894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman dyn_cast_or_null<RegisterSDNode>(Base_Reg.getNode())) 95894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return RegNode->getReg() == X86::RIP; 96894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 97894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 98894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 99894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman void setBaseReg(SDValue Reg) { 100894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman BaseType = RegBase; 101894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Base_Reg = Reg; 102894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 103894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 104894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman void dump() { 105894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman dbgs() << "X86ISelAddressMode " << this << '\n'; 106894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman dbgs() << "Base_Reg "; 107894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (Base_Reg.getNode() != 0) 108894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Base_Reg.getNode()->dump(); 109894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else 110894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman dbgs() << "nul"; 111894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman dbgs() << " Base.FrameIndex " << Base_FrameIndex << '\n' 112894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman << " Scale" << Scale << '\n' 113894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman << "IndexReg "; 114894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (IndexReg.getNode() != 0) 115894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman IndexReg.getNode()->dump(); 116894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else 117894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman dbgs() << "nul"; 118894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman dbgs() << " Disp " << Disp << '\n' 119894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman << "GV "; 120894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (GV) 121894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman GV->dump(); 122894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else 123894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman dbgs() << "nul"; 124894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman dbgs() << " CP "; 125894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (CP) 126894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CP->dump(); 127894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else 128894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman dbgs() << "nul"; 129894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman dbgs() << '\n' 130894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman << "ES "; 131894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (ES) 132894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman dbgs() << ES; 133894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else 134894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman dbgs() << "nul"; 135894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman dbgs() << " JT" << JT << " Align" << Align << '\n'; 136894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 137894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman }; 138894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman} 139894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 140894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Baumannamespace { 141894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman //===--------------------------------------------------------------------===// 142894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// ISel - X86 specific code to select X86 machine instructions for 143894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// SelectionDAG operations. 144894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// 145894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman class X86DAGToDAGISel : public SelectionDAGISel { 146894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// X86Lowering - This object fully describes how to lower LLVM code to an 147894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// X86-specific SelectionDAG. 148894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman const X86TargetLowering &X86Lowering; 149894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 150894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// Subtarget - Keep a pointer to the X86Subtarget around so that we can 151894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// make the right decision when generating code for different targets. 152894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman const X86Subtarget *Subtarget; 153894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 154894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// OptForSize - If true, selector should try to optimize for code size 155894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// instead of performance. 156894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman bool OptForSize; 157894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 158894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman public: 159894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman explicit X86DAGToDAGISel(X86TargetMachine &tm, CodeGenOpt::Level OptLevel) 160894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman : SelectionDAGISel(tm, OptLevel), 161894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman X86Lowering(*tm.getTargetLowering()), 162894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Subtarget(&tm.getSubtarget<X86Subtarget>()), 163894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman OptForSize(false) {} 164894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 165894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman virtual const char *getPassName() const { 166894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return "X86 DAG->DAG Instruction Selection"; 167894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 168894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 169894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman virtual void EmitFunctionEntryCode(); 170894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 171894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const; 172894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 173894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman virtual void PreprocessISelDAG(); 174894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 17519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman inline bool immSext8(SDNode *N) const { 17619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman return isInt<8>(cast<ConstantSDNode>(N)->getSExtValue()); 17719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman } 17819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 17919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // i64immSExt32 predicate - True if the 64-bit immediate fits in a 32-bit 18019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // sign extended field. 18119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman inline bool i64immSExt32(SDNode *N) const { 18219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman uint64_t v = cast<ConstantSDNode>(N)->getZExtValue(); 18319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman return (int64_t)v == (int32_t)v; 18419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman } 18519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 186894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman// Include the pieces autogenerated from the target description. 187894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman#include "X86GenDAGISel.inc" 188894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 189894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman private: 190894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDNode *Select(SDNode *N); 191894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDNode *SelectAtomic64(SDNode *Node, unsigned Opc); 192894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDNode *SelectAtomicLoadAdd(SDNode *Node, EVT NVT); 19319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SDNode *SelectAtomicLoadArith(SDNode *Node, EVT NVT); 194894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 19519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman bool FoldOffsetIntoAddress(uint64_t Offset, X86ISelAddressMode &AM); 19619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman bool MatchLoadInAddress(LoadSDNode *N, X86ISelAddressMode &AM); 197894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman bool MatchWrapper(SDValue N, X86ISelAddressMode &AM); 198894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman bool MatchAddress(SDValue N, X86ISelAddressMode &AM); 199894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman bool MatchAddressRecursively(SDValue N, X86ISelAddressMode &AM, 200894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman unsigned Depth); 201894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman bool MatchAddressBase(SDValue N, X86ISelAddressMode &AM); 20219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman bool SelectAddr(SDNode *Parent, SDValue N, SDValue &Base, 203894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue &Scale, SDValue &Index, SDValue &Disp, 204894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue &Segment); 20519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman bool SelectLEAAddr(SDValue N, SDValue &Base, 206894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue &Scale, SDValue &Index, SDValue &Disp, 207894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue &Segment); 20819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman bool SelectTLSADDRAddr(SDValue N, SDValue &Base, 209894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue &Scale, SDValue &Index, SDValue &Disp, 210894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue &Segment); 211894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman bool SelectScalarSSELoad(SDNode *Root, SDValue N, 212894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue &Base, SDValue &Scale, 213894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue &Index, SDValue &Disp, 214894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue &Segment, 215894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue &NodeWithChain); 216894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 217894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman bool TryFoldLoad(SDNode *P, SDValue N, 218894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue &Base, SDValue &Scale, 219894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue &Index, SDValue &Disp, 220894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue &Segment); 221894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 222894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for 223894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// inline asm expressions. 224894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, 225894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman char ConstraintCode, 226894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman std::vector<SDValue> &OutOps); 227894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 228894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman void EmitSpecialCodeForMain(MachineBasicBlock *BB, MachineFrameInfo *MFI); 229894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 230894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman inline void getAddressOperands(X86ISelAddressMode &AM, SDValue &Base, 231894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue &Scale, SDValue &Index, 232894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue &Disp, SDValue &Segment) { 233894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Base = (AM.BaseType == X86ISelAddressMode::FrameIndexBase) ? 234894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CurDAG->getTargetFrameIndex(AM.Base_FrameIndex, TLI.getPointerTy()) : 235894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Base_Reg; 236894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Scale = getI8Imm(AM.Scale); 237894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Index = AM.IndexReg; 238894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // These are 32-bit even in 64-bit mode since RIP relative offset 239894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // is 32-bit. 240894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (AM.GV) 241894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Disp = CurDAG->getTargetGlobalAddress(AM.GV, DebugLoc(), 242894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman MVT::i32, AM.Disp, 243894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.SymbolFlags); 244894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else if (AM.CP) 245894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i32, 246894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Align, AM.Disp, AM.SymbolFlags); 247894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else if (AM.ES) 248894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Disp = CurDAG->getTargetExternalSymbol(AM.ES, MVT::i32, AM.SymbolFlags); 249894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else if (AM.JT != -1) 250894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i32, AM.SymbolFlags); 251894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else if (AM.BlockAddr) 252894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Disp = CurDAG->getBlockAddress(AM.BlockAddr, MVT::i32, 253894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman true, AM.SymbolFlags); 254894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else 255894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Disp = CurDAG->getTargetConstant(AM.Disp, MVT::i32); 256894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 257894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (AM.Segment.getNode()) 258894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Segment = AM.Segment; 259894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else 260894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Segment = CurDAG->getRegister(0, MVT::i32); 261894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 262894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 263894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// getI8Imm - Return a target constant with the specified value, of type 264894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// i8. 265894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman inline SDValue getI8Imm(unsigned Imm) { 266894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return CurDAG->getTargetConstant(Imm, MVT::i8); 267894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 268894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 269894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// getI32Imm - Return a target constant with the specified value, of type 270894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// i32. 271894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman inline SDValue getI32Imm(unsigned Imm) { 272894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return CurDAG->getTargetConstant(Imm, MVT::i32); 273894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 274894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 275894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// getGlobalBaseReg - Return an SDNode that returns the value of 276894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// the global base register. Output instructions required to 277894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// initialize the global base register, if necessary. 278894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// 279894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDNode *getGlobalBaseReg(); 280894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 281894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// getTargetMachine - Return a reference to the TargetMachine, casted 282894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// to the target-specific type. 283894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman const X86TargetMachine &getTargetMachine() { 284894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return static_cast<const X86TargetMachine &>(TM); 285894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 286894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 287894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// getInstrInfo - Return a reference to the TargetInstrInfo, casted 288894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// to the target-specific type. 289894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman const X86InstrInfo *getInstrInfo() { 290894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return getTargetMachine().getInstrInfo(); 291894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 292894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman }; 293894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman} 294894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 295894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 296894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Baumanbool 297894018228b0e0bdbd7aa7e8f47d4a9458789ca82John BaumanX86DAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const { 298894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (OptLevel == CodeGenOpt::None) return false; 299894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 300894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (!N.hasOneUse()) 301894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 302894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 303894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (N.getOpcode() != ISD::LOAD) 304894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return true; 305894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 306894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // If N is a load, do additional profitability checks. 307894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (U == Root) { 308894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman switch (U->getOpcode()) { 309894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman default: break; 310894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86ISD::ADD: 311894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86ISD::SUB: 312894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86ISD::AND: 313894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86ISD::XOR: 314894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86ISD::OR: 315894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case ISD::ADD: 316894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case ISD::ADDC: 317894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case ISD::ADDE: 318894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case ISD::AND: 319894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case ISD::OR: 320894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case ISD::XOR: { 321894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Op1 = U->getOperand(1); 322894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 323894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // If the other operand is a 8-bit immediate we should fold the immediate 324894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // instead. This reduces code size. 325894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // e.g. 326894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // movl 4(%esp), %eax 327894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // addl $4, %eax 328894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // vs. 329894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // movl $4, %eax 330894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // addl 4(%esp), %eax 331894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // The former is 2 bytes shorter. In case where the increment is 1, then 332894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // the saving can be 4 bytes (by using incl %eax). 333894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (ConstantSDNode *Imm = dyn_cast<ConstantSDNode>(Op1)) 334894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (Imm->getAPIntValue().isSignedIntN(8)) 335894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 336894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 337894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // If the other operand is a TLS address, we should fold it instead. 338894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // This produces 339894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // movl %gs:0, %eax 340894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // leal i@NTPOFF(%eax), %eax 341894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // instead of 342894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // movl $i@NTPOFF, %eax 343894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // addl %gs:0, %eax 344894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // if the block also has an access to a second TLS address this will save 345894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // a load. 346894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // FIXME: This is probably also true for non TLS addresses. 347894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (Op1.getOpcode() == X86ISD::Wrapper) { 348894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Val = Op1.getOperand(0); 349894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (Val.getOpcode() == ISD::TargetGlobalTLSAddress) 350894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 351894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 352894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 353894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 354894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 355894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 356894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return true; 357894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman} 358894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 359894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// MoveBelowCallOrigChain - Replace the original chain operand of the call with 360894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// load's chain operand and move load below the call's chain operand. 361894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Baumanstatic void MoveBelowOrigChain(SelectionDAG *CurDAG, SDValue Load, 362894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Call, SDValue OrigChain) { 363894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SmallVector<SDValue, 8> Ops; 364894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Chain = OrigChain.getOperand(0); 365894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (Chain.getNode() == Load.getNode()) 366894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Ops.push_back(Load.getOperand(0)); 367894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else { 368894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman assert(Chain.getOpcode() == ISD::TokenFactor && 369894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman "Unexpected chain operand"); 370894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i) 371894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (Chain.getOperand(i).getNode() == Load.getNode()) 372894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Ops.push_back(Load.getOperand(0)); 373894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else 374894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Ops.push_back(Chain.getOperand(i)); 375894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue NewChain = 376894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CurDAG->getNode(ISD::TokenFactor, Load.getDebugLoc(), 377894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman MVT::Other, &Ops[0], Ops.size()); 378894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Ops.clear(); 379894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Ops.push_back(NewChain); 380894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 381894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman for (unsigned i = 1, e = OrigChain.getNumOperands(); i != e; ++i) 382894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Ops.push_back(OrigChain.getOperand(i)); 383894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CurDAG->UpdateNodeOperands(OrigChain.getNode(), &Ops[0], Ops.size()); 384894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CurDAG->UpdateNodeOperands(Load.getNode(), Call.getOperand(0), 385894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Load.getOperand(1), Load.getOperand(2)); 386894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Ops.clear(); 387894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Ops.push_back(SDValue(Load.getNode(), 1)); 388894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman for (unsigned i = 1, e = Call.getNode()->getNumOperands(); i != e; ++i) 389894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Ops.push_back(Call.getOperand(i)); 390894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CurDAG->UpdateNodeOperands(Call.getNode(), &Ops[0], Ops.size()); 391894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman} 392894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 393894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// isCalleeLoad - Return true if call address is a load and it can be 394894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// moved below CALLSEQ_START and the chains leading up to the call. 395894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// Return the CALLSEQ_START by reference as a second output. 396894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// In the case of a tail call, there isn't a callseq node between the call 397894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// chain and the load. 398894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Baumanstatic bool isCalleeLoad(SDValue Callee, SDValue &Chain, bool HasCallSeq) { 399894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (Callee.getNode() == Chain.getNode() || !Callee.hasOneUse()) 400894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 401894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman LoadSDNode *LD = dyn_cast<LoadSDNode>(Callee.getNode()); 402894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (!LD || 403894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman LD->isVolatile() || 404894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman LD->getAddressingMode() != ISD::UNINDEXED || 405894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman LD->getExtensionType() != ISD::NON_EXTLOAD) 406894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 407894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 408894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Now let's find the callseq_start. 409894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman while (HasCallSeq && Chain.getOpcode() != ISD::CALLSEQ_START) { 410894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (!Chain.hasOneUse()) 411894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 412894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Chain = Chain.getOperand(0); 413894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 414894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 415894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (!Chain.getNumOperands()) 416894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 417894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (Chain.getOperand(0).getNode() == Callee.getNode()) 418894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return true; 419894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (Chain.getOperand(0).getOpcode() == ISD::TokenFactor && 420894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Callee.getValue(1).isOperandOf(Chain.getOperand(0).getNode()) && 421894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Callee.getValue(1).hasOneUse()) 422894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return true; 423894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 424894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman} 425894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 426894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Baumanvoid X86DAGToDAGISel::PreprocessISelDAG() { 427894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // OptForSize is used in pattern predicates that isel is matching. 428894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman OptForSize = MF->getFunction()->hasFnAttr(Attribute::OptimizeForSize); 429894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 430894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(), 431894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman E = CurDAG->allnodes_end(); I != E; ) { 432894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDNode *N = I++; // Preincrement iterator to avoid invalidation issues. 433894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 434894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (OptLevel != CodeGenOpt::None && 435894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman (N->getOpcode() == X86ISD::CALL || 436894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman N->getOpcode() == X86ISD::TC_RETURN)) { 437894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// Also try moving call address load from outside callseq_start to just 438894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// before the call to allow it to be folded. 439894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// 440894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// [Load chain] 441894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// ^ 442894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// | 443894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// [Load] 444894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// ^ ^ 445894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// | | 446894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// / \-- 447894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// / | 448894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ///[CALLSEQ_START] | 449894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// ^ | 450894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// | | 451894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// [LOAD/C2Reg] | 452894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// | | 453894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// \ / 454894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// \ / 455894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman /// [CALL] 456894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman bool HasCallSeq = N->getOpcode() == X86ISD::CALL; 457894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Chain = N->getOperand(0); 458894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Load = N->getOperand(1); 459894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (!isCalleeLoad(Load, Chain, HasCallSeq)) 460894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman continue; 461894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman MoveBelowOrigChain(CurDAG, Load, SDValue(N, 0), Chain); 462894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ++NumLoadMoved; 463894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman continue; 464894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 465894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 466894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Lower fpround and fpextend nodes that target the FP stack to be store and 467894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // load to the stack. This is a gross hack. We would like to simply mark 468894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // these as being illegal, but when we do that, legalize produces these when 469894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // it expands calls, then expands these in the same legalize pass. We would 470894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // like dag combine to be able to hack on these between the call expansion 471894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // and the node legalization. As such this pass basically does "really 472894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // late" legalization of these inline with the X86 isel pass. 473894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // FIXME: This should only happen when not compiled with -O0. 474894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (N->getOpcode() != ISD::FP_ROUND && N->getOpcode() != ISD::FP_EXTEND) 475894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman continue; 476894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 477894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman EVT SrcVT = N->getOperand(0).getValueType(); 478894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman EVT DstVT = N->getValueType(0); 47919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 48019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // If any of the sources are vectors, no fp stack involved. 48119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (SrcVT.isVector() || DstVT.isVector()) 48219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman continue; 48319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 48419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // If the source and destination are SSE registers, then this is a legal 48519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // conversion that should not be lowered. 486894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman bool SrcIsSSE = X86Lowering.isScalarFPTypeInSSEReg(SrcVT); 487894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman bool DstIsSSE = X86Lowering.isScalarFPTypeInSSEReg(DstVT); 488894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (SrcIsSSE && DstIsSSE) 489894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman continue; 490894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 491894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (!SrcIsSSE && !DstIsSSE) { 492894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // If this is an FPStack extension, it is a noop. 493894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (N->getOpcode() == ISD::FP_EXTEND) 494894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman continue; 495894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // If this is a value-preserving FPStack truncation, it is a noop. 496894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (N->getConstantOperandVal(1)) 497894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman continue; 498894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 499894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 500894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Here we could have an FP stack truncation or an FPStack <-> SSE convert. 501894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // FPStack has extload and truncstore. SSE can fold direct loads into other 502894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // operations. Based on this, decide what we want to do. 503894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman EVT MemVT; 504894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (N->getOpcode() == ISD::FP_ROUND) 505894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman MemVT = DstVT; // FP_ROUND must use DstVT, we can't do a 'trunc load'. 506894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else 507894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman MemVT = SrcIsSSE ? SrcVT : DstVT; 508894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 509894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT); 510894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman DebugLoc dl = N->getDebugLoc(); 511894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 512894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // FIXME: optimize the case where the src/dest is a load or store? 513894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Store = CurDAG->getTruncStore(CurDAG->getEntryNode(), dl, 514894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman N->getOperand(0), 51519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman MemTmp, MachinePointerInfo(), MemVT, 516894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman false, false, 0); 51719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SDValue Result = CurDAG->getExtLoad(ISD::EXTLOAD, dl, DstVT, Store, MemTmp, 51819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman MachinePointerInfo(), 51919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman MemVT, false, false, 0); 520894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 521894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // We're about to replace all uses of the FP_ROUND/FP_EXTEND with the 522894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // extload we created. This will cause general havok on the dag because 523894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // anything below the conversion could be folded into other existing nodes. 524894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // To avoid invalidating 'I', back it up to the convert node. 525894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman --I; 526894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Result); 527894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 528894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Now that we did that, the node is dead. Increment the iterator to the 529894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // next node to process, then delete N. 530894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ++I; 531894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CurDAG->DeleteNode(N); 532894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 533894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman} 534894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 535894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 536894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// EmitSpecialCodeForMain - Emit any code that needs to be executed only in 537894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// the main function. 538894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Baumanvoid X86DAGToDAGISel::EmitSpecialCodeForMain(MachineBasicBlock *BB, 539894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman MachineFrameInfo *MFI) { 540894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman const TargetInstrInfo *TII = TM.getInstrInfo(); 54119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (Subtarget->isTargetCygMing()) { 54219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman unsigned CallOp = 54319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman Subtarget->is64Bit() ? X86::WINCALL64pcrel32 : X86::CALLpcrel32; 544894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman BuildMI(BB, DebugLoc(), 54519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman TII->get(CallOp)).addExternalSymbol("__main"); 54619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman } 547894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman} 548894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 549894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Baumanvoid X86DAGToDAGISel::EmitFunctionEntryCode() { 550894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // If this is main, emit special code for main. 551894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (const Function *Fn = MF->getFunction()) 552894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (Fn->hasExternalLinkage() && Fn->getName() == "main") 553894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman EmitSpecialCodeForMain(MF->begin(), MF->getFrameInfo()); 554894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman} 555894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 55619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Baumanstatic bool isDispSafeForFrameIndex(int64_t Val) { 55719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // On 64-bit platforms, we can run into an issue where a frame index 55819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // includes a displacement that, when added to the explicit displacement, 55919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // will overflow the displacement field. Assuming that the frame index 56019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // displacement fits into a 31-bit integer (which is only slightly more 56119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // aggressive than the current fundamental assumption that it fits into 56219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // a 32-bit integer), a 31-bit disp should always be safe. 56319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman return isInt<31>(Val); 56419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman} 565894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 56619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Baumanbool X86DAGToDAGISel::FoldOffsetIntoAddress(uint64_t Offset, 56719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86ISelAddressMode &AM) { 56819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman int64_t Val = AM.Disp + Offset; 56919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman CodeModel::Model M = TM.getCodeModel(); 57019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (Subtarget->is64Bit()) { 57119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (!X86::isOffsetSuitableForCodeModel(Val, M, 57219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman AM.hasSymbolicDisplacement())) 57319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman return true; 57419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // In addition to the checks required for a register base, check that 57519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // we do not try to use an unsafe Disp with a frame index. 57619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (AM.BaseType == X86ISelAddressMode::FrameIndexBase && 57719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman !isDispSafeForFrameIndex(Val)) 57819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman return true; 579894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 58019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman AM.Disp = Val; 58119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman return false; 582894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 583894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman} 584894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 58519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Baumanbool X86DAGToDAGISel::MatchLoadInAddress(LoadSDNode *N, X86ISelAddressMode &AM){ 58619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SDValue Address = N->getOperand(1); 58719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 58819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // load gs:0 -> GS segment register. 58919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // load fs:0 -> FS segment register. 59019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // 591894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // This optimization is valid because the GNU TLS model defines that 592894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // gs:0 (or fs:0 on X86-64) contains its own address. 593894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // For more information see http://people.redhat.com/drepper/tls.pdf 59419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Address)) 59519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (C->getSExtValue() == 0 && AM.Segment.getNode() == 0 && 59619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman Subtarget->isTargetELF()) 59719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman switch (N->getPointerInfo().getAddrSpace()) { 59819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman case 256: 59919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16); 60019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman return false; 60119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman case 257: 60219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16); 60319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman return false; 60419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman } 60519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 606894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return true; 607894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman} 608894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 609894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// MatchWrapper - Try to match X86ISD::Wrapper and X86ISD::WrapperRIP nodes 610894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// into an addressing mode. These wrap things that will resolve down into a 611894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// symbol reference. If no match is possible, this returns true, otherwise it 612894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// returns false. 613894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Baumanbool X86DAGToDAGISel::MatchWrapper(SDValue N, X86ISelAddressMode &AM) { 614894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // If the addressing mode already has a symbol as the displacement, we can 615894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // never match another symbol. 616894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (AM.hasSymbolicDisplacement()) 617894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return true; 618894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 619894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue N0 = N.getOperand(0); 620894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CodeModel::Model M = TM.getCodeModel(); 621894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 622894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Handle X86-64 rip-relative addresses. We check this before checking direct 623894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // folding because RIP is preferable to non-RIP accesses. 624894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (Subtarget->is64Bit() && 625894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Under X86-64 non-small code model, GV (and friends) are 64-bits, so 626894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // they cannot be folded into immediate fields. 627894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // FIXME: This can be improved for kernel and other models? 628894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman (M == CodeModel::Small || M == CodeModel::Kernel) && 629894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Base and index reg must be 0 in order to use %rip as base and lowering 630894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // must allow RIP. 631894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman !AM.hasBaseOrIndexReg() && N.getOpcode() == X86ISD::WrapperRIP) { 632894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(N0)) { 63319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86ISelAddressMode Backup = AM; 634894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.GV = G->getGlobal(); 635894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.SymbolFlags = G->getTargetFlags(); 63619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (FoldOffsetIntoAddress(G->getOffset(), AM)) { 63719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman AM = Backup; 63819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman return true; 63919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman } 640894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(N0)) { 64119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86ISelAddressMode Backup = AM; 642894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.CP = CP->getConstVal(); 643894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Align = CP->getAlignment(); 644894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.SymbolFlags = CP->getTargetFlags(); 64519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (FoldOffsetIntoAddress(CP->getOffset(), AM)) { 64619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman AM = Backup; 64719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman return true; 64819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman } 649894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(N0)) { 650894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.ES = S->getSymbol(); 651894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.SymbolFlags = S->getTargetFlags(); 652894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } else if (JumpTableSDNode *J = dyn_cast<JumpTableSDNode>(N0)) { 653894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.JT = J->getIndex(); 654894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.SymbolFlags = J->getTargetFlags(); 655894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } else { 656894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.BlockAddr = cast<BlockAddressSDNode>(N0)->getBlockAddress(); 657894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.SymbolFlags = cast<BlockAddressSDNode>(N0)->getTargetFlags(); 658894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 659894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 660894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (N.getOpcode() == X86ISD::WrapperRIP) 661894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.setBaseReg(CurDAG->getRegister(X86::RIP, MVT::i64)); 662894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 663894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 664894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 665894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Handle the case when globals fit in our immediate field: This is true for 666894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // X86-32 always and X86-64 when in -static -mcmodel=small mode. In 64-bit 667894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // mode, this results in a non-RIP-relative computation. 668894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (!Subtarget->is64Bit() || 669894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ((M == CodeModel::Small || M == CodeModel::Kernel) && 670894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman TM.getRelocationModel() == Reloc::Static)) { 671894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(N0)) { 672894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.GV = G->getGlobal(); 673894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Disp += G->getOffset(); 674894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.SymbolFlags = G->getTargetFlags(); 675894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(N0)) { 676894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.CP = CP->getConstVal(); 677894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Align = CP->getAlignment(); 678894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Disp += CP->getOffset(); 679894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.SymbolFlags = CP->getTargetFlags(); 680894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(N0)) { 681894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.ES = S->getSymbol(); 682894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.SymbolFlags = S->getTargetFlags(); 683894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } else if (JumpTableSDNode *J = dyn_cast<JumpTableSDNode>(N0)) { 684894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.JT = J->getIndex(); 685894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.SymbolFlags = J->getTargetFlags(); 686894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } else { 687894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.BlockAddr = cast<BlockAddressSDNode>(N0)->getBlockAddress(); 688894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.SymbolFlags = cast<BlockAddressSDNode>(N0)->getTargetFlags(); 689894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 690894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 691894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 692894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 693894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return true; 694894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman} 695894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 696894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// MatchAddress - Add the specified node to the specified addressing mode, 697894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// returning true if it cannot be done. This just pattern matches for the 698894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// addressing mode. 699894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Baumanbool X86DAGToDAGISel::MatchAddress(SDValue N, X86ISelAddressMode &AM) { 700894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (MatchAddressRecursively(N, AM, 0)) 701894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return true; 702894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 703894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Post-processing: Convert lea(,%reg,2) to lea(%reg,%reg), which has 704894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // a smaller encoding and avoids a scaled-index. 705894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (AM.Scale == 2 && 706894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.BaseType == X86ISelAddressMode::RegBase && 707894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Base_Reg.getNode() == 0) { 708894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Base_Reg = AM.IndexReg; 709894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Scale = 1; 710894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 711894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 712894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Post-processing: Convert foo to foo(%rip), even in non-PIC mode, 713894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // because it has a smaller encoding. 714894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // TODO: Which other code models can use this? 715894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (TM.getCodeModel() == CodeModel::Small && 716894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Subtarget->is64Bit() && 717894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Scale == 1 && 718894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.BaseType == X86ISelAddressMode::RegBase && 719894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Base_Reg.getNode() == 0 && 720894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.IndexReg.getNode() == 0 && 721894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.SymbolFlags == X86II::MO_NO_FLAG && 722894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.hasSymbolicDisplacement()) 723894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Base_Reg = CurDAG->getRegister(X86::RIP, MVT::i64); 724894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 725894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 726894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman} 727894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 728894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Baumanbool X86DAGToDAGISel::MatchAddressRecursively(SDValue N, X86ISelAddressMode &AM, 729894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman unsigned Depth) { 730894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman DebugLoc dl = N.getDebugLoc(); 731894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman DEBUG({ 732894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman dbgs() << "MatchAddress: "; 733894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.dump(); 734894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman }); 735894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Limit recursion. 736894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (Depth > 5) 737894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return MatchAddressBase(N, AM); 738894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 739894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // If this is already a %rip relative address, we can only merge immediates 740894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // into it. Instead of handling this in every case, we handle it here. 741894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // RIP relative addressing: %rip + 32-bit displacement! 742894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (AM.isRIPRelative()) { 743894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // FIXME: JumpTable and ExternalSymbol address currently don't like 744894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // displacements. It isn't very important, but this should be fixed for 745894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // consistency. 746894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (!AM.ES && AM.JT != -1) return true; 747894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 74819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(N)) 74919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (!FoldOffsetIntoAddress(Cst->getSExtValue(), AM)) 750894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 751894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return true; 752894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 753894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 754894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman switch (N.getOpcode()) { 755894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman default: break; 756894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case ISD::Constant: { 757894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman uint64_t Val = cast<ConstantSDNode>(N)->getSExtValue(); 75819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (!FoldOffsetIntoAddress(Val, AM)) 759894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 760894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman break; 761894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 762894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 763894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86ISD::Wrapper: 764894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86ISD::WrapperRIP: 765894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (!MatchWrapper(N, AM)) 766894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 767894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman break; 768894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 769894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case ISD::LOAD: 77019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (!MatchLoadInAddress(cast<LoadSDNode>(N), AM)) 771894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 772894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman break; 773894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 774894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case ISD::FrameIndex: 77519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (AM.BaseType == X86ISelAddressMode::RegBase && 77619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman AM.Base_Reg.getNode() == 0 && 77719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman (!Subtarget->is64Bit() || isDispSafeForFrameIndex(AM.Disp))) { 778894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.BaseType = X86ISelAddressMode::FrameIndexBase; 779894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Base_FrameIndex = cast<FrameIndexSDNode>(N)->getIndex(); 780894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 781894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 782894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman break; 783894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 784894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case ISD::SHL: 785894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (AM.IndexReg.getNode() != 0 || AM.Scale != 1) 786894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman break; 787894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 788894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (ConstantSDNode 789894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman *CN = dyn_cast<ConstantSDNode>(N.getNode()->getOperand(1))) { 790894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman unsigned Val = CN->getZExtValue(); 791894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Note that we handle x<<1 as (,x,2) rather than (x,x) here so 792894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // that the base operand remains free for further matching. If 793894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // the base doesn't end up getting used, a post-processing step 794894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // in MatchAddress turns (,x,2) into (x,x), which is cheaper. 795894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (Val == 1 || Val == 2 || Val == 3) { 796894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Scale = 1 << Val; 797894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue ShVal = N.getNode()->getOperand(0); 798894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 799894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Okay, we know that we have a scale by now. However, if the scaled 800894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // value is an add of something and a constant, we can fold the 801894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // constant into the disp field here. 80219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (CurDAG->isBaseWithConstantOffset(ShVal)) { 803894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.IndexReg = ShVal.getNode()->getOperand(0); 804894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ConstantSDNode *AddVal = 805894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman cast<ConstantSDNode>(ShVal.getNode()->getOperand(1)); 80619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman uint64_t Disp = AddVal->getSExtValue() << Val; 80719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (!FoldOffsetIntoAddress(Disp, AM)) 80819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman return false; 809894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 81019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 81119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman AM.IndexReg = ShVal; 812894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 813894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 814894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman break; 815894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 816894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 817894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case ISD::SMUL_LOHI: 818894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case ISD::UMUL_LOHI: 819894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // A mul_lohi where we need the low part can be folded as a plain multiply. 820894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (N.getResNo() != 0) break; 821894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // FALL THROUGH 822894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case ISD::MUL: 823894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86ISD::MUL_IMM: 824894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // X*[3,5,9] -> X+X*[2,4,8] 825894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (AM.BaseType == X86ISelAddressMode::RegBase && 826894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Base_Reg.getNode() == 0 && 827894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.IndexReg.getNode() == 0) { 828894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (ConstantSDNode 829894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman *CN = dyn_cast<ConstantSDNode>(N.getNode()->getOperand(1))) 830894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (CN->getZExtValue() == 3 || CN->getZExtValue() == 5 || 831894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CN->getZExtValue() == 9) { 832894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Scale = unsigned(CN->getZExtValue())-1; 833894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 834894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue MulVal = N.getNode()->getOperand(0); 835894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Reg; 836894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 837894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Okay, we know that we have a scale by now. However, if the scaled 838894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // value is an add of something and a constant, we can fold the 839894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // constant into the disp field here. 840894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (MulVal.getNode()->getOpcode() == ISD::ADD && MulVal.hasOneUse() && 841894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman isa<ConstantSDNode>(MulVal.getNode()->getOperand(1))) { 842894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Reg = MulVal.getNode()->getOperand(0); 843894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ConstantSDNode *AddVal = 844894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman cast<ConstantSDNode>(MulVal.getNode()->getOperand(1)); 84519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman uint64_t Disp = AddVal->getSExtValue() * CN->getZExtValue(); 84619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (FoldOffsetIntoAddress(Disp, AM)) 847894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Reg = N.getNode()->getOperand(0); 848894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } else { 849894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Reg = N.getNode()->getOperand(0); 850894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 851894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 852894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.IndexReg = AM.Base_Reg = Reg; 853894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 854894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 855894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 856894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman break; 857894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 858894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case ISD::SUB: { 859894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Given A-B, if A can be completely folded into the address and 860894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // the index field with the index field unused, use -B as the index. 861894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // This is a win if a has multiple parts that can be folded into 862894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // the address. Also, this saves a mov if the base register has 863894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // other uses, since it avoids a two-address sub instruction, however 864894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // it costs an additional mov if the index register has other uses. 865894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 866894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Add an artificial use to this node so that we can keep track of 867894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // it if it gets CSE'd with a different node. 868894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman HandleSDNode Handle(N); 869894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 870894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Test if the LHS of the sub can be folded. 871894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman X86ISelAddressMode Backup = AM; 872894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (MatchAddressRecursively(N.getNode()->getOperand(0), AM, Depth+1)) { 873894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM = Backup; 874894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman break; 875894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 876894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Test if the index field is free for use. 877894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (AM.IndexReg.getNode() || AM.isRIPRelative()) { 878894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM = Backup; 879894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman break; 880894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 881894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 882894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman int Cost = 0; 883894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue RHS = Handle.getValue().getNode()->getOperand(1); 884894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // If the RHS involves a register with multiple uses, this 885894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // transformation incurs an extra mov, due to the neg instruction 886894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // clobbering its operand. 887894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (!RHS.getNode()->hasOneUse() || 888894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman RHS.getNode()->getOpcode() == ISD::CopyFromReg || 889894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman RHS.getNode()->getOpcode() == ISD::TRUNCATE || 890894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman RHS.getNode()->getOpcode() == ISD::ANY_EXTEND || 891894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman (RHS.getNode()->getOpcode() == ISD::ZERO_EXTEND && 892894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman RHS.getNode()->getOperand(0).getValueType() == MVT::i32)) 893894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ++Cost; 894894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // If the base is a register with multiple uses, this 895894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // transformation may save a mov. 896894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if ((AM.BaseType == X86ISelAddressMode::RegBase && 897894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Base_Reg.getNode() && 898894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman !AM.Base_Reg.getNode()->hasOneUse()) || 899894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.BaseType == X86ISelAddressMode::FrameIndexBase) 900894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman --Cost; 901894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // If the folded LHS was interesting, this transformation saves 902894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // address arithmetic. 903894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if ((AM.hasSymbolicDisplacement() && !Backup.hasSymbolicDisplacement()) + 904894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ((AM.Disp != 0) && (Backup.Disp == 0)) + 905894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman (AM.Segment.getNode() && !Backup.Segment.getNode()) >= 2) 906894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman --Cost; 907894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // If it doesn't look like it may be an overall win, don't do it. 908894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (Cost >= 0) { 909894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM = Backup; 910894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman break; 911894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 912894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 913894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Ok, the transformation is legal and appears profitable. Go for it. 914894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Zero = CurDAG->getConstant(0, N.getValueType()); 915894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Neg = CurDAG->getNode(ISD::SUB, dl, N.getValueType(), Zero, RHS); 916894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.IndexReg = Neg; 917894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Scale = 1; 918894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 919894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Insert the new nodes into the topological ordering. 920894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (Zero.getNode()->getNodeId() == -1 || 921894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Zero.getNode()->getNodeId() > N.getNode()->getNodeId()) { 922894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CurDAG->RepositionNode(N.getNode(), Zero.getNode()); 923894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Zero.getNode()->setNodeId(N.getNode()->getNodeId()); 924894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 925894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (Neg.getNode()->getNodeId() == -1 || 926894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Neg.getNode()->getNodeId() > N.getNode()->getNodeId()) { 927894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CurDAG->RepositionNode(N.getNode(), Neg.getNode()); 928894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Neg.getNode()->setNodeId(N.getNode()->getNodeId()); 929894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 930894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 931894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 932894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 933894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case ISD::ADD: { 934894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Add an artificial use to this node so that we can keep track of 935894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // it if it gets CSE'd with a different node. 936894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman HandleSDNode Handle(N); 937894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 938894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman X86ISelAddressMode Backup = AM; 93919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (!MatchAddressRecursively(N.getOperand(0), AM, Depth+1) && 94019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman !MatchAddressRecursively(Handle.getValue().getOperand(1), AM, Depth+1)) 941894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 942894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM = Backup; 94319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 944894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Try again after commuting the operands. 94519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (!MatchAddressRecursively(Handle.getValue().getOperand(1), AM, Depth+1)&& 94619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman !MatchAddressRecursively(Handle.getValue().getOperand(0), AM, Depth+1)) 947894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 948894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM = Backup; 949894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 950894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // If we couldn't fold both operands into the address at the same time, 951894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // see if we can just put each operand into a register and fold at least 952894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // the add. 953894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (AM.BaseType == X86ISelAddressMode::RegBase && 954894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman !AM.Base_Reg.getNode() && 955894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman !AM.IndexReg.getNode()) { 95619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman N = Handle.getValue(); 95719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman AM.Base_Reg = N.getOperand(0); 95819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman AM.IndexReg = N.getOperand(1); 959894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Scale = 1; 960894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 961894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 96219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman N = Handle.getValue(); 963894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman break; 964894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 965894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 966894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case ISD::OR: 967894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Handle "X | C" as "X + C" iff X is known to have C bits clear. 96819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (CurDAG->isBaseWithConstantOffset(N)) { 969894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman X86ISelAddressMode Backup = AM; 970894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ConstantSDNode *CN = cast<ConstantSDNode>(N.getOperand(1)); 971894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 972894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Start with the LHS as an addr mode. 973894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (!MatchAddressRecursively(N.getOperand(0), AM, Depth+1) && 97419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman !FoldOffsetIntoAddress(CN->getSExtValue(), AM)) 975894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 976894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM = Backup; 977894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 978894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman break; 979894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 980894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case ISD::AND: { 981894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Perform some heroic transforms on an and of a constant-count shift 982894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // with a constant to enable use of the scaled offset field. 983894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 984894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Shift = N.getOperand(0); 985894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (Shift.getNumOperands() != 2) break; 986894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 987894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Scale must not be used already. 988894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (AM.IndexReg.getNode() != 0 || AM.Scale != 1) break; 989894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 990894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue X = Shift.getOperand(0); 991894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N.getOperand(1)); 992894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ConstantSDNode *C1 = dyn_cast<ConstantSDNode>(Shift.getOperand(1)); 993894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (!C1 || !C2) break; 994894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 995894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Handle "(X >> (8-C1)) & C2" as "(X >> 8) & 0xff)" if safe. This 996894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // allows us to convert the shift and and into an h-register extract and 997894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // a scaled index. 998894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (Shift.getOpcode() == ISD::SRL && Shift.hasOneUse()) { 999894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman unsigned ScaleLog = 8 - C1->getZExtValue(); 1000894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (ScaleLog > 0 && ScaleLog < 4 && 1001894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman C2->getZExtValue() == (UINT64_C(0xff) << ScaleLog)) { 1002894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Eight = CurDAG->getConstant(8, MVT::i8); 1003894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Mask = CurDAG->getConstant(0xff, N.getValueType()); 1004894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Srl = CurDAG->getNode(ISD::SRL, dl, N.getValueType(), 1005894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman X, Eight); 1006894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue And = CurDAG->getNode(ISD::AND, dl, N.getValueType(), 1007894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Srl, Mask); 1008894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue ShlCount = CurDAG->getConstant(ScaleLog, MVT::i8); 1009894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Shl = CurDAG->getNode(ISD::SHL, dl, N.getValueType(), 1010894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman And, ShlCount); 1011894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1012894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Insert the new nodes into the topological ordering. 1013894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (Eight.getNode()->getNodeId() == -1 || 1014894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Eight.getNode()->getNodeId() > X.getNode()->getNodeId()) { 1015894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CurDAG->RepositionNode(X.getNode(), Eight.getNode()); 1016894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Eight.getNode()->setNodeId(X.getNode()->getNodeId()); 1017894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1018894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (Mask.getNode()->getNodeId() == -1 || 1019894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Mask.getNode()->getNodeId() > X.getNode()->getNodeId()) { 1020894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CurDAG->RepositionNode(X.getNode(), Mask.getNode()); 1021894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Mask.getNode()->setNodeId(X.getNode()->getNodeId()); 1022894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1023894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (Srl.getNode()->getNodeId() == -1 || 1024894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Srl.getNode()->getNodeId() > Shift.getNode()->getNodeId()) { 1025894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CurDAG->RepositionNode(Shift.getNode(), Srl.getNode()); 1026894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Srl.getNode()->setNodeId(Shift.getNode()->getNodeId()); 1027894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1028894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (And.getNode()->getNodeId() == -1 || 1029894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman And.getNode()->getNodeId() > N.getNode()->getNodeId()) { 1030894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CurDAG->RepositionNode(N.getNode(), And.getNode()); 1031894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman And.getNode()->setNodeId(N.getNode()->getNodeId()); 1032894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1033894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (ShlCount.getNode()->getNodeId() == -1 || 1034894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ShlCount.getNode()->getNodeId() > X.getNode()->getNodeId()) { 1035894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CurDAG->RepositionNode(X.getNode(), ShlCount.getNode()); 1036894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ShlCount.getNode()->setNodeId(N.getNode()->getNodeId()); 1037894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1038894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (Shl.getNode()->getNodeId() == -1 || 1039894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Shl.getNode()->getNodeId() > N.getNode()->getNodeId()) { 1040894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CurDAG->RepositionNode(N.getNode(), Shl.getNode()); 1041894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Shl.getNode()->setNodeId(N.getNode()->getNodeId()); 1042894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1043894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CurDAG->ReplaceAllUsesWith(N, Shl); 1044894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.IndexReg = And; 1045894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Scale = (1 << ScaleLog); 1046894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 1047894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1048894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1049894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1050894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Handle "(X << C1) & C2" as "(X & (C2>>C1)) << C1" if safe and if this 1051894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // allows us to fold the shift into this addressing mode. 1052894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (Shift.getOpcode() != ISD::SHL) break; 1053894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1054894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Not likely to be profitable if either the AND or SHIFT node has more 1055894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // than one use (unless all uses are for address computation). Besides, 1056894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // isel mechanism requires their node ids to be reused. 1057894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (!N.hasOneUse() || !Shift.hasOneUse()) 1058894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman break; 1059894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1060894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Verify that the shift amount is something we can fold. 1061894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman unsigned ShiftCst = C1->getZExtValue(); 1062894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (ShiftCst != 1 && ShiftCst != 2 && ShiftCst != 3) 1063894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman break; 1064894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1065894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Get the new AND mask, this folds to a constant. 1066894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue NewANDMask = CurDAG->getNode(ISD::SRL, dl, N.getValueType(), 1067894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue(C2, 0), SDValue(C1, 0)); 1068894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue NewAND = CurDAG->getNode(ISD::AND, dl, N.getValueType(), X, 1069894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman NewANDMask); 1070894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue NewSHIFT = CurDAG->getNode(ISD::SHL, dl, N.getValueType(), 1071894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman NewAND, SDValue(C1, 0)); 1072894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1073894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Insert the new nodes into the topological ordering. 1074894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (C1->getNodeId() > X.getNode()->getNodeId()) { 1075894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CurDAG->RepositionNode(X.getNode(), C1); 1076894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman C1->setNodeId(X.getNode()->getNodeId()); 1077894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1078894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (NewANDMask.getNode()->getNodeId() == -1 || 1079894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman NewANDMask.getNode()->getNodeId() > X.getNode()->getNodeId()) { 1080894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CurDAG->RepositionNode(X.getNode(), NewANDMask.getNode()); 1081894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman NewANDMask.getNode()->setNodeId(X.getNode()->getNodeId()); 1082894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1083894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (NewAND.getNode()->getNodeId() == -1 || 1084894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman NewAND.getNode()->getNodeId() > Shift.getNode()->getNodeId()) { 1085894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CurDAG->RepositionNode(Shift.getNode(), NewAND.getNode()); 1086894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman NewAND.getNode()->setNodeId(Shift.getNode()->getNodeId()); 1087894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1088894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (NewSHIFT.getNode()->getNodeId() == -1 || 1089894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman NewSHIFT.getNode()->getNodeId() > N.getNode()->getNodeId()) { 1090894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CurDAG->RepositionNode(N.getNode(), NewSHIFT.getNode()); 1091894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman NewSHIFT.getNode()->setNodeId(N.getNode()->getNodeId()); 1092894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1093894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1094894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CurDAG->ReplaceAllUsesWith(N, NewSHIFT); 1095894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1096894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Scale = 1 << ShiftCst; 1097894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.IndexReg = NewAND; 1098894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 1099894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1100894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1101894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1102894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return MatchAddressBase(N, AM); 1103894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman} 1104894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1105894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// MatchAddressBase - Helper for MatchAddress. Add the specified node to the 1106894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// specified addressing mode without any further recursion. 1107894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Baumanbool X86DAGToDAGISel::MatchAddressBase(SDValue N, X86ISelAddressMode &AM) { 1108894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Is the base register already occupied? 1109894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (AM.BaseType != X86ISelAddressMode::RegBase || AM.Base_Reg.getNode()) { 1110894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // If so, check to see if the scale index register is set. 1111894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (AM.IndexReg.getNode() == 0) { 1112894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.IndexReg = N; 1113894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Scale = 1; 1114894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 1115894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1116894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1117894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Otherwise, we cannot select it. 1118894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return true; 1119894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1120894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1121894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Default, generate it as a register. 1122894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.BaseType = X86ISelAddressMode::RegBase; 1123894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Base_Reg = N; 1124894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 1125894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman} 1126894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1127894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// SelectAddr - returns true if it is able pattern match an addressing mode. 1128894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// It returns the operands which make up the maximal addressing mode it can 1129894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// match by reference. 113019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman/// 113119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman/// Parent is the parent node of the addr operand that is being matched. It 113219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman/// is always a load, store, atomic node, or null. It is only null when 113319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman/// checking memory operands for inline asm nodes. 113419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Baumanbool X86DAGToDAGISel::SelectAddr(SDNode *Parent, SDValue N, SDValue &Base, 1135894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue &Scale, SDValue &Index, 1136894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue &Disp, SDValue &Segment) { 1137894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman X86ISelAddressMode AM; 113819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 113919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (Parent && 114019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // This list of opcodes are all the nodes that have an "addr:$ptr" operand 114119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // that are not a MemSDNode, and thus don't have proper addrspace info. 114219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman Parent->getOpcode() != ISD::INTRINSIC_W_CHAIN && // unaligned loads, fixme 114319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman Parent->getOpcode() != ISD::INTRINSIC_VOID && // nontemporal stores 114419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman Parent->getOpcode() != X86ISD::TLSCALL) { // Fixme 114519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman unsigned AddrSpace = 114619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman cast<MemSDNode>(Parent)->getPointerInfo().getAddrSpace(); 114719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // AddrSpace 256 -> GS, 257 -> FS. 114819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (AddrSpace == 256) 114919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16); 115019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (AddrSpace == 257) 115119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16); 115219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman } 115319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 1154894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (MatchAddress(N, AM)) 1155894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 1156894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1157894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman EVT VT = N.getValueType(); 1158894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (AM.BaseType == X86ISelAddressMode::RegBase) { 1159894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (!AM.Base_Reg.getNode()) 1160894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Base_Reg = CurDAG->getRegister(0, VT); 1161894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1162894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1163894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (!AM.IndexReg.getNode()) 1164894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.IndexReg = CurDAG->getRegister(0, VT); 1165894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1166894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman getAddressOperands(AM, Base, Scale, Index, Disp, Segment); 1167894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return true; 1168894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman} 1169894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1170894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// SelectScalarSSELoad - Match a scalar SSE load. In particular, we want to 1171894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// match a load whose top elements are either undef or zeros. The load flavor 1172894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// is derived from the type of N, which is either v4f32 or v2f64. 1173894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// 1174894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// We also return: 1175894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// PatternChainNode: this is the matched node that has a chain input and 1176894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// output. 1177894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Baumanbool X86DAGToDAGISel::SelectScalarSSELoad(SDNode *Root, 1178894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue N, SDValue &Base, 1179894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue &Scale, SDValue &Index, 1180894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue &Disp, SDValue &Segment, 1181894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue &PatternNodeWithChain) { 1182894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (N.getOpcode() == ISD::SCALAR_TO_VECTOR) { 1183894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman PatternNodeWithChain = N.getOperand(0); 1184894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (ISD::isNON_EXTLoad(PatternNodeWithChain.getNode()) && 1185894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman PatternNodeWithChain.hasOneUse() && 1186894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman IsProfitableToFold(N.getOperand(0), N.getNode(), Root) && 1187894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman IsLegalToFold(N.getOperand(0), N.getNode(), Root, OptLevel)) { 1188894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman LoadSDNode *LD = cast<LoadSDNode>(PatternNodeWithChain); 118919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (!SelectAddr(LD, LD->getBasePtr(), Base, Scale, Index, Disp, Segment)) 1190894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 1191894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return true; 1192894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1193894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1194894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1195894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Also handle the case where we explicitly require zeros in the top 1196894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // elements. This is a vector shuffle from the zero vector. 1197894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (N.getOpcode() == X86ISD::VZEXT_MOVL && N.getNode()->hasOneUse() && 1198894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Check to see if the top elements are all zeros (or bitcast of zeros). 1199894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman N.getOperand(0).getOpcode() == ISD::SCALAR_TO_VECTOR && 1200894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman N.getOperand(0).getNode()->hasOneUse() && 1201894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ISD::isNON_EXTLoad(N.getOperand(0).getOperand(0).getNode()) && 1202894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman N.getOperand(0).getOperand(0).hasOneUse() && 1203894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman IsProfitableToFold(N.getOperand(0), N.getNode(), Root) && 1204894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman IsLegalToFold(N.getOperand(0), N.getNode(), Root, OptLevel)) { 1205894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Okay, this is a zero extending load. Fold it. 1206894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman LoadSDNode *LD = cast<LoadSDNode>(N.getOperand(0).getOperand(0)); 120719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (!SelectAddr(LD, LD->getBasePtr(), Base, Scale, Index, Disp, Segment)) 1208894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 1209894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman PatternNodeWithChain = SDValue(LD, 0); 1210894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return true; 1211894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1212894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 1213894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman} 1214894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1215894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1216894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// SelectLEAAddr - it calls SelectAddr and determines if the maximal addressing 1217894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// mode it matches can be cost effectively emitted as an LEA instruction. 121819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Baumanbool X86DAGToDAGISel::SelectLEAAddr(SDValue N, 1219894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue &Base, SDValue &Scale, 1220894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue &Index, SDValue &Disp, 1221894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue &Segment) { 1222894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman X86ISelAddressMode AM; 1223894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1224894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Set AM.Segment to prevent MatchAddress from using one. LEA doesn't support 1225894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // segments. 1226894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Copy = AM.Segment; 1227894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue T = CurDAG->getRegister(0, MVT::i32); 1228894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Segment = T; 1229894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (MatchAddress(N, AM)) 1230894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 1231894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman assert (T == AM.Segment); 1232894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Segment = Copy; 1233894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1234894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman EVT VT = N.getValueType(); 1235894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman unsigned Complexity = 0; 1236894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (AM.BaseType == X86ISelAddressMode::RegBase) 1237894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (AM.Base_Reg.getNode()) 1238894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Complexity = 1; 1239894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else 1240894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Base_Reg = CurDAG->getRegister(0, VT); 1241894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else if (AM.BaseType == X86ISelAddressMode::FrameIndexBase) 1242894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Complexity = 4; 1243894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1244894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (AM.IndexReg.getNode()) 1245894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Complexity++; 1246894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else 1247894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.IndexReg = CurDAG->getRegister(0, VT); 1248894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1249894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Don't match just leal(,%reg,2). It's cheaper to do addl %reg, %reg, or with 1250894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // a simple shift. 1251894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (AM.Scale > 1) 1252894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Complexity++; 1253894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1254894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // FIXME: We are artificially lowering the criteria to turn ADD %reg, $GA 1255894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // to a LEA. This is determined with some expermentation but is by no means 1256894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // optimal (especially for code size consideration). LEA is nice because of 1257894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // its three-address nature. Tweak the cost function again when we can run 1258894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // convertToThreeAddress() at register allocation time. 1259894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (AM.hasSymbolicDisplacement()) { 1260894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // For X86-64, we should always use lea to materialize RIP relative 1261894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // addresses. 1262894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (Subtarget->is64Bit()) 1263894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Complexity = 4; 1264894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else 1265894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Complexity += 2; 1266894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1267894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1268894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (AM.Disp && (AM.Base_Reg.getNode() || AM.IndexReg.getNode())) 1269894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Complexity++; 1270894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1271894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // If it isn't worth using an LEA, reject it. 1272894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (Complexity <= 2) 1273894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 1274894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1275894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman getAddressOperands(AM, Base, Scale, Index, Disp, Segment); 1276894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return true; 1277894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman} 1278894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1279894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// SelectTLSADDRAddr - This is only run on TargetGlobalTLSAddress nodes. 128019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Baumanbool X86DAGToDAGISel::SelectTLSADDRAddr(SDValue N, SDValue &Base, 1281894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue &Scale, SDValue &Index, 1282894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue &Disp, SDValue &Segment) { 1283894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman assert(N.getOpcode() == ISD::TargetGlobalTLSAddress); 1284894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N); 1285894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1286894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman X86ISelAddressMode AM; 1287894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.GV = GA->getGlobal(); 1288894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Disp += GA->getOffset(); 1289894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Base_Reg = CurDAG->getRegister(0, N.getValueType()); 1290894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.SymbolFlags = GA->getTargetFlags(); 1291894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1292894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (N.getValueType() == MVT::i32) { 1293894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.Scale = 1; 1294894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.IndexReg = CurDAG->getRegister(X86::EBX, MVT::i32); 1295894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } else { 1296894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman AM.IndexReg = CurDAG->getRegister(0, MVT::i64); 1297894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1298894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1299894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman getAddressOperands(AM, Base, Scale, Index, Disp, Segment); 1300894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return true; 1301894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman} 1302894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1303894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1304894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Baumanbool X86DAGToDAGISel::TryFoldLoad(SDNode *P, SDValue N, 1305894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue &Base, SDValue &Scale, 1306894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue &Index, SDValue &Disp, 1307894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue &Segment) { 1308894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (!ISD::isNON_EXTLoad(N.getNode()) || 1309894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman !IsProfitableToFold(N, P, P) || 1310894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman !IsLegalToFold(N, P, P, OptLevel)) 1311894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 1312894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 131319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman return SelectAddr(N.getNode(), 131419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman N.getOperand(1), Base, Scale, Index, Disp, Segment); 1315894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman} 1316894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1317894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// getGlobalBaseReg - Return an SDNode that returns the value of 1318894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// the global base register. Output instructions required to 1319894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// initialize the global base register, if necessary. 1320894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// 1321894018228b0e0bdbd7aa7e8f47d4a9458789ca82John BaumanSDNode *X86DAGToDAGISel::getGlobalBaseReg() { 1322894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman unsigned GlobalBaseReg = getInstrInfo()->getGlobalBaseReg(MF); 1323894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return CurDAG->getRegister(GlobalBaseReg, TLI.getPointerTy()).getNode(); 1324894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman} 1325894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1326894018228b0e0bdbd7aa7e8f47d4a9458789ca82John BaumanSDNode *X86DAGToDAGISel::SelectAtomic64(SDNode *Node, unsigned Opc) { 1327894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Chain = Node->getOperand(0); 1328894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue In1 = Node->getOperand(1); 1329894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue In2L = Node->getOperand(2); 1330894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue In2H = Node->getOperand(3); 1331894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 133219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (!SelectAddr(Node, In1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) 1333894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return NULL; 1334894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 1335894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman MemOp[0] = cast<MemSDNode>(Node)->getMemOperand(); 1336894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman const SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, In2L, In2H, Chain}; 1337894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDNode *ResNode = CurDAG->getMachineNode(Opc, Node->getDebugLoc(), 1338894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman MVT::i32, MVT::i32, MVT::Other, Ops, 1339894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman array_lengthof(Ops)); 1340894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman cast<MachineSDNode>(ResNode)->setMemRefs(MemOp, MemOp + 1); 1341894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return ResNode; 1342894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman} 1343894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 134419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman// FIXME: Figure out some way to unify this with the 'or' and other code 134519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman// below. 1346894018228b0e0bdbd7aa7e8f47d4a9458789ca82John BaumanSDNode *X86DAGToDAGISel::SelectAtomicLoadAdd(SDNode *Node, EVT NVT) { 1347894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (Node->hasAnyUseOfValue(0)) 1348894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return 0; 1349894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1350894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Optimize common patterns for __sync_add_and_fetch and 1351894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // __sync_sub_and_fetch where the result is not used. This allows us 1352894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // to use "lock" version of add, sub, inc, dec instructions. 1353894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // FIXME: Do not use special instructions but instead add the "lock" 1354894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // prefix to the target node somehow. The extra information will then be 1355894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // transferred to machine instruction and it denotes the prefix. 1356894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Chain = Node->getOperand(0); 1357894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Ptr = Node->getOperand(1); 1358894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Val = Node->getOperand(2); 1359894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 136019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (!SelectAddr(Node, Ptr, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) 1361894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return 0; 1362894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1363894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman bool isInc = false, isDec = false, isSub = false, isCN = false; 1364894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val); 136519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (CN && CN->getSExtValue() == (int32_t)CN->getSExtValue()) { 1366894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman isCN = true; 1367894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman int64_t CNVal = CN->getSExtValue(); 1368894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (CNVal == 1) 1369894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman isInc = true; 1370894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else if (CNVal == -1) 1371894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman isDec = true; 1372894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else if (CNVal >= 0) 1373894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Val = CurDAG->getTargetConstant(CNVal, NVT); 1374894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else { 1375894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman isSub = true; 1376894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Val = CurDAG->getTargetConstant(-CNVal, NVT); 1377894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1378894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } else if (Val.hasOneUse() && 1379894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Val.getOpcode() == ISD::SUB && 1380894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman X86::isZeroNode(Val.getOperand(0))) { 1381894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman isSub = true; 1382894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Val = Val.getOperand(1); 1383894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1384894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 138519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman DebugLoc dl = Node->getDebugLoc(); 1386894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman unsigned Opc = 0; 1387894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman switch (NVT.getSimpleVT().SimpleTy) { 1388894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman default: return 0; 1389894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i8: 1390894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (isInc) 1391894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_INC8m; 1392894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else if (isDec) 1393894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_DEC8m; 1394894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else if (isSub) { 1395894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (isCN) 1396894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_SUB8mi; 1397894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else 1398894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_SUB8mr; 1399894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } else { 1400894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (isCN) 1401894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_ADD8mi; 1402894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else 1403894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_ADD8mr; 1404894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1405894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman break; 1406894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i16: 1407894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (isInc) 1408894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_INC16m; 1409894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else if (isDec) 1410894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_DEC16m; 1411894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else if (isSub) { 1412894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (isCN) { 141319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (immSext8(Val.getNode())) 1414894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_SUB16mi8; 1415894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else 1416894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_SUB16mi; 1417894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } else 1418894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_SUB16mr; 1419894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } else { 1420894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (isCN) { 142119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (immSext8(Val.getNode())) 1422894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_ADD16mi8; 1423894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else 1424894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_ADD16mi; 1425894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } else 1426894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_ADD16mr; 1427894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1428894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman break; 1429894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i32: 1430894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (isInc) 1431894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_INC32m; 1432894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else if (isDec) 1433894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_DEC32m; 1434894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else if (isSub) { 1435894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (isCN) { 143619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (immSext8(Val.getNode())) 1437894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_SUB32mi8; 1438894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else 1439894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_SUB32mi; 1440894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } else 1441894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_SUB32mr; 1442894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } else { 1443894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (isCN) { 144419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (immSext8(Val.getNode())) 1445894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_ADD32mi8; 1446894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else 1447894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_ADD32mi; 1448894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } else 1449894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_ADD32mr; 1450894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1451894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman break; 1452894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i64: 1453894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (isInc) 1454894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_INC64m; 1455894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else if (isDec) 1456894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_DEC64m; 1457894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else if (isSub) { 1458894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_SUB64mr; 1459894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (isCN) { 146019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (immSext8(Val.getNode())) 1461894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_SUB64mi8; 146219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman else if (i64immSExt32(Val.getNode())) 1463894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_SUB64mi32; 1464894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1465894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } else { 1466894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_ADD64mr; 1467894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (isCN) { 146819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (immSext8(Val.getNode())) 1469894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_ADD64mi8; 147019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman else if (i64immSExt32(Val.getNode())) 1471894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Opc = X86::LOCK_ADD64mi32; 1472894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1473894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1474894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman break; 1475894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1476894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1477894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Undef = SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, 1478894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman dl, NVT), 0); 1479894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 1480894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman MemOp[0] = cast<MemSDNode>(Node)->getMemOperand(); 1481894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (isInc || isDec) { 1482894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain }; 1483894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Ret = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 6), 0); 1484894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman cast<MachineSDNode>(Ret)->setMemRefs(MemOp, MemOp + 1); 1485894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue RetVals[] = { Undef, Ret }; 1486894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return CurDAG->getMergeValues(RetVals, 2, dl).getNode(); 1487894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } else { 1488894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Val, Chain }; 1489894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Ret = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 7), 0); 1490894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman cast<MachineSDNode>(Ret)->setMemRefs(MemOp, MemOp + 1); 1491894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue RetVals[] = { Undef, Ret }; 1492894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return CurDAG->getMergeValues(RetVals, 2, dl).getNode(); 1493894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1494894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman} 1495894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 149619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Baumanenum AtomicOpc { 149719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman OR, 149819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman AND, 149919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman XOR, 150019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman AtomicOpcEnd 150119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman}; 150219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 150319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Baumanenum AtomicSz { 150419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman ConstantI8, 150519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman I8, 150619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SextConstantI16, 150719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman ConstantI16, 150819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman I16, 150919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SextConstantI32, 151019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman ConstantI32, 151119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman I32, 151219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SextConstantI64, 151319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman ConstantI64, 151419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman I64, 151519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman AtomicSzEnd 151619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman}; 151719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 151819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Baumanstatic const unsigned int AtomicOpcTbl[AtomicOpcEnd][AtomicSzEnd] = { 151919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman { 152019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_OR8mi, 152119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_OR8mr, 152219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_OR16mi8, 152319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_OR16mi, 152419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_OR16mr, 152519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_OR32mi8, 152619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_OR32mi, 152719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_OR32mr, 152819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_OR64mi8, 152919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_OR64mi32, 153019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_OR64mr 153119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman }, 153219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman { 153319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_AND8mi, 153419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_AND8mr, 153519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_AND16mi8, 153619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_AND16mi, 153719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_AND16mr, 153819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_AND32mi8, 153919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_AND32mi, 154019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_AND32mr, 154119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_AND64mi8, 154219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_AND64mi32, 154319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_AND64mr 154419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman }, 154519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman { 154619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_XOR8mi, 154719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_XOR8mr, 154819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_XOR16mi8, 154919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_XOR16mi, 155019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_XOR16mr, 155119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_XOR32mi8, 155219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_XOR32mi, 155319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_XOR32mr, 155419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_XOR64mi8, 155519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_XOR64mi32, 155619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman X86::LOCK_XOR64mr 155719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman } 155819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman}; 155919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 156019bac1e08be200c31efd26f0f5fd144c9b3eefd3John BaumanSDNode *X86DAGToDAGISel::SelectAtomicLoadArith(SDNode *Node, EVT NVT) { 156119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (Node->hasAnyUseOfValue(0)) 156219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman return 0; 156319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 156419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // Optimize common patterns for __sync_or_and_fetch and similar arith 156519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // operations where the result is not used. This allows us to use the "lock" 156619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // version of the arithmetic instruction. 156719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // FIXME: Same as for 'add' and 'sub', try to merge those down here. 156819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SDValue Chain = Node->getOperand(0); 156919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SDValue Ptr = Node->getOperand(1); 157019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SDValue Val = Node->getOperand(2); 157119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 157219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (!SelectAddr(Node, Ptr, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) 157319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman return 0; 157419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 157519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // Which index into the table. 157619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman enum AtomicOpc Op; 157719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman switch (Node->getOpcode()) { 157819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman case ISD::ATOMIC_LOAD_OR: 157919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman Op = OR; 158019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman break; 158119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman case ISD::ATOMIC_LOAD_AND: 158219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman Op = AND; 158319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman break; 158419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman case ISD::ATOMIC_LOAD_XOR: 158519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman Op = XOR; 158619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman break; 158719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman default: 158819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman return 0; 158919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman } 159019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 159119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman bool isCN = false; 159219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val); 159319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (CN && (int32_t)CN->getSExtValue() == CN->getSExtValue()) { 159419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman isCN = true; 159519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman Val = CurDAG->getTargetConstant(CN->getSExtValue(), NVT); 159619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman } 159719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 159819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman unsigned Opc = 0; 159919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman switch (NVT.getSimpleVT().SimpleTy) { 160019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman default: return 0; 160119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman case MVT::i8: 160219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (isCN) 160319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman Opc = AtomicOpcTbl[Op][ConstantI8]; 160419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman else 160519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman Opc = AtomicOpcTbl[Op][I8]; 160619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman break; 160719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman case MVT::i16: 160819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (isCN) { 160919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (immSext8(Val.getNode())) 161019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman Opc = AtomicOpcTbl[Op][SextConstantI16]; 161119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman else 161219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman Opc = AtomicOpcTbl[Op][ConstantI16]; 161319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman } else 161419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman Opc = AtomicOpcTbl[Op][I16]; 161519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman break; 161619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman case MVT::i32: 161719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (isCN) { 161819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (immSext8(Val.getNode())) 161919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman Opc = AtomicOpcTbl[Op][SextConstantI32]; 162019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman else 162119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman Opc = AtomicOpcTbl[Op][ConstantI32]; 162219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman } else 162319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman Opc = AtomicOpcTbl[Op][I32]; 162419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman break; 162519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman case MVT::i64: 162619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman Opc = AtomicOpcTbl[Op][I64]; 162719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (isCN) { 162819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (immSext8(Val.getNode())) 162919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman Opc = AtomicOpcTbl[Op][SextConstantI64]; 163019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman else if (i64immSExt32(Val.getNode())) 163119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman Opc = AtomicOpcTbl[Op][ConstantI64]; 163219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman } 163319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman break; 163419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman } 163519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 163619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman assert(Opc != 0 && "Invalid arith lock transform!"); 163719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 163819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman DebugLoc dl = Node->getDebugLoc(); 163919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SDValue Undef = SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, 164019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman dl, NVT), 0); 164119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 164219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman MemOp[0] = cast<MemSDNode>(Node)->getMemOperand(); 164319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Val, Chain }; 164419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SDValue Ret = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 7), 0); 164519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman cast<MachineSDNode>(Ret)->setMemRefs(MemOp, MemOp + 1); 164619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SDValue RetVals[] = { Undef, Ret }; 164719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman return CurDAG->getMergeValues(RetVals, 2, dl).getNode(); 164819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman} 164919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 1650894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// HasNoSignedComparisonUses - Test whether the given X86ISD::CMP node has 1651894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// any uses which require the SF or OF bits to be accurate. 1652894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Baumanstatic bool HasNoSignedComparisonUses(SDNode *N) { 1653894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Examine each user of the node. 1654894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman for (SDNode::use_iterator UI = N->use_begin(), 1655894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman UE = N->use_end(); UI != UE; ++UI) { 1656894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Only examine CopyToReg uses. 1657894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (UI->getOpcode() != ISD::CopyToReg) 1658894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 1659894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Only examine CopyToReg uses that copy to EFLAGS. 1660894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (cast<RegisterSDNode>(UI->getOperand(1))->getReg() != 1661894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman X86::EFLAGS) 1662894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 1663894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Examine each user of the CopyToReg use. 1664894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman for (SDNode::use_iterator FlagUI = UI->use_begin(), 1665894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman FlagUE = UI->use_end(); FlagUI != FlagUE; ++FlagUI) { 1666894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Only examine the Flag result. 1667894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (FlagUI.getUse().getResNo() != 1) continue; 1668894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Anything unusual: assume conservatively. 1669894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (!FlagUI->isMachineOpcode()) return false; 1670894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Examine the opcode of the user. 1671894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman switch (FlagUI->getMachineOpcode()) { 1672894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // These comparisons don't treat the most significant bit specially. 1673894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::SETAr: case X86::SETAEr: case X86::SETBr: case X86::SETBEr: 1674894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::SETEr: case X86::SETNEr: case X86::SETPr: case X86::SETNPr: 1675894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::SETAm: case X86::SETAEm: case X86::SETBm: case X86::SETBEm: 1676894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::SETEm: case X86::SETNEm: case X86::SETPm: case X86::SETNPm: 1677894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::JA_4: case X86::JAE_4: case X86::JB_4: case X86::JBE_4: 1678894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::JE_4: case X86::JNE_4: case X86::JP_4: case X86::JNP_4: 1679894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::CMOVA16rr: case X86::CMOVA16rm: 1680894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::CMOVA32rr: case X86::CMOVA32rm: 1681894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::CMOVA64rr: case X86::CMOVA64rm: 1682894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::CMOVAE16rr: case X86::CMOVAE16rm: 1683894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::CMOVAE32rr: case X86::CMOVAE32rm: 1684894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::CMOVAE64rr: case X86::CMOVAE64rm: 1685894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::CMOVB16rr: case X86::CMOVB16rm: 1686894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::CMOVB32rr: case X86::CMOVB32rm: 1687894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::CMOVB64rr: case X86::CMOVB64rm: 1688894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::CMOVBE16rr: case X86::CMOVBE16rm: 1689894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::CMOVBE32rr: case X86::CMOVBE32rm: 1690894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::CMOVBE64rr: case X86::CMOVBE64rm: 1691894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::CMOVE16rr: case X86::CMOVE16rm: 1692894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::CMOVE32rr: case X86::CMOVE32rm: 1693894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::CMOVE64rr: case X86::CMOVE64rm: 1694894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::CMOVNE16rr: case X86::CMOVNE16rm: 1695894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::CMOVNE32rr: case X86::CMOVNE32rm: 1696894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::CMOVNE64rr: case X86::CMOVNE64rm: 1697894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::CMOVNP16rr: case X86::CMOVNP16rm: 1698894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::CMOVNP32rr: case X86::CMOVNP32rm: 1699894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::CMOVNP64rr: case X86::CMOVNP64rm: 1700894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::CMOVP16rr: case X86::CMOVP16rm: 1701894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::CMOVP32rr: case X86::CMOVP32rm: 1702894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86::CMOVP64rr: case X86::CMOVP64rm: 1703894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman continue; 1704894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Anything else: assume conservatively. 1705894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman default: return false; 1706894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1707894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1708894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1709894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return true; 1710894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman} 1711894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1712894018228b0e0bdbd7aa7e8f47d4a9458789ca82John BaumanSDNode *X86DAGToDAGISel::Select(SDNode *Node) { 1713894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman EVT NVT = Node->getValueType(0); 1714894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman unsigned Opc, MOpc; 1715894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman unsigned Opcode = Node->getOpcode(); 1716894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman DebugLoc dl = Node->getDebugLoc(); 1717894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1718894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman DEBUG(dbgs() << "Selecting: "; Node->dump(CurDAG); dbgs() << '\n'); 1719894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1720894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (Node->isMachineOpcode()) { 1721894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << '\n'); 1722894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return NULL; // Already selected. 1723894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1724894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1725894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman switch (Opcode) { 1726894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman default: break; 1727894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86ISD::GlobalBaseReg: 1728894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return getGlobalBaseReg(); 1729894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1730894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86ISD::ATOMOR64_DAG: 1731894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return SelectAtomic64(Node, X86::ATOMOR6432); 1732894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86ISD::ATOMXOR64_DAG: 1733894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return SelectAtomic64(Node, X86::ATOMXOR6432); 1734894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86ISD::ATOMADD64_DAG: 1735894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return SelectAtomic64(Node, X86::ATOMADD6432); 1736894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86ISD::ATOMSUB64_DAG: 1737894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return SelectAtomic64(Node, X86::ATOMSUB6432); 1738894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86ISD::ATOMNAND64_DAG: 1739894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return SelectAtomic64(Node, X86::ATOMNAND6432); 1740894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86ISD::ATOMAND64_DAG: 1741894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return SelectAtomic64(Node, X86::ATOMAND6432); 1742894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86ISD::ATOMSWAP64_DAG: 1743894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return SelectAtomic64(Node, X86::ATOMSWAP6432); 1744894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1745894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case ISD::ATOMIC_LOAD_ADD: { 1746894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDNode *RetVal = SelectAtomicLoadAdd(Node, NVT); 1747894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (RetVal) 1748894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return RetVal; 1749894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman break; 1750894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 175119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman case ISD::ATOMIC_LOAD_XOR: 175219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman case ISD::ATOMIC_LOAD_AND: 175319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman case ISD::ATOMIC_LOAD_OR: { 175419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SDNode *RetVal = SelectAtomicLoadArith(Node, NVT); 175519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (RetVal) 175619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman return RetVal; 175719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman break; 175819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman } 175919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman case ISD::AND: 176019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman case ISD::OR: 176119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman case ISD::XOR: { 176219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // For operations of the form (x << C1) op C2, check if we can use a smaller 176319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // encoding for C2 by transforming it into (x op (C2>>C1)) << C1. 176419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SDValue N0 = Node->getOperand(0); 176519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SDValue N1 = Node->getOperand(1); 176619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 176719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (N0->getOpcode() != ISD::SHL || !N0->hasOneUse()) 176819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman break; 176919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 177019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // i8 is unshrinkable, i16 should be promoted to i32. 177119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (NVT != MVT::i32 && NVT != MVT::i64) 177219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman break; 177319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 177419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(N1); 177519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman ConstantSDNode *ShlCst = dyn_cast<ConstantSDNode>(N0->getOperand(1)); 177619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (!Cst || !ShlCst) 177719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman break; 177819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 177919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman int64_t Val = Cst->getSExtValue(); 178019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman uint64_t ShlVal = ShlCst->getZExtValue(); 178119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 178219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // Make sure that we don't change the operation by removing bits. 178319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // This only matters for OR and XOR, AND is unaffected. 178419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (Opcode != ISD::AND && ((Val >> ShlVal) << ShlVal) != Val) 178519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman break; 1786894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 178719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman unsigned ShlOp, Op = 0; 178819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman EVT CstVT = NVT; 178919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 179019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // Check the minimum bitwidth for the new constant. 179119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // TODO: AND32ri is the same as AND64ri32 with zext imm. 179219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // TODO: MOV32ri+OR64r is cheaper than MOV64ri64+OR64rr 179319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // TODO: Using 16 and 8 bit operations is also possible for or32 & xor32. 179419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (!isInt<8>(Val) && isInt<8>(Val >> ShlVal)) 179519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman CstVT = MVT::i8; 179619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman else if (!isInt<32>(Val) && isInt<32>(Val >> ShlVal)) 179719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman CstVT = MVT::i32; 179819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 179919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // Bail if there is no smaller encoding. 180019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (NVT == CstVT) 180119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman break; 180219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 180319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman switch (NVT.getSimpleVT().SimpleTy) { 180419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman default: llvm_unreachable("Unsupported VT!"); 180519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman case MVT::i32: 180619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman assert(CstVT == MVT::i8); 180719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman ShlOp = X86::SHL32ri; 180819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 180919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman switch (Opcode) { 181019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman case ISD::AND: Op = X86::AND32ri8; break; 181119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman case ISD::OR: Op = X86::OR32ri8; break; 181219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman case ISD::XOR: Op = X86::XOR32ri8; break; 181319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman } 181419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman break; 181519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman case MVT::i64: 181619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman assert(CstVT == MVT::i8 || CstVT == MVT::i32); 181719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman ShlOp = X86::SHL64ri; 181819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 181919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman switch (Opcode) { 182019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman case ISD::AND: Op = CstVT==MVT::i8? X86::AND64ri8 : X86::AND64ri32; break; 182119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman case ISD::OR: Op = CstVT==MVT::i8? X86::OR64ri8 : X86::OR64ri32; break; 182219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman case ISD::XOR: Op = CstVT==MVT::i8? X86::XOR64ri8 : X86::XOR64ri32; break; 182319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman } 182419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman break; 182519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman } 182619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 182719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // Emit the smaller op and the shift. 182819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SDValue NewCst = CurDAG->getTargetConstant(Val >> ShlVal, CstVT); 182919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SDNode *New = CurDAG->getMachineNode(Op, dl, NVT, N0->getOperand(0),NewCst); 183019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman return CurDAG->SelectNodeTo(Node, ShlOp, NVT, SDValue(New, 0), 183119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman getI8Imm(ShlVal)); 183219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman break; 183319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman } 183419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman case X86ISD::UMUL: { 183519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SDValue N0 = Node->getOperand(0); 183619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SDValue N1 = Node->getOperand(1); 183719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 183819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman unsigned LoReg; 183919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman switch (NVT.getSimpleVT().SimpleTy) { 184019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman default: llvm_unreachable("Unsupported VT!"); 184119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman case MVT::i8: LoReg = X86::AL; Opc = X86::MUL8r; break; 184219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman case MVT::i16: LoReg = X86::AX; Opc = X86::MUL16r; break; 184319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman case MVT::i32: LoReg = X86::EAX; Opc = X86::MUL32r; break; 184419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman case MVT::i64: LoReg = X86::RAX; Opc = X86::MUL64r; break; 184519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman } 184619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 184719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg, 184819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman N0, SDValue()).getValue(1); 184919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 185019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SDVTList VTs = CurDAG->getVTList(NVT, NVT, MVT::i32); 185119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SDValue Ops[] = {N1, InFlag}; 185219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops, 2); 185319bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 185419bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman ReplaceUses(SDValue(Node, 0), SDValue(CNode, 0)); 185519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman ReplaceUses(SDValue(Node, 1), SDValue(CNode, 1)); 185619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman ReplaceUses(SDValue(Node, 2), SDValue(CNode, 2)); 185719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman return NULL; 185819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman } 185919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 1860894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case ISD::SMUL_LOHI: 1861894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case ISD::UMUL_LOHI: { 1862894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue N0 = Node->getOperand(0); 1863894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue N1 = Node->getOperand(1); 1864894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1865894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman bool isSigned = Opcode == ISD::SMUL_LOHI; 1866894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (!isSigned) { 1867894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman switch (NVT.getSimpleVT().SimpleTy) { 1868894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman default: llvm_unreachable("Unsupported VT!"); 1869894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i8: Opc = X86::MUL8r; MOpc = X86::MUL8m; break; 1870894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i16: Opc = X86::MUL16r; MOpc = X86::MUL16m; break; 1871894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i32: Opc = X86::MUL32r; MOpc = X86::MUL32m; break; 1872894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i64: Opc = X86::MUL64r; MOpc = X86::MUL64m; break; 1873894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1874894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } else { 1875894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman switch (NVT.getSimpleVT().SimpleTy) { 1876894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman default: llvm_unreachable("Unsupported VT!"); 1877894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i8: Opc = X86::IMUL8r; MOpc = X86::IMUL8m; break; 1878894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i16: Opc = X86::IMUL16r; MOpc = X86::IMUL16m; break; 1879894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i32: Opc = X86::IMUL32r; MOpc = X86::IMUL32m; break; 1880894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i64: Opc = X86::IMUL64r; MOpc = X86::IMUL64m; break; 1881894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1882894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1883894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1884894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman unsigned LoReg, HiReg; 1885894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman switch (NVT.getSimpleVT().SimpleTy) { 1886894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman default: llvm_unreachable("Unsupported VT!"); 1887894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i8: LoReg = X86::AL; HiReg = X86::AH; break; 1888894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i16: LoReg = X86::AX; HiReg = X86::DX; break; 1889894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i32: LoReg = X86::EAX; HiReg = X86::EDX; break; 1890894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i64: LoReg = X86::RAX; HiReg = X86::RDX; break; 1891894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1892894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1893894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 1894894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman bool foldedLoad = TryFoldLoad(Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4); 1895894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Multiply is commmutative. 1896894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (!foldedLoad) { 1897894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman foldedLoad = TryFoldLoad(Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4); 1898894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (foldedLoad) 1899894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman std::swap(N0, N1); 1900894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1901894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1902894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg, 1903894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman N0, SDValue()).getValue(1); 1904894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1905894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (foldedLoad) { 1906894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0), 1907894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman InFlag }; 1908894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDNode *CNode = 190919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman CurDAG->getMachineNode(MOpc, dl, MVT::Other, MVT::Glue, Ops, 1910894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman array_lengthof(Ops)); 1911894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman InFlag = SDValue(CNode, 1); 191219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 1913894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Update the chain. 1914894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ReplaceUses(N1.getValue(1), SDValue(CNode, 0)); 1915894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } else { 191619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SDNode *CNode = CurDAG->getMachineNode(Opc, dl, MVT::Glue, N1, InFlag); 191719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman InFlag = SDValue(CNode, 0); 1918894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1919894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1920894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Prevent use of AH in a REX instruction by referencing AX instead. 1921894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (HiReg == X86::AH && Subtarget->is64Bit() && 1922894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman !SDValue(Node, 1).use_empty()) { 1923894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 1924894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman X86::AX, MVT::i16, InFlag); 1925894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman InFlag = Result.getValue(2); 1926894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Get the low part if needed. Don't use getCopyFromReg for aliasing 1927894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // registers. 1928894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (!SDValue(Node, 0).use_empty()) 1929894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ReplaceUses(SDValue(Node, 1), 1930894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result)); 1931894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1932894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Shift AX down 8 bits. 1933894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Result = SDValue(CurDAG->getMachineNode(X86::SHR16ri, dl, MVT::i16, 1934894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Result, 1935894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CurDAG->getTargetConstant(8, MVT::i8)), 0); 1936894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Then truncate it down to i8. 1937894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ReplaceUses(SDValue(Node, 1), 1938894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result)); 1939894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1940894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Copy the low half of the result, if it is needed. 1941894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (!SDValue(Node, 0).use_empty()) { 1942894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 1943894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman LoReg, NVT, InFlag); 1944894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman InFlag = Result.getValue(2); 1945894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ReplaceUses(SDValue(Node, 0), Result); 1946894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); dbgs() << '\n'); 1947894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1948894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Copy the high half of the result, if it is needed. 1949894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (!SDValue(Node, 1).use_empty()) { 1950894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 1951894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman HiReg, NVT, InFlag); 1952894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman InFlag = Result.getValue(2); 1953894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ReplaceUses(SDValue(Node, 1), Result); 1954894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); dbgs() << '\n'); 1955894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 195619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman 1957894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return NULL; 1958894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1959894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1960894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case ISD::SDIVREM: 1961894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case ISD::UDIVREM: { 1962894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue N0 = Node->getOperand(0); 1963894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue N1 = Node->getOperand(1); 1964894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1965894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman bool isSigned = Opcode == ISD::SDIVREM; 1966894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (!isSigned) { 1967894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman switch (NVT.getSimpleVT().SimpleTy) { 1968894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman default: llvm_unreachable("Unsupported VT!"); 1969894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i8: Opc = X86::DIV8r; MOpc = X86::DIV8m; break; 1970894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i16: Opc = X86::DIV16r; MOpc = X86::DIV16m; break; 1971894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i32: Opc = X86::DIV32r; MOpc = X86::DIV32m; break; 1972894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i64: Opc = X86::DIV64r; MOpc = X86::DIV64m; break; 1973894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1974894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } else { 1975894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman switch (NVT.getSimpleVT().SimpleTy) { 1976894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman default: llvm_unreachable("Unsupported VT!"); 1977894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i8: Opc = X86::IDIV8r; MOpc = X86::IDIV8m; break; 1978894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i16: Opc = X86::IDIV16r; MOpc = X86::IDIV16m; break; 1979894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i32: Opc = X86::IDIV32r; MOpc = X86::IDIV32m; break; 1980894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i64: Opc = X86::IDIV64r; MOpc = X86::IDIV64m; break; 1981894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1982894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 1983894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 1984894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman unsigned LoReg, HiReg, ClrReg; 1985894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman unsigned ClrOpcode, SExtOpcode; 1986894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman switch (NVT.getSimpleVT().SimpleTy) { 1987894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman default: llvm_unreachable("Unsupported VT!"); 1988894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i8: 1989894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman LoReg = X86::AL; ClrReg = HiReg = X86::AH; 1990894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ClrOpcode = 0; 1991894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SExtOpcode = X86::CBW; 1992894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman break; 1993894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i16: 1994894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman LoReg = X86::AX; HiReg = X86::DX; 1995894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ClrOpcode = X86::MOV16r0; ClrReg = X86::DX; 1996894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SExtOpcode = X86::CWD; 1997894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman break; 1998894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i32: 1999894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman LoReg = X86::EAX; ClrReg = HiReg = X86::EDX; 2000894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ClrOpcode = X86::MOV32r0; 2001894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SExtOpcode = X86::CDQ; 2002894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman break; 2003894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i64: 2004894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman LoReg = X86::RAX; ClrReg = HiReg = X86::RDX; 2005894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ClrOpcode = X86::MOV64r0; 2006894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SExtOpcode = X86::CQO; 2007894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman break; 2008894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 2009894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 2010894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 2011894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman bool foldedLoad = TryFoldLoad(Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4); 2012894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman bool signBitIsZero = CurDAG->SignBitIsZero(N0); 2013894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 2014894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue InFlag; 2015894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (NVT == MVT::i8 && (!isSigned || signBitIsZero)) { 2016894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Special case for div8, just use a move with zero extension to AX to 2017894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // clear the upper 8 bits (AH). 2018894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Move, Chain; 2019894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (TryFoldLoad(Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) { 2020894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N0.getOperand(0) }; 2021894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Move = 202219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SDValue(CurDAG->getMachineNode(X86::MOVZX32rm8, dl, MVT::i32, 2023894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman MVT::Other, Ops, 2024894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman array_lengthof(Ops)), 0); 2025894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Chain = Move.getValue(1); 2026894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ReplaceUses(N0.getValue(1), Chain); 2027894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } else { 2028894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Move = 202919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SDValue(CurDAG->getMachineNode(X86::MOVZX32rr8, dl, MVT::i32, N0),0); 2030894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Chain = CurDAG->getEntryNode(); 2031894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 203219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman Chain = CurDAG->getCopyToReg(Chain, dl, X86::EAX, Move, SDValue()); 2033894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman InFlag = Chain.getValue(1); 2034894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } else { 2035894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman InFlag = 2036894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, 2037894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman LoReg, N0, SDValue()).getValue(1); 2038894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (isSigned && !signBitIsZero) { 2039894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Sign extend the low part into the high part. 2040894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman InFlag = 204119bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SDValue(CurDAG->getMachineNode(SExtOpcode, dl, MVT::Glue, InFlag),0); 2042894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } else { 2043894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Zero out the high part, effectively zero extending the input. 2044894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue ClrNode = 2045894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue(CurDAG->getMachineNode(ClrOpcode, dl, NVT), 0); 2046894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ClrReg, 2047894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ClrNode, InFlag).getValue(1); 2048894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 2049894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 2050894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 2051894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (foldedLoad) { 2052894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0), 2053894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman InFlag }; 2054894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDNode *CNode = 205519bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman CurDAG->getMachineNode(MOpc, dl, MVT::Other, MVT::Glue, Ops, 2056894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman array_lengthof(Ops)); 2057894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman InFlag = SDValue(CNode, 1); 2058894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Update the chain. 2059894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ReplaceUses(N1.getValue(1), SDValue(CNode, 0)); 2060894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } else { 2061894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman InFlag = 206219bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Glue, N1, InFlag), 0); 2063894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 2064894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 2065894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Prevent use of AH in a REX instruction by referencing AX instead. 2066894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Shift it down 8 bits. 2067894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (HiReg == X86::AH && Subtarget->is64Bit() && 2068894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman !SDValue(Node, 1).use_empty()) { 2069894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 2070894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman X86::AX, MVT::i16, InFlag); 2071894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman InFlag = Result.getValue(2); 2072894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 2073894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // If we also need AL (the quotient), get it by extracting a subreg from 2074894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Result. The fast register allocator does not like multiple CopyFromReg 2075894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // nodes using aliasing registers. 2076894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (!SDValue(Node, 0).use_empty()) 2077894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ReplaceUses(SDValue(Node, 0), 2078894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result)); 2079894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 2080894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Shift AX right by 8 bits instead of using AH. 2081894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Result = SDValue(CurDAG->getMachineNode(X86::SHR16ri, dl, MVT::i16, 2082894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Result, 2083894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CurDAG->getTargetConstant(8, MVT::i8)), 2084894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 0); 2085894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ReplaceUses(SDValue(Node, 1), 2086894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result)); 2087894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 2088894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Copy the division (low) result, if it is needed. 2089894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (!SDValue(Node, 0).use_empty()) { 2090894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 2091894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman LoReg, NVT, InFlag); 2092894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman InFlag = Result.getValue(2); 2093894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ReplaceUses(SDValue(Node, 0), Result); 2094894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); dbgs() << '\n'); 2095894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 2096894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Copy the remainder (high) result, if it is needed. 2097894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (!SDValue(Node, 1).use_empty()) { 2098894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 2099894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman HiReg, NVT, InFlag); 2100894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman InFlag = Result.getValue(2); 2101894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ReplaceUses(SDValue(Node, 1), Result); 2102894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); dbgs() << '\n'); 2103894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 2104894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return NULL; 2105894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 2106894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 2107894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case X86ISD::CMP: { 2108894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue N0 = Node->getOperand(0); 2109894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue N1 = Node->getOperand(1); 2110894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 2111894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Look for (X86cmp (and $op, $imm), 0) and see if we can convert it to 2112894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // use a smaller encoding. 2113894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && 2114894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman HasNoSignedComparisonUses(Node)) 2115894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Look past the truncate if CMP is the only use of it. 2116894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman N0 = N0.getOperand(0); 2117894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (N0.getNode()->getOpcode() == ISD::AND && N0.getNode()->hasOneUse() && 2118894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman N0.getValueType() != MVT::i8 && 2119894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman X86::isZeroNode(N1)) { 2120894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getNode()->getOperand(1)); 2121894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (!C) break; 2122894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 2123894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // For example, convert "testl %eax, $8" to "testb %al, $8" 2124894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if ((C->getZExtValue() & ~UINT64_C(0xff)) == 0 && 2125894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman (!(C->getZExtValue() & 0x80) || 2126894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman HasNoSignedComparisonUses(Node))) { 2127894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Imm = CurDAG->getTargetConstant(C->getZExtValue(), MVT::i8); 2128894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Reg = N0.getNode()->getOperand(0); 2129894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 2130894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // On x86-32, only the ABCD registers have 8-bit subregisters. 2131894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (!Subtarget->is64Bit()) { 2132894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman TargetRegisterClass *TRC = 0; 2133894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman switch (N0.getValueType().getSimpleVT().SimpleTy) { 2134894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i32: TRC = &X86::GR32_ABCDRegClass; break; 2135894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i16: TRC = &X86::GR16_ABCDRegClass; break; 2136894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman default: llvm_unreachable("Unsupported TEST operand type!"); 2137894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 2138894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue RC = CurDAG->getTargetConstant(TRC->getID(), MVT::i32); 2139894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Reg = SDValue(CurDAG->getMachineNode(X86::COPY_TO_REGCLASS, dl, 2140894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Reg.getValueType(), Reg, RC), 0); 2141894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 2142894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 2143894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Extract the l-register. 2144894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, 2145894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman MVT::i8, Reg); 2146894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 2147894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Emit a testb. 2148894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return CurDAG->getMachineNode(X86::TEST8ri, dl, MVT::i32, Subreg, Imm); 2149894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 2150894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 2151894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // For example, "testl %eax, $2048" to "testb %ah, $8". 2152894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if ((C->getZExtValue() & ~UINT64_C(0xff00)) == 0 && 2153894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman (!(C->getZExtValue() & 0x8000) || 2154894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman HasNoSignedComparisonUses(Node))) { 2155894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Shift the immediate right by 8 bits. 2156894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue ShiftedImm = CurDAG->getTargetConstant(C->getZExtValue() >> 8, 2157894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman MVT::i8); 2158894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Reg = N0.getNode()->getOperand(0); 2159894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 2160894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Put the value in an ABCD register. 2161894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman TargetRegisterClass *TRC = 0; 2162894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman switch (N0.getValueType().getSimpleVT().SimpleTy) { 2163894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i64: TRC = &X86::GR64_ABCDRegClass; break; 2164894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i32: TRC = &X86::GR32_ABCDRegClass; break; 2165894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case MVT::i16: TRC = &X86::GR16_ABCDRegClass; break; 2166894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman default: llvm_unreachable("Unsupported TEST operand type!"); 2167894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 2168894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue RC = CurDAG->getTargetConstant(TRC->getID(), MVT::i32); 2169894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Reg = SDValue(CurDAG->getMachineNode(X86::COPY_TO_REGCLASS, dl, 2170894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Reg.getValueType(), Reg, RC), 0); 2171894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 2172894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Extract the h-register. 2173894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_8bit_hi, dl, 2174894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman MVT::i8, Reg); 2175894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 217619bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // Emit a testb. The EXTRACT_SUBREG becomes a COPY that can only 217719bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // target GR8_NOREX registers, so make sure the register class is 217819bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman // forced. 217919bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman return CurDAG->getMachineNode(X86::TEST8ri_NOREX, dl, MVT::i32, 2180894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Subreg, ShiftedImm); 2181894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 2182894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 2183894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // For example, "testl %eax, $32776" to "testw %ax, $32776". 2184894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if ((C->getZExtValue() & ~UINT64_C(0xffff)) == 0 && 2185894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman N0.getValueType() != MVT::i16 && 2186894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman (!(C->getZExtValue() & 0x8000) || 2187894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman HasNoSignedComparisonUses(Node))) { 2188894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Imm = CurDAG->getTargetConstant(C->getZExtValue(), MVT::i16); 2189894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Reg = N0.getNode()->getOperand(0); 2190894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 2191894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Extract the 16-bit subregister. 2192894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_16bit, dl, 2193894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman MVT::i16, Reg); 2194894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 2195894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Emit a testw. 2196894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return CurDAG->getMachineNode(X86::TEST16ri, dl, MVT::i32, Subreg, Imm); 2197894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 2198894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 2199894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // For example, "testq %rax, $268468232" to "testl %eax, $268468232". 2200894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if ((C->getZExtValue() & ~UINT64_C(0xffffffff)) == 0 && 2201894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman N0.getValueType() == MVT::i64 && 2202894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman (!(C->getZExtValue() & 0x80000000) || 2203894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman HasNoSignedComparisonUses(Node))) { 2204894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Imm = CurDAG->getTargetConstant(C->getZExtValue(), MVT::i32); 2205894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Reg = N0.getNode()->getOperand(0); 2206894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 2207894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Extract the 32-bit subregister. 2208894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_32bit, dl, 2209894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman MVT::i32, Reg); 2210894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 2211894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman // Emit a testl. 2212894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return CurDAG->getMachineNode(X86::TEST32ri, dl, MVT::i32, Subreg, Imm); 2213894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 2214894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 2215894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman break; 2216894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 2217894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 2218894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 2219894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDNode *ResNode = SelectCode(Node); 2220894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 2221894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman DEBUG(dbgs() << "=> "; 2222894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman if (ResNode == NULL || ResNode == Node) 2223894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman Node->dump(CurDAG); 2224894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman else 2225894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman ResNode->dump(CurDAG); 2226894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman dbgs() << '\n'); 2227894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 2228894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return ResNode; 2229894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman} 2230894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 2231894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Baumanbool X86DAGToDAGISel:: 2232894018228b0e0bdbd7aa7e8f47d4a9458789ca82John BaumanSelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode, 2233894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman std::vector<SDValue> &OutOps) { 2234894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman SDValue Op0, Op1, Op2, Op3, Op4; 2235894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman switch (ConstraintCode) { 2236894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case 'o': // offsetable ?? 2237894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case 'v': // not offsetable ?? 2238894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman default: return true; 2239894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman case 'm': // memory 224019bac1e08be200c31efd26f0f5fd144c9b3eefd3John Bauman if (!SelectAddr(0, Op, Op0, Op1, Op2, Op3, Op4)) 2241894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return true; 2242894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman break; 2243894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman } 2244894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 2245894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman OutOps.push_back(Op0); 2246894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman OutOps.push_back(Op1); 2247894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman OutOps.push_back(Op2); 2248894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman OutOps.push_back(Op3); 2249894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman OutOps.push_back(Op4); 2250894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return false; 2251894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman} 2252894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman 2253894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// createX86ISelDag - This pass converts a legalized DAG into a 2254894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// X86-specific DAG, ready for instruction scheduling. 2255894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman/// 2256894018228b0e0bdbd7aa7e8f47d4a9458789ca82John BaumanFunctionPass *llvm::createX86ISelDag(X86TargetMachine &TM, 2257894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman llvm::CodeGenOpt::Level OptLevel) { 2258894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman return new X86DAGToDAGISel(TM, OptLevel); 2259894018228b0e0bdbd7aa7e8f47d4a9458789ca82John Bauman} 2260