X86ISelDAGToDAG.cpp revision b38fe4b52de05a35c0693f4fd117f908a2abadb7
1//===- X86ISelDAGToDAG.cpp - A DAG pattern matching inst selector for X86 -===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file defines a DAG pattern matching instruction selector for X86, 11// converting from a legalized dag to a X86 dag. 12// 13//===----------------------------------------------------------------------===// 14 15#define DEBUG_TYPE "x86-isel" 16#include "X86.h" 17#include "X86InstrBuilder.h" 18#include "X86MachineFunctionInfo.h" 19#include "X86RegisterInfo.h" 20#include "X86Subtarget.h" 21#include "X86TargetMachine.h" 22#include "llvm/Instructions.h" 23#include "llvm/Intrinsics.h" 24#include "llvm/Support/CFG.h" 25#include "llvm/Type.h" 26#include "llvm/CodeGen/MachineConstantPool.h" 27#include "llvm/CodeGen/MachineFunction.h" 28#include "llvm/CodeGen/MachineFrameInfo.h" 29#include "llvm/CodeGen/MachineInstrBuilder.h" 30#include "llvm/CodeGen/MachineRegisterInfo.h" 31#include "llvm/CodeGen/SelectionDAGISel.h" 32#include "llvm/Target/TargetMachine.h" 33#include "llvm/Target/TargetOptions.h" 34#include "llvm/Support/Debug.h" 35#include "llvm/Support/ErrorHandling.h" 36#include "llvm/Support/MathExtras.h" 37#include "llvm/Support/raw_ostream.h" 38#include "llvm/ADT/SmallPtrSet.h" 39#include "llvm/ADT/Statistic.h" 40using namespace llvm; 41 42STATISTIC(NumLoadMoved, "Number of loads moved below TokenFactor"); 43 44//===----------------------------------------------------------------------===// 45// Pattern Matcher Implementation 46//===----------------------------------------------------------------------===// 47 48namespace { 49 /// X86ISelAddressMode - This corresponds to X86AddressMode, but uses 50 /// SDValue's instead of register numbers for the leaves of the matched 51 /// tree. 52 struct X86ISelAddressMode { 53 enum { 54 RegBase, 55 FrameIndexBase 56 } BaseType; 57 58 // This is really a union, discriminated by BaseType! 59 SDValue Base_Reg; 60 int Base_FrameIndex; 61 62 unsigned Scale; 63 SDValue IndexReg; 64 int32_t Disp; 65 SDValue Segment; 66 const GlobalValue *GV; 67 const Constant *CP; 68 const BlockAddress *BlockAddr; 69 const char *ES; 70 int JT; 71 unsigned Align; // CP alignment. 72 unsigned char SymbolFlags; // X86II::MO_* 73 74 X86ISelAddressMode() 75 : BaseType(RegBase), Base_FrameIndex(0), Scale(1), IndexReg(), Disp(0), 76 Segment(), GV(0), CP(0), BlockAddr(0), ES(0), JT(-1), Align(0), 77 SymbolFlags(X86II::MO_NO_FLAG) { 78 } 79 80 bool hasSymbolicDisplacement() const { 81 return GV != 0 || CP != 0 || ES != 0 || JT != -1 || BlockAddr != 0; 82 } 83 84 bool hasBaseOrIndexReg() const { 85 return IndexReg.getNode() != 0 || Base_Reg.getNode() != 0; 86 } 87 88 /// isRIPRelative - Return true if this addressing mode is already RIP 89 /// relative. 90 bool isRIPRelative() const { 91 if (BaseType != RegBase) return false; 92 if (RegisterSDNode *RegNode = 93 dyn_cast_or_null<RegisterSDNode>(Base_Reg.getNode())) 94 return RegNode->getReg() == X86::RIP; 95 return false; 96 } 97 98 void setBaseReg(SDValue Reg) { 99 BaseType = RegBase; 100 Base_Reg = Reg; 101 } 102 103 void dump() { 104 dbgs() << "X86ISelAddressMode " << this << '\n'; 105 dbgs() << "Base_Reg "; 106 if (Base_Reg.getNode() != 0) 107 Base_Reg.getNode()->dump(); 108 else 109 dbgs() << "nul"; 110 dbgs() << " Base.FrameIndex " << Base_FrameIndex << '\n' 111 << " Scale" << Scale << '\n' 112 << "IndexReg "; 113 if (IndexReg.getNode() != 0) 114 IndexReg.getNode()->dump(); 115 else 116 dbgs() << "nul"; 117 dbgs() << " Disp " << Disp << '\n' 118 << "GV "; 119 if (GV) 120 GV->dump(); 121 else 122 dbgs() << "nul"; 123 dbgs() << " CP "; 124 if (CP) 125 CP->dump(); 126 else 127 dbgs() << "nul"; 128 dbgs() << '\n' 129 << "ES "; 130 if (ES) 131 dbgs() << ES; 132 else 133 dbgs() << "nul"; 134 dbgs() << " JT" << JT << " Align" << Align << '\n'; 135 } 136 }; 137} 138 139namespace { 140 //===--------------------------------------------------------------------===// 141 /// ISel - X86 specific code to select X86 machine instructions for 142 /// SelectionDAG operations. 143 /// 144 class X86DAGToDAGISel : public SelectionDAGISel { 145 /// X86Lowering - This object fully describes how to lower LLVM code to an 146 /// X86-specific SelectionDAG. 147 const X86TargetLowering &X86Lowering; 148 149 /// Subtarget - Keep a pointer to the X86Subtarget around so that we can 150 /// make the right decision when generating code for different targets. 151 const X86Subtarget *Subtarget; 152 153 /// OptForSize - If true, selector should try to optimize for code size 154 /// instead of performance. 155 bool OptForSize; 156 157 public: 158 explicit X86DAGToDAGISel(X86TargetMachine &tm, CodeGenOpt::Level OptLevel) 159 : SelectionDAGISel(tm, OptLevel), 160 X86Lowering(*tm.getTargetLowering()), 161 Subtarget(&tm.getSubtarget<X86Subtarget>()), 162 OptForSize(false) {} 163 164 virtual const char *getPassName() const { 165 return "X86 DAG->DAG Instruction Selection"; 166 } 167 168 virtual void EmitFunctionEntryCode(); 169 170 virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const; 171 172 virtual void PreprocessISelDAG(); 173 174 inline bool immSext8(SDNode *N) const { 175 return isInt<8>(cast<ConstantSDNode>(N)->getSExtValue()); 176 } 177 178 // i64immSExt32 predicate - True if the 64-bit immediate fits in a 32-bit 179 // sign extended field. 180 inline bool i64immSExt32(SDNode *N) const { 181 uint64_t v = cast<ConstantSDNode>(N)->getZExtValue(); 182 return (int64_t)v == (int32_t)v; 183 } 184 185// Include the pieces autogenerated from the target description. 186#include "X86GenDAGISel.inc" 187 188 private: 189 SDNode *Select(SDNode *N); 190 SDNode *SelectAtomic64(SDNode *Node, unsigned Opc); 191 SDNode *SelectAtomicLoadAdd(SDNode *Node, EVT NVT); 192 SDNode *SelectAtomicLoadOr(SDNode *Node, EVT NVT); 193 194 bool MatchLoadInAddress(LoadSDNode *N, X86ISelAddressMode &AM); 195 bool MatchWrapper(SDValue N, X86ISelAddressMode &AM); 196 bool MatchAddress(SDValue N, X86ISelAddressMode &AM); 197 bool MatchAddressRecursively(SDValue N, X86ISelAddressMode &AM, 198 unsigned Depth); 199 bool MatchAddressBase(SDValue N, X86ISelAddressMode &AM); 200 bool SelectAddr(SDNode *Parent, SDValue N, SDValue &Base, 201 SDValue &Scale, SDValue &Index, SDValue &Disp, 202 SDValue &Segment); 203 bool SelectLEAAddr(SDValue N, SDValue &Base, 204 SDValue &Scale, SDValue &Index, SDValue &Disp, 205 SDValue &Segment); 206 bool SelectTLSADDRAddr(SDValue N, SDValue &Base, 207 SDValue &Scale, SDValue &Index, SDValue &Disp, 208 SDValue &Segment); 209 bool SelectScalarSSELoad(SDNode *Root, SDValue N, 210 SDValue &Base, SDValue &Scale, 211 SDValue &Index, SDValue &Disp, 212 SDValue &Segment, 213 SDValue &NodeWithChain); 214 215 bool TryFoldLoad(SDNode *P, SDValue N, 216 SDValue &Base, SDValue &Scale, 217 SDValue &Index, SDValue &Disp, 218 SDValue &Segment); 219 220 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for 221 /// inline asm expressions. 222 virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op, 223 char ConstraintCode, 224 std::vector<SDValue> &OutOps); 225 226 void EmitSpecialCodeForMain(MachineBasicBlock *BB, MachineFrameInfo *MFI); 227 228 inline void getAddressOperands(X86ISelAddressMode &AM, SDValue &Base, 229 SDValue &Scale, SDValue &Index, 230 SDValue &Disp, SDValue &Segment) { 231 Base = (AM.BaseType == X86ISelAddressMode::FrameIndexBase) ? 232 CurDAG->getTargetFrameIndex(AM.Base_FrameIndex, TLI.getPointerTy()) : 233 AM.Base_Reg; 234 Scale = getI8Imm(AM.Scale); 235 Index = AM.IndexReg; 236 // These are 32-bit even in 64-bit mode since RIP relative offset 237 // is 32-bit. 238 if (AM.GV) 239 Disp = CurDAG->getTargetGlobalAddress(AM.GV, DebugLoc(), 240 MVT::i32, AM.Disp, 241 AM.SymbolFlags); 242 else if (AM.CP) 243 Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i32, 244 AM.Align, AM.Disp, AM.SymbolFlags); 245 else if (AM.ES) 246 Disp = CurDAG->getTargetExternalSymbol(AM.ES, MVT::i32, AM.SymbolFlags); 247 else if (AM.JT != -1) 248 Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i32, AM.SymbolFlags); 249 else if (AM.BlockAddr) 250 Disp = CurDAG->getBlockAddress(AM.BlockAddr, MVT::i32, 251 true, AM.SymbolFlags); 252 else 253 Disp = CurDAG->getTargetConstant(AM.Disp, MVT::i32); 254 255 if (AM.Segment.getNode()) 256 Segment = AM.Segment; 257 else 258 Segment = CurDAG->getRegister(0, MVT::i32); 259 } 260 261 /// getI8Imm - Return a target constant with the specified value, of type 262 /// i8. 263 inline SDValue getI8Imm(unsigned Imm) { 264 return CurDAG->getTargetConstant(Imm, MVT::i8); 265 } 266 267 /// getI32Imm - Return a target constant with the specified value, of type 268 /// i32. 269 inline SDValue getI32Imm(unsigned Imm) { 270 return CurDAG->getTargetConstant(Imm, MVT::i32); 271 } 272 273 /// getGlobalBaseReg - Return an SDNode that returns the value of 274 /// the global base register. Output instructions required to 275 /// initialize the global base register, if necessary. 276 /// 277 SDNode *getGlobalBaseReg(); 278 279 /// getTargetMachine - Return a reference to the TargetMachine, casted 280 /// to the target-specific type. 281 const X86TargetMachine &getTargetMachine() { 282 return static_cast<const X86TargetMachine &>(TM); 283 } 284 285 /// getInstrInfo - Return a reference to the TargetInstrInfo, casted 286 /// to the target-specific type. 287 const X86InstrInfo *getInstrInfo() { 288 return getTargetMachine().getInstrInfo(); 289 } 290 }; 291} 292 293 294bool 295X86DAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const { 296 if (OptLevel == CodeGenOpt::None) return false; 297 298 if (!N.hasOneUse()) 299 return false; 300 301 if (N.getOpcode() != ISD::LOAD) 302 return true; 303 304 // If N is a load, do additional profitability checks. 305 if (U == Root) { 306 switch (U->getOpcode()) { 307 default: break; 308 case X86ISD::ADD: 309 case X86ISD::SUB: 310 case X86ISD::AND: 311 case X86ISD::XOR: 312 case X86ISD::OR: 313 case ISD::ADD: 314 case ISD::ADDC: 315 case ISD::ADDE: 316 case ISD::AND: 317 case ISD::OR: 318 case ISD::XOR: { 319 SDValue Op1 = U->getOperand(1); 320 321 // If the other operand is a 8-bit immediate we should fold the immediate 322 // instead. This reduces code size. 323 // e.g. 324 // movl 4(%esp), %eax 325 // addl $4, %eax 326 // vs. 327 // movl $4, %eax 328 // addl 4(%esp), %eax 329 // The former is 2 bytes shorter. In case where the increment is 1, then 330 // the saving can be 4 bytes (by using incl %eax). 331 if (ConstantSDNode *Imm = dyn_cast<ConstantSDNode>(Op1)) 332 if (Imm->getAPIntValue().isSignedIntN(8)) 333 return false; 334 335 // If the other operand is a TLS address, we should fold it instead. 336 // This produces 337 // movl %gs:0, %eax 338 // leal i@NTPOFF(%eax), %eax 339 // instead of 340 // movl $i@NTPOFF, %eax 341 // addl %gs:0, %eax 342 // if the block also has an access to a second TLS address this will save 343 // a load. 344 // FIXME: This is probably also true for non TLS addresses. 345 if (Op1.getOpcode() == X86ISD::Wrapper) { 346 SDValue Val = Op1.getOperand(0); 347 if (Val.getOpcode() == ISD::TargetGlobalTLSAddress) 348 return false; 349 } 350 } 351 } 352 } 353 354 return true; 355} 356 357/// MoveBelowCallOrigChain - Replace the original chain operand of the call with 358/// load's chain operand and move load below the call's chain operand. 359static void MoveBelowOrigChain(SelectionDAG *CurDAG, SDValue Load, 360 SDValue Call, SDValue OrigChain) { 361 SmallVector<SDValue, 8> Ops; 362 SDValue Chain = OrigChain.getOperand(0); 363 if (Chain.getNode() == Load.getNode()) 364 Ops.push_back(Load.getOperand(0)); 365 else { 366 assert(Chain.getOpcode() == ISD::TokenFactor && 367 "Unexpected chain operand"); 368 for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i) 369 if (Chain.getOperand(i).getNode() == Load.getNode()) 370 Ops.push_back(Load.getOperand(0)); 371 else 372 Ops.push_back(Chain.getOperand(i)); 373 SDValue NewChain = 374 CurDAG->getNode(ISD::TokenFactor, Load.getDebugLoc(), 375 MVT::Other, &Ops[0], Ops.size()); 376 Ops.clear(); 377 Ops.push_back(NewChain); 378 } 379 for (unsigned i = 1, e = OrigChain.getNumOperands(); i != e; ++i) 380 Ops.push_back(OrigChain.getOperand(i)); 381 CurDAG->UpdateNodeOperands(OrigChain.getNode(), &Ops[0], Ops.size()); 382 CurDAG->UpdateNodeOperands(Load.getNode(), Call.getOperand(0), 383 Load.getOperand(1), Load.getOperand(2)); 384 Ops.clear(); 385 Ops.push_back(SDValue(Load.getNode(), 1)); 386 for (unsigned i = 1, e = Call.getNode()->getNumOperands(); i != e; ++i) 387 Ops.push_back(Call.getOperand(i)); 388 CurDAG->UpdateNodeOperands(Call.getNode(), &Ops[0], Ops.size()); 389} 390 391/// isCalleeLoad - Return true if call address is a load and it can be 392/// moved below CALLSEQ_START and the chains leading up to the call. 393/// Return the CALLSEQ_START by reference as a second output. 394/// In the case of a tail call, there isn't a callseq node between the call 395/// chain and the load. 396static bool isCalleeLoad(SDValue Callee, SDValue &Chain, bool HasCallSeq) { 397 if (Callee.getNode() == Chain.getNode() || !Callee.hasOneUse()) 398 return false; 399 LoadSDNode *LD = dyn_cast<LoadSDNode>(Callee.getNode()); 400 if (!LD || 401 LD->isVolatile() || 402 LD->getAddressingMode() != ISD::UNINDEXED || 403 LD->getExtensionType() != ISD::NON_EXTLOAD) 404 return false; 405 406 // Now let's find the callseq_start. 407 while (HasCallSeq && Chain.getOpcode() != ISD::CALLSEQ_START) { 408 if (!Chain.hasOneUse()) 409 return false; 410 Chain = Chain.getOperand(0); 411 } 412 413 if (!Chain.getNumOperands()) 414 return false; 415 if (Chain.getOperand(0).getNode() == Callee.getNode()) 416 return true; 417 if (Chain.getOperand(0).getOpcode() == ISD::TokenFactor && 418 Callee.getValue(1).isOperandOf(Chain.getOperand(0).getNode()) && 419 Callee.getValue(1).hasOneUse()) 420 return true; 421 return false; 422} 423 424void X86DAGToDAGISel::PreprocessISelDAG() { 425 // OptForSize is used in pattern predicates that isel is matching. 426 OptForSize = MF->getFunction()->hasFnAttr(Attribute::OptimizeForSize); 427 428 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(), 429 E = CurDAG->allnodes_end(); I != E; ) { 430 SDNode *N = I++; // Preincrement iterator to avoid invalidation issues. 431 432 if (OptLevel != CodeGenOpt::None && 433 (N->getOpcode() == X86ISD::CALL || 434 N->getOpcode() == X86ISD::TC_RETURN)) { 435 /// Also try moving call address load from outside callseq_start to just 436 /// before the call to allow it to be folded. 437 /// 438 /// [Load chain] 439 /// ^ 440 /// | 441 /// [Load] 442 /// ^ ^ 443 /// | | 444 /// / \-- 445 /// / | 446 ///[CALLSEQ_START] | 447 /// ^ | 448 /// | | 449 /// [LOAD/C2Reg] | 450 /// | | 451 /// \ / 452 /// \ / 453 /// [CALL] 454 bool HasCallSeq = N->getOpcode() == X86ISD::CALL; 455 SDValue Chain = N->getOperand(0); 456 SDValue Load = N->getOperand(1); 457 if (!isCalleeLoad(Load, Chain, HasCallSeq)) 458 continue; 459 MoveBelowOrigChain(CurDAG, Load, SDValue(N, 0), Chain); 460 ++NumLoadMoved; 461 continue; 462 } 463 464 // Lower fpround and fpextend nodes that target the FP stack to be store and 465 // load to the stack. This is a gross hack. We would like to simply mark 466 // these as being illegal, but when we do that, legalize produces these when 467 // it expands calls, then expands these in the same legalize pass. We would 468 // like dag combine to be able to hack on these between the call expansion 469 // and the node legalization. As such this pass basically does "really 470 // late" legalization of these inline with the X86 isel pass. 471 // FIXME: This should only happen when not compiled with -O0. 472 if (N->getOpcode() != ISD::FP_ROUND && N->getOpcode() != ISD::FP_EXTEND) 473 continue; 474 475 // If the source and destination are SSE registers, then this is a legal 476 // conversion that should not be lowered. 477 EVT SrcVT = N->getOperand(0).getValueType(); 478 EVT DstVT = N->getValueType(0); 479 bool SrcIsSSE = X86Lowering.isScalarFPTypeInSSEReg(SrcVT); 480 bool DstIsSSE = X86Lowering.isScalarFPTypeInSSEReg(DstVT); 481 if (SrcIsSSE && DstIsSSE) 482 continue; 483 484 if (!SrcIsSSE && !DstIsSSE) { 485 // If this is an FPStack extension, it is a noop. 486 if (N->getOpcode() == ISD::FP_EXTEND) 487 continue; 488 // If this is a value-preserving FPStack truncation, it is a noop. 489 if (N->getConstantOperandVal(1)) 490 continue; 491 } 492 493 // Here we could have an FP stack truncation or an FPStack <-> SSE convert. 494 // FPStack has extload and truncstore. SSE can fold direct loads into other 495 // operations. Based on this, decide what we want to do. 496 EVT MemVT; 497 if (N->getOpcode() == ISD::FP_ROUND) 498 MemVT = DstVT; // FP_ROUND must use DstVT, we can't do a 'trunc load'. 499 else 500 MemVT = SrcIsSSE ? SrcVT : DstVT; 501 502 SDValue MemTmp = CurDAG->CreateStackTemporary(MemVT); 503 DebugLoc dl = N->getDebugLoc(); 504 505 // FIXME: optimize the case where the src/dest is a load or store? 506 SDValue Store = CurDAG->getTruncStore(CurDAG->getEntryNode(), dl, 507 N->getOperand(0), 508 MemTmp, MachinePointerInfo(), MemVT, 509 false, false, 0); 510 SDValue Result = CurDAG->getExtLoad(ISD::EXTLOAD, dl, DstVT, Store, MemTmp, 511 MachinePointerInfo(), 512 MemVT, false, false, 0); 513 514 // We're about to replace all uses of the FP_ROUND/FP_EXTEND with the 515 // extload we created. This will cause general havok on the dag because 516 // anything below the conversion could be folded into other existing nodes. 517 // To avoid invalidating 'I', back it up to the convert node. 518 --I; 519 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Result); 520 521 // Now that we did that, the node is dead. Increment the iterator to the 522 // next node to process, then delete N. 523 ++I; 524 CurDAG->DeleteNode(N); 525 } 526} 527 528 529/// EmitSpecialCodeForMain - Emit any code that needs to be executed only in 530/// the main function. 531void X86DAGToDAGISel::EmitSpecialCodeForMain(MachineBasicBlock *BB, 532 MachineFrameInfo *MFI) { 533 const TargetInstrInfo *TII = TM.getInstrInfo(); 534 if (Subtarget->isTargetCygMing()) { 535 unsigned CallOp = 536 Subtarget->is64Bit() ? X86::WINCALL64pcrel32 : X86::CALLpcrel32; 537 BuildMI(BB, DebugLoc(), 538 TII->get(CallOp)).addExternalSymbol("__main"); 539 } 540} 541 542void X86DAGToDAGISel::EmitFunctionEntryCode() { 543 // If this is main, emit special code for main. 544 if (const Function *Fn = MF->getFunction()) 545 if (Fn->hasExternalLinkage() && Fn->getName() == "main") 546 EmitSpecialCodeForMain(MF->begin(), MF->getFrameInfo()); 547} 548 549 550bool X86DAGToDAGISel::MatchLoadInAddress(LoadSDNode *N, X86ISelAddressMode &AM){ 551 SDValue Address = N->getOperand(1); 552 553 // load gs:0 -> GS segment register. 554 // load fs:0 -> FS segment register. 555 // 556 // This optimization is valid because the GNU TLS model defines that 557 // gs:0 (or fs:0 on X86-64) contains its own address. 558 // For more information see http://people.redhat.com/drepper/tls.pdf 559 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Address)) 560 if (C->getSExtValue() == 0 && AM.Segment.getNode() == 0 && 561 Subtarget->isTargetELF()) 562 switch (N->getPointerInfo().getAddrSpace()) { 563 case 256: 564 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16); 565 return false; 566 case 257: 567 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16); 568 return false; 569 } 570 571 return true; 572} 573 574/// MatchWrapper - Try to match X86ISD::Wrapper and X86ISD::WrapperRIP nodes 575/// into an addressing mode. These wrap things that will resolve down into a 576/// symbol reference. If no match is possible, this returns true, otherwise it 577/// returns false. 578bool X86DAGToDAGISel::MatchWrapper(SDValue N, X86ISelAddressMode &AM) { 579 // If the addressing mode already has a symbol as the displacement, we can 580 // never match another symbol. 581 if (AM.hasSymbolicDisplacement()) 582 return true; 583 584 SDValue N0 = N.getOperand(0); 585 CodeModel::Model M = TM.getCodeModel(); 586 587 // Handle X86-64 rip-relative addresses. We check this before checking direct 588 // folding because RIP is preferable to non-RIP accesses. 589 if (Subtarget->is64Bit() && 590 // Under X86-64 non-small code model, GV (and friends) are 64-bits, so 591 // they cannot be folded into immediate fields. 592 // FIXME: This can be improved for kernel and other models? 593 (M == CodeModel::Small || M == CodeModel::Kernel) && 594 // Base and index reg must be 0 in order to use %rip as base and lowering 595 // must allow RIP. 596 !AM.hasBaseOrIndexReg() && N.getOpcode() == X86ISD::WrapperRIP) { 597 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(N0)) { 598 int64_t Offset = AM.Disp + G->getOffset(); 599 if (!X86::isOffsetSuitableForCodeModel(Offset, M)) return true; 600 AM.GV = G->getGlobal(); 601 AM.Disp = Offset; 602 AM.SymbolFlags = G->getTargetFlags(); 603 } else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(N0)) { 604 int64_t Offset = AM.Disp + CP->getOffset(); 605 if (!X86::isOffsetSuitableForCodeModel(Offset, M)) return true; 606 AM.CP = CP->getConstVal(); 607 AM.Align = CP->getAlignment(); 608 AM.Disp = Offset; 609 AM.SymbolFlags = CP->getTargetFlags(); 610 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(N0)) { 611 AM.ES = S->getSymbol(); 612 AM.SymbolFlags = S->getTargetFlags(); 613 } else if (JumpTableSDNode *J = dyn_cast<JumpTableSDNode>(N0)) { 614 AM.JT = J->getIndex(); 615 AM.SymbolFlags = J->getTargetFlags(); 616 } else { 617 AM.BlockAddr = cast<BlockAddressSDNode>(N0)->getBlockAddress(); 618 AM.SymbolFlags = cast<BlockAddressSDNode>(N0)->getTargetFlags(); 619 } 620 621 if (N.getOpcode() == X86ISD::WrapperRIP) 622 AM.setBaseReg(CurDAG->getRegister(X86::RIP, MVT::i64)); 623 return false; 624 } 625 626 // Handle the case when globals fit in our immediate field: This is true for 627 // X86-32 always and X86-64 when in -static -mcmodel=small mode. In 64-bit 628 // mode, this results in a non-RIP-relative computation. 629 if (!Subtarget->is64Bit() || 630 ((M == CodeModel::Small || M == CodeModel::Kernel) && 631 TM.getRelocationModel() == Reloc::Static)) { 632 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(N0)) { 633 AM.GV = G->getGlobal(); 634 AM.Disp += G->getOffset(); 635 AM.SymbolFlags = G->getTargetFlags(); 636 } else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(N0)) { 637 AM.CP = CP->getConstVal(); 638 AM.Align = CP->getAlignment(); 639 AM.Disp += CP->getOffset(); 640 AM.SymbolFlags = CP->getTargetFlags(); 641 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(N0)) { 642 AM.ES = S->getSymbol(); 643 AM.SymbolFlags = S->getTargetFlags(); 644 } else if (JumpTableSDNode *J = dyn_cast<JumpTableSDNode>(N0)) { 645 AM.JT = J->getIndex(); 646 AM.SymbolFlags = J->getTargetFlags(); 647 } else { 648 AM.BlockAddr = cast<BlockAddressSDNode>(N0)->getBlockAddress(); 649 AM.SymbolFlags = cast<BlockAddressSDNode>(N0)->getTargetFlags(); 650 } 651 return false; 652 } 653 654 return true; 655} 656 657/// MatchAddress - Add the specified node to the specified addressing mode, 658/// returning true if it cannot be done. This just pattern matches for the 659/// addressing mode. 660bool X86DAGToDAGISel::MatchAddress(SDValue N, X86ISelAddressMode &AM) { 661 if (MatchAddressRecursively(N, AM, 0)) 662 return true; 663 664 // Post-processing: Convert lea(,%reg,2) to lea(%reg,%reg), which has 665 // a smaller encoding and avoids a scaled-index. 666 if (AM.Scale == 2 && 667 AM.BaseType == X86ISelAddressMode::RegBase && 668 AM.Base_Reg.getNode() == 0) { 669 AM.Base_Reg = AM.IndexReg; 670 AM.Scale = 1; 671 } 672 673 // Post-processing: Convert foo to foo(%rip), even in non-PIC mode, 674 // because it has a smaller encoding. 675 // TODO: Which other code models can use this? 676 if (TM.getCodeModel() == CodeModel::Small && 677 Subtarget->is64Bit() && 678 AM.Scale == 1 && 679 AM.BaseType == X86ISelAddressMode::RegBase && 680 AM.Base_Reg.getNode() == 0 && 681 AM.IndexReg.getNode() == 0 && 682 AM.SymbolFlags == X86II::MO_NO_FLAG && 683 AM.hasSymbolicDisplacement()) 684 AM.Base_Reg = CurDAG->getRegister(X86::RIP, MVT::i64); 685 686 return false; 687} 688 689bool X86DAGToDAGISel::MatchAddressRecursively(SDValue N, X86ISelAddressMode &AM, 690 unsigned Depth) { 691 bool is64Bit = Subtarget->is64Bit(); 692 DebugLoc dl = N.getDebugLoc(); 693 DEBUG({ 694 dbgs() << "MatchAddress: "; 695 AM.dump(); 696 }); 697 // Limit recursion. 698 if (Depth > 5) 699 return MatchAddressBase(N, AM); 700 701 CodeModel::Model M = TM.getCodeModel(); 702 703 // If this is already a %rip relative address, we can only merge immediates 704 // into it. Instead of handling this in every case, we handle it here. 705 // RIP relative addressing: %rip + 32-bit displacement! 706 if (AM.isRIPRelative()) { 707 // FIXME: JumpTable and ExternalSymbol address currently don't like 708 // displacements. It isn't very important, but this should be fixed for 709 // consistency. 710 if (!AM.ES && AM.JT != -1) return true; 711 712 if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(N)) { 713 int64_t Val = AM.Disp + Cst->getSExtValue(); 714 if (X86::isOffsetSuitableForCodeModel(Val, M, 715 AM.hasSymbolicDisplacement())) { 716 AM.Disp = Val; 717 return false; 718 } 719 } 720 return true; 721 } 722 723 switch (N.getOpcode()) { 724 default: break; 725 case ISD::Constant: { 726 uint64_t Val = cast<ConstantSDNode>(N)->getSExtValue(); 727 if (!is64Bit || 728 X86::isOffsetSuitableForCodeModel(AM.Disp + Val, M, 729 AM.hasSymbolicDisplacement())) { 730 AM.Disp += Val; 731 return false; 732 } 733 break; 734 } 735 736 case X86ISD::Wrapper: 737 case X86ISD::WrapperRIP: 738 if (!MatchWrapper(N, AM)) 739 return false; 740 break; 741 742 case ISD::LOAD: 743 if (!MatchLoadInAddress(cast<LoadSDNode>(N), AM)) 744 return false; 745 break; 746 747 case ISD::FrameIndex: 748 if (AM.BaseType == X86ISelAddressMode::RegBase 749 && AM.Base_Reg.getNode() == 0) { 750 AM.BaseType = X86ISelAddressMode::FrameIndexBase; 751 AM.Base_FrameIndex = cast<FrameIndexSDNode>(N)->getIndex(); 752 return false; 753 } 754 break; 755 756 case ISD::SHL: 757 if (AM.IndexReg.getNode() != 0 || AM.Scale != 1) 758 break; 759 760 if (ConstantSDNode 761 *CN = dyn_cast<ConstantSDNode>(N.getNode()->getOperand(1))) { 762 unsigned Val = CN->getZExtValue(); 763 // Note that we handle x<<1 as (,x,2) rather than (x,x) here so 764 // that the base operand remains free for further matching. If 765 // the base doesn't end up getting used, a post-processing step 766 // in MatchAddress turns (,x,2) into (x,x), which is cheaper. 767 if (Val == 1 || Val == 2 || Val == 3) { 768 AM.Scale = 1 << Val; 769 SDValue ShVal = N.getNode()->getOperand(0); 770 771 // Okay, we know that we have a scale by now. However, if the scaled 772 // value is an add of something and a constant, we can fold the 773 // constant into the disp field here. 774 if (CurDAG->isBaseWithConstantOffset(ShVal)) { 775 AM.IndexReg = ShVal.getNode()->getOperand(0); 776 ConstantSDNode *AddVal = 777 cast<ConstantSDNode>(ShVal.getNode()->getOperand(1)); 778 uint64_t Disp = AM.Disp + (AddVal->getSExtValue() << Val); 779 if (!is64Bit || 780 X86::isOffsetSuitableForCodeModel(Disp, M, 781 AM.hasSymbolicDisplacement())) 782 AM.Disp = Disp; 783 else 784 AM.IndexReg = ShVal; 785 } else { 786 AM.IndexReg = ShVal; 787 } 788 return false; 789 } 790 break; 791 } 792 793 case ISD::SMUL_LOHI: 794 case ISD::UMUL_LOHI: 795 // A mul_lohi where we need the low part can be folded as a plain multiply. 796 if (N.getResNo() != 0) break; 797 // FALL THROUGH 798 case ISD::MUL: 799 case X86ISD::MUL_IMM: 800 // X*[3,5,9] -> X+X*[2,4,8] 801 if (AM.BaseType == X86ISelAddressMode::RegBase && 802 AM.Base_Reg.getNode() == 0 && 803 AM.IndexReg.getNode() == 0) { 804 if (ConstantSDNode 805 *CN = dyn_cast<ConstantSDNode>(N.getNode()->getOperand(1))) 806 if (CN->getZExtValue() == 3 || CN->getZExtValue() == 5 || 807 CN->getZExtValue() == 9) { 808 AM.Scale = unsigned(CN->getZExtValue())-1; 809 810 SDValue MulVal = N.getNode()->getOperand(0); 811 SDValue Reg; 812 813 // Okay, we know that we have a scale by now. However, if the scaled 814 // value is an add of something and a constant, we can fold the 815 // constant into the disp field here. 816 if (MulVal.getNode()->getOpcode() == ISD::ADD && MulVal.hasOneUse() && 817 isa<ConstantSDNode>(MulVal.getNode()->getOperand(1))) { 818 Reg = MulVal.getNode()->getOperand(0); 819 ConstantSDNode *AddVal = 820 cast<ConstantSDNode>(MulVal.getNode()->getOperand(1)); 821 uint64_t Disp = AM.Disp + AddVal->getSExtValue() * 822 CN->getZExtValue(); 823 if (!is64Bit || 824 X86::isOffsetSuitableForCodeModel(Disp, M, 825 AM.hasSymbolicDisplacement())) 826 AM.Disp = Disp; 827 else 828 Reg = N.getNode()->getOperand(0); 829 } else { 830 Reg = N.getNode()->getOperand(0); 831 } 832 833 AM.IndexReg = AM.Base_Reg = Reg; 834 return false; 835 } 836 } 837 break; 838 839 case ISD::SUB: { 840 // Given A-B, if A can be completely folded into the address and 841 // the index field with the index field unused, use -B as the index. 842 // This is a win if a has multiple parts that can be folded into 843 // the address. Also, this saves a mov if the base register has 844 // other uses, since it avoids a two-address sub instruction, however 845 // it costs an additional mov if the index register has other uses. 846 847 // Add an artificial use to this node so that we can keep track of 848 // it if it gets CSE'd with a different node. 849 HandleSDNode Handle(N); 850 851 // Test if the LHS of the sub can be folded. 852 X86ISelAddressMode Backup = AM; 853 if (MatchAddressRecursively(N.getNode()->getOperand(0), AM, Depth+1)) { 854 AM = Backup; 855 break; 856 } 857 // Test if the index field is free for use. 858 if (AM.IndexReg.getNode() || AM.isRIPRelative()) { 859 AM = Backup; 860 break; 861 } 862 863 int Cost = 0; 864 SDValue RHS = Handle.getValue().getNode()->getOperand(1); 865 // If the RHS involves a register with multiple uses, this 866 // transformation incurs an extra mov, due to the neg instruction 867 // clobbering its operand. 868 if (!RHS.getNode()->hasOneUse() || 869 RHS.getNode()->getOpcode() == ISD::CopyFromReg || 870 RHS.getNode()->getOpcode() == ISD::TRUNCATE || 871 RHS.getNode()->getOpcode() == ISD::ANY_EXTEND || 872 (RHS.getNode()->getOpcode() == ISD::ZERO_EXTEND && 873 RHS.getNode()->getOperand(0).getValueType() == MVT::i32)) 874 ++Cost; 875 // If the base is a register with multiple uses, this 876 // transformation may save a mov. 877 if ((AM.BaseType == X86ISelAddressMode::RegBase && 878 AM.Base_Reg.getNode() && 879 !AM.Base_Reg.getNode()->hasOneUse()) || 880 AM.BaseType == X86ISelAddressMode::FrameIndexBase) 881 --Cost; 882 // If the folded LHS was interesting, this transformation saves 883 // address arithmetic. 884 if ((AM.hasSymbolicDisplacement() && !Backup.hasSymbolicDisplacement()) + 885 ((AM.Disp != 0) && (Backup.Disp == 0)) + 886 (AM.Segment.getNode() && !Backup.Segment.getNode()) >= 2) 887 --Cost; 888 // If it doesn't look like it may be an overall win, don't do it. 889 if (Cost >= 0) { 890 AM = Backup; 891 break; 892 } 893 894 // Ok, the transformation is legal and appears profitable. Go for it. 895 SDValue Zero = CurDAG->getConstant(0, N.getValueType()); 896 SDValue Neg = CurDAG->getNode(ISD::SUB, dl, N.getValueType(), Zero, RHS); 897 AM.IndexReg = Neg; 898 AM.Scale = 1; 899 900 // Insert the new nodes into the topological ordering. 901 if (Zero.getNode()->getNodeId() == -1 || 902 Zero.getNode()->getNodeId() > N.getNode()->getNodeId()) { 903 CurDAG->RepositionNode(N.getNode(), Zero.getNode()); 904 Zero.getNode()->setNodeId(N.getNode()->getNodeId()); 905 } 906 if (Neg.getNode()->getNodeId() == -1 || 907 Neg.getNode()->getNodeId() > N.getNode()->getNodeId()) { 908 CurDAG->RepositionNode(N.getNode(), Neg.getNode()); 909 Neg.getNode()->setNodeId(N.getNode()->getNodeId()); 910 } 911 return false; 912 } 913 914 case ISD::ADD: { 915 // Add an artificial use to this node so that we can keep track of 916 // it if it gets CSE'd with a different node. 917 HandleSDNode Handle(N); 918 919 X86ISelAddressMode Backup = AM; 920 if (!MatchAddressRecursively(N.getOperand(0), AM, Depth+1) && 921 !MatchAddressRecursively(Handle.getValue().getOperand(1), AM, Depth+1)) 922 return false; 923 AM = Backup; 924 925 // Try again after commuting the operands. 926 if (!MatchAddressRecursively(Handle.getValue().getOperand(1), AM, Depth+1)&& 927 !MatchAddressRecursively(Handle.getValue().getOperand(0), AM, Depth+1)) 928 return false; 929 AM = Backup; 930 931 // If we couldn't fold both operands into the address at the same time, 932 // see if we can just put each operand into a register and fold at least 933 // the add. 934 if (AM.BaseType == X86ISelAddressMode::RegBase && 935 !AM.Base_Reg.getNode() && 936 !AM.IndexReg.getNode()) { 937 N = Handle.getValue(); 938 AM.Base_Reg = N.getOperand(0); 939 AM.IndexReg = N.getOperand(1); 940 AM.Scale = 1; 941 return false; 942 } 943 N = Handle.getValue(); 944 break; 945 } 946 947 case ISD::OR: 948 // Handle "X | C" as "X + C" iff X is known to have C bits clear. 949 if (CurDAG->isBaseWithConstantOffset(N)) { 950 X86ISelAddressMode Backup = AM; 951 ConstantSDNode *CN = cast<ConstantSDNode>(N.getOperand(1)); 952 uint64_t Offset = CN->getSExtValue(); 953 954 // Start with the LHS as an addr mode. 955 if (!MatchAddressRecursively(N.getOperand(0), AM, Depth+1) && 956 // Address could not have picked a GV address for the displacement. 957 AM.GV == NULL && 958 // On x86-64, the resultant disp must fit in 32-bits. 959 (!is64Bit || 960 X86::isOffsetSuitableForCodeModel(AM.Disp + Offset, M, 961 AM.hasSymbolicDisplacement()))) { 962 AM.Disp += Offset; 963 return false; 964 } 965 AM = Backup; 966 } 967 break; 968 969 case ISD::AND: { 970 // Perform some heroic transforms on an and of a constant-count shift 971 // with a constant to enable use of the scaled offset field. 972 973 SDValue Shift = N.getOperand(0); 974 if (Shift.getNumOperands() != 2) break; 975 976 // Scale must not be used already. 977 if (AM.IndexReg.getNode() != 0 || AM.Scale != 1) break; 978 979 SDValue X = Shift.getOperand(0); 980 ConstantSDNode *C2 = dyn_cast<ConstantSDNode>(N.getOperand(1)); 981 ConstantSDNode *C1 = dyn_cast<ConstantSDNode>(Shift.getOperand(1)); 982 if (!C1 || !C2) break; 983 984 // Handle "(X >> (8-C1)) & C2" as "(X >> 8) & 0xff)" if safe. This 985 // allows us to convert the shift and and into an h-register extract and 986 // a scaled index. 987 if (Shift.getOpcode() == ISD::SRL && Shift.hasOneUse()) { 988 unsigned ScaleLog = 8 - C1->getZExtValue(); 989 if (ScaleLog > 0 && ScaleLog < 4 && 990 C2->getZExtValue() == (UINT64_C(0xff) << ScaleLog)) { 991 SDValue Eight = CurDAG->getConstant(8, MVT::i8); 992 SDValue Mask = CurDAG->getConstant(0xff, N.getValueType()); 993 SDValue Srl = CurDAG->getNode(ISD::SRL, dl, N.getValueType(), 994 X, Eight); 995 SDValue And = CurDAG->getNode(ISD::AND, dl, N.getValueType(), 996 Srl, Mask); 997 SDValue ShlCount = CurDAG->getConstant(ScaleLog, MVT::i8); 998 SDValue Shl = CurDAG->getNode(ISD::SHL, dl, N.getValueType(), 999 And, ShlCount); 1000 1001 // Insert the new nodes into the topological ordering. 1002 if (Eight.getNode()->getNodeId() == -1 || 1003 Eight.getNode()->getNodeId() > X.getNode()->getNodeId()) { 1004 CurDAG->RepositionNode(X.getNode(), Eight.getNode()); 1005 Eight.getNode()->setNodeId(X.getNode()->getNodeId()); 1006 } 1007 if (Mask.getNode()->getNodeId() == -1 || 1008 Mask.getNode()->getNodeId() > X.getNode()->getNodeId()) { 1009 CurDAG->RepositionNode(X.getNode(), Mask.getNode()); 1010 Mask.getNode()->setNodeId(X.getNode()->getNodeId()); 1011 } 1012 if (Srl.getNode()->getNodeId() == -1 || 1013 Srl.getNode()->getNodeId() > Shift.getNode()->getNodeId()) { 1014 CurDAG->RepositionNode(Shift.getNode(), Srl.getNode()); 1015 Srl.getNode()->setNodeId(Shift.getNode()->getNodeId()); 1016 } 1017 if (And.getNode()->getNodeId() == -1 || 1018 And.getNode()->getNodeId() > N.getNode()->getNodeId()) { 1019 CurDAG->RepositionNode(N.getNode(), And.getNode()); 1020 And.getNode()->setNodeId(N.getNode()->getNodeId()); 1021 } 1022 if (ShlCount.getNode()->getNodeId() == -1 || 1023 ShlCount.getNode()->getNodeId() > X.getNode()->getNodeId()) { 1024 CurDAG->RepositionNode(X.getNode(), ShlCount.getNode()); 1025 ShlCount.getNode()->setNodeId(N.getNode()->getNodeId()); 1026 } 1027 if (Shl.getNode()->getNodeId() == -1 || 1028 Shl.getNode()->getNodeId() > N.getNode()->getNodeId()) { 1029 CurDAG->RepositionNode(N.getNode(), Shl.getNode()); 1030 Shl.getNode()->setNodeId(N.getNode()->getNodeId()); 1031 } 1032 CurDAG->ReplaceAllUsesWith(N, Shl); 1033 AM.IndexReg = And; 1034 AM.Scale = (1 << ScaleLog); 1035 return false; 1036 } 1037 } 1038 1039 // Handle "(X << C1) & C2" as "(X & (C2>>C1)) << C1" if safe and if this 1040 // allows us to fold the shift into this addressing mode. 1041 if (Shift.getOpcode() != ISD::SHL) break; 1042 1043 // Not likely to be profitable if either the AND or SHIFT node has more 1044 // than one use (unless all uses are for address computation). Besides, 1045 // isel mechanism requires their node ids to be reused. 1046 if (!N.hasOneUse() || !Shift.hasOneUse()) 1047 break; 1048 1049 // Verify that the shift amount is something we can fold. 1050 unsigned ShiftCst = C1->getZExtValue(); 1051 if (ShiftCst != 1 && ShiftCst != 2 && ShiftCst != 3) 1052 break; 1053 1054 // Get the new AND mask, this folds to a constant. 1055 SDValue NewANDMask = CurDAG->getNode(ISD::SRL, dl, N.getValueType(), 1056 SDValue(C2, 0), SDValue(C1, 0)); 1057 SDValue NewAND = CurDAG->getNode(ISD::AND, dl, N.getValueType(), X, 1058 NewANDMask); 1059 SDValue NewSHIFT = CurDAG->getNode(ISD::SHL, dl, N.getValueType(), 1060 NewAND, SDValue(C1, 0)); 1061 1062 // Insert the new nodes into the topological ordering. 1063 if (C1->getNodeId() > X.getNode()->getNodeId()) { 1064 CurDAG->RepositionNode(X.getNode(), C1); 1065 C1->setNodeId(X.getNode()->getNodeId()); 1066 } 1067 if (NewANDMask.getNode()->getNodeId() == -1 || 1068 NewANDMask.getNode()->getNodeId() > X.getNode()->getNodeId()) { 1069 CurDAG->RepositionNode(X.getNode(), NewANDMask.getNode()); 1070 NewANDMask.getNode()->setNodeId(X.getNode()->getNodeId()); 1071 } 1072 if (NewAND.getNode()->getNodeId() == -1 || 1073 NewAND.getNode()->getNodeId() > Shift.getNode()->getNodeId()) { 1074 CurDAG->RepositionNode(Shift.getNode(), NewAND.getNode()); 1075 NewAND.getNode()->setNodeId(Shift.getNode()->getNodeId()); 1076 } 1077 if (NewSHIFT.getNode()->getNodeId() == -1 || 1078 NewSHIFT.getNode()->getNodeId() > N.getNode()->getNodeId()) { 1079 CurDAG->RepositionNode(N.getNode(), NewSHIFT.getNode()); 1080 NewSHIFT.getNode()->setNodeId(N.getNode()->getNodeId()); 1081 } 1082 1083 CurDAG->ReplaceAllUsesWith(N, NewSHIFT); 1084 1085 AM.Scale = 1 << ShiftCst; 1086 AM.IndexReg = NewAND; 1087 return false; 1088 } 1089 } 1090 1091 return MatchAddressBase(N, AM); 1092} 1093 1094/// MatchAddressBase - Helper for MatchAddress. Add the specified node to the 1095/// specified addressing mode without any further recursion. 1096bool X86DAGToDAGISel::MatchAddressBase(SDValue N, X86ISelAddressMode &AM) { 1097 // Is the base register already occupied? 1098 if (AM.BaseType != X86ISelAddressMode::RegBase || AM.Base_Reg.getNode()) { 1099 // If so, check to see if the scale index register is set. 1100 if (AM.IndexReg.getNode() == 0) { 1101 AM.IndexReg = N; 1102 AM.Scale = 1; 1103 return false; 1104 } 1105 1106 // Otherwise, we cannot select it. 1107 return true; 1108 } 1109 1110 // Default, generate it as a register. 1111 AM.BaseType = X86ISelAddressMode::RegBase; 1112 AM.Base_Reg = N; 1113 return false; 1114} 1115 1116/// SelectAddr - returns true if it is able pattern match an addressing mode. 1117/// It returns the operands which make up the maximal addressing mode it can 1118/// match by reference. 1119/// 1120/// Parent is the parent node of the addr operand that is being matched. It 1121/// is always a load, store, atomic node, or null. It is only null when 1122/// checking memory operands for inline asm nodes. 1123bool X86DAGToDAGISel::SelectAddr(SDNode *Parent, SDValue N, SDValue &Base, 1124 SDValue &Scale, SDValue &Index, 1125 SDValue &Disp, SDValue &Segment) { 1126 X86ISelAddressMode AM; 1127 1128 if (Parent && 1129 // This list of opcodes are all the nodes that have an "addr:$ptr" operand 1130 // that are not a MemSDNode, and thus don't have proper addrspace info. 1131 Parent->getOpcode() != ISD::INTRINSIC_W_CHAIN && // unaligned loads, fixme 1132 Parent->getOpcode() != ISD::INTRINSIC_VOID && // nontemporal stores 1133 Parent->getOpcode() != X86ISD::TLSCALL) { // Fixme 1134 unsigned AddrSpace = 1135 cast<MemSDNode>(Parent)->getPointerInfo().getAddrSpace(); 1136 // AddrSpace 256 -> GS, 257 -> FS. 1137 if (AddrSpace == 256) 1138 AM.Segment = CurDAG->getRegister(X86::GS, MVT::i16); 1139 if (AddrSpace == 257) 1140 AM.Segment = CurDAG->getRegister(X86::FS, MVT::i16); 1141 } 1142 1143 if (MatchAddress(N, AM)) 1144 return false; 1145 1146 EVT VT = N.getValueType(); 1147 if (AM.BaseType == X86ISelAddressMode::RegBase) { 1148 if (!AM.Base_Reg.getNode()) 1149 AM.Base_Reg = CurDAG->getRegister(0, VT); 1150 } 1151 1152 if (!AM.IndexReg.getNode()) 1153 AM.IndexReg = CurDAG->getRegister(0, VT); 1154 1155 getAddressOperands(AM, Base, Scale, Index, Disp, Segment); 1156 return true; 1157} 1158 1159/// SelectScalarSSELoad - Match a scalar SSE load. In particular, we want to 1160/// match a load whose top elements are either undef or zeros. The load flavor 1161/// is derived from the type of N, which is either v4f32 or v2f64. 1162/// 1163/// We also return: 1164/// PatternChainNode: this is the matched node that has a chain input and 1165/// output. 1166bool X86DAGToDAGISel::SelectScalarSSELoad(SDNode *Root, 1167 SDValue N, SDValue &Base, 1168 SDValue &Scale, SDValue &Index, 1169 SDValue &Disp, SDValue &Segment, 1170 SDValue &PatternNodeWithChain) { 1171 if (N.getOpcode() == ISD::SCALAR_TO_VECTOR) { 1172 PatternNodeWithChain = N.getOperand(0); 1173 if (ISD::isNON_EXTLoad(PatternNodeWithChain.getNode()) && 1174 PatternNodeWithChain.hasOneUse() && 1175 IsProfitableToFold(N.getOperand(0), N.getNode(), Root) && 1176 IsLegalToFold(N.getOperand(0), N.getNode(), Root, OptLevel)) { 1177 LoadSDNode *LD = cast<LoadSDNode>(PatternNodeWithChain); 1178 if (!SelectAddr(LD, LD->getBasePtr(), Base, Scale, Index, Disp, Segment)) 1179 return false; 1180 return true; 1181 } 1182 } 1183 1184 // Also handle the case where we explicitly require zeros in the top 1185 // elements. This is a vector shuffle from the zero vector. 1186 if (N.getOpcode() == X86ISD::VZEXT_MOVL && N.getNode()->hasOneUse() && 1187 // Check to see if the top elements are all zeros (or bitcast of zeros). 1188 N.getOperand(0).getOpcode() == ISD::SCALAR_TO_VECTOR && 1189 N.getOperand(0).getNode()->hasOneUse() && 1190 ISD::isNON_EXTLoad(N.getOperand(0).getOperand(0).getNode()) && 1191 N.getOperand(0).getOperand(0).hasOneUse() && 1192 IsProfitableToFold(N.getOperand(0), N.getNode(), Root) && 1193 IsLegalToFold(N.getOperand(0), N.getNode(), Root, OptLevel)) { 1194 // Okay, this is a zero extending load. Fold it. 1195 LoadSDNode *LD = cast<LoadSDNode>(N.getOperand(0).getOperand(0)); 1196 if (!SelectAddr(LD, LD->getBasePtr(), Base, Scale, Index, Disp, Segment)) 1197 return false; 1198 PatternNodeWithChain = SDValue(LD, 0); 1199 return true; 1200 } 1201 return false; 1202} 1203 1204 1205/// SelectLEAAddr - it calls SelectAddr and determines if the maximal addressing 1206/// mode it matches can be cost effectively emitted as an LEA instruction. 1207bool X86DAGToDAGISel::SelectLEAAddr(SDValue N, 1208 SDValue &Base, SDValue &Scale, 1209 SDValue &Index, SDValue &Disp, 1210 SDValue &Segment) { 1211 X86ISelAddressMode AM; 1212 1213 // Set AM.Segment to prevent MatchAddress from using one. LEA doesn't support 1214 // segments. 1215 SDValue Copy = AM.Segment; 1216 SDValue T = CurDAG->getRegister(0, MVT::i32); 1217 AM.Segment = T; 1218 if (MatchAddress(N, AM)) 1219 return false; 1220 assert (T == AM.Segment); 1221 AM.Segment = Copy; 1222 1223 EVT VT = N.getValueType(); 1224 unsigned Complexity = 0; 1225 if (AM.BaseType == X86ISelAddressMode::RegBase) 1226 if (AM.Base_Reg.getNode()) 1227 Complexity = 1; 1228 else 1229 AM.Base_Reg = CurDAG->getRegister(0, VT); 1230 else if (AM.BaseType == X86ISelAddressMode::FrameIndexBase) 1231 Complexity = 4; 1232 1233 if (AM.IndexReg.getNode()) 1234 Complexity++; 1235 else 1236 AM.IndexReg = CurDAG->getRegister(0, VT); 1237 1238 // Don't match just leal(,%reg,2). It's cheaper to do addl %reg, %reg, or with 1239 // a simple shift. 1240 if (AM.Scale > 1) 1241 Complexity++; 1242 1243 // FIXME: We are artificially lowering the criteria to turn ADD %reg, $GA 1244 // to a LEA. This is determined with some expermentation but is by no means 1245 // optimal (especially for code size consideration). LEA is nice because of 1246 // its three-address nature. Tweak the cost function again when we can run 1247 // convertToThreeAddress() at register allocation time. 1248 if (AM.hasSymbolicDisplacement()) { 1249 // For X86-64, we should always use lea to materialize RIP relative 1250 // addresses. 1251 if (Subtarget->is64Bit()) 1252 Complexity = 4; 1253 else 1254 Complexity += 2; 1255 } 1256 1257 if (AM.Disp && (AM.Base_Reg.getNode() || AM.IndexReg.getNode())) 1258 Complexity++; 1259 1260 // If it isn't worth using an LEA, reject it. 1261 if (Complexity <= 2) 1262 return false; 1263 1264 getAddressOperands(AM, Base, Scale, Index, Disp, Segment); 1265 return true; 1266} 1267 1268/// SelectTLSADDRAddr - This is only run on TargetGlobalTLSAddress nodes. 1269bool X86DAGToDAGISel::SelectTLSADDRAddr(SDValue N, SDValue &Base, 1270 SDValue &Scale, SDValue &Index, 1271 SDValue &Disp, SDValue &Segment) { 1272 assert(N.getOpcode() == ISD::TargetGlobalTLSAddress); 1273 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N); 1274 1275 X86ISelAddressMode AM; 1276 AM.GV = GA->getGlobal(); 1277 AM.Disp += GA->getOffset(); 1278 AM.Base_Reg = CurDAG->getRegister(0, N.getValueType()); 1279 AM.SymbolFlags = GA->getTargetFlags(); 1280 1281 if (N.getValueType() == MVT::i32) { 1282 AM.Scale = 1; 1283 AM.IndexReg = CurDAG->getRegister(X86::EBX, MVT::i32); 1284 } else { 1285 AM.IndexReg = CurDAG->getRegister(0, MVT::i64); 1286 } 1287 1288 getAddressOperands(AM, Base, Scale, Index, Disp, Segment); 1289 return true; 1290} 1291 1292 1293bool X86DAGToDAGISel::TryFoldLoad(SDNode *P, SDValue N, 1294 SDValue &Base, SDValue &Scale, 1295 SDValue &Index, SDValue &Disp, 1296 SDValue &Segment) { 1297 if (!ISD::isNON_EXTLoad(N.getNode()) || 1298 !IsProfitableToFold(N, P, P) || 1299 !IsLegalToFold(N, P, P, OptLevel)) 1300 return false; 1301 1302 return SelectAddr(N.getNode(), 1303 N.getOperand(1), Base, Scale, Index, Disp, Segment); 1304} 1305 1306/// getGlobalBaseReg - Return an SDNode that returns the value of 1307/// the global base register. Output instructions required to 1308/// initialize the global base register, if necessary. 1309/// 1310SDNode *X86DAGToDAGISel::getGlobalBaseReg() { 1311 unsigned GlobalBaseReg = getInstrInfo()->getGlobalBaseReg(MF); 1312 return CurDAG->getRegister(GlobalBaseReg, TLI.getPointerTy()).getNode(); 1313} 1314 1315SDNode *X86DAGToDAGISel::SelectAtomic64(SDNode *Node, unsigned Opc) { 1316 SDValue Chain = Node->getOperand(0); 1317 SDValue In1 = Node->getOperand(1); 1318 SDValue In2L = Node->getOperand(2); 1319 SDValue In2H = Node->getOperand(3); 1320 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 1321 if (!SelectAddr(Node, In1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) 1322 return NULL; 1323 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 1324 MemOp[0] = cast<MemSDNode>(Node)->getMemOperand(); 1325 const SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, In2L, In2H, Chain}; 1326 SDNode *ResNode = CurDAG->getMachineNode(Opc, Node->getDebugLoc(), 1327 MVT::i32, MVT::i32, MVT::Other, Ops, 1328 array_lengthof(Ops)); 1329 cast<MachineSDNode>(ResNode)->setMemRefs(MemOp, MemOp + 1); 1330 return ResNode; 1331} 1332 1333// FIXME: Figure out some way to unify this with the 'or' and other code 1334// below. 1335SDNode *X86DAGToDAGISel::SelectAtomicLoadAdd(SDNode *Node, EVT NVT) { 1336 if (Node->hasAnyUseOfValue(0)) 1337 return 0; 1338 1339 // Optimize common patterns for __sync_add_and_fetch and 1340 // __sync_sub_and_fetch where the result is not used. This allows us 1341 // to use "lock" version of add, sub, inc, dec instructions. 1342 // FIXME: Do not use special instructions but instead add the "lock" 1343 // prefix to the target node somehow. The extra information will then be 1344 // transferred to machine instruction and it denotes the prefix. 1345 SDValue Chain = Node->getOperand(0); 1346 SDValue Ptr = Node->getOperand(1); 1347 SDValue Val = Node->getOperand(2); 1348 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 1349 if (!SelectAddr(Node, Ptr, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) 1350 return 0; 1351 1352 bool isInc = false, isDec = false, isSub = false, isCN = false; 1353 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val); 1354 if (CN) { 1355 isCN = true; 1356 int64_t CNVal = CN->getSExtValue(); 1357 if (CNVal == 1) 1358 isInc = true; 1359 else if (CNVal == -1) 1360 isDec = true; 1361 else if (CNVal >= 0) 1362 Val = CurDAG->getTargetConstant(CNVal, NVT); 1363 else { 1364 isSub = true; 1365 Val = CurDAG->getTargetConstant(-CNVal, NVT); 1366 } 1367 } else if (Val.hasOneUse() && 1368 Val.getOpcode() == ISD::SUB && 1369 X86::isZeroNode(Val.getOperand(0))) { 1370 isSub = true; 1371 Val = Val.getOperand(1); 1372 } 1373 1374 unsigned Opc = 0; 1375 switch (NVT.getSimpleVT().SimpleTy) { 1376 default: return 0; 1377 case MVT::i8: 1378 if (isInc) 1379 Opc = X86::LOCK_INC8m; 1380 else if (isDec) 1381 Opc = X86::LOCK_DEC8m; 1382 else if (isSub) { 1383 if (isCN) 1384 Opc = X86::LOCK_SUB8mi; 1385 else 1386 Opc = X86::LOCK_SUB8mr; 1387 } else { 1388 if (isCN) 1389 Opc = X86::LOCK_ADD8mi; 1390 else 1391 Opc = X86::LOCK_ADD8mr; 1392 } 1393 break; 1394 case MVT::i16: 1395 if (isInc) 1396 Opc = X86::LOCK_INC16m; 1397 else if (isDec) 1398 Opc = X86::LOCK_DEC16m; 1399 else if (isSub) { 1400 if (isCN) { 1401 if (immSext8(Val.getNode())) 1402 Opc = X86::LOCK_SUB16mi8; 1403 else 1404 Opc = X86::LOCK_SUB16mi; 1405 } else 1406 Opc = X86::LOCK_SUB16mr; 1407 } else { 1408 if (isCN) { 1409 if (immSext8(Val.getNode())) 1410 Opc = X86::LOCK_ADD16mi8; 1411 else 1412 Opc = X86::LOCK_ADD16mi; 1413 } else 1414 Opc = X86::LOCK_ADD16mr; 1415 } 1416 break; 1417 case MVT::i32: 1418 if (isInc) 1419 Opc = X86::LOCK_INC32m; 1420 else if (isDec) 1421 Opc = X86::LOCK_DEC32m; 1422 else if (isSub) { 1423 if (isCN) { 1424 if (immSext8(Val.getNode())) 1425 Opc = X86::LOCK_SUB32mi8; 1426 else 1427 Opc = X86::LOCK_SUB32mi; 1428 } else 1429 Opc = X86::LOCK_SUB32mr; 1430 } else { 1431 if (isCN) { 1432 if (immSext8(Val.getNode())) 1433 Opc = X86::LOCK_ADD32mi8; 1434 else 1435 Opc = X86::LOCK_ADD32mi; 1436 } else 1437 Opc = X86::LOCK_ADD32mr; 1438 } 1439 break; 1440 case MVT::i64: 1441 if (isInc) 1442 Opc = X86::LOCK_INC64m; 1443 else if (isDec) 1444 Opc = X86::LOCK_DEC64m; 1445 else if (isSub) { 1446 Opc = X86::LOCK_SUB64mr; 1447 if (isCN) { 1448 if (immSext8(Val.getNode())) 1449 Opc = X86::LOCK_SUB64mi8; 1450 else if (i64immSExt32(Val.getNode())) 1451 Opc = X86::LOCK_SUB64mi32; 1452 } 1453 } else { 1454 Opc = X86::LOCK_ADD64mr; 1455 if (isCN) { 1456 if (immSext8(Val.getNode())) 1457 Opc = X86::LOCK_ADD64mi8; 1458 else if (i64immSExt32(Val.getNode())) 1459 Opc = X86::LOCK_ADD64mi32; 1460 } 1461 } 1462 break; 1463 } 1464 1465 DebugLoc dl = Node->getDebugLoc(); 1466 SDValue Undef = SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, 1467 dl, NVT), 0); 1468 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 1469 MemOp[0] = cast<MemSDNode>(Node)->getMemOperand(); 1470 if (isInc || isDec) { 1471 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain }; 1472 SDValue Ret = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 6), 0); 1473 cast<MachineSDNode>(Ret)->setMemRefs(MemOp, MemOp + 1); 1474 SDValue RetVals[] = { Undef, Ret }; 1475 return CurDAG->getMergeValues(RetVals, 2, dl).getNode(); 1476 } else { 1477 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Val, Chain }; 1478 SDValue Ret = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 7), 0); 1479 cast<MachineSDNode>(Ret)->setMemRefs(MemOp, MemOp + 1); 1480 SDValue RetVals[] = { Undef, Ret }; 1481 return CurDAG->getMergeValues(RetVals, 2, dl).getNode(); 1482 } 1483} 1484 1485SDNode *X86DAGToDAGISel::SelectAtomicLoadOr(SDNode *Node, EVT NVT) { 1486 if (Node->hasAnyUseOfValue(0)) 1487 return 0; 1488 1489 // Optimize common patterns for __sync_or_and_fetch where the result 1490 // is not used. This allows us to use the "lock" version of the or 1491 // instruction. 1492 // FIXME: Same as for 'add' and 'sub'. 1493 SDValue Chain = Node->getOperand(0); 1494 SDValue Ptr = Node->getOperand(1); 1495 SDValue Val = Node->getOperand(2); 1496 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 1497 if (!SelectAddr(Node, Ptr, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) 1498 return 0; 1499 1500 bool isCN = false; 1501 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val); 1502 if (CN) { 1503 isCN = true; 1504 Val = CurDAG->getTargetConstant(CN->getSExtValue(), NVT); 1505 } 1506 1507 unsigned Opc = 0; 1508 switch (NVT.getSimpleVT().SimpleTy) { 1509 default: return 0; 1510 case MVT::i8: 1511 if (isCN) 1512 Opc = X86::LOCK_OR8mi; 1513 else 1514 Opc = X86::LOCK_OR8mr; 1515 break; 1516 case MVT::i16: 1517 if (isCN) { 1518 if (immSext8(Val.getNode())) 1519 Opc = X86::LOCK_OR16mi8; 1520 else 1521 Opc = X86::LOCK_OR16mi; 1522 } else 1523 Opc = X86::LOCK_OR16mr; 1524 break; 1525 case MVT::i32: 1526 if (isCN) { 1527 if (immSext8(Val.getNode())) 1528 Opc = X86::LOCK_OR32mi8; 1529 else 1530 Opc = X86::LOCK_OR32mi; 1531 } else 1532 Opc = X86::LOCK_OR32mr; 1533 break; 1534 case MVT::i64: 1535 if (isCN) { 1536 if (immSext8(Val.getNode())) 1537 Opc = X86::LOCK_OR64mi8; 1538 else if (i64immSExt32(Val.getNode())) 1539 Opc = X86::LOCK_OR64mi32; 1540 } else 1541 Opc = X86::LOCK_OR64mr; 1542 break; 1543 } 1544 1545 DebugLoc dl = Node->getDebugLoc(); 1546 SDValue Undef = SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, 1547 dl, NVT), 0); 1548 MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); 1549 MemOp[0] = cast<MemSDNode>(Node)->getMemOperand(); 1550 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Val, Chain }; 1551 SDValue Ret = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 7), 0); 1552 cast<MachineSDNode>(Ret)->setMemRefs(MemOp, MemOp + 1); 1553 SDValue RetVals[] = { Undef, Ret }; 1554 return CurDAG->getMergeValues(RetVals, 2, dl).getNode(); 1555} 1556 1557/// HasNoSignedComparisonUses - Test whether the given X86ISD::CMP node has 1558/// any uses which require the SF or OF bits to be accurate. 1559static bool HasNoSignedComparisonUses(SDNode *N) { 1560 // Examine each user of the node. 1561 for (SDNode::use_iterator UI = N->use_begin(), 1562 UE = N->use_end(); UI != UE; ++UI) { 1563 // Only examine CopyToReg uses. 1564 if (UI->getOpcode() != ISD::CopyToReg) 1565 return false; 1566 // Only examine CopyToReg uses that copy to EFLAGS. 1567 if (cast<RegisterSDNode>(UI->getOperand(1))->getReg() != 1568 X86::EFLAGS) 1569 return false; 1570 // Examine each user of the CopyToReg use. 1571 for (SDNode::use_iterator FlagUI = UI->use_begin(), 1572 FlagUE = UI->use_end(); FlagUI != FlagUE; ++FlagUI) { 1573 // Only examine the Flag result. 1574 if (FlagUI.getUse().getResNo() != 1) continue; 1575 // Anything unusual: assume conservatively. 1576 if (!FlagUI->isMachineOpcode()) return false; 1577 // Examine the opcode of the user. 1578 switch (FlagUI->getMachineOpcode()) { 1579 // These comparisons don't treat the most significant bit specially. 1580 case X86::SETAr: case X86::SETAEr: case X86::SETBr: case X86::SETBEr: 1581 case X86::SETEr: case X86::SETNEr: case X86::SETPr: case X86::SETNPr: 1582 case X86::SETAm: case X86::SETAEm: case X86::SETBm: case X86::SETBEm: 1583 case X86::SETEm: case X86::SETNEm: case X86::SETPm: case X86::SETNPm: 1584 case X86::JA_4: case X86::JAE_4: case X86::JB_4: case X86::JBE_4: 1585 case X86::JE_4: case X86::JNE_4: case X86::JP_4: case X86::JNP_4: 1586 case X86::CMOVA16rr: case X86::CMOVA16rm: 1587 case X86::CMOVA32rr: case X86::CMOVA32rm: 1588 case X86::CMOVA64rr: case X86::CMOVA64rm: 1589 case X86::CMOVAE16rr: case X86::CMOVAE16rm: 1590 case X86::CMOVAE32rr: case X86::CMOVAE32rm: 1591 case X86::CMOVAE64rr: case X86::CMOVAE64rm: 1592 case X86::CMOVB16rr: case X86::CMOVB16rm: 1593 case X86::CMOVB32rr: case X86::CMOVB32rm: 1594 case X86::CMOVB64rr: case X86::CMOVB64rm: 1595 case X86::CMOVBE16rr: case X86::CMOVBE16rm: 1596 case X86::CMOVBE32rr: case X86::CMOVBE32rm: 1597 case X86::CMOVBE64rr: case X86::CMOVBE64rm: 1598 case X86::CMOVE16rr: case X86::CMOVE16rm: 1599 case X86::CMOVE32rr: case X86::CMOVE32rm: 1600 case X86::CMOVE64rr: case X86::CMOVE64rm: 1601 case X86::CMOVNE16rr: case X86::CMOVNE16rm: 1602 case X86::CMOVNE32rr: case X86::CMOVNE32rm: 1603 case X86::CMOVNE64rr: case X86::CMOVNE64rm: 1604 case X86::CMOVNP16rr: case X86::CMOVNP16rm: 1605 case X86::CMOVNP32rr: case X86::CMOVNP32rm: 1606 case X86::CMOVNP64rr: case X86::CMOVNP64rm: 1607 case X86::CMOVP16rr: case X86::CMOVP16rm: 1608 case X86::CMOVP32rr: case X86::CMOVP32rm: 1609 case X86::CMOVP64rr: case X86::CMOVP64rm: 1610 continue; 1611 // Anything else: assume conservatively. 1612 default: return false; 1613 } 1614 } 1615 } 1616 return true; 1617} 1618 1619SDNode *X86DAGToDAGISel::Select(SDNode *Node) { 1620 EVT NVT = Node->getValueType(0); 1621 unsigned Opc, MOpc; 1622 unsigned Opcode = Node->getOpcode(); 1623 DebugLoc dl = Node->getDebugLoc(); 1624 1625 DEBUG(dbgs() << "Selecting: "; Node->dump(CurDAG); dbgs() << '\n'); 1626 1627 if (Node->isMachineOpcode()) { 1628 DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << '\n'); 1629 return NULL; // Already selected. 1630 } 1631 1632 switch (Opcode) { 1633 default: break; 1634 case X86ISD::GlobalBaseReg: 1635 return getGlobalBaseReg(); 1636 1637 case X86ISD::ATOMOR64_DAG: 1638 return SelectAtomic64(Node, X86::ATOMOR6432); 1639 case X86ISD::ATOMXOR64_DAG: 1640 return SelectAtomic64(Node, X86::ATOMXOR6432); 1641 case X86ISD::ATOMADD64_DAG: 1642 return SelectAtomic64(Node, X86::ATOMADD6432); 1643 case X86ISD::ATOMSUB64_DAG: 1644 return SelectAtomic64(Node, X86::ATOMSUB6432); 1645 case X86ISD::ATOMNAND64_DAG: 1646 return SelectAtomic64(Node, X86::ATOMNAND6432); 1647 case X86ISD::ATOMAND64_DAG: 1648 return SelectAtomic64(Node, X86::ATOMAND6432); 1649 case X86ISD::ATOMSWAP64_DAG: 1650 return SelectAtomic64(Node, X86::ATOMSWAP6432); 1651 1652 case ISD::ATOMIC_LOAD_ADD: { 1653 SDNode *RetVal = SelectAtomicLoadAdd(Node, NVT); 1654 if (RetVal) 1655 return RetVal; 1656 break; 1657 } 1658 case ISD::ATOMIC_LOAD_OR: { 1659 SDNode *RetVal = SelectAtomicLoadOr(Node, NVT); 1660 if (RetVal) 1661 return RetVal; 1662 break; 1663 } 1664 case ISD::AND: 1665 case ISD::OR: 1666 case ISD::XOR: { 1667 // For operations of the form (x << C1) op C2, check if we can use a smaller 1668 // encoding for C2 by transforming it into (x op (C2>>C1)) << C1. 1669 SDValue N0 = Node->getOperand(0); 1670 SDValue N1 = Node->getOperand(1); 1671 1672 if (N0->getOpcode() != ISD::SHL || !N0->hasOneUse()) 1673 break; 1674 1675 // i8 is unshrinkable, i16 should be promoted to i32. 1676 if (NVT != MVT::i32 && NVT != MVT::i64) 1677 break; 1678 1679 ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(N1); 1680 ConstantSDNode *ShlCst = dyn_cast<ConstantSDNode>(N0->getOperand(1)); 1681 if (!Cst || !ShlCst) 1682 break; 1683 1684 int64_t Val = Cst->getSExtValue(); 1685 uint64_t ShlVal = ShlCst->getZExtValue(); 1686 1687 // Make sure that we don't change the operation by removing bits. 1688 // This only matters for OR and XOR, AND is unaffected. 1689 if (Opcode != ISD::AND && ((Val >> ShlVal) << ShlVal) != Val) 1690 break; 1691 1692 unsigned ShlOp, Op = 0; 1693 EVT CstVT = NVT; 1694 1695 // Check the minimum bitwidth for the new constant. 1696 // TODO: AND32ri is the same as AND64ri32 with zext imm. 1697 // TODO: MOV32ri+OR64r is cheaper than MOV64ri64+OR64rr 1698 // TODO: Using 16 and 8 bit operations is also possible for or32 & xor32. 1699 if (!isInt<8>(Val) && isInt<8>(Val >> ShlVal)) 1700 CstVT = MVT::i8; 1701 else if (!isInt<32>(Val) && isInt<32>(Val >> ShlVal)) 1702 CstVT = MVT::i32; 1703 1704 // Bail if there is no smaller encoding. 1705 if (NVT == CstVT) 1706 break; 1707 1708 switch (NVT.getSimpleVT().SimpleTy) { 1709 default: llvm_unreachable("Unsupported VT!"); 1710 case MVT::i32: 1711 assert(CstVT == MVT::i8); 1712 ShlOp = X86::SHL32ri; 1713 1714 switch (Opcode) { 1715 case ISD::AND: Op = X86::AND32ri8; break; 1716 case ISD::OR: Op = X86::OR32ri8; break; 1717 case ISD::XOR: Op = X86::XOR32ri8; break; 1718 } 1719 break; 1720 case MVT::i64: 1721 assert(CstVT == MVT::i8 || CstVT == MVT::i32); 1722 ShlOp = X86::SHL64ri; 1723 1724 switch (Opcode) { 1725 case ISD::AND: Op = CstVT==MVT::i8? X86::AND64ri8 : X86::AND64ri32; break; 1726 case ISD::OR: Op = CstVT==MVT::i8? X86::OR64ri8 : X86::OR64ri32; break; 1727 case ISD::XOR: Op = CstVT==MVT::i8? X86::XOR64ri8 : X86::XOR64ri32; break; 1728 } 1729 break; 1730 } 1731 1732 // Emit the smaller op and the shift. 1733 SDValue NewCst = CurDAG->getTargetConstant(Val >> ShlVal, CstVT); 1734 SDNode *New = CurDAG->getMachineNode(Op, dl, NVT, N0->getOperand(0),NewCst); 1735 return CurDAG->SelectNodeTo(Node, ShlOp, NVT, SDValue(New, 0), 1736 getI8Imm(ShlVal)); 1737 break; 1738 } 1739 case X86ISD::UMUL: { 1740 SDValue N0 = Node->getOperand(0); 1741 SDValue N1 = Node->getOperand(1); 1742 1743 unsigned LoReg; 1744 switch (NVT.getSimpleVT().SimpleTy) { 1745 default: llvm_unreachable("Unsupported VT!"); 1746 case MVT::i8: LoReg = X86::AL; Opc = X86::MUL8r; break; 1747 case MVT::i16: LoReg = X86::AX; Opc = X86::MUL16r; break; 1748 case MVT::i32: LoReg = X86::EAX; Opc = X86::MUL32r; break; 1749 case MVT::i64: LoReg = X86::RAX; Opc = X86::MUL64r; break; 1750 } 1751 1752 SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg, 1753 N0, SDValue()).getValue(1); 1754 1755 SDVTList VTs = CurDAG->getVTList(NVT, NVT, MVT::i32); 1756 SDValue Ops[] = {N1, InFlag}; 1757 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops, 2); 1758 1759 ReplaceUses(SDValue(Node, 0), SDValue(CNode, 0)); 1760 ReplaceUses(SDValue(Node, 1), SDValue(CNode, 1)); 1761 ReplaceUses(SDValue(Node, 2), SDValue(CNode, 2)); 1762 return NULL; 1763 } 1764 1765 case ISD::SMUL_LOHI: 1766 case ISD::UMUL_LOHI: { 1767 SDValue N0 = Node->getOperand(0); 1768 SDValue N1 = Node->getOperand(1); 1769 1770 bool isSigned = Opcode == ISD::SMUL_LOHI; 1771 if (!isSigned) { 1772 switch (NVT.getSimpleVT().SimpleTy) { 1773 default: llvm_unreachable("Unsupported VT!"); 1774 case MVT::i8: Opc = X86::MUL8r; MOpc = X86::MUL8m; break; 1775 case MVT::i16: Opc = X86::MUL16r; MOpc = X86::MUL16m; break; 1776 case MVT::i32: Opc = X86::MUL32r; MOpc = X86::MUL32m; break; 1777 case MVT::i64: Opc = X86::MUL64r; MOpc = X86::MUL64m; break; 1778 } 1779 } else { 1780 switch (NVT.getSimpleVT().SimpleTy) { 1781 default: llvm_unreachable("Unsupported VT!"); 1782 case MVT::i8: Opc = X86::IMUL8r; MOpc = X86::IMUL8m; break; 1783 case MVT::i16: Opc = X86::IMUL16r; MOpc = X86::IMUL16m; break; 1784 case MVT::i32: Opc = X86::IMUL32r; MOpc = X86::IMUL32m; break; 1785 case MVT::i64: Opc = X86::IMUL64r; MOpc = X86::IMUL64m; break; 1786 } 1787 } 1788 1789 unsigned LoReg, HiReg; 1790 switch (NVT.getSimpleVT().SimpleTy) { 1791 default: llvm_unreachable("Unsupported VT!"); 1792 case MVT::i8: LoReg = X86::AL; HiReg = X86::AH; break; 1793 case MVT::i16: LoReg = X86::AX; HiReg = X86::DX; break; 1794 case MVT::i32: LoReg = X86::EAX; HiReg = X86::EDX; break; 1795 case MVT::i64: LoReg = X86::RAX; HiReg = X86::RDX; break; 1796 } 1797 1798 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 1799 bool foldedLoad = TryFoldLoad(Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4); 1800 // Multiply is commmutative. 1801 if (!foldedLoad) { 1802 foldedLoad = TryFoldLoad(Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4); 1803 if (foldedLoad) 1804 std::swap(N0, N1); 1805 } 1806 1807 SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg, 1808 N0, SDValue()).getValue(1); 1809 1810 if (foldedLoad) { 1811 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0), 1812 InFlag }; 1813 SDNode *CNode = 1814 CurDAG->getMachineNode(MOpc, dl, MVT::Other, MVT::Glue, Ops, 1815 array_lengthof(Ops)); 1816 InFlag = SDValue(CNode, 1); 1817 1818 // Update the chain. 1819 ReplaceUses(N1.getValue(1), SDValue(CNode, 0)); 1820 } else { 1821 SDNode *CNode = CurDAG->getMachineNode(Opc, dl, MVT::Glue, N1, InFlag); 1822 InFlag = SDValue(CNode, 0); 1823 } 1824 1825 // Prevent use of AH in a REX instruction by referencing AX instead. 1826 if (HiReg == X86::AH && Subtarget->is64Bit() && 1827 !SDValue(Node, 1).use_empty()) { 1828 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 1829 X86::AX, MVT::i16, InFlag); 1830 InFlag = Result.getValue(2); 1831 // Get the low part if needed. Don't use getCopyFromReg for aliasing 1832 // registers. 1833 if (!SDValue(Node, 0).use_empty()) 1834 ReplaceUses(SDValue(Node, 1), 1835 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result)); 1836 1837 // Shift AX down 8 bits. 1838 Result = SDValue(CurDAG->getMachineNode(X86::SHR16ri, dl, MVT::i16, 1839 Result, 1840 CurDAG->getTargetConstant(8, MVT::i8)), 0); 1841 // Then truncate it down to i8. 1842 ReplaceUses(SDValue(Node, 1), 1843 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result)); 1844 } 1845 // Copy the low half of the result, if it is needed. 1846 if (!SDValue(Node, 0).use_empty()) { 1847 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 1848 LoReg, NVT, InFlag); 1849 InFlag = Result.getValue(2); 1850 ReplaceUses(SDValue(Node, 0), Result); 1851 DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); dbgs() << '\n'); 1852 } 1853 // Copy the high half of the result, if it is needed. 1854 if (!SDValue(Node, 1).use_empty()) { 1855 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 1856 HiReg, NVT, InFlag); 1857 InFlag = Result.getValue(2); 1858 ReplaceUses(SDValue(Node, 1), Result); 1859 DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); dbgs() << '\n'); 1860 } 1861 1862 return NULL; 1863 } 1864 1865 case ISD::SDIVREM: 1866 case ISD::UDIVREM: { 1867 SDValue N0 = Node->getOperand(0); 1868 SDValue N1 = Node->getOperand(1); 1869 1870 bool isSigned = Opcode == ISD::SDIVREM; 1871 if (!isSigned) { 1872 switch (NVT.getSimpleVT().SimpleTy) { 1873 default: llvm_unreachable("Unsupported VT!"); 1874 case MVT::i8: Opc = X86::DIV8r; MOpc = X86::DIV8m; break; 1875 case MVT::i16: Opc = X86::DIV16r; MOpc = X86::DIV16m; break; 1876 case MVT::i32: Opc = X86::DIV32r; MOpc = X86::DIV32m; break; 1877 case MVT::i64: Opc = X86::DIV64r; MOpc = X86::DIV64m; break; 1878 } 1879 } else { 1880 switch (NVT.getSimpleVT().SimpleTy) { 1881 default: llvm_unreachable("Unsupported VT!"); 1882 case MVT::i8: Opc = X86::IDIV8r; MOpc = X86::IDIV8m; break; 1883 case MVT::i16: Opc = X86::IDIV16r; MOpc = X86::IDIV16m; break; 1884 case MVT::i32: Opc = X86::IDIV32r; MOpc = X86::IDIV32m; break; 1885 case MVT::i64: Opc = X86::IDIV64r; MOpc = X86::IDIV64m; break; 1886 } 1887 } 1888 1889 unsigned LoReg, HiReg, ClrReg; 1890 unsigned ClrOpcode, SExtOpcode; 1891 switch (NVT.getSimpleVT().SimpleTy) { 1892 default: llvm_unreachable("Unsupported VT!"); 1893 case MVT::i8: 1894 LoReg = X86::AL; ClrReg = HiReg = X86::AH; 1895 ClrOpcode = 0; 1896 SExtOpcode = X86::CBW; 1897 break; 1898 case MVT::i16: 1899 LoReg = X86::AX; HiReg = X86::DX; 1900 ClrOpcode = X86::MOV16r0; ClrReg = X86::DX; 1901 SExtOpcode = X86::CWD; 1902 break; 1903 case MVT::i32: 1904 LoReg = X86::EAX; ClrReg = HiReg = X86::EDX; 1905 ClrOpcode = X86::MOV32r0; 1906 SExtOpcode = X86::CDQ; 1907 break; 1908 case MVT::i64: 1909 LoReg = X86::RAX; ClrReg = HiReg = X86::RDX; 1910 ClrOpcode = X86::MOV64r0; 1911 SExtOpcode = X86::CQO; 1912 break; 1913 } 1914 1915 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4; 1916 bool foldedLoad = TryFoldLoad(Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4); 1917 bool signBitIsZero = CurDAG->SignBitIsZero(N0); 1918 1919 SDValue InFlag; 1920 if (NVT == MVT::i8 && (!isSigned || signBitIsZero)) { 1921 // Special case for div8, just use a move with zero extension to AX to 1922 // clear the upper 8 bits (AH). 1923 SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Move, Chain; 1924 if (TryFoldLoad(Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) { 1925 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N0.getOperand(0) }; 1926 Move = 1927 SDValue(CurDAG->getMachineNode(X86::MOVZX16rm8, dl, MVT::i16, 1928 MVT::Other, Ops, 1929 array_lengthof(Ops)), 0); 1930 Chain = Move.getValue(1); 1931 ReplaceUses(N0.getValue(1), Chain); 1932 } else { 1933 Move = 1934 SDValue(CurDAG->getMachineNode(X86::MOVZX16rr8, dl, MVT::i16, N0),0); 1935 Chain = CurDAG->getEntryNode(); 1936 } 1937 Chain = CurDAG->getCopyToReg(Chain, dl, X86::AX, Move, SDValue()); 1938 InFlag = Chain.getValue(1); 1939 } else { 1940 InFlag = 1941 CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, 1942 LoReg, N0, SDValue()).getValue(1); 1943 if (isSigned && !signBitIsZero) { 1944 // Sign extend the low part into the high part. 1945 InFlag = 1946 SDValue(CurDAG->getMachineNode(SExtOpcode, dl, MVT::Glue, InFlag),0); 1947 } else { 1948 // Zero out the high part, effectively zero extending the input. 1949 SDValue ClrNode = 1950 SDValue(CurDAG->getMachineNode(ClrOpcode, dl, NVT), 0); 1951 InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, ClrReg, 1952 ClrNode, InFlag).getValue(1); 1953 } 1954 } 1955 1956 if (foldedLoad) { 1957 SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0), 1958 InFlag }; 1959 SDNode *CNode = 1960 CurDAG->getMachineNode(MOpc, dl, MVT::Other, MVT::Glue, Ops, 1961 array_lengthof(Ops)); 1962 InFlag = SDValue(CNode, 1); 1963 // Update the chain. 1964 ReplaceUses(N1.getValue(1), SDValue(CNode, 0)); 1965 } else { 1966 InFlag = 1967 SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Glue, N1, InFlag), 0); 1968 } 1969 1970 // Prevent use of AH in a REX instruction by referencing AX instead. 1971 // Shift it down 8 bits. 1972 if (HiReg == X86::AH && Subtarget->is64Bit() && 1973 !SDValue(Node, 1).use_empty()) { 1974 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 1975 X86::AX, MVT::i16, InFlag); 1976 InFlag = Result.getValue(2); 1977 1978 // If we also need AL (the quotient), get it by extracting a subreg from 1979 // Result. The fast register allocator does not like multiple CopyFromReg 1980 // nodes using aliasing registers. 1981 if (!SDValue(Node, 0).use_empty()) 1982 ReplaceUses(SDValue(Node, 0), 1983 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result)); 1984 1985 // Shift AX right by 8 bits instead of using AH. 1986 Result = SDValue(CurDAG->getMachineNode(X86::SHR16ri, dl, MVT::i16, 1987 Result, 1988 CurDAG->getTargetConstant(8, MVT::i8)), 1989 0); 1990 ReplaceUses(SDValue(Node, 1), 1991 CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, MVT::i8, Result)); 1992 } 1993 // Copy the division (low) result, if it is needed. 1994 if (!SDValue(Node, 0).use_empty()) { 1995 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 1996 LoReg, NVT, InFlag); 1997 InFlag = Result.getValue(2); 1998 ReplaceUses(SDValue(Node, 0), Result); 1999 DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); dbgs() << '\n'); 2000 } 2001 // Copy the remainder (high) result, if it is needed. 2002 if (!SDValue(Node, 1).use_empty()) { 2003 SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, 2004 HiReg, NVT, InFlag); 2005 InFlag = Result.getValue(2); 2006 ReplaceUses(SDValue(Node, 1), Result); 2007 DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); dbgs() << '\n'); 2008 } 2009 return NULL; 2010 } 2011 2012 case X86ISD::CMP: { 2013 SDValue N0 = Node->getOperand(0); 2014 SDValue N1 = Node->getOperand(1); 2015 2016 // Look for (X86cmp (and $op, $imm), 0) and see if we can convert it to 2017 // use a smaller encoding. 2018 if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && 2019 HasNoSignedComparisonUses(Node)) 2020 // Look past the truncate if CMP is the only use of it. 2021 N0 = N0.getOperand(0); 2022 if (N0.getNode()->getOpcode() == ISD::AND && N0.getNode()->hasOneUse() && 2023 N0.getValueType() != MVT::i8 && 2024 X86::isZeroNode(N1)) { 2025 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getNode()->getOperand(1)); 2026 if (!C) break; 2027 2028 // For example, convert "testl %eax, $8" to "testb %al, $8" 2029 if ((C->getZExtValue() & ~UINT64_C(0xff)) == 0 && 2030 (!(C->getZExtValue() & 0x80) || 2031 HasNoSignedComparisonUses(Node))) { 2032 SDValue Imm = CurDAG->getTargetConstant(C->getZExtValue(), MVT::i8); 2033 SDValue Reg = N0.getNode()->getOperand(0); 2034 2035 // On x86-32, only the ABCD registers have 8-bit subregisters. 2036 if (!Subtarget->is64Bit()) { 2037 TargetRegisterClass *TRC = 0; 2038 switch (N0.getValueType().getSimpleVT().SimpleTy) { 2039 case MVT::i32: TRC = &X86::GR32_ABCDRegClass; break; 2040 case MVT::i16: TRC = &X86::GR16_ABCDRegClass; break; 2041 default: llvm_unreachable("Unsupported TEST operand type!"); 2042 } 2043 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), MVT::i32); 2044 Reg = SDValue(CurDAG->getMachineNode(X86::COPY_TO_REGCLASS, dl, 2045 Reg.getValueType(), Reg, RC), 0); 2046 } 2047 2048 // Extract the l-register. 2049 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_8bit, dl, 2050 MVT::i8, Reg); 2051 2052 // Emit a testb. 2053 return CurDAG->getMachineNode(X86::TEST8ri, dl, MVT::i32, Subreg, Imm); 2054 } 2055 2056 // For example, "testl %eax, $2048" to "testb %ah, $8". 2057 if ((C->getZExtValue() & ~UINT64_C(0xff00)) == 0 && 2058 (!(C->getZExtValue() & 0x8000) || 2059 HasNoSignedComparisonUses(Node))) { 2060 // Shift the immediate right by 8 bits. 2061 SDValue ShiftedImm = CurDAG->getTargetConstant(C->getZExtValue() >> 8, 2062 MVT::i8); 2063 SDValue Reg = N0.getNode()->getOperand(0); 2064 2065 // Put the value in an ABCD register. 2066 TargetRegisterClass *TRC = 0; 2067 switch (N0.getValueType().getSimpleVT().SimpleTy) { 2068 case MVT::i64: TRC = &X86::GR64_ABCDRegClass; break; 2069 case MVT::i32: TRC = &X86::GR32_ABCDRegClass; break; 2070 case MVT::i16: TRC = &X86::GR16_ABCDRegClass; break; 2071 default: llvm_unreachable("Unsupported TEST operand type!"); 2072 } 2073 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), MVT::i32); 2074 Reg = SDValue(CurDAG->getMachineNode(X86::COPY_TO_REGCLASS, dl, 2075 Reg.getValueType(), Reg, RC), 0); 2076 2077 // Extract the h-register. 2078 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_8bit_hi, dl, 2079 MVT::i8, Reg); 2080 2081 // Emit a testb. No special NOREX tricks are needed since there's 2082 // only one GPR operand! 2083 return CurDAG->getMachineNode(X86::TEST8ri, dl, MVT::i32, 2084 Subreg, ShiftedImm); 2085 } 2086 2087 // For example, "testl %eax, $32776" to "testw %ax, $32776". 2088 if ((C->getZExtValue() & ~UINT64_C(0xffff)) == 0 && 2089 N0.getValueType() != MVT::i16 && 2090 (!(C->getZExtValue() & 0x8000) || 2091 HasNoSignedComparisonUses(Node))) { 2092 SDValue Imm = CurDAG->getTargetConstant(C->getZExtValue(), MVT::i16); 2093 SDValue Reg = N0.getNode()->getOperand(0); 2094 2095 // Extract the 16-bit subregister. 2096 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_16bit, dl, 2097 MVT::i16, Reg); 2098 2099 // Emit a testw. 2100 return CurDAG->getMachineNode(X86::TEST16ri, dl, MVT::i32, Subreg, Imm); 2101 } 2102 2103 // For example, "testq %rax, $268468232" to "testl %eax, $268468232". 2104 if ((C->getZExtValue() & ~UINT64_C(0xffffffff)) == 0 && 2105 N0.getValueType() == MVT::i64 && 2106 (!(C->getZExtValue() & 0x80000000) || 2107 HasNoSignedComparisonUses(Node))) { 2108 SDValue Imm = CurDAG->getTargetConstant(C->getZExtValue(), MVT::i32); 2109 SDValue Reg = N0.getNode()->getOperand(0); 2110 2111 // Extract the 32-bit subregister. 2112 SDValue Subreg = CurDAG->getTargetExtractSubreg(X86::sub_32bit, dl, 2113 MVT::i32, Reg); 2114 2115 // Emit a testl. 2116 return CurDAG->getMachineNode(X86::TEST32ri, dl, MVT::i32, Subreg, Imm); 2117 } 2118 } 2119 break; 2120 } 2121 } 2122 2123 SDNode *ResNode = SelectCode(Node); 2124 2125 DEBUG(dbgs() << "=> "; 2126 if (ResNode == NULL || ResNode == Node) 2127 Node->dump(CurDAG); 2128 else 2129 ResNode->dump(CurDAG); 2130 dbgs() << '\n'); 2131 2132 return ResNode; 2133} 2134 2135bool X86DAGToDAGISel:: 2136SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode, 2137 std::vector<SDValue> &OutOps) { 2138 SDValue Op0, Op1, Op2, Op3, Op4; 2139 switch (ConstraintCode) { 2140 case 'o': // offsetable ?? 2141 case 'v': // not offsetable ?? 2142 default: return true; 2143 case 'm': // memory 2144 if (!SelectAddr(0, Op, Op0, Op1, Op2, Op3, Op4)) 2145 return true; 2146 break; 2147 } 2148 2149 OutOps.push_back(Op0); 2150 OutOps.push_back(Op1); 2151 OutOps.push_back(Op2); 2152 OutOps.push_back(Op3); 2153 OutOps.push_back(Op4); 2154 return false; 2155} 2156 2157/// createX86ISelDag - This pass converts a legalized DAG into a 2158/// X86-specific DAG, ready for instruction scheduling. 2159/// 2160FunctionPass *llvm::createX86ISelDag(X86TargetMachine &TM, 2161 llvm::CodeGenOpt::Level OptLevel) { 2162 return new X86DAGToDAGISel(TM, OptLevel); 2163} 2164