X86ISelDAGToDAG.cpp revision ac0b6ae358944ae8b2b5a11dc08f52c3ed89f2da
1//===- X86ISelDAGToDAG.cpp - A DAG pattern matching inst selector for X86 -===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file was developed by the Evan Cheng and is distributed under 6// the University of Illinois Open Source License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file defines a DAG pattern matching instruction selector for X86, 11// converting from a legalized dag to a X86 dag. 12// 13//===----------------------------------------------------------------------===// 14 15#define DEBUG_TYPE "x86-isel" 16#include "X86.h" 17#include "X86InstrBuilder.h" 18#include "X86ISelLowering.h" 19#include "X86RegisterInfo.h" 20#include "X86Subtarget.h" 21#include "X86TargetMachine.h" 22#include "llvm/GlobalValue.h" 23#include "llvm/Instructions.h" 24#include "llvm/Intrinsics.h" 25#include "llvm/Support/CFG.h" 26#include "llvm/CodeGen/MachineConstantPool.h" 27#include "llvm/CodeGen/MachineFunction.h" 28#include "llvm/CodeGen/MachineFrameInfo.h" 29#include "llvm/CodeGen/MachineInstrBuilder.h" 30#include "llvm/CodeGen/SSARegMap.h" 31#include "llvm/CodeGen/SelectionDAGISel.h" 32#include "llvm/Target/TargetMachine.h" 33#include "llvm/Support/Compiler.h" 34#include "llvm/Support/Debug.h" 35#include "llvm/Support/MathExtras.h" 36#include "llvm/ADT/Statistic.h" 37#include <iostream> 38#include <queue> 39#include <set> 40using namespace llvm; 41 42//===----------------------------------------------------------------------===// 43// Pattern Matcher Implementation 44//===----------------------------------------------------------------------===// 45 46namespace { 47 /// X86ISelAddressMode - This corresponds to X86AddressMode, but uses 48 /// SDOperand's instead of register numbers for the leaves of the matched 49 /// tree. 50 struct X86ISelAddressMode { 51 enum { 52 RegBase, 53 FrameIndexBase 54 } BaseType; 55 56 struct { // This is really a union, discriminated by BaseType! 57 SDOperand Reg; 58 int FrameIndex; 59 } Base; 60 61 bool isRIPRel; // RIP relative? 62 unsigned Scale; 63 SDOperand IndexReg; 64 unsigned Disp; 65 GlobalValue *GV; 66 Constant *CP; 67 const char *ES; 68 int JT; 69 unsigned Align; // CP alignment. 70 71 X86ISelAddressMode() 72 : BaseType(RegBase), isRIPRel(false), Scale(1), IndexReg(), Disp(0), 73 GV(0), CP(0), ES(0), JT(-1), Align(0) { 74 } 75 }; 76} 77 78namespace { 79 Statistic 80 NumFPKill("x86-codegen", "Number of FP_REG_KILL instructions added"); 81 82 Statistic 83 NumLoadMoved("x86-codegen", "Number of loads moved below TokenFactor"); 84 85 //===--------------------------------------------------------------------===// 86 /// ISel - X86 specific code to select X86 machine instructions for 87 /// SelectionDAG operations. 88 /// 89 class VISIBILITY_HIDDEN X86DAGToDAGISel : public SelectionDAGISel { 90 /// ContainsFPCode - Every instruction we select that uses or defines a FP 91 /// register should set this to true. 92 bool ContainsFPCode; 93 94 /// FastISel - Enable fast(er) instruction selection. 95 /// 96 bool FastISel; 97 98 /// TM - Keep a reference to X86TargetMachine. 99 /// 100 X86TargetMachine &TM; 101 102 /// X86Lowering - This object fully describes how to lower LLVM code to an 103 /// X86-specific SelectionDAG. 104 X86TargetLowering X86Lowering; 105 106 /// Subtarget - Keep a pointer to the X86Subtarget around so that we can 107 /// make the right decision when generating code for different targets. 108 const X86Subtarget *Subtarget; 109 110 /// GlobalBaseReg - keeps track of the virtual register mapped onto global 111 /// base register. 112 unsigned GlobalBaseReg; 113 114 public: 115 X86DAGToDAGISel(X86TargetMachine &tm, bool fast) 116 : SelectionDAGISel(X86Lowering), 117 ContainsFPCode(false), FastISel(fast), TM(tm), 118 X86Lowering(*TM.getTargetLowering()), 119 Subtarget(&TM.getSubtarget<X86Subtarget>()) {} 120 121 virtual bool runOnFunction(Function &Fn) { 122 // Make sure we re-emit a set of the global base reg if necessary 123 GlobalBaseReg = 0; 124 return SelectionDAGISel::runOnFunction(Fn); 125 } 126 127 virtual const char *getPassName() const { 128 return "X86 DAG->DAG Instruction Selection"; 129 } 130 131 /// InstructionSelectBasicBlock - This callback is invoked by 132 /// SelectionDAGISel when it has created a SelectionDAG for us to codegen. 133 virtual void InstructionSelectBasicBlock(SelectionDAG &DAG); 134 135 virtual void EmitFunctionEntryCode(Function &Fn, MachineFunction &MF); 136 137 virtual bool CanBeFoldedBy(SDNode *N, SDNode *U, SDNode *Root); 138 139// Include the pieces autogenerated from the target description. 140#include "X86GenDAGISel.inc" 141 142 private: 143 SDNode *Select(SDOperand N); 144 145 bool MatchAddress(SDOperand N, X86ISelAddressMode &AM, bool isRoot = true); 146 bool SelectAddr(SDOperand Op, SDOperand N, SDOperand &Base, 147 SDOperand &Scale, SDOperand &Index, SDOperand &Disp); 148 bool SelectLEAAddr(SDOperand Op, SDOperand N, SDOperand &Base, 149 SDOperand &Scale, SDOperand &Index, SDOperand &Disp); 150 bool SelectScalarSSELoad(SDOperand Op, SDOperand Pred, 151 SDOperand N, SDOperand &Base, SDOperand &Scale, 152 SDOperand &Index, SDOperand &Disp, 153 SDOperand &InChain, SDOperand &OutChain); 154 bool TryFoldLoad(SDOperand P, SDOperand N, 155 SDOperand &Base, SDOperand &Scale, 156 SDOperand &Index, SDOperand &Disp); 157 void InstructionSelectPreprocess(SelectionDAG &DAG); 158 159 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for 160 /// inline asm expressions. 161 virtual bool SelectInlineAsmMemoryOperand(const SDOperand &Op, 162 char ConstraintCode, 163 std::vector<SDOperand> &OutOps, 164 SelectionDAG &DAG); 165 166 void EmitSpecialCodeForMain(MachineBasicBlock *BB, MachineFrameInfo *MFI); 167 168 inline void getAddressOperands(X86ISelAddressMode &AM, SDOperand &Base, 169 SDOperand &Scale, SDOperand &Index, 170 SDOperand &Disp) { 171 Base = (AM.BaseType == X86ISelAddressMode::FrameIndexBase) ? 172 CurDAG->getTargetFrameIndex(AM.Base.FrameIndex, TLI.getPointerTy()) : 173 AM.Base.Reg; 174 Scale = getI8Imm(AM.Scale); 175 Index = AM.IndexReg; 176 // These are 32-bit even in 64-bit mode since RIP relative offset 177 // is 32-bit. 178 if (AM.GV) 179 Disp = CurDAG->getTargetGlobalAddress(AM.GV, MVT::i32, AM.Disp); 180 else if (AM.CP) 181 Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i32, AM.Align, AM.Disp); 182 else if (AM.ES) 183 Disp = CurDAG->getTargetExternalSymbol(AM.ES, MVT::i32); 184 else if (AM.JT != -1) 185 Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i32); 186 else 187 Disp = getI32Imm(AM.Disp); 188 } 189 190 /// getI8Imm - Return a target constant with the specified value, of type 191 /// i8. 192 inline SDOperand getI8Imm(unsigned Imm) { 193 return CurDAG->getTargetConstant(Imm, MVT::i8); 194 } 195 196 /// getI16Imm - Return a target constant with the specified value, of type 197 /// i16. 198 inline SDOperand getI16Imm(unsigned Imm) { 199 return CurDAG->getTargetConstant(Imm, MVT::i16); 200 } 201 202 /// getI32Imm - Return a target constant with the specified value, of type 203 /// i32. 204 inline SDOperand getI32Imm(unsigned Imm) { 205 return CurDAG->getTargetConstant(Imm, MVT::i32); 206 } 207 208 /// getGlobalBaseReg - insert code into the entry mbb to materialize the PIC 209 /// base register. Return the virtual register that holds this value. 210 SDNode *getGlobalBaseReg(); 211 212#ifndef NDEBUG 213 unsigned Indent; 214#endif 215 }; 216} 217 218static SDNode *findFlagUse(SDNode *N) { 219 unsigned FlagResNo = N->getNumValues()-1; 220 for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) { 221 SDNode *User = *I; 222 for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) { 223 SDOperand Op = User->getOperand(i); 224 if (Op.Val == N && Op.ResNo == FlagResNo) 225 return User; 226 } 227 } 228 return NULL; 229} 230 231static void findNonImmUse(SDNode *Use, SDNode* Def, SDNode *ImmedUse, 232 SDNode *Root, SDNode *Skip, bool &found, 233 std::set<SDNode *> &Visited) { 234 if (found || 235 Use->getNodeId() > Def->getNodeId() || 236 !Visited.insert(Use).second) 237 return; 238 239 for (unsigned i = 0, e = Use->getNumOperands(); !found && i != e; ++i) { 240 SDNode *N = Use->getOperand(i).Val; 241 if (N == Skip) 242 continue; 243 if (N == Def) { 244 if (Use == ImmedUse) 245 continue; // Immediate use is ok. 246 if (Use == Root) { 247 assert(Use->getOpcode() == ISD::STORE || 248 Use->getOpcode() == X86ISD::CMP); 249 continue; 250 } 251 found = true; 252 break; 253 } 254 findNonImmUse(N, Def, ImmedUse, Root, Skip, found, Visited); 255 } 256} 257 258/// isNonImmUse - Start searching from Root up the DAG to check is Def can 259/// be reached. Return true if that's the case. However, ignore direct uses 260/// by ImmedUse (which would be U in the example illustrated in 261/// CanBeFoldedBy) and by Root (which can happen in the store case). 262/// FIXME: to be really generic, we should allow direct use by any node 263/// that is being folded. But realisticly since we only fold loads which 264/// have one non-chain use, we only need to watch out for load/op/store 265/// and load/op/cmp case where the root (store / cmp) may reach the load via 266/// its chain operand. 267static inline bool isNonImmUse(SDNode *Root, SDNode *Def, SDNode *ImmedUse, 268 SDNode *Skip = NULL) { 269 std::set<SDNode *> Visited; 270 bool found = false; 271 findNonImmUse(Root, Def, ImmedUse, Root, Skip, found, Visited); 272 return found; 273} 274 275 276bool X86DAGToDAGISel::CanBeFoldedBy(SDNode *N, SDNode *U, SDNode *Root) { 277 if (FastISel) return false; 278 279 // If U use can somehow reach N through another path then U can't fold N or 280 // it will create a cycle. e.g. In the following diagram, U can reach N 281 // through X. If N is folded into into U, then X is both a predecessor and 282 // a successor of U. 283 // 284 // [ N ] 285 // ^ ^ 286 // | | 287 // / \--- 288 // / [X] 289 // | ^ 290 // [U]--------| 291 292 if (isNonImmUse(Root, N, U)) 293 return false; 294 295 // If U produces a flag, then it gets (even more) interesting. Since it 296 // would have been "glued" together with its flag use, we need to check if 297 // it might reach N: 298 // 299 // [ N ] 300 // ^ ^ 301 // | | 302 // [U] \-- 303 // ^ [TF] 304 // | ^ 305 // | | 306 // \ / 307 // [FU] 308 // 309 // If FU (flag use) indirectly reach N (the load), and U fold N (call it 310 // NU), then TF is a predecessor of FU and a successor of NU. But since 311 // NU and FU are flagged together, this effectively creates a cycle. 312 bool HasFlagUse = false; 313 MVT::ValueType VT = Root->getValueType(Root->getNumValues()-1); 314 while ((VT == MVT::Flag && !Root->use_empty())) { 315 SDNode *FU = findFlagUse(Root); 316 if (FU == NULL) 317 break; 318 else { 319 Root = FU; 320 HasFlagUse = true; 321 } 322 VT = Root->getValueType(Root->getNumValues()-1); 323 } 324 325 if (HasFlagUse) 326 return !isNonImmUse(Root, N, Root, U); 327 return true; 328} 329 330/// MoveBelowTokenFactor - Replace TokenFactor operand with load's chain operand 331/// and move load below the TokenFactor. Replace store's chain operand with 332/// load's chain result. 333static void MoveBelowTokenFactor(SelectionDAG &DAG, SDOperand Load, 334 SDOperand Store, SDOperand TF) { 335 std::vector<SDOperand> Ops; 336 for (unsigned i = 0, e = TF.Val->getNumOperands(); i != e; ++i) 337 if (Load.Val == TF.Val->getOperand(i).Val) 338 Ops.push_back(Load.Val->getOperand(0)); 339 else 340 Ops.push_back(TF.Val->getOperand(i)); 341 DAG.UpdateNodeOperands(TF, &Ops[0], Ops.size()); 342 DAG.UpdateNodeOperands(Load, TF, Load.getOperand(1), Load.getOperand(2)); 343 DAG.UpdateNodeOperands(Store, Load.getValue(1), Store.getOperand(1), 344 Store.getOperand(2), Store.getOperand(3)); 345} 346 347/// InstructionSelectPreprocess - Preprocess the DAG to allow the instruction 348/// selector to pick more load-modify-store instructions. This is a common 349/// case: 350/// 351/// [Load chain] 352/// ^ 353/// | 354/// [Load] 355/// ^ ^ 356/// | | 357/// / \- 358/// / | 359/// [TokenFactor] [Op] 360/// ^ ^ 361/// | | 362/// \ / 363/// \ / 364/// [Store] 365/// 366/// The fact the store's chain operand != load's chain will prevent the 367/// (store (op (load))) instruction from being selected. We can transform it to: 368/// 369/// [Load chain] 370/// ^ 371/// | 372/// [TokenFactor] 373/// ^ 374/// | 375/// [Load] 376/// ^ ^ 377/// | | 378/// | \- 379/// | | 380/// | [Op] 381/// | ^ 382/// | | 383/// \ / 384/// \ / 385/// [Store] 386void X86DAGToDAGISel::InstructionSelectPreprocess(SelectionDAG &DAG) { 387 for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(), 388 E = DAG.allnodes_end(); I != E; ++I) { 389 if (!ISD::isNON_TRUNCStore(I)) 390 continue; 391 SDOperand Chain = I->getOperand(0); 392 if (Chain.Val->getOpcode() != ISD::TokenFactor) 393 continue; 394 395 SDOperand N1 = I->getOperand(1); 396 SDOperand N2 = I->getOperand(2); 397 if (MVT::isFloatingPoint(N1.getValueType()) || 398 MVT::isVector(N1.getValueType()) || 399 !N1.hasOneUse()) 400 continue; 401 402 bool RModW = false; 403 SDOperand Load; 404 unsigned Opcode = N1.Val->getOpcode(); 405 switch (Opcode) { 406 case ISD::ADD: 407 case ISD::MUL: 408 case ISD::AND: 409 case ISD::OR: 410 case ISD::XOR: 411 case ISD::ADDC: 412 case ISD::ADDE: { 413 SDOperand N10 = N1.getOperand(0); 414 SDOperand N11 = N1.getOperand(1); 415 if (ISD::isNON_EXTLoad(N10.Val)) 416 RModW = true; 417 else if (ISD::isNON_EXTLoad(N11.Val)) { 418 RModW = true; 419 std::swap(N10, N11); 420 } 421 RModW = RModW && N10.Val->isOperand(Chain.Val) && N10.hasOneUse() && 422 (N10.getOperand(1) == N2) && 423 (N10.Val->getValueType(0) == N1.getValueType()); 424 if (RModW) 425 Load = N10; 426 break; 427 } 428 case ISD::SUB: 429 case ISD::SHL: 430 case ISD::SRA: 431 case ISD::SRL: 432 case ISD::ROTL: 433 case ISD::ROTR: 434 case ISD::SUBC: 435 case ISD::SUBE: 436 case X86ISD::SHLD: 437 case X86ISD::SHRD: { 438 SDOperand N10 = N1.getOperand(0); 439 if (ISD::isNON_EXTLoad(N10.Val)) 440 RModW = N10.Val->isOperand(Chain.Val) && N10.hasOneUse() && 441 (N10.getOperand(1) == N2) && 442 (N10.Val->getValueType(0) == N1.getValueType()); 443 if (RModW) 444 Load = N10; 445 break; 446 } 447 } 448 449 if (RModW) { 450 MoveBelowTokenFactor(DAG, Load, SDOperand(I, 0), Chain); 451 ++NumLoadMoved; 452 } 453 } 454} 455 456/// InstructionSelectBasicBlock - This callback is invoked by SelectionDAGISel 457/// when it has created a SelectionDAG for us to codegen. 458void X86DAGToDAGISel::InstructionSelectBasicBlock(SelectionDAG &DAG) { 459 DEBUG(BB->dump()); 460 MachineFunction::iterator FirstMBB = BB; 461 462 if (!FastISel) 463 InstructionSelectPreprocess(DAG); 464 465 // Codegen the basic block. 466#ifndef NDEBUG 467 DOUT << "===== Instruction selection begins:\n"; 468 Indent = 0; 469#endif 470 DAG.setRoot(SelectRoot(DAG.getRoot())); 471#ifndef NDEBUG 472 DOUT << "===== Instruction selection ends:\n"; 473#endif 474 475 DAG.RemoveDeadNodes(); 476 477 // Emit machine code to BB. 478 ScheduleAndEmitDAG(DAG); 479 480 // If we are emitting FP stack code, scan the basic block to determine if this 481 // block defines any FP values. If so, put an FP_REG_KILL instruction before 482 // the terminator of the block. 483 if (!Subtarget->hasSSE2()) { 484 // Note that FP stack instructions *are* used in SSE code when returning 485 // values, but these are not live out of the basic block, so we don't need 486 // an FP_REG_KILL in this case either. 487 bool ContainsFPCode = false; 488 489 // Scan all of the machine instructions in these MBBs, checking for FP 490 // stores. 491 MachineFunction::iterator MBBI = FirstMBB; 492 do { 493 for (MachineBasicBlock::iterator I = MBBI->begin(), E = MBBI->end(); 494 !ContainsFPCode && I != E; ++I) { 495 for (unsigned op = 0, e = I->getNumOperands(); op != e; ++op) { 496 if (I->getOperand(op).isRegister() && I->getOperand(op).isDef() && 497 MRegisterInfo::isVirtualRegister(I->getOperand(op).getReg()) && 498 RegMap->getRegClass(I->getOperand(0).getReg()) == 499 X86::RFPRegisterClass) { 500 ContainsFPCode = true; 501 break; 502 } 503 } 504 } 505 } while (!ContainsFPCode && &*(MBBI++) != BB); 506 507 // Check PHI nodes in successor blocks. These PHI's will be lowered to have 508 // a copy of the input value in this block. 509 if (!ContainsFPCode) { 510 // Final check, check LLVM BB's that are successors to the LLVM BB 511 // corresponding to BB for FP PHI nodes. 512 const BasicBlock *LLVMBB = BB->getBasicBlock(); 513 const PHINode *PN; 514 for (succ_const_iterator SI = succ_begin(LLVMBB), E = succ_end(LLVMBB); 515 !ContainsFPCode && SI != E; ++SI) { 516 for (BasicBlock::const_iterator II = SI->begin(); 517 (PN = dyn_cast<PHINode>(II)); ++II) { 518 if (PN->getType()->isFloatingPoint()) { 519 ContainsFPCode = true; 520 break; 521 } 522 } 523 } 524 } 525 526 // Finally, if we found any FP code, emit the FP_REG_KILL instruction. 527 if (ContainsFPCode) { 528 BuildMI(*BB, BB->getFirstTerminator(), 529 TM.getInstrInfo()->get(X86::FP_REG_KILL)); 530 ++NumFPKill; 531 } 532 } 533} 534 535/// EmitSpecialCodeForMain - Emit any code that needs to be executed only in 536/// the main function. 537void X86DAGToDAGISel::EmitSpecialCodeForMain(MachineBasicBlock *BB, 538 MachineFrameInfo *MFI) { 539 const TargetInstrInfo *TII = TM.getInstrInfo(); 540 if (Subtarget->isTargetCygwin()) 541 BuildMI(BB, TII->get(X86::CALLpcrel32)).addExternalSymbol("__main"); 542 543 // Switch the FPU to 64-bit precision mode for better compatibility and speed. 544 int CWFrameIdx = MFI->CreateStackObject(2, 2); 545 addFrameReference(BuildMI(BB, TII->get(X86::FNSTCW16m)), CWFrameIdx); 546 547 // Set the high part to be 64-bit precision. 548 addFrameReference(BuildMI(BB, TII->get(X86::MOV8mi)), 549 CWFrameIdx, 1).addImm(2); 550 551 // Reload the modified control word now. 552 addFrameReference(BuildMI(BB, TII->get(X86::FLDCW16m)), CWFrameIdx); 553} 554 555void X86DAGToDAGISel::EmitFunctionEntryCode(Function &Fn, MachineFunction &MF) { 556 // If this is main, emit special code for main. 557 MachineBasicBlock *BB = MF.begin(); 558 if (Fn.hasExternalLinkage() && Fn.getName() == "main") 559 EmitSpecialCodeForMain(BB, MF.getFrameInfo()); 560} 561 562/// MatchAddress - Add the specified node to the specified addressing mode, 563/// returning true if it cannot be done. This just pattern matches for the 564/// addressing mode 565bool X86DAGToDAGISel::MatchAddress(SDOperand N, X86ISelAddressMode &AM, 566 bool isRoot) { 567 // RIP relative addressing: %rip + 32-bit displacement! 568 if (AM.isRIPRel) { 569 if (!AM.ES && AM.JT != -1 && N.getOpcode() == ISD::Constant) { 570 int64_t Val = cast<ConstantSDNode>(N)->getSignExtended(); 571 if (isInt32(AM.Disp + Val)) { 572 AM.Disp += Val; 573 return false; 574 } 575 } 576 return true; 577 } 578 579 int id = N.Val->getNodeId(); 580 bool Available = isSelected(id); 581 582 switch (N.getOpcode()) { 583 default: break; 584 case ISD::Constant: { 585 int64_t Val = cast<ConstantSDNode>(N)->getSignExtended(); 586 if (isInt32(AM.Disp + Val)) { 587 AM.Disp += Val; 588 return false; 589 } 590 break; 591 } 592 593 case X86ISD::Wrapper: { 594 bool is64Bit = Subtarget->is64Bit(); 595 // Under X86-64 non-small code model, GV (and friends) are 64-bits. 596 if (is64Bit && TM.getCodeModel() != CodeModel::Small) 597 break; 598 if (AM.GV != 0 || AM.CP != 0 || AM.ES != 0 || AM.JT != -1) 599 break; 600 // If value is available in a register both base and index components have 601 // been picked, we can't fit the result available in the register in the 602 // addressing mode. Duplicate GlobalAddress or ConstantPool as displacement. 603 if (!Available || (AM.Base.Reg.Val && AM.IndexReg.Val)) { 604 bool isStatic = TM.getRelocationModel() == Reloc::Static; 605 SDOperand N0 = N.getOperand(0); 606 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(N0)) { 607 GlobalValue *GV = G->getGlobal(); 608 bool isAbs32 = !is64Bit || 609 (isStatic && !(GV->isExternal() || GV->hasWeakLinkage() || 610 GV->hasLinkOnceLinkage())); 611 if (isAbs32 || isRoot) { 612 AM.GV = G->getGlobal(); 613 AM.Disp += G->getOffset(); 614 AM.isRIPRel = !isAbs32; 615 return false; 616 } 617 } else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(N0)) { 618 if (!is64Bit || isStatic || isRoot) { 619 AM.CP = CP->getConstVal(); 620 AM.Align = CP->getAlignment(); 621 AM.Disp += CP->getOffset(); 622 AM.isRIPRel = !isStatic; 623 return false; 624 } 625 } else if (ExternalSymbolSDNode *S =dyn_cast<ExternalSymbolSDNode>(N0)) { 626 if (isStatic || isRoot) { 627 AM.ES = S->getSymbol(); 628 AM.isRIPRel = !isStatic; 629 return false; 630 } 631 } else if (JumpTableSDNode *J = dyn_cast<JumpTableSDNode>(N0)) { 632 if (isStatic || isRoot) { 633 AM.JT = J->getIndex(); 634 AM.isRIPRel = !isStatic; 635 return false; 636 } 637 } 638 } 639 break; 640 } 641 642 case ISD::FrameIndex: 643 if (AM.BaseType == X86ISelAddressMode::RegBase && AM.Base.Reg.Val == 0) { 644 AM.BaseType = X86ISelAddressMode::FrameIndexBase; 645 AM.Base.FrameIndex = cast<FrameIndexSDNode>(N)->getIndex(); 646 return false; 647 } 648 break; 649 650 case ISD::SHL: 651 if (!Available && AM.IndexReg.Val == 0 && AM.Scale == 1) 652 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N.Val->getOperand(1))) { 653 unsigned Val = CN->getValue(); 654 if (Val == 1 || Val == 2 || Val == 3) { 655 AM.Scale = 1 << Val; 656 SDOperand ShVal = N.Val->getOperand(0); 657 658 // Okay, we know that we have a scale by now. However, if the scaled 659 // value is an add of something and a constant, we can fold the 660 // constant into the disp field here. 661 if (ShVal.Val->getOpcode() == ISD::ADD && ShVal.hasOneUse() && 662 isa<ConstantSDNode>(ShVal.Val->getOperand(1))) { 663 AM.IndexReg = ShVal.Val->getOperand(0); 664 ConstantSDNode *AddVal = 665 cast<ConstantSDNode>(ShVal.Val->getOperand(1)); 666 uint64_t Disp = AM.Disp + (AddVal->getValue() << Val); 667 if (isInt32(Disp)) 668 AM.Disp = Disp; 669 else 670 AM.IndexReg = ShVal; 671 } else { 672 AM.IndexReg = ShVal; 673 } 674 return false; 675 } 676 } 677 break; 678 679 case ISD::MUL: 680 // X*[3,5,9] -> X+X*[2,4,8] 681 if (!Available && 682 AM.BaseType == X86ISelAddressMode::RegBase && 683 AM.Base.Reg.Val == 0 && 684 AM.IndexReg.Val == 0) 685 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N.Val->getOperand(1))) 686 if (CN->getValue() == 3 || CN->getValue() == 5 || CN->getValue() == 9) { 687 AM.Scale = unsigned(CN->getValue())-1; 688 689 SDOperand MulVal = N.Val->getOperand(0); 690 SDOperand Reg; 691 692 // Okay, we know that we have a scale by now. However, if the scaled 693 // value is an add of something and a constant, we can fold the 694 // constant into the disp field here. 695 if (MulVal.Val->getOpcode() == ISD::ADD && MulVal.hasOneUse() && 696 isa<ConstantSDNode>(MulVal.Val->getOperand(1))) { 697 Reg = MulVal.Val->getOperand(0); 698 ConstantSDNode *AddVal = 699 cast<ConstantSDNode>(MulVal.Val->getOperand(1)); 700 uint64_t Disp = AM.Disp + AddVal->getValue() * CN->getValue(); 701 if (isInt32(Disp)) 702 AM.Disp = Disp; 703 else 704 Reg = N.Val->getOperand(0); 705 } else { 706 Reg = N.Val->getOperand(0); 707 } 708 709 AM.IndexReg = AM.Base.Reg = Reg; 710 return false; 711 } 712 break; 713 714 case ISD::ADD: { 715 if (!Available) { 716 X86ISelAddressMode Backup = AM; 717 if (!MatchAddress(N.Val->getOperand(0), AM, false) && 718 !MatchAddress(N.Val->getOperand(1), AM, false)) 719 return false; 720 AM = Backup; 721 if (!MatchAddress(N.Val->getOperand(1), AM, false) && 722 !MatchAddress(N.Val->getOperand(0), AM, false)) 723 return false; 724 AM = Backup; 725 } 726 break; 727 } 728 729 case ISD::OR: { 730 if (!Available) { 731 X86ISelAddressMode Backup = AM; 732 // Look for (x << c1) | c2 where (c2 < c1) 733 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N.Val->getOperand(0)); 734 if (CN && !MatchAddress(N.Val->getOperand(1), AM, false)) { 735 if (AM.GV == NULL && AM.Disp == 0 && CN->getValue() < AM.Scale) { 736 AM.Disp = CN->getValue(); 737 return false; 738 } 739 } 740 AM = Backup; 741 CN = dyn_cast<ConstantSDNode>(N.Val->getOperand(1)); 742 if (CN && !MatchAddress(N.Val->getOperand(0), AM, false)) { 743 if (AM.GV == NULL && AM.Disp == 0 && CN->getValue() < AM.Scale) { 744 AM.Disp = CN->getValue(); 745 return false; 746 } 747 } 748 AM = Backup; 749 } 750 break; 751 } 752 } 753 754 // Is the base register already occupied? 755 if (AM.BaseType != X86ISelAddressMode::RegBase || AM.Base.Reg.Val) { 756 // If so, check to see if the scale index register is set. 757 if (AM.IndexReg.Val == 0) { 758 AM.IndexReg = N; 759 AM.Scale = 1; 760 return false; 761 } 762 763 // Otherwise, we cannot select it. 764 return true; 765 } 766 767 // Default, generate it as a register. 768 AM.BaseType = X86ISelAddressMode::RegBase; 769 AM.Base.Reg = N; 770 return false; 771} 772 773/// SelectAddr - returns true if it is able pattern match an addressing mode. 774/// It returns the operands which make up the maximal addressing mode it can 775/// match by reference. 776bool X86DAGToDAGISel::SelectAddr(SDOperand Op, SDOperand N, SDOperand &Base, 777 SDOperand &Scale, SDOperand &Index, 778 SDOperand &Disp) { 779 X86ISelAddressMode AM; 780 if (MatchAddress(N, AM)) 781 return false; 782 783 MVT::ValueType VT = N.getValueType(); 784 if (AM.BaseType == X86ISelAddressMode::RegBase) { 785 if (!AM.Base.Reg.Val) 786 AM.Base.Reg = CurDAG->getRegister(0, VT); 787 } 788 789 if (!AM.IndexReg.Val) 790 AM.IndexReg = CurDAG->getRegister(0, VT); 791 792 getAddressOperands(AM, Base, Scale, Index, Disp); 793 return true; 794} 795 796/// isZeroNode - Returns true if Elt is a constant zero or a floating point 797/// constant +0.0. 798static inline bool isZeroNode(SDOperand Elt) { 799 return ((isa<ConstantSDNode>(Elt) && 800 cast<ConstantSDNode>(Elt)->getValue() == 0) || 801 (isa<ConstantFPSDNode>(Elt) && 802 cast<ConstantFPSDNode>(Elt)->isExactlyValue(0.0))); 803} 804 805 806/// SelectScalarSSELoad - Match a scalar SSE load. In particular, we want to 807/// match a load whose top elements are either undef or zeros. The load flavor 808/// is derived from the type of N, which is either v4f32 or v2f64. 809bool X86DAGToDAGISel::SelectScalarSSELoad(SDOperand Op, SDOperand Pred, 810 SDOperand N, SDOperand &Base, 811 SDOperand &Scale, SDOperand &Index, 812 SDOperand &Disp, SDOperand &InChain, 813 SDOperand &OutChain) { 814 if (N.getOpcode() == ISD::SCALAR_TO_VECTOR) { 815 InChain = N.getOperand(0).getValue(1); 816 if (ISD::isNON_EXTLoad(InChain.Val) && 817 InChain.getValue(0).hasOneUse() && 818 N.hasOneUse() && 819 CanBeFoldedBy(N.Val, Pred.Val, Op.Val)) { 820 LoadSDNode *LD = cast<LoadSDNode>(InChain); 821 if (!SelectAddr(Op, LD->getBasePtr(), Base, Scale, Index, Disp)) 822 return false; 823 OutChain = LD->getChain(); 824 return true; 825 } 826 } 827 828 // Also handle the case where we explicitly require zeros in the top 829 // elements. This is a vector shuffle from the zero vector. 830 if (N.getOpcode() == ISD::VECTOR_SHUFFLE && N.Val->hasOneUse() && 831 N.getOperand(0).getOpcode() == ISD::BUILD_VECTOR && 832 N.getOperand(1).getOpcode() == ISD::SCALAR_TO_VECTOR && 833 N.getOperand(1).Val->hasOneUse() && 834 ISD::isNON_EXTLoad(N.getOperand(1).getOperand(0).Val) && 835 N.getOperand(1).getOperand(0).hasOneUse()) { 836 // Check to see if the BUILD_VECTOR is building a zero vector. 837 SDOperand BV = N.getOperand(0); 838 for (unsigned i = 0, e = BV.getNumOperands(); i != e; ++i) 839 if (!isZeroNode(BV.getOperand(i)) && 840 BV.getOperand(i).getOpcode() != ISD::UNDEF) 841 return false; // Not a zero/undef vector. 842 // Check to see if the shuffle mask is 4/L/L/L or 2/L, where L is something 843 // from the LHS. 844 unsigned VecWidth = BV.getNumOperands(); 845 SDOperand ShufMask = N.getOperand(2); 846 assert(ShufMask.getOpcode() == ISD::BUILD_VECTOR && "Invalid shuf mask!"); 847 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(ShufMask.getOperand(0))) { 848 if (C->getValue() == VecWidth) { 849 for (unsigned i = 1; i != VecWidth; ++i) { 850 if (ShufMask.getOperand(i).getOpcode() == ISD::UNDEF) { 851 // ok. 852 } else { 853 ConstantSDNode *C = cast<ConstantSDNode>(ShufMask.getOperand(i)); 854 if (C->getValue() >= VecWidth) return false; 855 } 856 } 857 } 858 859 // Okay, this is a zero extending load. Fold it. 860 LoadSDNode *LD = cast<LoadSDNode>(N.getOperand(1).getOperand(0)); 861 if (!SelectAddr(Op, LD->getBasePtr(), Base, Scale, Index, Disp)) 862 return false; 863 OutChain = LD->getChain(); 864 InChain = SDOperand(LD, 1); 865 return true; 866 } 867 } 868 return false; 869} 870 871 872/// SelectLEAAddr - it calls SelectAddr and determines if the maximal addressing 873/// mode it matches can be cost effectively emitted as an LEA instruction. 874bool X86DAGToDAGISel::SelectLEAAddr(SDOperand Op, SDOperand N, 875 SDOperand &Base, SDOperand &Scale, 876 SDOperand &Index, SDOperand &Disp) { 877 X86ISelAddressMode AM; 878 if (MatchAddress(N, AM)) 879 return false; 880 881 MVT::ValueType VT = N.getValueType(); 882 unsigned Complexity = 0; 883 if (AM.BaseType == X86ISelAddressMode::RegBase) 884 if (AM.Base.Reg.Val) 885 Complexity = 1; 886 else 887 AM.Base.Reg = CurDAG->getRegister(0, VT); 888 else if (AM.BaseType == X86ISelAddressMode::FrameIndexBase) 889 Complexity = 4; 890 891 if (AM.IndexReg.Val) 892 Complexity++; 893 else 894 AM.IndexReg = CurDAG->getRegister(0, VT); 895 896 if (AM.Scale > 2) 897 Complexity += 2; 898 // Don't match just leal(,%reg,2). It's cheaper to do addl %reg, %reg 899 else if (AM.Scale > 1) 900 Complexity++; 901 902 // FIXME: We are artificially lowering the criteria to turn ADD %reg, $GA 903 // to a LEA. This is determined with some expermentation but is by no means 904 // optimal (especially for code size consideration). LEA is nice because of 905 // its three-address nature. Tweak the cost function again when we can run 906 // convertToThreeAddress() at register allocation time. 907 if (AM.GV || AM.CP || AM.ES || AM.JT != -1) { 908 // For X86-64, we should always use lea to materialize RIP relative 909 // addresses. 910 if (Subtarget->is64Bit()) 911 Complexity = 4; 912 else 913 Complexity += 2; 914 } 915 916 if (AM.Disp && (AM.Base.Reg.Val || AM.IndexReg.Val)) 917 Complexity++; 918 919 if (Complexity > 2) { 920 getAddressOperands(AM, Base, Scale, Index, Disp); 921 return true; 922 } 923 return false; 924} 925 926bool X86DAGToDAGISel::TryFoldLoad(SDOperand P, SDOperand N, 927 SDOperand &Base, SDOperand &Scale, 928 SDOperand &Index, SDOperand &Disp) { 929 if (ISD::isNON_EXTLoad(N.Val) && 930 N.hasOneUse() && 931 CanBeFoldedBy(N.Val, P.Val, P.Val)) 932 return SelectAddr(P, N.getOperand(1), Base, Scale, Index, Disp); 933 return false; 934} 935 936/// getGlobalBaseReg - Output the instructions required to put the 937/// base address to use for accessing globals into a register. 938/// 939SDNode *X86DAGToDAGISel::getGlobalBaseReg() { 940 assert(!Subtarget->is64Bit() && "X86-64 PIC uses RIP relative addressing"); 941 if (!GlobalBaseReg) { 942 // Insert the set of GlobalBaseReg into the first MBB of the function 943 MachineBasicBlock &FirstMBB = BB->getParent()->front(); 944 MachineBasicBlock::iterator MBBI = FirstMBB.begin(); 945 SSARegMap *RegMap = BB->getParent()->getSSARegMap(); 946 GlobalBaseReg = RegMap->createVirtualRegister(X86::GR32RegisterClass); 947 const TargetInstrInfo *TII = TM.getInstrInfo(); 948 BuildMI(FirstMBB, MBBI, TII->get(X86::MovePCtoStack)); 949 BuildMI(FirstMBB, MBBI, TII->get(X86::POP32r), GlobalBaseReg); 950 } 951 return CurDAG->getRegister(GlobalBaseReg, TLI.getPointerTy()).Val; 952} 953 954static SDNode *FindCallStartFromCall(SDNode *Node) { 955 if (Node->getOpcode() == ISD::CALLSEQ_START) return Node; 956 assert(Node->getOperand(0).getValueType() == MVT::Other && 957 "Node doesn't have a token chain argument!"); 958 return FindCallStartFromCall(Node->getOperand(0).Val); 959} 960 961SDNode *X86DAGToDAGISel::Select(SDOperand N) { 962 SDNode *Node = N.Val; 963 MVT::ValueType NVT = Node->getValueType(0); 964 unsigned Opc, MOpc; 965 unsigned Opcode = Node->getOpcode(); 966 967#ifndef NDEBUG 968 DOUT << std::string(Indent, ' ') << "Selecting: "; 969 DEBUG(Node->dump(CurDAG)); 970 DOUT << "\n"; 971 Indent += 2; 972#endif 973 974 if (Opcode >= ISD::BUILTIN_OP_END && Opcode < X86ISD::FIRST_NUMBER) { 975#ifndef NDEBUG 976 DOUT << std::string(Indent-2, ' ') << "== "; 977 DEBUG(Node->dump(CurDAG)); 978 DOUT << "\n"; 979 Indent -= 2; 980#endif 981 return NULL; // Already selected. 982 } 983 984 switch (Opcode) { 985 default: break; 986 case X86ISD::GlobalBaseReg: 987 return getGlobalBaseReg(); 988 989 case ISD::ADD: { 990 // Turn ADD X, c to MOV32ri X+c. This cannot be done with tblgen'd 991 // code and is matched first so to prevent it from being turned into 992 // LEA32r X+c. 993 // In 64-bit mode, use LEA to take advantage of RIP-relative addressing. 994 MVT::ValueType PtrVT = TLI.getPointerTy(); 995 SDOperand N0 = N.getOperand(0); 996 SDOperand N1 = N.getOperand(1); 997 if (N.Val->getValueType(0) == PtrVT && 998 N0.getOpcode() == X86ISD::Wrapper && 999 N1.getOpcode() == ISD::Constant) { 1000 unsigned Offset = (unsigned)cast<ConstantSDNode>(N1)->getValue(); 1001 SDOperand C(0, 0); 1002 // TODO: handle ExternalSymbolSDNode. 1003 if (GlobalAddressSDNode *G = 1004 dyn_cast<GlobalAddressSDNode>(N0.getOperand(0))) { 1005 C = CurDAG->getTargetGlobalAddress(G->getGlobal(), PtrVT, 1006 G->getOffset() + Offset); 1007 } else if (ConstantPoolSDNode *CP = 1008 dyn_cast<ConstantPoolSDNode>(N0.getOperand(0))) { 1009 C = CurDAG->getTargetConstantPool(CP->getConstVal(), PtrVT, 1010 CP->getAlignment(), 1011 CP->getOffset()+Offset); 1012 } 1013 1014 if (C.Val) { 1015 if (Subtarget->is64Bit()) { 1016 SDOperand Ops[] = { CurDAG->getRegister(0, PtrVT), getI8Imm(1), 1017 CurDAG->getRegister(0, PtrVT), C }; 1018 return CurDAG->SelectNodeTo(N.Val, X86::LEA64r, MVT::i64, Ops, 4); 1019 } else 1020 return CurDAG->SelectNodeTo(N.Val, X86::MOV32ri, PtrVT, C); 1021 } 1022 } 1023 1024 // Other cases are handled by auto-generated code. 1025 break; 1026 } 1027 1028 case ISD::MULHU: 1029 case ISD::MULHS: { 1030 if (Opcode == ISD::MULHU) 1031 switch (NVT) { 1032 default: assert(0 && "Unsupported VT!"); 1033 case MVT::i8: Opc = X86::MUL8r; MOpc = X86::MUL8m; break; 1034 case MVT::i16: Opc = X86::MUL16r; MOpc = X86::MUL16m; break; 1035 case MVT::i32: Opc = X86::MUL32r; MOpc = X86::MUL32m; break; 1036 case MVT::i64: Opc = X86::MUL64r; MOpc = X86::MUL64m; break; 1037 } 1038 else 1039 switch (NVT) { 1040 default: assert(0 && "Unsupported VT!"); 1041 case MVT::i8: Opc = X86::IMUL8r; MOpc = X86::IMUL8m; break; 1042 case MVT::i16: Opc = X86::IMUL16r; MOpc = X86::IMUL16m; break; 1043 case MVT::i32: Opc = X86::IMUL32r; MOpc = X86::IMUL32m; break; 1044 case MVT::i64: Opc = X86::IMUL64r; MOpc = X86::IMUL64m; break; 1045 } 1046 1047 unsigned LoReg, HiReg; 1048 switch (NVT) { 1049 default: assert(0 && "Unsupported VT!"); 1050 case MVT::i8: LoReg = X86::AL; HiReg = X86::AH; break; 1051 case MVT::i16: LoReg = X86::AX; HiReg = X86::DX; break; 1052 case MVT::i32: LoReg = X86::EAX; HiReg = X86::EDX; break; 1053 case MVT::i64: LoReg = X86::RAX; HiReg = X86::RDX; break; 1054 } 1055 1056 SDOperand N0 = Node->getOperand(0); 1057 SDOperand N1 = Node->getOperand(1); 1058 1059 bool foldedLoad = false; 1060 SDOperand Tmp0, Tmp1, Tmp2, Tmp3; 1061 foldedLoad = TryFoldLoad(N, N1, Tmp0, Tmp1, Tmp2, Tmp3); 1062 // MULHU and MULHS are commmutative 1063 if (!foldedLoad) { 1064 foldedLoad = TryFoldLoad(N, N0, Tmp0, Tmp1, Tmp2, Tmp3); 1065 if (foldedLoad) { 1066 N0 = Node->getOperand(1); 1067 N1 = Node->getOperand(0); 1068 } 1069 } 1070 1071 SDOperand Chain; 1072 if (foldedLoad) { 1073 Chain = N1.getOperand(0); 1074 AddToISelQueue(Chain); 1075 } else 1076 Chain = CurDAG->getEntryNode(); 1077 1078 SDOperand InFlag(0, 0); 1079 AddToISelQueue(N0); 1080 Chain = CurDAG->getCopyToReg(Chain, CurDAG->getRegister(LoReg, NVT), 1081 N0, InFlag); 1082 InFlag = Chain.getValue(1); 1083 1084 if (foldedLoad) { 1085 AddToISelQueue(Tmp0); 1086 AddToISelQueue(Tmp1); 1087 AddToISelQueue(Tmp2); 1088 AddToISelQueue(Tmp3); 1089 SDOperand Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Chain, InFlag }; 1090 SDNode *CNode = 1091 CurDAG->getTargetNode(MOpc, MVT::Other, MVT::Flag, Ops, 6); 1092 Chain = SDOperand(CNode, 0); 1093 InFlag = SDOperand(CNode, 1); 1094 } else { 1095 AddToISelQueue(N1); 1096 InFlag = 1097 SDOperand(CurDAG->getTargetNode(Opc, MVT::Flag, N1, InFlag), 0); 1098 } 1099 1100 SDOperand Result = CurDAG->getCopyFromReg(Chain, HiReg, NVT, InFlag); 1101 ReplaceUses(N.getValue(0), Result); 1102 if (foldedLoad) 1103 ReplaceUses(N1.getValue(1), Result.getValue(1)); 1104 1105#ifndef NDEBUG 1106 DOUT << std::string(Indent-2, ' ') << "=> "; 1107 DEBUG(Result.Val->dump(CurDAG)); 1108 DOUT << "\n"; 1109 Indent -= 2; 1110#endif 1111 return NULL; 1112 } 1113 1114 case ISD::SDIV: 1115 case ISD::UDIV: 1116 case ISD::SREM: 1117 case ISD::UREM: { 1118 bool isSigned = Opcode == ISD::SDIV || Opcode == ISD::SREM; 1119 bool isDiv = Opcode == ISD::SDIV || Opcode == ISD::UDIV; 1120 if (!isSigned) 1121 switch (NVT) { 1122 default: assert(0 && "Unsupported VT!"); 1123 case MVT::i8: Opc = X86::DIV8r; MOpc = X86::DIV8m; break; 1124 case MVT::i16: Opc = X86::DIV16r; MOpc = X86::DIV16m; break; 1125 case MVT::i32: Opc = X86::DIV32r; MOpc = X86::DIV32m; break; 1126 case MVT::i64: Opc = X86::DIV64r; MOpc = X86::DIV64m; break; 1127 } 1128 else 1129 switch (NVT) { 1130 default: assert(0 && "Unsupported VT!"); 1131 case MVT::i8: Opc = X86::IDIV8r; MOpc = X86::IDIV8m; break; 1132 case MVT::i16: Opc = X86::IDIV16r; MOpc = X86::IDIV16m; break; 1133 case MVT::i32: Opc = X86::IDIV32r; MOpc = X86::IDIV32m; break; 1134 case MVT::i64: Opc = X86::IDIV64r; MOpc = X86::IDIV64m; break; 1135 } 1136 1137 unsigned LoReg, HiReg; 1138 unsigned ClrOpcode, SExtOpcode; 1139 switch (NVT) { 1140 default: assert(0 && "Unsupported VT!"); 1141 case MVT::i8: 1142 LoReg = X86::AL; HiReg = X86::AH; 1143 ClrOpcode = 0; 1144 SExtOpcode = X86::CBW; 1145 break; 1146 case MVT::i16: 1147 LoReg = X86::AX; HiReg = X86::DX; 1148 ClrOpcode = X86::MOV16r0; 1149 SExtOpcode = X86::CWD; 1150 break; 1151 case MVT::i32: 1152 LoReg = X86::EAX; HiReg = X86::EDX; 1153 ClrOpcode = X86::MOV32r0; 1154 SExtOpcode = X86::CDQ; 1155 break; 1156 case MVT::i64: 1157 LoReg = X86::RAX; HiReg = X86::RDX; 1158 ClrOpcode = X86::MOV64r0; 1159 SExtOpcode = X86::CQO; 1160 break; 1161 } 1162 1163 SDOperand N0 = Node->getOperand(0); 1164 SDOperand N1 = Node->getOperand(1); 1165 SDOperand InFlag(0, 0); 1166 if (NVT == MVT::i8 && !isSigned) { 1167 // Special case for div8, just use a move with zero extension to AX to 1168 // clear the upper 8 bits (AH). 1169 SDOperand Tmp0, Tmp1, Tmp2, Tmp3, Move, Chain; 1170 if (TryFoldLoad(N, N0, Tmp0, Tmp1, Tmp2, Tmp3)) { 1171 SDOperand Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, N0.getOperand(0) }; 1172 AddToISelQueue(N0.getOperand(0)); 1173 AddToISelQueue(Tmp0); 1174 AddToISelQueue(Tmp1); 1175 AddToISelQueue(Tmp2); 1176 AddToISelQueue(Tmp3); 1177 Move = 1178 SDOperand(CurDAG->getTargetNode(X86::MOVZX16rm8, MVT::i16, MVT::Other, 1179 Ops, 5), 0); 1180 Chain = Move.getValue(1); 1181 ReplaceUses(N0.getValue(1), Chain); 1182 } else { 1183 AddToISelQueue(N0); 1184 Move = 1185 SDOperand(CurDAG->getTargetNode(X86::MOVZX16rr8, MVT::i16, N0), 0); 1186 Chain = CurDAG->getEntryNode(); 1187 } 1188 Chain = CurDAG->getCopyToReg(Chain, X86::AX, Move, InFlag); 1189 InFlag = Chain.getValue(1); 1190 } else { 1191 AddToISelQueue(N0); 1192 InFlag = 1193 CurDAG->getCopyToReg(CurDAG->getEntryNode(), LoReg, N0, 1194 InFlag).getValue(1); 1195 if (isSigned) { 1196 // Sign extend the low part into the high part. 1197 InFlag = 1198 SDOperand(CurDAG->getTargetNode(SExtOpcode, MVT::Flag, InFlag), 0); 1199 } else { 1200 // Zero out the high part, effectively zero extending the input. 1201 SDOperand ClrNode = SDOperand(CurDAG->getTargetNode(ClrOpcode, NVT), 0); 1202 InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), HiReg, ClrNode, 1203 InFlag).getValue(1); 1204 } 1205 } 1206 1207 SDOperand Tmp0, Tmp1, Tmp2, Tmp3, Chain; 1208 bool foldedLoad = TryFoldLoad(N, N1, Tmp0, Tmp1, Tmp2, Tmp3); 1209 if (foldedLoad) { 1210 AddToISelQueue(N1.getOperand(0)); 1211 AddToISelQueue(Tmp0); 1212 AddToISelQueue(Tmp1); 1213 AddToISelQueue(Tmp2); 1214 AddToISelQueue(Tmp3); 1215 SDOperand Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, N1.getOperand(0), InFlag }; 1216 SDNode *CNode = 1217 CurDAG->getTargetNode(MOpc, MVT::Other, MVT::Flag, Ops, 6); 1218 Chain = SDOperand(CNode, 0); 1219 InFlag = SDOperand(CNode, 1); 1220 } else { 1221 AddToISelQueue(N1); 1222 Chain = CurDAG->getEntryNode(); 1223 InFlag = 1224 SDOperand(CurDAG->getTargetNode(Opc, MVT::Flag, N1, InFlag), 0); 1225 } 1226 1227 SDOperand Result = 1228 CurDAG->getCopyFromReg(Chain, isDiv ? LoReg : HiReg, NVT, InFlag); 1229 ReplaceUses(N.getValue(0), Result); 1230 if (foldedLoad) 1231 ReplaceUses(N1.getValue(1), Result.getValue(1)); 1232 1233#ifndef NDEBUG 1234 DOUT << std::string(Indent-2, ' ') << "=> "; 1235 DEBUG(Result.Val->dump(CurDAG)); 1236 DOUT << "\n"; 1237 Indent -= 2; 1238#endif 1239 1240 return NULL; 1241 } 1242 1243 case ISD::TRUNCATE: { 1244 if (!Subtarget->is64Bit() && NVT == MVT::i8) { 1245 unsigned Opc2; 1246 MVT::ValueType VT; 1247 switch (Node->getOperand(0).getValueType()) { 1248 default: assert(0 && "Unknown truncate!"); 1249 case MVT::i16: 1250 Opc = X86::MOV16to16_; 1251 VT = MVT::i16; 1252 Opc2 = X86::TRUNC_16_to8; 1253 break; 1254 case MVT::i32: 1255 Opc = X86::MOV32to32_; 1256 VT = MVT::i32; 1257 Opc2 = X86::TRUNC_32_to8; 1258 break; 1259 } 1260 1261 AddToISelQueue(Node->getOperand(0)); 1262 SDOperand Tmp = 1263 SDOperand(CurDAG->getTargetNode(Opc, VT, Node->getOperand(0)), 0); 1264 SDNode *ResNode = CurDAG->getTargetNode(Opc2, NVT, Tmp); 1265 1266#ifndef NDEBUG 1267 DOUT << std::string(Indent-2, ' ') << "=> "; 1268 DEBUG(ResNode->dump(CurDAG)); 1269 DOUT << "\n"; 1270 Indent -= 2; 1271#endif 1272 return ResNode; 1273 } 1274 1275 break; 1276 } 1277 } 1278 1279 SDNode *ResNode = SelectCode(N); 1280 1281#ifndef NDEBUG 1282 DOUT << std::string(Indent-2, ' ') << "=> "; 1283 if (ResNode == NULL || ResNode == N.Val) 1284 DEBUG(N.Val->dump(CurDAG)); 1285 else 1286 DEBUG(ResNode->dump(CurDAG)); 1287 DOUT << "\n"; 1288 Indent -= 2; 1289#endif 1290 1291 return ResNode; 1292} 1293 1294bool X86DAGToDAGISel:: 1295SelectInlineAsmMemoryOperand(const SDOperand &Op, char ConstraintCode, 1296 std::vector<SDOperand> &OutOps, SelectionDAG &DAG){ 1297 SDOperand Op0, Op1, Op2, Op3; 1298 switch (ConstraintCode) { 1299 case 'o': // offsetable ?? 1300 case 'v': // not offsetable ?? 1301 default: return true; 1302 case 'm': // memory 1303 if (!SelectAddr(Op, Op, Op0, Op1, Op2, Op3)) 1304 return true; 1305 break; 1306 } 1307 1308 OutOps.push_back(Op0); 1309 OutOps.push_back(Op1); 1310 OutOps.push_back(Op2); 1311 OutOps.push_back(Op3); 1312 AddToISelQueue(Op0); 1313 AddToISelQueue(Op1); 1314 AddToISelQueue(Op2); 1315 AddToISelQueue(Op3); 1316 return false; 1317} 1318 1319/// createX86ISelDag - This pass converts a legalized DAG into a 1320/// X86-specific DAG, ready for instruction scheduling. 1321/// 1322FunctionPass *llvm::createX86ISelDag(X86TargetMachine &TM, bool Fast) { 1323 return new X86DAGToDAGISel(TM, Fast); 1324} 1325