SelectionDAGISel.cpp revision 2332b9f16fe17d1886566729b2241b8cd90f9916
1//===-- SelectionDAGISel.cpp - Implement the SelectionDAGISel class -------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file was developed by the LLVM research group and is distributed under 6// the University of Illinois Open Source License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This implements the SelectionDAGISel class. 11// 12//===----------------------------------------------------------------------===// 13 14#define DEBUG_TYPE "isel" 15#include "llvm/CodeGen/SelectionDAGISel.h" 16#include "llvm/CodeGen/ScheduleDAG.h" 17#include "llvm/CallingConv.h" 18#include "llvm/Constants.h" 19#include "llvm/DerivedTypes.h" 20#include "llvm/Function.h" 21#include "llvm/GlobalVariable.h" 22#include "llvm/InlineAsm.h" 23#include "llvm/Instructions.h" 24#include "llvm/Intrinsics.h" 25#include "llvm/CodeGen/IntrinsicLowering.h" 26#include "llvm/CodeGen/MachineDebugInfo.h" 27#include "llvm/CodeGen/MachineFunction.h" 28#include "llvm/CodeGen/MachineFrameInfo.h" 29#include "llvm/CodeGen/MachineInstrBuilder.h" 30#include "llvm/CodeGen/SelectionDAG.h" 31#include "llvm/CodeGen/SSARegMap.h" 32#include "llvm/Target/MRegisterInfo.h" 33#include "llvm/Target/TargetData.h" 34#include "llvm/Target/TargetFrameInfo.h" 35#include "llvm/Target/TargetInstrInfo.h" 36#include "llvm/Target/TargetLowering.h" 37#include "llvm/Target/TargetMachine.h" 38#include "llvm/Transforms/Utils/BasicBlockUtils.h" 39#include "llvm/Support/CommandLine.h" 40#include "llvm/Support/MathExtras.h" 41#include "llvm/Support/Debug.h" 42#include <map> 43#include <set> 44#include <iostream> 45#include <algorithm> 46using namespace llvm; 47 48#ifndef NDEBUG 49static cl::opt<bool> 50ViewISelDAGs("view-isel-dags", cl::Hidden, 51 cl::desc("Pop up a window to show isel dags as they are selected")); 52static cl::opt<bool> 53ViewSchedDAGs("view-sched-dags", cl::Hidden, 54 cl::desc("Pop up a window to show sched dags as they are processed")); 55#else 56static const bool ViewISelDAGs = 0; 57static const bool ViewSchedDAGs = 0; 58#endif 59 60// Scheduling heuristics 61enum SchedHeuristics { 62 defaultScheduling, // Let the target specify its preference. 63 noScheduling, // No scheduling, emit breadth first sequence. 64 simpleScheduling, // Two pass, min. critical path, max. utilization. 65 simpleNoItinScheduling, // Same as above exact using generic latency. 66 listSchedulingBURR, // Bottom up reg reduction list scheduling. 67 listSchedulingTD // Top-down list scheduler. 68}; 69 70namespace { 71 cl::opt<SchedHeuristics> 72 ISHeuristic( 73 "sched", 74 cl::desc("Choose scheduling style"), 75 cl::init(defaultScheduling), 76 cl::values( 77 clEnumValN(defaultScheduling, "default", 78 "Target preferred scheduling style"), 79 clEnumValN(noScheduling, "none", 80 "No scheduling: breadth first sequencing"), 81 clEnumValN(simpleScheduling, "simple", 82 "Simple two pass scheduling: minimize critical path " 83 "and maximize processor utilization"), 84 clEnumValN(simpleNoItinScheduling, "simple-noitin", 85 "Simple two pass scheduling: Same as simple " 86 "except using generic latency"), 87 clEnumValN(listSchedulingBURR, "list-burr", 88 "Bottom up register reduction list scheduling"), 89 clEnumValN(listSchedulingTD, "list-td", 90 "Top-down list scheduler"), 91 clEnumValEnd)); 92} // namespace 93 94namespace { 95 /// RegsForValue - This struct represents the physical registers that a 96 /// particular value is assigned and the type information about the value. 97 /// This is needed because values can be promoted into larger registers and 98 /// expanded into multiple smaller registers than the value. 99 struct RegsForValue { 100 /// Regs - This list hold the register (for legal and promoted values) 101 /// or register set (for expanded values) that the value should be assigned 102 /// to. 103 std::vector<unsigned> Regs; 104 105 /// RegVT - The value type of each register. 106 /// 107 MVT::ValueType RegVT; 108 109 /// ValueVT - The value type of the LLVM value, which may be promoted from 110 /// RegVT or made from merging the two expanded parts. 111 MVT::ValueType ValueVT; 112 113 RegsForValue() : RegVT(MVT::Other), ValueVT(MVT::Other) {} 114 115 RegsForValue(unsigned Reg, MVT::ValueType regvt, MVT::ValueType valuevt) 116 : RegVT(regvt), ValueVT(valuevt) { 117 Regs.push_back(Reg); 118 } 119 RegsForValue(const std::vector<unsigned> ®s, 120 MVT::ValueType regvt, MVT::ValueType valuevt) 121 : Regs(regs), RegVT(regvt), ValueVT(valuevt) { 122 } 123 124 /// getCopyFromRegs - Emit a series of CopyFromReg nodes that copies from 125 /// this value and returns the result as a ValueVT value. This uses 126 /// Chain/Flag as the input and updates them for the output Chain/Flag. 127 SDOperand getCopyFromRegs(SelectionDAG &DAG, 128 SDOperand &Chain, SDOperand &Flag) const; 129 130 /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the 131 /// specified value into the registers specified by this object. This uses 132 /// Chain/Flag as the input and updates them for the output Chain/Flag. 133 void getCopyToRegs(SDOperand Val, SelectionDAG &DAG, 134 SDOperand &Chain, SDOperand &Flag) const; 135 136 /// AddInlineAsmOperands - Add this value to the specified inlineasm node 137 /// operand list. This adds the code marker and includes the number of 138 /// values added into it. 139 void AddInlineAsmOperands(unsigned Code, SelectionDAG &DAG, 140 std::vector<SDOperand> &Ops) const; 141 }; 142} 143 144namespace llvm { 145 //===--------------------------------------------------------------------===// 146 /// FunctionLoweringInfo - This contains information that is global to a 147 /// function that is used when lowering a region of the function. 148 class FunctionLoweringInfo { 149 public: 150 TargetLowering &TLI; 151 Function &Fn; 152 MachineFunction &MF; 153 SSARegMap *RegMap; 154 155 FunctionLoweringInfo(TargetLowering &TLI, Function &Fn,MachineFunction &MF); 156 157 /// MBBMap - A mapping from LLVM basic blocks to their machine code entry. 158 std::map<const BasicBlock*, MachineBasicBlock *> MBBMap; 159 160 /// ValueMap - Since we emit code for the function a basic block at a time, 161 /// we must remember which virtual registers hold the values for 162 /// cross-basic-block values. 163 std::map<const Value*, unsigned> ValueMap; 164 165 /// StaticAllocaMap - Keep track of frame indices for fixed sized allocas in 166 /// the entry block. This allows the allocas to be efficiently referenced 167 /// anywhere in the function. 168 std::map<const AllocaInst*, int> StaticAllocaMap; 169 170 unsigned MakeReg(MVT::ValueType VT) { 171 return RegMap->createVirtualRegister(TLI.getRegClassFor(VT)); 172 } 173 174 unsigned CreateRegForValue(const Value *V); 175 176 unsigned InitializeRegForValue(const Value *V) { 177 unsigned &R = ValueMap[V]; 178 assert(R == 0 && "Already initialized this value register!"); 179 return R = CreateRegForValue(V); 180 } 181 }; 182} 183 184/// isUsedOutsideOfDefiningBlock - Return true if this instruction is used by 185/// PHI nodes or outside of the basic block that defines it. 186static bool isUsedOutsideOfDefiningBlock(Instruction *I) { 187 if (isa<PHINode>(I)) return true; 188 BasicBlock *BB = I->getParent(); 189 for (Value::use_iterator UI = I->use_begin(), E = I->use_end(); UI != E; ++UI) 190 if (cast<Instruction>(*UI)->getParent() != BB || isa<PHINode>(*UI)) 191 return true; 192 return false; 193} 194 195/// isOnlyUsedInEntryBlock - If the specified argument is only used in the 196/// entry block, return true. 197static bool isOnlyUsedInEntryBlock(Argument *A) { 198 BasicBlock *Entry = A->getParent()->begin(); 199 for (Value::use_iterator UI = A->use_begin(), E = A->use_end(); UI != E; ++UI) 200 if (cast<Instruction>(*UI)->getParent() != Entry) 201 return false; // Use not in entry block. 202 return true; 203} 204 205FunctionLoweringInfo::FunctionLoweringInfo(TargetLowering &tli, 206 Function &fn, MachineFunction &mf) 207 : TLI(tli), Fn(fn), MF(mf), RegMap(MF.getSSARegMap()) { 208 209 // Create a vreg for each argument register that is not dead and is used 210 // outside of the entry block for the function. 211 for (Function::arg_iterator AI = Fn.arg_begin(), E = Fn.arg_end(); 212 AI != E; ++AI) 213 if (!isOnlyUsedInEntryBlock(AI)) 214 InitializeRegForValue(AI); 215 216 // Initialize the mapping of values to registers. This is only set up for 217 // instruction values that are used outside of the block that defines 218 // them. 219 Function::iterator BB = Fn.begin(), EB = Fn.end(); 220 for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I) 221 if (AllocaInst *AI = dyn_cast<AllocaInst>(I)) 222 if (ConstantUInt *CUI = dyn_cast<ConstantUInt>(AI->getArraySize())) { 223 const Type *Ty = AI->getAllocatedType(); 224 uint64_t TySize = TLI.getTargetData().getTypeSize(Ty); 225 unsigned Align = 226 std::max((unsigned)TLI.getTargetData().getTypeAlignment(Ty), 227 AI->getAlignment()); 228 229 // If the alignment of the value is smaller than the size of the value, 230 // and if the size of the value is particularly small (<= 8 bytes), 231 // round up to the size of the value for potentially better performance. 232 // 233 // FIXME: This could be made better with a preferred alignment hook in 234 // TargetData. It serves primarily to 8-byte align doubles for X86. 235 if (Align < TySize && TySize <= 8) Align = TySize; 236 TySize *= CUI->getValue(); // Get total allocated size. 237 if (TySize == 0) TySize = 1; // Don't create zero-sized stack objects. 238 StaticAllocaMap[AI] = 239 MF.getFrameInfo()->CreateStackObject((unsigned)TySize, Align); 240 } 241 242 for (; BB != EB; ++BB) 243 for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I) 244 if (!I->use_empty() && isUsedOutsideOfDefiningBlock(I)) 245 if (!isa<AllocaInst>(I) || 246 !StaticAllocaMap.count(cast<AllocaInst>(I))) 247 InitializeRegForValue(I); 248 249 // Create an initial MachineBasicBlock for each LLVM BasicBlock in F. This 250 // also creates the initial PHI MachineInstrs, though none of the input 251 // operands are populated. 252 for (BB = Fn.begin(), EB = Fn.end(); BB != EB; ++BB) { 253 MachineBasicBlock *MBB = new MachineBasicBlock(BB); 254 MBBMap[BB] = MBB; 255 MF.getBasicBlockList().push_back(MBB); 256 257 // Create Machine PHI nodes for LLVM PHI nodes, lowering them as 258 // appropriate. 259 PHINode *PN; 260 for (BasicBlock::iterator I = BB->begin(); 261 (PN = dyn_cast<PHINode>(I)); ++I) 262 if (!PN->use_empty()) { 263 unsigned NumElements = 264 TLI.getNumElements(TLI.getValueType(PN->getType())); 265 unsigned PHIReg = ValueMap[PN]; 266 assert(PHIReg &&"PHI node does not have an assigned virtual register!"); 267 for (unsigned i = 0; i != NumElements; ++i) 268 BuildMI(MBB, TargetInstrInfo::PHI, PN->getNumOperands(), PHIReg+i); 269 } 270 } 271} 272 273/// CreateRegForValue - Allocate the appropriate number of virtual registers of 274/// the correctly promoted or expanded types. Assign these registers 275/// consecutive vreg numbers and return the first assigned number. 276unsigned FunctionLoweringInfo::CreateRegForValue(const Value *V) { 277 MVT::ValueType VT = TLI.getValueType(V->getType()); 278 279 // The number of multiples of registers that we need, to, e.g., split up 280 // a <2 x int64> -> 4 x i32 registers. 281 unsigned NumVectorRegs = 1; 282 283 // If this is a packed type, figure out what type it will decompose into 284 // and how many of the elements it will use. 285 if (VT == MVT::Vector) { 286 const PackedType *PTy = cast<PackedType>(V->getType()); 287 unsigned NumElts = PTy->getNumElements(); 288 MVT::ValueType EltTy = TLI.getValueType(PTy->getElementType()); 289 290 // Divide the input until we get to a supported size. This will always 291 // end with a scalar if the target doesn't support vectors. 292 while (NumElts > 1 && !TLI.isTypeLegal(getVectorType(EltTy, NumElts))) { 293 NumElts >>= 1; 294 NumVectorRegs <<= 1; 295 } 296 if (NumElts == 1) 297 VT = EltTy; 298 else 299 VT = getVectorType(EltTy, NumElts); 300 } 301 302 // The common case is that we will only create one register for this 303 // value. If we have that case, create and return the virtual register. 304 unsigned NV = TLI.getNumElements(VT); 305 if (NV == 1) { 306 // If we are promoting this value, pick the next largest supported type. 307 MVT::ValueType PromotedType = TLI.getTypeToTransformTo(VT); 308 unsigned Reg = MakeReg(PromotedType); 309 // If this is a vector of supported or promoted types (e.g. 4 x i16), 310 // create all of the registers. 311 for (unsigned i = 1; i != NumVectorRegs; ++i) 312 MakeReg(PromotedType); 313 return Reg; 314 } 315 316 // If this value is represented with multiple target registers, make sure 317 // to create enough consecutive registers of the right (smaller) type. 318 unsigned NT = VT-1; // Find the type to use. 319 while (TLI.getNumElements((MVT::ValueType)NT) != 1) 320 --NT; 321 322 unsigned R = MakeReg((MVT::ValueType)NT); 323 for (unsigned i = 1; i != NV*NumVectorRegs; ++i) 324 MakeReg((MVT::ValueType)NT); 325 return R; 326} 327 328//===----------------------------------------------------------------------===// 329/// SelectionDAGLowering - This is the common target-independent lowering 330/// implementation that is parameterized by a TargetLowering object. 331/// Also, targets can overload any lowering method. 332/// 333namespace llvm { 334class SelectionDAGLowering { 335 MachineBasicBlock *CurMBB; 336 337 std::map<const Value*, SDOperand> NodeMap; 338 339 /// PendingLoads - Loads are not emitted to the program immediately. We bunch 340 /// them up and then emit token factor nodes when possible. This allows us to 341 /// get simple disambiguation between loads without worrying about alias 342 /// analysis. 343 std::vector<SDOperand> PendingLoads; 344 345public: 346 // TLI - This is information that describes the available target features we 347 // need for lowering. This indicates when operations are unavailable, 348 // implemented with a libcall, etc. 349 TargetLowering &TLI; 350 SelectionDAG &DAG; 351 const TargetData &TD; 352 353 /// FuncInfo - Information about the function as a whole. 354 /// 355 FunctionLoweringInfo &FuncInfo; 356 357 SelectionDAGLowering(SelectionDAG &dag, TargetLowering &tli, 358 FunctionLoweringInfo &funcinfo) 359 : TLI(tli), DAG(dag), TD(DAG.getTarget().getTargetData()), 360 FuncInfo(funcinfo) { 361 } 362 363 /// getRoot - Return the current virtual root of the Selection DAG. 364 /// 365 SDOperand getRoot() { 366 if (PendingLoads.empty()) 367 return DAG.getRoot(); 368 369 if (PendingLoads.size() == 1) { 370 SDOperand Root = PendingLoads[0]; 371 DAG.setRoot(Root); 372 PendingLoads.clear(); 373 return Root; 374 } 375 376 // Otherwise, we have to make a token factor node. 377 SDOperand Root = DAG.getNode(ISD::TokenFactor, MVT::Other, PendingLoads); 378 PendingLoads.clear(); 379 DAG.setRoot(Root); 380 return Root; 381 } 382 383 void visit(Instruction &I) { visit(I.getOpcode(), I); } 384 385 void visit(unsigned Opcode, User &I) { 386 switch (Opcode) { 387 default: assert(0 && "Unknown instruction type encountered!"); 388 abort(); 389 // Build the switch statement using the Instruction.def file. 390#define HANDLE_INST(NUM, OPCODE, CLASS) \ 391 case Instruction::OPCODE:return visit##OPCODE((CLASS&)I); 392#include "llvm/Instruction.def" 393 } 394 } 395 396 void setCurrentBasicBlock(MachineBasicBlock *MBB) { CurMBB = MBB; } 397 398 SDOperand getLoadFrom(const Type *Ty, SDOperand Ptr, 399 SDOperand SrcValue, SDOperand Root, 400 bool isVolatile); 401 402 SDOperand getIntPtrConstant(uint64_t Val) { 403 return DAG.getConstant(Val, TLI.getPointerTy()); 404 } 405 406 SDOperand getValue(const Value *V); 407 408 const SDOperand &setValue(const Value *V, SDOperand NewN) { 409 SDOperand &N = NodeMap[V]; 410 assert(N.Val == 0 && "Already set a value for this node!"); 411 return N = NewN; 412 } 413 414 RegsForValue GetRegistersForValue(const std::string &ConstrCode, 415 MVT::ValueType VT, 416 bool OutReg, bool InReg, 417 std::set<unsigned> &OutputRegs, 418 std::set<unsigned> &InputRegs); 419 420 // Terminator instructions. 421 void visitRet(ReturnInst &I); 422 void visitBr(BranchInst &I); 423 void visitUnreachable(UnreachableInst &I) { /* noop */ } 424 425 // These all get lowered before this pass. 426 void visitSwitch(SwitchInst &I) { assert(0 && "TODO"); } 427 void visitInvoke(InvokeInst &I) { assert(0 && "TODO"); } 428 void visitUnwind(UnwindInst &I) { assert(0 && "TODO"); } 429 430 // 431 void visitBinary(User &I, unsigned IntOp, unsigned FPOp, unsigned VecOp); 432 void visitShift(User &I, unsigned Opcode); 433 void visitAdd(User &I) { 434 visitBinary(I, ISD::ADD, ISD::FADD, ISD::VADD); 435 } 436 void visitSub(User &I); 437 void visitMul(User &I) { 438 visitBinary(I, ISD::MUL, ISD::FMUL, ISD::VMUL); 439 } 440 void visitDiv(User &I) { 441 const Type *Ty = I.getType(); 442 visitBinary(I, 443 Ty->isSigned() ? ISD::SDIV : ISD::UDIV, ISD::FDIV, 444 Ty->isSigned() ? ISD::VSDIV : ISD::VUDIV); 445 } 446 void visitRem(User &I) { 447 const Type *Ty = I.getType(); 448 visitBinary(I, Ty->isSigned() ? ISD::SREM : ISD::UREM, ISD::FREM, 0); 449 } 450 void visitAnd(User &I) { visitBinary(I, ISD::AND, 0, ISD::VAND); } 451 void visitOr (User &I) { visitBinary(I, ISD::OR, 0, ISD::VOR); } 452 void visitXor(User &I) { visitBinary(I, ISD::XOR, 0, ISD::VXOR); } 453 void visitShl(User &I) { visitShift(I, ISD::SHL); } 454 void visitShr(User &I) { 455 visitShift(I, I.getType()->isUnsigned() ? ISD::SRL : ISD::SRA); 456 } 457 458 void visitSetCC(User &I, ISD::CondCode SignedOpc, ISD::CondCode UnsignedOpc); 459 void visitSetEQ(User &I) { visitSetCC(I, ISD::SETEQ, ISD::SETEQ); } 460 void visitSetNE(User &I) { visitSetCC(I, ISD::SETNE, ISD::SETNE); } 461 void visitSetLE(User &I) { visitSetCC(I, ISD::SETLE, ISD::SETULE); } 462 void visitSetGE(User &I) { visitSetCC(I, ISD::SETGE, ISD::SETUGE); } 463 void visitSetLT(User &I) { visitSetCC(I, ISD::SETLT, ISD::SETULT); } 464 void visitSetGT(User &I) { visitSetCC(I, ISD::SETGT, ISD::SETUGT); } 465 466 void visitExtractElement(ExtractElementInst &I) { assert(0 && "TODO"); } 467 void visitInsertElement(InsertElementInst &I); 468 469 void visitGetElementPtr(User &I); 470 void visitCast(User &I); 471 void visitSelect(User &I); 472 // 473 474 void visitMalloc(MallocInst &I); 475 void visitFree(FreeInst &I); 476 void visitAlloca(AllocaInst &I); 477 void visitLoad(LoadInst &I); 478 void visitStore(StoreInst &I); 479 void visitPHI(PHINode &I) { } // PHI nodes are handled specially. 480 void visitCall(CallInst &I); 481 void visitInlineAsm(CallInst &I); 482 const char *visitIntrinsicCall(CallInst &I, unsigned Intrinsic); 483 484 void visitVAStart(CallInst &I); 485 void visitVAArg(VAArgInst &I); 486 void visitVAEnd(CallInst &I); 487 void visitVACopy(CallInst &I); 488 void visitFrameReturnAddress(CallInst &I, bool isFrameAddress); 489 490 void visitMemIntrinsic(CallInst &I, unsigned Op); 491 492 void visitUserOp1(Instruction &I) { 493 assert(0 && "UserOp1 should not exist at instruction selection time!"); 494 abort(); 495 } 496 void visitUserOp2(Instruction &I) { 497 assert(0 && "UserOp2 should not exist at instruction selection time!"); 498 abort(); 499 } 500}; 501} // end namespace llvm 502 503SDOperand SelectionDAGLowering::getValue(const Value *V) { 504 SDOperand &N = NodeMap[V]; 505 if (N.Val) return N; 506 507 const Type *VTy = V->getType(); 508 MVT::ValueType VT = TLI.getValueType(VTy); 509 if (Constant *C = const_cast<Constant*>(dyn_cast<Constant>(V))) { 510 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) { 511 visit(CE->getOpcode(), *CE); 512 assert(N.Val && "visit didn't populate the ValueMap!"); 513 return N; 514 } else if (GlobalValue *GV = dyn_cast<GlobalValue>(C)) { 515 return N = DAG.getGlobalAddress(GV, VT); 516 } else if (isa<ConstantPointerNull>(C)) { 517 return N = DAG.getConstant(0, TLI.getPointerTy()); 518 } else if (isa<UndefValue>(C)) { 519 if (!isa<PackedType>(VTy)) 520 return N = DAG.getNode(ISD::UNDEF, VT); 521 522 // Create a VBUILD_VECTOR of undef nodes. 523 const PackedType *PTy = cast<PackedType>(VTy); 524 unsigned NumElements = PTy->getNumElements(); 525 MVT::ValueType PVT = TLI.getValueType(PTy->getElementType()); 526 527 std::vector<SDOperand> Ops; 528 Ops.assign(NumElements, DAG.getNode(ISD::UNDEF, PVT)); 529 530 // Create a VConstant node with generic Vector type. 531 Ops.push_back(DAG.getConstant(NumElements, MVT::i32)); 532 Ops.push_back(DAG.getValueType(PVT)); 533 return N = DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, Ops); 534 } else if (ConstantFP *CFP = dyn_cast<ConstantFP>(C)) { 535 return N = DAG.getConstantFP(CFP->getValue(), VT); 536 } else if (const PackedType *PTy = dyn_cast<PackedType>(VTy)) { 537 unsigned NumElements = PTy->getNumElements(); 538 MVT::ValueType PVT = TLI.getValueType(PTy->getElementType()); 539 540 // Now that we know the number and type of the elements, push a 541 // Constant or ConstantFP node onto the ops list for each element of 542 // the packed constant. 543 std::vector<SDOperand> Ops; 544 if (ConstantPacked *CP = dyn_cast<ConstantPacked>(C)) { 545 if (MVT::isFloatingPoint(PVT)) { 546 for (unsigned i = 0; i != NumElements; ++i) { 547 const ConstantFP *El = cast<ConstantFP>(CP->getOperand(i)); 548 Ops.push_back(DAG.getConstantFP(El->getValue(), PVT)); 549 } 550 } else { 551 for (unsigned i = 0; i != NumElements; ++i) { 552 const ConstantIntegral *El = 553 cast<ConstantIntegral>(CP->getOperand(i)); 554 Ops.push_back(DAG.getConstant(El->getRawValue(), PVT)); 555 } 556 } 557 } else { 558 assert(isa<ConstantAggregateZero>(C) && "Unknown packed constant!"); 559 SDOperand Op; 560 if (MVT::isFloatingPoint(PVT)) 561 Op = DAG.getConstantFP(0, PVT); 562 else 563 Op = DAG.getConstant(0, PVT); 564 Ops.assign(NumElements, Op); 565 } 566 567 // Create a VBUILD_VECTOR node with generic Vector type. 568 Ops.push_back(DAG.getConstant(NumElements, MVT::i32)); 569 Ops.push_back(DAG.getValueType(PVT)); 570 return N = DAG.getNode(ISD::VBUILD_VECTOR, MVT::Vector, Ops); 571 } else { 572 // Canonicalize all constant ints to be unsigned. 573 return N = DAG.getConstant(cast<ConstantIntegral>(C)->getRawValue(),VT); 574 } 575 } 576 577 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) { 578 std::map<const AllocaInst*, int>::iterator SI = 579 FuncInfo.StaticAllocaMap.find(AI); 580 if (SI != FuncInfo.StaticAllocaMap.end()) 581 return DAG.getFrameIndex(SI->second, TLI.getPointerTy()); 582 } 583 584 std::map<const Value*, unsigned>::const_iterator VMI = 585 FuncInfo.ValueMap.find(V); 586 assert(VMI != FuncInfo.ValueMap.end() && "Value not in map!"); 587 588 unsigned InReg = VMI->second; 589 590 // If this type is not legal, make it so now. 591 if (VT == MVT::Vector) { 592 // FIXME: We only handle legal vectors right now. We need a VBUILD_VECTOR 593 const PackedType *PTy = cast<PackedType>(VTy); 594 unsigned NumElements = PTy->getNumElements(); 595 MVT::ValueType PVT = TLI.getValueType(PTy->getElementType()); 596 MVT::ValueType TVT = MVT::getVectorType(PVT, NumElements); 597 assert(TLI.isTypeLegal(TVT) && 598 "FIXME: Cannot handle illegal vector types here yet!"); 599 VT = TVT; 600 } 601 602 MVT::ValueType DestVT = TLI.getTypeToTransformTo(VT); 603 604 N = DAG.getCopyFromReg(DAG.getEntryNode(), InReg, DestVT); 605 if (DestVT < VT) { 606 // Source must be expanded. This input value is actually coming from the 607 // register pair VMI->second and VMI->second+1. 608 N = DAG.getNode(ISD::BUILD_PAIR, VT, N, 609 DAG.getCopyFromReg(DAG.getEntryNode(), InReg+1, DestVT)); 610 } else { 611 if (DestVT > VT) { // Promotion case 612 if (MVT::isFloatingPoint(VT)) 613 N = DAG.getNode(ISD::FP_ROUND, VT, N); 614 else 615 N = DAG.getNode(ISD::TRUNCATE, VT, N); 616 } 617 } 618 619 return N; 620} 621 622 623void SelectionDAGLowering::visitRet(ReturnInst &I) { 624 if (I.getNumOperands() == 0) { 625 DAG.setRoot(DAG.getNode(ISD::RET, MVT::Other, getRoot())); 626 return; 627 } 628 std::vector<SDOperand> NewValues; 629 NewValues.push_back(getRoot()); 630 for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i) { 631 SDOperand RetOp = getValue(I.getOperand(i)); 632 633 // If this is an integer return value, we need to promote it ourselves to 634 // the full width of a register, since LegalizeOp will use ANY_EXTEND rather 635 // than sign/zero. 636 if (MVT::isInteger(RetOp.getValueType()) && 637 RetOp.getValueType() < MVT::i64) { 638 MVT::ValueType TmpVT; 639 if (TLI.getTypeAction(MVT::i32) == TargetLowering::Promote) 640 TmpVT = TLI.getTypeToTransformTo(MVT::i32); 641 else 642 TmpVT = MVT::i32; 643 644 if (I.getOperand(i)->getType()->isSigned()) 645 RetOp = DAG.getNode(ISD::SIGN_EXTEND, TmpVT, RetOp); 646 else 647 RetOp = DAG.getNode(ISD::ZERO_EXTEND, TmpVT, RetOp); 648 } 649 NewValues.push_back(RetOp); 650 } 651 DAG.setRoot(DAG.getNode(ISD::RET, MVT::Other, NewValues)); 652} 653 654void SelectionDAGLowering::visitBr(BranchInst &I) { 655 // Update machine-CFG edges. 656 MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)]; 657 658 // Figure out which block is immediately after the current one. 659 MachineBasicBlock *NextBlock = 0; 660 MachineFunction::iterator BBI = CurMBB; 661 if (++BBI != CurMBB->getParent()->end()) 662 NextBlock = BBI; 663 664 if (I.isUnconditional()) { 665 // If this is not a fall-through branch, emit the branch. 666 if (Succ0MBB != NextBlock) 667 DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, getRoot(), 668 DAG.getBasicBlock(Succ0MBB))); 669 } else { 670 MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)]; 671 672 SDOperand Cond = getValue(I.getCondition()); 673 if (Succ1MBB == NextBlock) { 674 // If the condition is false, fall through. This means we should branch 675 // if the condition is true to Succ #0. 676 DAG.setRoot(DAG.getNode(ISD::BRCOND, MVT::Other, getRoot(), 677 Cond, DAG.getBasicBlock(Succ0MBB))); 678 } else if (Succ0MBB == NextBlock) { 679 // If the condition is true, fall through. This means we should branch if 680 // the condition is false to Succ #1. Invert the condition first. 681 SDOperand True = DAG.getConstant(1, Cond.getValueType()); 682 Cond = DAG.getNode(ISD::XOR, Cond.getValueType(), Cond, True); 683 DAG.setRoot(DAG.getNode(ISD::BRCOND, MVT::Other, getRoot(), 684 Cond, DAG.getBasicBlock(Succ1MBB))); 685 } else { 686 std::vector<SDOperand> Ops; 687 Ops.push_back(getRoot()); 688 // If the false case is the current basic block, then this is a self 689 // loop. We do not want to emit "Loop: ... brcond Out; br Loop", as it 690 // adds an extra instruction in the loop. Instead, invert the 691 // condition and emit "Loop: ... br!cond Loop; br Out. 692 if (CurMBB == Succ1MBB) { 693 std::swap(Succ0MBB, Succ1MBB); 694 SDOperand True = DAG.getConstant(1, Cond.getValueType()); 695 Cond = DAG.getNode(ISD::XOR, Cond.getValueType(), Cond, True); 696 } 697 SDOperand True = DAG.getNode(ISD::BRCOND, MVT::Other, getRoot(), Cond, 698 DAG.getBasicBlock(Succ0MBB)); 699 DAG.setRoot(DAG.getNode(ISD::BR, MVT::Other, True, 700 DAG.getBasicBlock(Succ1MBB))); 701 } 702 } 703} 704 705void SelectionDAGLowering::visitSub(User &I) { 706 // -0.0 - X --> fneg 707 if (I.getType()->isFloatingPoint()) { 708 if (ConstantFP *CFP = dyn_cast<ConstantFP>(I.getOperand(0))) 709 if (CFP->isExactlyValue(-0.0)) { 710 SDOperand Op2 = getValue(I.getOperand(1)); 711 setValue(&I, DAG.getNode(ISD::FNEG, Op2.getValueType(), Op2)); 712 return; 713 } 714 } 715 visitBinary(I, ISD::SUB, ISD::FSUB, ISD::VSUB); 716} 717 718void SelectionDAGLowering::visitBinary(User &I, unsigned IntOp, unsigned FPOp, 719 unsigned VecOp) { 720 const Type *Ty = I.getType(); 721 SDOperand Op1 = getValue(I.getOperand(0)); 722 SDOperand Op2 = getValue(I.getOperand(1)); 723 724 if (Ty->isIntegral()) { 725 setValue(&I, DAG.getNode(IntOp, Op1.getValueType(), Op1, Op2)); 726 } else if (Ty->isFloatingPoint()) { 727 setValue(&I, DAG.getNode(FPOp, Op1.getValueType(), Op1, Op2)); 728 } else { 729 const PackedType *PTy = cast<PackedType>(Ty); 730 SDOperand Num = DAG.getConstant(PTy->getNumElements(), MVT::i32); 731 SDOperand Typ = DAG.getValueType(TLI.getValueType(PTy->getElementType())); 732 setValue(&I, DAG.getNode(VecOp, MVT::Vector, Op1, Op2, Num, Typ)); 733 } 734} 735 736void SelectionDAGLowering::visitShift(User &I, unsigned Opcode) { 737 SDOperand Op1 = getValue(I.getOperand(0)); 738 SDOperand Op2 = getValue(I.getOperand(1)); 739 740 Op2 = DAG.getNode(ISD::ANY_EXTEND, TLI.getShiftAmountTy(), Op2); 741 742 setValue(&I, DAG.getNode(Opcode, Op1.getValueType(), Op1, Op2)); 743} 744 745void SelectionDAGLowering::visitSetCC(User &I,ISD::CondCode SignedOpcode, 746 ISD::CondCode UnsignedOpcode) { 747 SDOperand Op1 = getValue(I.getOperand(0)); 748 SDOperand Op2 = getValue(I.getOperand(1)); 749 ISD::CondCode Opcode = SignedOpcode; 750 if (I.getOperand(0)->getType()->isUnsigned()) 751 Opcode = UnsignedOpcode; 752 setValue(&I, DAG.getSetCC(MVT::i1, Op1, Op2, Opcode)); 753} 754 755void SelectionDAGLowering::visitSelect(User &I) { 756 SDOperand Cond = getValue(I.getOperand(0)); 757 SDOperand TrueVal = getValue(I.getOperand(1)); 758 SDOperand FalseVal = getValue(I.getOperand(2)); 759 setValue(&I, DAG.getNode(ISD::SELECT, TrueVal.getValueType(), Cond, 760 TrueVal, FalseVal)); 761} 762 763void SelectionDAGLowering::visitCast(User &I) { 764 SDOperand N = getValue(I.getOperand(0)); 765 MVT::ValueType SrcVT = TLI.getValueType(I.getOperand(0)->getType()); 766 MVT::ValueType DestVT = TLI.getValueType(I.getType()); 767 768 if (N.getValueType() == DestVT) { 769 setValue(&I, N); // noop cast. 770 } else if (DestVT == MVT::i1) { 771 // Cast to bool is a comparison against zero, not truncation to zero. 772 SDOperand Zero = isInteger(SrcVT) ? DAG.getConstant(0, N.getValueType()) : 773 DAG.getConstantFP(0.0, N.getValueType()); 774 setValue(&I, DAG.getSetCC(MVT::i1, N, Zero, ISD::SETNE)); 775 } else if (isInteger(SrcVT)) { 776 if (isInteger(DestVT)) { // Int -> Int cast 777 if (DestVT < SrcVT) // Truncating cast? 778 setValue(&I, DAG.getNode(ISD::TRUNCATE, DestVT, N)); 779 else if (I.getOperand(0)->getType()->isSigned()) 780 setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, DestVT, N)); 781 else 782 setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, DestVT, N)); 783 } else { // Int -> FP cast 784 if (I.getOperand(0)->getType()->isSigned()) 785 setValue(&I, DAG.getNode(ISD::SINT_TO_FP, DestVT, N)); 786 else 787 setValue(&I, DAG.getNode(ISD::UINT_TO_FP, DestVT, N)); 788 } 789 } else if (isFloatingPoint(SrcVT)) { 790 if (isFloatingPoint(DestVT)) { // FP -> FP cast 791 if (DestVT < SrcVT) // Rounding cast? 792 setValue(&I, DAG.getNode(ISD::FP_ROUND, DestVT, N)); 793 else 794 setValue(&I, DAG.getNode(ISD::FP_EXTEND, DestVT, N)); 795 } else { // FP -> Int cast. 796 if (I.getType()->isSigned()) 797 setValue(&I, DAG.getNode(ISD::FP_TO_SINT, DestVT, N)); 798 else 799 setValue(&I, DAG.getNode(ISD::FP_TO_UINT, DestVT, N)); 800 } 801 } else { 802 assert(0 && "Cannot bitconvert vectors yet!"); 803#if 0 804 const PackedType *SrcTy = cast<PackedType>(I.getOperand(0)->getType()); 805 const PackedType *DstTy = cast<PackedType>(I.getType()); 806 807 unsigned SrcNumElements = SrcTy->getNumElements(); 808 MVT::ValueType SrcPVT = TLI.getValueType(SrcTy->getElementType()); 809 MVT::ValueType SrcTVT = MVT::getVectorType(SrcPVT, SrcNumElements); 810 811 unsigned DstNumElements = DstTy->getNumElements(); 812 MVT::ValueType DstPVT = TLI.getValueType(DstTy->getElementType()); 813 MVT::ValueType DstTVT = MVT::getVectorType(DstPVT, DstNumElements); 814 815 // If the input and output type are legal, convert this to a bit convert of 816 // the SrcTVT/DstTVT types. 817 if (SrcTVT != MVT::Other && DstTVT != MVT::Other && 818 TLI.isTypeLegal(SrcTVT) && TLI.isTypeLegal(DstTVT)) { 819 assert(N.getValueType() == SrcTVT); 820 setValue(&I, DAG.getNode(ISD::BIT_CONVERT, DstTVT, N)); 821 } else { 822 // Otherwise, convert this directly into a store/load. 823 // FIXME: add a VBIT_CONVERT node that we could use to automatically turn 824 // 8xFloat -> 8xInt casts into two 4xFloat -> 4xInt casts. 825 // Create the stack frame object. 826 uint64_t ByteSize = TD.getTypeSize(SrcTy); 827 assert(ByteSize == TD.getTypeSize(DstTy) && "Not a bit_convert!"); 828 MachineFrameInfo *FrameInfo = DAG.getMachineFunction().getFrameInfo(); 829 int FrameIdx = FrameInfo->CreateStackObject(ByteSize, ByteSize); 830 SDOperand FIPtr = DAG.getFrameIndex(FrameIdx, TLI.getPointerTy()); 831 832 // Emit a store to the stack slot. 833 SDOperand Store = DAG.getNode(ISD::STORE, MVT::Other, DAG.getEntryNode(), 834 N, FIPtr, DAG.getSrcValue(NULL)); 835 // Result is a load from the stack slot. 836 SDOperand Val = 837 getLoadFrom(DstTy, FIPtr, DAG.getSrcValue(NULL), Store, false); 838 setValue(&I, Val); 839 } 840#endif 841 } 842} 843 844void SelectionDAGLowering::visitInsertElement(InsertElementInst &I) { 845 SDOperand InVec = getValue(I.getOperand(0)); 846 SDOperand InVal = getValue(I.getOperand(1)); 847 SDOperand InIdx = DAG.getNode(ISD::ZERO_EXTEND, TLI.getPointerTy(), 848 getValue(I.getOperand(2))); 849 850 SDOperand Num = *(InVec.Val->op_end()-2); 851 SDOperand Typ = *(InVec.Val->op_end()-1); 852 setValue(&I, DAG.getNode(ISD::VINSERT_VECTOR_ELT, MVT::Vector, 853 InVec, InVal, InIdx, Num, Typ)); 854} 855 856 857void SelectionDAGLowering::visitGetElementPtr(User &I) { 858 SDOperand N = getValue(I.getOperand(0)); 859 const Type *Ty = I.getOperand(0)->getType(); 860 const Type *UIntPtrTy = TD.getIntPtrType(); 861 862 for (GetElementPtrInst::op_iterator OI = I.op_begin()+1, E = I.op_end(); 863 OI != E; ++OI) { 864 Value *Idx = *OI; 865 if (const StructType *StTy = dyn_cast<StructType>(Ty)) { 866 unsigned Field = cast<ConstantUInt>(Idx)->getValue(); 867 if (Field) { 868 // N = N + Offset 869 uint64_t Offset = TD.getStructLayout(StTy)->MemberOffsets[Field]; 870 N = DAG.getNode(ISD::ADD, N.getValueType(), N, 871 getIntPtrConstant(Offset)); 872 } 873 Ty = StTy->getElementType(Field); 874 } else { 875 Ty = cast<SequentialType>(Ty)->getElementType(); 876 877 // If this is a constant subscript, handle it quickly. 878 if (ConstantInt *CI = dyn_cast<ConstantInt>(Idx)) { 879 if (CI->getRawValue() == 0) continue; 880 881 uint64_t Offs; 882 if (ConstantSInt *CSI = dyn_cast<ConstantSInt>(CI)) 883 Offs = (int64_t)TD.getTypeSize(Ty)*CSI->getValue(); 884 else 885 Offs = TD.getTypeSize(Ty)*cast<ConstantUInt>(CI)->getValue(); 886 N = DAG.getNode(ISD::ADD, N.getValueType(), N, getIntPtrConstant(Offs)); 887 continue; 888 } 889 890 // N = N + Idx * ElementSize; 891 uint64_t ElementSize = TD.getTypeSize(Ty); 892 SDOperand IdxN = getValue(Idx); 893 894 // If the index is smaller or larger than intptr_t, truncate or extend 895 // it. 896 if (IdxN.getValueType() < N.getValueType()) { 897 if (Idx->getType()->isSigned()) 898 IdxN = DAG.getNode(ISD::SIGN_EXTEND, N.getValueType(), IdxN); 899 else 900 IdxN = DAG.getNode(ISD::ZERO_EXTEND, N.getValueType(), IdxN); 901 } else if (IdxN.getValueType() > N.getValueType()) 902 IdxN = DAG.getNode(ISD::TRUNCATE, N.getValueType(), IdxN); 903 904 // If this is a multiply by a power of two, turn it into a shl 905 // immediately. This is a very common case. 906 if (isPowerOf2_64(ElementSize)) { 907 unsigned Amt = Log2_64(ElementSize); 908 IdxN = DAG.getNode(ISD::SHL, N.getValueType(), IdxN, 909 DAG.getConstant(Amt, TLI.getShiftAmountTy())); 910 N = DAG.getNode(ISD::ADD, N.getValueType(), N, IdxN); 911 continue; 912 } 913 914 SDOperand Scale = getIntPtrConstant(ElementSize); 915 IdxN = DAG.getNode(ISD::MUL, N.getValueType(), IdxN, Scale); 916 N = DAG.getNode(ISD::ADD, N.getValueType(), N, IdxN); 917 } 918 } 919 setValue(&I, N); 920} 921 922void SelectionDAGLowering::visitAlloca(AllocaInst &I) { 923 // If this is a fixed sized alloca in the entry block of the function, 924 // allocate it statically on the stack. 925 if (FuncInfo.StaticAllocaMap.count(&I)) 926 return; // getValue will auto-populate this. 927 928 const Type *Ty = I.getAllocatedType(); 929 uint64_t TySize = TLI.getTargetData().getTypeSize(Ty); 930 unsigned Align = std::max((unsigned)TLI.getTargetData().getTypeAlignment(Ty), 931 I.getAlignment()); 932 933 SDOperand AllocSize = getValue(I.getArraySize()); 934 MVT::ValueType IntPtr = TLI.getPointerTy(); 935 if (IntPtr < AllocSize.getValueType()) 936 AllocSize = DAG.getNode(ISD::TRUNCATE, IntPtr, AllocSize); 937 else if (IntPtr > AllocSize.getValueType()) 938 AllocSize = DAG.getNode(ISD::ZERO_EXTEND, IntPtr, AllocSize); 939 940 AllocSize = DAG.getNode(ISD::MUL, IntPtr, AllocSize, 941 getIntPtrConstant(TySize)); 942 943 // Handle alignment. If the requested alignment is less than or equal to the 944 // stack alignment, ignore it and round the size of the allocation up to the 945 // stack alignment size. If the size is greater than the stack alignment, we 946 // note this in the DYNAMIC_STACKALLOC node. 947 unsigned StackAlign = 948 TLI.getTargetMachine().getFrameInfo()->getStackAlignment(); 949 if (Align <= StackAlign) { 950 Align = 0; 951 // Add SA-1 to the size. 952 AllocSize = DAG.getNode(ISD::ADD, AllocSize.getValueType(), AllocSize, 953 getIntPtrConstant(StackAlign-1)); 954 // Mask out the low bits for alignment purposes. 955 AllocSize = DAG.getNode(ISD::AND, AllocSize.getValueType(), AllocSize, 956 getIntPtrConstant(~(uint64_t)(StackAlign-1))); 957 } 958 959 std::vector<MVT::ValueType> VTs; 960 VTs.push_back(AllocSize.getValueType()); 961 VTs.push_back(MVT::Other); 962 std::vector<SDOperand> Ops; 963 Ops.push_back(getRoot()); 964 Ops.push_back(AllocSize); 965 Ops.push_back(getIntPtrConstant(Align)); 966 SDOperand DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, VTs, Ops); 967 DAG.setRoot(setValue(&I, DSA).getValue(1)); 968 969 // Inform the Frame Information that we have just allocated a variable-sized 970 // object. 971 CurMBB->getParent()->getFrameInfo()->CreateVariableSizedObject(); 972} 973 974void SelectionDAGLowering::visitLoad(LoadInst &I) { 975 SDOperand Ptr = getValue(I.getOperand(0)); 976 977 SDOperand Root; 978 if (I.isVolatile()) 979 Root = getRoot(); 980 else { 981 // Do not serialize non-volatile loads against each other. 982 Root = DAG.getRoot(); 983 } 984 985 setValue(&I, getLoadFrom(I.getType(), Ptr, DAG.getSrcValue(I.getOperand(0)), 986 Root, I.isVolatile())); 987} 988 989SDOperand SelectionDAGLowering::getLoadFrom(const Type *Ty, SDOperand Ptr, 990 SDOperand SrcValue, SDOperand Root, 991 bool isVolatile) { 992 SDOperand L; 993 if (const PackedType *PTy = dyn_cast<PackedType>(Ty)) { 994 MVT::ValueType PVT = TLI.getValueType(PTy->getElementType()); 995 L = DAG.getVecLoad(PTy->getNumElements(), PVT, Root, Ptr, SrcValue); 996 } else { 997 L = DAG.getLoad(TLI.getValueType(Ty), Root, Ptr, SrcValue); 998 } 999 1000 if (isVolatile) 1001 DAG.setRoot(L.getValue(1)); 1002 else 1003 PendingLoads.push_back(L.getValue(1)); 1004 1005 return L; 1006} 1007 1008 1009void SelectionDAGLowering::visitStore(StoreInst &I) { 1010 Value *SrcV = I.getOperand(0); 1011 SDOperand Src = getValue(SrcV); 1012 SDOperand Ptr = getValue(I.getOperand(1)); 1013 DAG.setRoot(DAG.getNode(ISD::STORE, MVT::Other, getRoot(), Src, Ptr, 1014 DAG.getSrcValue(I.getOperand(1)))); 1015} 1016 1017/// visitIntrinsicCall - Lower the call to the specified intrinsic function. If 1018/// we want to emit this as a call to a named external function, return the name 1019/// otherwise lower it and return null. 1020const char * 1021SelectionDAGLowering::visitIntrinsicCall(CallInst &I, unsigned Intrinsic) { 1022 switch (Intrinsic) { 1023 case Intrinsic::vastart: visitVAStart(I); return 0; 1024 case Intrinsic::vaend: visitVAEnd(I); return 0; 1025 case Intrinsic::vacopy: visitVACopy(I); return 0; 1026 case Intrinsic::returnaddress: visitFrameReturnAddress(I, false); return 0; 1027 case Intrinsic::frameaddress: visitFrameReturnAddress(I, true); return 0; 1028 case Intrinsic::setjmp: 1029 return "_setjmp"+!TLI.usesUnderscoreSetJmpLongJmp(); 1030 break; 1031 case Intrinsic::longjmp: 1032 return "_longjmp"+!TLI.usesUnderscoreSetJmpLongJmp(); 1033 break; 1034 case Intrinsic::memcpy_i32: 1035 case Intrinsic::memcpy_i64: 1036 visitMemIntrinsic(I, ISD::MEMCPY); 1037 return 0; 1038 case Intrinsic::memset_i32: 1039 case Intrinsic::memset_i64: 1040 visitMemIntrinsic(I, ISD::MEMSET); 1041 return 0; 1042 case Intrinsic::memmove_i32: 1043 case Intrinsic::memmove_i64: 1044 visitMemIntrinsic(I, ISD::MEMMOVE); 1045 return 0; 1046 1047 case Intrinsic::dbg_stoppoint: { 1048 MachineDebugInfo *DebugInfo = DAG.getMachineDebugInfo(); 1049 if (DebugInfo && DebugInfo->Verify(I.getOperand(3))) { 1050 std::vector<SDOperand> Ops; 1051 1052 // Input Chain 1053 Ops.push_back(getRoot()); 1054 1055 // line number 1056 Ops.push_back(getValue(I.getOperand(1))); 1057 1058 // column 1059 Ops.push_back(getValue(I.getOperand(2))); 1060 1061 DebugInfoDesc *DD = DebugInfo->getDescFor(I.getOperand(3)); 1062 assert(DD && "Not a debug information descriptor"); 1063 CompileUnitDesc *CompileUnit = dyn_cast<CompileUnitDesc>(DD); 1064 assert(CompileUnit && "Not a compile unit"); 1065 Ops.push_back(DAG.getString(CompileUnit->getFileName())); 1066 Ops.push_back(DAG.getString(CompileUnit->getDirectory())); 1067 1068 if (Ops.size() == 5) // Found filename/workingdir. 1069 DAG.setRoot(DAG.getNode(ISD::LOCATION, MVT::Other, Ops)); 1070 } 1071 1072 setValue(&I, DAG.getNode(ISD::UNDEF, TLI.getValueType(I.getType()))); 1073 return 0; 1074 } 1075 case Intrinsic::dbg_region_start: 1076 if (I.getType() != Type::VoidTy) 1077 setValue(&I, DAG.getNode(ISD::UNDEF, TLI.getValueType(I.getType()))); 1078 return 0; 1079 case Intrinsic::dbg_region_end: 1080 if (I.getType() != Type::VoidTy) 1081 setValue(&I, DAG.getNode(ISD::UNDEF, TLI.getValueType(I.getType()))); 1082 return 0; 1083 case Intrinsic::dbg_func_start: 1084 if (I.getType() != Type::VoidTy) 1085 setValue(&I, DAG.getNode(ISD::UNDEF, TLI.getValueType(I.getType()))); 1086 return 0; 1087 1088 case Intrinsic::isunordered_f32: 1089 case Intrinsic::isunordered_f64: 1090 setValue(&I, DAG.getSetCC(MVT::i1,getValue(I.getOperand(1)), 1091 getValue(I.getOperand(2)), ISD::SETUO)); 1092 return 0; 1093 1094 case Intrinsic::sqrt_f32: 1095 case Intrinsic::sqrt_f64: 1096 setValue(&I, DAG.getNode(ISD::FSQRT, 1097 getValue(I.getOperand(1)).getValueType(), 1098 getValue(I.getOperand(1)))); 1099 return 0; 1100 case Intrinsic::pcmarker: { 1101 SDOperand Tmp = getValue(I.getOperand(1)); 1102 DAG.setRoot(DAG.getNode(ISD::PCMARKER, MVT::Other, getRoot(), Tmp)); 1103 return 0; 1104 } 1105 case Intrinsic::readcyclecounter: { 1106 std::vector<MVT::ValueType> VTs; 1107 VTs.push_back(MVT::i64); 1108 VTs.push_back(MVT::Other); 1109 std::vector<SDOperand> Ops; 1110 Ops.push_back(getRoot()); 1111 SDOperand Tmp = DAG.getNode(ISD::READCYCLECOUNTER, VTs, Ops); 1112 setValue(&I, Tmp); 1113 DAG.setRoot(Tmp.getValue(1)); 1114 return 0; 1115 } 1116 case Intrinsic::bswap_i16: 1117 case Intrinsic::bswap_i32: 1118 case Intrinsic::bswap_i64: 1119 setValue(&I, DAG.getNode(ISD::BSWAP, 1120 getValue(I.getOperand(1)).getValueType(), 1121 getValue(I.getOperand(1)))); 1122 return 0; 1123 case Intrinsic::cttz_i8: 1124 case Intrinsic::cttz_i16: 1125 case Intrinsic::cttz_i32: 1126 case Intrinsic::cttz_i64: 1127 setValue(&I, DAG.getNode(ISD::CTTZ, 1128 getValue(I.getOperand(1)).getValueType(), 1129 getValue(I.getOperand(1)))); 1130 return 0; 1131 case Intrinsic::ctlz_i8: 1132 case Intrinsic::ctlz_i16: 1133 case Intrinsic::ctlz_i32: 1134 case Intrinsic::ctlz_i64: 1135 setValue(&I, DAG.getNode(ISD::CTLZ, 1136 getValue(I.getOperand(1)).getValueType(), 1137 getValue(I.getOperand(1)))); 1138 return 0; 1139 case Intrinsic::ctpop_i8: 1140 case Intrinsic::ctpop_i16: 1141 case Intrinsic::ctpop_i32: 1142 case Intrinsic::ctpop_i64: 1143 setValue(&I, DAG.getNode(ISD::CTPOP, 1144 getValue(I.getOperand(1)).getValueType(), 1145 getValue(I.getOperand(1)))); 1146 return 0; 1147 case Intrinsic::stacksave: { 1148 std::vector<MVT::ValueType> VTs; 1149 VTs.push_back(TLI.getPointerTy()); 1150 VTs.push_back(MVT::Other); 1151 std::vector<SDOperand> Ops; 1152 Ops.push_back(getRoot()); 1153 SDOperand Tmp = DAG.getNode(ISD::STACKSAVE, VTs, Ops); 1154 setValue(&I, Tmp); 1155 DAG.setRoot(Tmp.getValue(1)); 1156 return 0; 1157 } 1158 case Intrinsic::stackrestore: { 1159 SDOperand Tmp = getValue(I.getOperand(1)); 1160 DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, MVT::Other, getRoot(), Tmp)); 1161 return 0; 1162 } 1163 case Intrinsic::prefetch: 1164 // FIXME: Currently discarding prefetches. 1165 return 0; 1166 default: 1167 std::cerr << I; 1168 assert(0 && "This intrinsic is not implemented yet!"); 1169 return 0; 1170 } 1171} 1172 1173 1174void SelectionDAGLowering::visitCall(CallInst &I) { 1175 const char *RenameFn = 0; 1176 if (Function *F = I.getCalledFunction()) { 1177 if (F->isExternal()) 1178 if (unsigned IID = F->getIntrinsicID()) { 1179 RenameFn = visitIntrinsicCall(I, IID); 1180 if (!RenameFn) 1181 return; 1182 } else { // Not an LLVM intrinsic. 1183 const std::string &Name = F->getName(); 1184 if (Name[0] == 'c' && (Name == "copysign" || Name == "copysignf")) { 1185 if (I.getNumOperands() == 3 && // Basic sanity checks. 1186 I.getOperand(1)->getType()->isFloatingPoint() && 1187 I.getType() == I.getOperand(1)->getType() && 1188 I.getType() == I.getOperand(2)->getType()) { 1189 SDOperand LHS = getValue(I.getOperand(1)); 1190 SDOperand RHS = getValue(I.getOperand(2)); 1191 setValue(&I, DAG.getNode(ISD::FCOPYSIGN, LHS.getValueType(), 1192 LHS, RHS)); 1193 return; 1194 } 1195 } else if (Name[0] == 'f' && (Name == "fabs" || Name == "fabsf")) { 1196 if (I.getNumOperands() == 2 && // Basic sanity checks. 1197 I.getOperand(1)->getType()->isFloatingPoint() && 1198 I.getType() == I.getOperand(1)->getType()) { 1199 SDOperand Tmp = getValue(I.getOperand(1)); 1200 setValue(&I, DAG.getNode(ISD::FABS, Tmp.getValueType(), Tmp)); 1201 return; 1202 } 1203 } else if (Name[0] == 's' && (Name == "sin" || Name == "sinf")) { 1204 if (I.getNumOperands() == 2 && // Basic sanity checks. 1205 I.getOperand(1)->getType()->isFloatingPoint() && 1206 I.getType() == I.getOperand(1)->getType()) { 1207 SDOperand Tmp = getValue(I.getOperand(1)); 1208 setValue(&I, DAG.getNode(ISD::FSIN, Tmp.getValueType(), Tmp)); 1209 return; 1210 } 1211 } else if (Name[0] == 'c' && (Name == "cos" || Name == "cosf")) { 1212 if (I.getNumOperands() == 2 && // Basic sanity checks. 1213 I.getOperand(1)->getType()->isFloatingPoint() && 1214 I.getType() == I.getOperand(1)->getType()) { 1215 SDOperand Tmp = getValue(I.getOperand(1)); 1216 setValue(&I, DAG.getNode(ISD::FCOS, Tmp.getValueType(), Tmp)); 1217 return; 1218 } 1219 } 1220 } 1221 } else if (isa<InlineAsm>(I.getOperand(0))) { 1222 visitInlineAsm(I); 1223 return; 1224 } 1225 1226 SDOperand Callee; 1227 if (!RenameFn) 1228 Callee = getValue(I.getOperand(0)); 1229 else 1230 Callee = DAG.getExternalSymbol(RenameFn, TLI.getPointerTy()); 1231 std::vector<std::pair<SDOperand, const Type*> > Args; 1232 Args.reserve(I.getNumOperands()); 1233 for (unsigned i = 1, e = I.getNumOperands(); i != e; ++i) { 1234 Value *Arg = I.getOperand(i); 1235 SDOperand ArgNode = getValue(Arg); 1236 Args.push_back(std::make_pair(ArgNode, Arg->getType())); 1237 } 1238 1239 const PointerType *PT = cast<PointerType>(I.getCalledValue()->getType()); 1240 const FunctionType *FTy = cast<FunctionType>(PT->getElementType()); 1241 1242 std::pair<SDOperand,SDOperand> Result = 1243 TLI.LowerCallTo(getRoot(), I.getType(), FTy->isVarArg(), I.getCallingConv(), 1244 I.isTailCall(), Callee, Args, DAG); 1245 if (I.getType() != Type::VoidTy) 1246 setValue(&I, Result.first); 1247 DAG.setRoot(Result.second); 1248} 1249 1250SDOperand RegsForValue::getCopyFromRegs(SelectionDAG &DAG, 1251 SDOperand &Chain, SDOperand &Flag)const{ 1252 SDOperand Val = DAG.getCopyFromReg(Chain, Regs[0], RegVT, Flag); 1253 Chain = Val.getValue(1); 1254 Flag = Val.getValue(2); 1255 1256 // If the result was expanded, copy from the top part. 1257 if (Regs.size() > 1) { 1258 assert(Regs.size() == 2 && 1259 "Cannot expand to more than 2 elts yet!"); 1260 SDOperand Hi = DAG.getCopyFromReg(Chain, Regs[1], RegVT, Flag); 1261 Chain = Val.getValue(1); 1262 Flag = Val.getValue(2); 1263 if (DAG.getTargetLoweringInfo().isLittleEndian()) 1264 return DAG.getNode(ISD::BUILD_PAIR, ValueVT, Val, Hi); 1265 else 1266 return DAG.getNode(ISD::BUILD_PAIR, ValueVT, Hi, Val); 1267 } 1268 1269 // Otherwise, if the return value was promoted, truncate it to the 1270 // appropriate type. 1271 if (RegVT == ValueVT) 1272 return Val; 1273 1274 if (MVT::isInteger(RegVT)) 1275 return DAG.getNode(ISD::TRUNCATE, ValueVT, Val); 1276 else 1277 return DAG.getNode(ISD::FP_ROUND, ValueVT, Val); 1278} 1279 1280/// getCopyToRegs - Emit a series of CopyToReg nodes that copies the 1281/// specified value into the registers specified by this object. This uses 1282/// Chain/Flag as the input and updates them for the output Chain/Flag. 1283void RegsForValue::getCopyToRegs(SDOperand Val, SelectionDAG &DAG, 1284 SDOperand &Chain, SDOperand &Flag) const { 1285 if (Regs.size() == 1) { 1286 // If there is a single register and the types differ, this must be 1287 // a promotion. 1288 if (RegVT != ValueVT) { 1289 if (MVT::isInteger(RegVT)) 1290 Val = DAG.getNode(ISD::ANY_EXTEND, RegVT, Val); 1291 else 1292 Val = DAG.getNode(ISD::FP_EXTEND, RegVT, Val); 1293 } 1294 Chain = DAG.getCopyToReg(Chain, Regs[0], Val, Flag); 1295 Flag = Chain.getValue(1); 1296 } else { 1297 std::vector<unsigned> R(Regs); 1298 if (!DAG.getTargetLoweringInfo().isLittleEndian()) 1299 std::reverse(R.begin(), R.end()); 1300 1301 for (unsigned i = 0, e = R.size(); i != e; ++i) { 1302 SDOperand Part = DAG.getNode(ISD::EXTRACT_ELEMENT, RegVT, Val, 1303 DAG.getConstant(i, MVT::i32)); 1304 Chain = DAG.getCopyToReg(Chain, R[i], Part, Flag); 1305 Flag = Chain.getValue(1); 1306 } 1307 } 1308} 1309 1310/// AddInlineAsmOperands - Add this value to the specified inlineasm node 1311/// operand list. This adds the code marker and includes the number of 1312/// values added into it. 1313void RegsForValue::AddInlineAsmOperands(unsigned Code, SelectionDAG &DAG, 1314 std::vector<SDOperand> &Ops) const { 1315 Ops.push_back(DAG.getConstant(Code | (Regs.size() << 3), MVT::i32)); 1316 for (unsigned i = 0, e = Regs.size(); i != e; ++i) 1317 Ops.push_back(DAG.getRegister(Regs[i], RegVT)); 1318} 1319 1320/// isAllocatableRegister - If the specified register is safe to allocate, 1321/// i.e. it isn't a stack pointer or some other special register, return the 1322/// register class for the register. Otherwise, return null. 1323static const TargetRegisterClass * 1324isAllocatableRegister(unsigned Reg, MachineFunction &MF, 1325 const TargetLowering &TLI, const MRegisterInfo *MRI) { 1326 for (MRegisterInfo::regclass_iterator RCI = MRI->regclass_begin(), 1327 E = MRI->regclass_end(); RCI != E; ++RCI) { 1328 const TargetRegisterClass *RC = *RCI; 1329 // If none of the the value types for this register class are valid, we 1330 // can't use it. For example, 64-bit reg classes on 32-bit targets. 1331 bool isLegal = false; 1332 for (TargetRegisterClass::vt_iterator I = RC->vt_begin(), E = RC->vt_end(); 1333 I != E; ++I) { 1334 if (TLI.isTypeLegal(*I)) { 1335 isLegal = true; 1336 break; 1337 } 1338 } 1339 1340 if (!isLegal) continue; 1341 1342 // NOTE: This isn't ideal. In particular, this might allocate the 1343 // frame pointer in functions that need it (due to them not being taken 1344 // out of allocation, because a variable sized allocation hasn't been seen 1345 // yet). This is a slight code pessimization, but should still work. 1346 for (TargetRegisterClass::iterator I = RC->allocation_order_begin(MF), 1347 E = RC->allocation_order_end(MF); I != E; ++I) 1348 if (*I == Reg) 1349 return RC; 1350 } 1351 return 0; 1352} 1353 1354RegsForValue SelectionDAGLowering:: 1355GetRegistersForValue(const std::string &ConstrCode, 1356 MVT::ValueType VT, bool isOutReg, bool isInReg, 1357 std::set<unsigned> &OutputRegs, 1358 std::set<unsigned> &InputRegs) { 1359 std::pair<unsigned, const TargetRegisterClass*> PhysReg = 1360 TLI.getRegForInlineAsmConstraint(ConstrCode, VT); 1361 std::vector<unsigned> Regs; 1362 1363 unsigned NumRegs = VT != MVT::Other ? TLI.getNumElements(VT) : 1; 1364 MVT::ValueType RegVT; 1365 MVT::ValueType ValueVT = VT; 1366 1367 if (PhysReg.first) { 1368 if (VT == MVT::Other) 1369 ValueVT = *PhysReg.second->vt_begin(); 1370 RegVT = VT; 1371 1372 // This is a explicit reference to a physical register. 1373 Regs.push_back(PhysReg.first); 1374 1375 // If this is an expanded reference, add the rest of the regs to Regs. 1376 if (NumRegs != 1) { 1377 RegVT = *PhysReg.second->vt_begin(); 1378 TargetRegisterClass::iterator I = PhysReg.second->begin(); 1379 TargetRegisterClass::iterator E = PhysReg.second->end(); 1380 for (; *I != PhysReg.first; ++I) 1381 assert(I != E && "Didn't find reg!"); 1382 1383 // Already added the first reg. 1384 --NumRegs; ++I; 1385 for (; NumRegs; --NumRegs, ++I) { 1386 assert(I != E && "Ran out of registers to allocate!"); 1387 Regs.push_back(*I); 1388 } 1389 } 1390 return RegsForValue(Regs, RegVT, ValueVT); 1391 } 1392 1393 // This is a reference to a register class. Allocate NumRegs consecutive, 1394 // available, registers from the class. 1395 std::vector<unsigned> RegClassRegs = 1396 TLI.getRegClassForInlineAsmConstraint(ConstrCode, VT); 1397 1398 const MRegisterInfo *MRI = DAG.getTarget().getRegisterInfo(); 1399 MachineFunction &MF = *CurMBB->getParent(); 1400 unsigned NumAllocated = 0; 1401 for (unsigned i = 0, e = RegClassRegs.size(); i != e; ++i) { 1402 unsigned Reg = RegClassRegs[i]; 1403 // See if this register is available. 1404 if ((isOutReg && OutputRegs.count(Reg)) || // Already used. 1405 (isInReg && InputRegs.count(Reg))) { // Already used. 1406 // Make sure we find consecutive registers. 1407 NumAllocated = 0; 1408 continue; 1409 } 1410 1411 // Check to see if this register is allocatable (i.e. don't give out the 1412 // stack pointer). 1413 const TargetRegisterClass *RC = isAllocatableRegister(Reg, MF, TLI, MRI); 1414 if (!RC) { 1415 // Make sure we find consecutive registers. 1416 NumAllocated = 0; 1417 continue; 1418 } 1419 1420 // Okay, this register is good, we can use it. 1421 ++NumAllocated; 1422 1423 // If we allocated enough consecutive 1424 if (NumAllocated == NumRegs) { 1425 unsigned RegStart = (i-NumAllocated)+1; 1426 unsigned RegEnd = i+1; 1427 // Mark all of the allocated registers used. 1428 for (unsigned i = RegStart; i != RegEnd; ++i) { 1429 unsigned Reg = RegClassRegs[i]; 1430 Regs.push_back(Reg); 1431 if (isOutReg) OutputRegs.insert(Reg); // Mark reg used. 1432 if (isInReg) InputRegs.insert(Reg); // Mark reg used. 1433 } 1434 1435 return RegsForValue(Regs, *RC->vt_begin(), VT); 1436 } 1437 } 1438 1439 // Otherwise, we couldn't allocate enough registers for this. 1440 return RegsForValue(); 1441} 1442 1443 1444/// visitInlineAsm - Handle a call to an InlineAsm object. 1445/// 1446void SelectionDAGLowering::visitInlineAsm(CallInst &I) { 1447 InlineAsm *IA = cast<InlineAsm>(I.getOperand(0)); 1448 1449 SDOperand AsmStr = DAG.getTargetExternalSymbol(IA->getAsmString().c_str(), 1450 MVT::Other); 1451 1452 // Note, we treat inline asms both with and without side-effects as the same. 1453 // If an inline asm doesn't have side effects and doesn't access memory, we 1454 // could not choose to not chain it. 1455 bool hasSideEffects = IA->hasSideEffects(); 1456 1457 std::vector<InlineAsm::ConstraintInfo> Constraints = IA->ParseConstraints(); 1458 std::vector<MVT::ValueType> ConstraintVTs; 1459 1460 /// AsmNodeOperands - A list of pairs. The first element is a register, the 1461 /// second is a bitfield where bit #0 is set if it is a use and bit #1 is set 1462 /// if it is a def of that register. 1463 std::vector<SDOperand> AsmNodeOperands; 1464 AsmNodeOperands.push_back(SDOperand()); // reserve space for input chain 1465 AsmNodeOperands.push_back(AsmStr); 1466 1467 SDOperand Chain = getRoot(); 1468 SDOperand Flag; 1469 1470 // We fully assign registers here at isel time. This is not optimal, but 1471 // should work. For register classes that correspond to LLVM classes, we 1472 // could let the LLVM RA do its thing, but we currently don't. Do a prepass 1473 // over the constraints, collecting fixed registers that we know we can't use. 1474 std::set<unsigned> OutputRegs, InputRegs; 1475 unsigned OpNum = 1; 1476 for (unsigned i = 0, e = Constraints.size(); i != e; ++i) { 1477 assert(Constraints[i].Codes.size() == 1 && "Only handles one code so far!"); 1478 std::string &ConstraintCode = Constraints[i].Codes[0]; 1479 1480 MVT::ValueType OpVT; 1481 1482 // Compute the value type for each operand and add it to ConstraintVTs. 1483 switch (Constraints[i].Type) { 1484 case InlineAsm::isOutput: 1485 if (!Constraints[i].isIndirectOutput) { 1486 assert(I.getType() != Type::VoidTy && "Bad inline asm!"); 1487 OpVT = TLI.getValueType(I.getType()); 1488 } else { 1489 const Type *OpTy = I.getOperand(OpNum)->getType(); 1490 OpVT = TLI.getValueType(cast<PointerType>(OpTy)->getElementType()); 1491 OpNum++; // Consumes a call operand. 1492 } 1493 break; 1494 case InlineAsm::isInput: 1495 OpVT = TLI.getValueType(I.getOperand(OpNum)->getType()); 1496 OpNum++; // Consumes a call operand. 1497 break; 1498 case InlineAsm::isClobber: 1499 OpVT = MVT::Other; 1500 break; 1501 } 1502 1503 ConstraintVTs.push_back(OpVT); 1504 1505 if (TLI.getRegForInlineAsmConstraint(ConstraintCode, OpVT).first == 0) 1506 continue; // Not assigned a fixed reg. 1507 1508 // Build a list of regs that this operand uses. This always has a single 1509 // element for promoted/expanded operands. 1510 RegsForValue Regs = GetRegistersForValue(ConstraintCode, OpVT, 1511 false, false, 1512 OutputRegs, InputRegs); 1513 1514 switch (Constraints[i].Type) { 1515 case InlineAsm::isOutput: 1516 // We can't assign any other output to this register. 1517 OutputRegs.insert(Regs.Regs.begin(), Regs.Regs.end()); 1518 // If this is an early-clobber output, it cannot be assigned to the same 1519 // value as the input reg. 1520 if (Constraints[i].isEarlyClobber || Constraints[i].hasMatchingInput) 1521 InputRegs.insert(Regs.Regs.begin(), Regs.Regs.end()); 1522 break; 1523 case InlineAsm::isInput: 1524 // We can't assign any other input to this register. 1525 InputRegs.insert(Regs.Regs.begin(), Regs.Regs.end()); 1526 break; 1527 case InlineAsm::isClobber: 1528 // Clobbered regs cannot be used as inputs or outputs. 1529 InputRegs.insert(Regs.Regs.begin(), Regs.Regs.end()); 1530 OutputRegs.insert(Regs.Regs.begin(), Regs.Regs.end()); 1531 break; 1532 } 1533 } 1534 1535 // Loop over all of the inputs, copying the operand values into the 1536 // appropriate registers and processing the output regs. 1537 RegsForValue RetValRegs; 1538 std::vector<std::pair<RegsForValue, Value*> > IndirectStoresToEmit; 1539 OpNum = 1; 1540 1541 for (unsigned i = 0, e = Constraints.size(); i != e; ++i) { 1542 assert(Constraints[i].Codes.size() == 1 && "Only handles one code so far!"); 1543 std::string &ConstraintCode = Constraints[i].Codes[0]; 1544 1545 switch (Constraints[i].Type) { 1546 case InlineAsm::isOutput: { 1547 TargetLowering::ConstraintType CTy = TargetLowering::C_RegisterClass; 1548 if (ConstraintCode.size() == 1) // not a physreg name. 1549 CTy = TLI.getConstraintType(ConstraintCode[0]); 1550 1551 if (CTy == TargetLowering::C_Memory) { 1552 // Memory output. 1553 SDOperand InOperandVal = getValue(I.getOperand(OpNum)); 1554 1555 // Check that the operand (the address to store to) isn't a float. 1556 if (!MVT::isInteger(InOperandVal.getValueType())) 1557 assert(0 && "MATCH FAIL!"); 1558 1559 if (!Constraints[i].isIndirectOutput) 1560 assert(0 && "MATCH FAIL!"); 1561 1562 OpNum++; // Consumes a call operand. 1563 1564 // Extend/truncate to the right pointer type if needed. 1565 MVT::ValueType PtrType = TLI.getPointerTy(); 1566 if (InOperandVal.getValueType() < PtrType) 1567 InOperandVal = DAG.getNode(ISD::ZERO_EXTEND, PtrType, InOperandVal); 1568 else if (InOperandVal.getValueType() > PtrType) 1569 InOperandVal = DAG.getNode(ISD::TRUNCATE, PtrType, InOperandVal); 1570 1571 // Add information to the INLINEASM node to know about this output. 1572 unsigned ResOpType = 4/*MEM*/ | (1 << 3); 1573 AsmNodeOperands.push_back(DAG.getConstant(ResOpType, MVT::i32)); 1574 AsmNodeOperands.push_back(InOperandVal); 1575 break; 1576 } 1577 1578 // Otherwise, this is a register output. 1579 assert(CTy == TargetLowering::C_RegisterClass && "Unknown op type!"); 1580 1581 // If this is an early-clobber output, or if there is an input 1582 // constraint that matches this, we need to reserve the input register 1583 // so no other inputs allocate to it. 1584 bool UsesInputRegister = false; 1585 if (Constraints[i].isEarlyClobber || Constraints[i].hasMatchingInput) 1586 UsesInputRegister = true; 1587 1588 // Copy the output from the appropriate register. Find a register that 1589 // we can use. 1590 RegsForValue Regs = 1591 GetRegistersForValue(ConstraintCode, ConstraintVTs[i], 1592 true, UsesInputRegister, 1593 OutputRegs, InputRegs); 1594 assert(!Regs.Regs.empty() && "Couldn't allocate output reg!"); 1595 1596 if (!Constraints[i].isIndirectOutput) { 1597 assert(RetValRegs.Regs.empty() && 1598 "Cannot have multiple output constraints yet!"); 1599 assert(I.getType() != Type::VoidTy && "Bad inline asm!"); 1600 RetValRegs = Regs; 1601 } else { 1602 IndirectStoresToEmit.push_back(std::make_pair(Regs, 1603 I.getOperand(OpNum))); 1604 OpNum++; // Consumes a call operand. 1605 } 1606 1607 // Add information to the INLINEASM node to know that this register is 1608 // set. 1609 Regs.AddInlineAsmOperands(2 /*REGDEF*/, DAG, AsmNodeOperands); 1610 break; 1611 } 1612 case InlineAsm::isInput: { 1613 SDOperand InOperandVal = getValue(I.getOperand(OpNum)); 1614 OpNum++; // Consumes a call operand. 1615 1616 if (isdigit(ConstraintCode[0])) { // Matching constraint? 1617 // If this is required to match an output register we have already set, 1618 // just use its register. 1619 unsigned OperandNo = atoi(ConstraintCode.c_str()); 1620 1621 // Scan until we find the definition we already emitted of this operand. 1622 // When we find it, create a RegsForValue operand. 1623 unsigned CurOp = 2; // The first operand. 1624 for (; OperandNo; --OperandNo) { 1625 // Advance to the next operand. 1626 unsigned NumOps = 1627 cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getValue(); 1628 assert((NumOps & 7) == 2 /*REGDEF*/ && 1629 "Skipped past definitions?"); 1630 CurOp += (NumOps>>3)+1; 1631 } 1632 1633 unsigned NumOps = 1634 cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getValue(); 1635 assert((NumOps & 7) == 2 /*REGDEF*/ && 1636 "Skipped past definitions?"); 1637 1638 // Add NumOps>>3 registers to MatchedRegs. 1639 RegsForValue MatchedRegs; 1640 MatchedRegs.ValueVT = InOperandVal.getValueType(); 1641 MatchedRegs.RegVT = AsmNodeOperands[CurOp+1].getValueType(); 1642 for (unsigned i = 0, e = NumOps>>3; i != e; ++i) { 1643 unsigned Reg=cast<RegisterSDNode>(AsmNodeOperands[++CurOp])->getReg(); 1644 MatchedRegs.Regs.push_back(Reg); 1645 } 1646 1647 // Use the produced MatchedRegs object to 1648 MatchedRegs.getCopyToRegs(InOperandVal, DAG, Chain, Flag); 1649 MatchedRegs.AddInlineAsmOperands(1 /*REGUSE*/, DAG, AsmNodeOperands); 1650 break; 1651 } 1652 1653 TargetLowering::ConstraintType CTy = TargetLowering::C_RegisterClass; 1654 if (ConstraintCode.size() == 1) // not a physreg name. 1655 CTy = TLI.getConstraintType(ConstraintCode[0]); 1656 1657 if (CTy == TargetLowering::C_Other) { 1658 if (!TLI.isOperandValidForConstraint(InOperandVal, ConstraintCode[0])) 1659 assert(0 && "MATCH FAIL!"); 1660 1661 // Add information to the INLINEASM node to know about this input. 1662 unsigned ResOpType = 3 /*IMM*/ | (1 << 3); 1663 AsmNodeOperands.push_back(DAG.getConstant(ResOpType, MVT::i32)); 1664 AsmNodeOperands.push_back(InOperandVal); 1665 break; 1666 } else if (CTy == TargetLowering::C_Memory) { 1667 // Memory input. 1668 1669 // Check that the operand isn't a float. 1670 if (!MVT::isInteger(InOperandVal.getValueType())) 1671 assert(0 && "MATCH FAIL!"); 1672 1673 // Extend/truncate to the right pointer type if needed. 1674 MVT::ValueType PtrType = TLI.getPointerTy(); 1675 if (InOperandVal.getValueType() < PtrType) 1676 InOperandVal = DAG.getNode(ISD::ZERO_EXTEND, PtrType, InOperandVal); 1677 else if (InOperandVal.getValueType() > PtrType) 1678 InOperandVal = DAG.getNode(ISD::TRUNCATE, PtrType, InOperandVal); 1679 1680 // Add information to the INLINEASM node to know about this input. 1681 unsigned ResOpType = 4/*MEM*/ | (1 << 3); 1682 AsmNodeOperands.push_back(DAG.getConstant(ResOpType, MVT::i32)); 1683 AsmNodeOperands.push_back(InOperandVal); 1684 break; 1685 } 1686 1687 assert(CTy == TargetLowering::C_RegisterClass && "Unknown op type!"); 1688 1689 // Copy the input into the appropriate registers. 1690 RegsForValue InRegs = 1691 GetRegistersForValue(ConstraintCode, ConstraintVTs[i], 1692 false, true, OutputRegs, InputRegs); 1693 // FIXME: should be match fail. 1694 assert(!InRegs.Regs.empty() && "Couldn't allocate input reg!"); 1695 1696 InRegs.getCopyToRegs(InOperandVal, DAG, Chain, Flag); 1697 1698 InRegs.AddInlineAsmOperands(1/*REGUSE*/, DAG, AsmNodeOperands); 1699 break; 1700 } 1701 case InlineAsm::isClobber: { 1702 RegsForValue ClobberedRegs = 1703 GetRegistersForValue(ConstraintCode, MVT::Other, false, false, 1704 OutputRegs, InputRegs); 1705 // Add the clobbered value to the operand list, so that the register 1706 // allocator is aware that the physreg got clobbered. 1707 if (!ClobberedRegs.Regs.empty()) 1708 ClobberedRegs.AddInlineAsmOperands(2/*REGDEF*/, DAG, AsmNodeOperands); 1709 break; 1710 } 1711 } 1712 } 1713 1714 // Finish up input operands. 1715 AsmNodeOperands[0] = Chain; 1716 if (Flag.Val) AsmNodeOperands.push_back(Flag); 1717 1718 std::vector<MVT::ValueType> VTs; 1719 VTs.push_back(MVT::Other); 1720 VTs.push_back(MVT::Flag); 1721 Chain = DAG.getNode(ISD::INLINEASM, VTs, AsmNodeOperands); 1722 Flag = Chain.getValue(1); 1723 1724 // If this asm returns a register value, copy the result from that register 1725 // and set it as the value of the call. 1726 if (!RetValRegs.Regs.empty()) 1727 setValue(&I, RetValRegs.getCopyFromRegs(DAG, Chain, Flag)); 1728 1729 std::vector<std::pair<SDOperand, Value*> > StoresToEmit; 1730 1731 // Process indirect outputs, first output all of the flagged copies out of 1732 // physregs. 1733 for (unsigned i = 0, e = IndirectStoresToEmit.size(); i != e; ++i) { 1734 RegsForValue &OutRegs = IndirectStoresToEmit[i].first; 1735 Value *Ptr = IndirectStoresToEmit[i].second; 1736 SDOperand OutVal = OutRegs.getCopyFromRegs(DAG, Chain, Flag); 1737 StoresToEmit.push_back(std::make_pair(OutVal, Ptr)); 1738 } 1739 1740 // Emit the non-flagged stores from the physregs. 1741 std::vector<SDOperand> OutChains; 1742 for (unsigned i = 0, e = StoresToEmit.size(); i != e; ++i) 1743 OutChains.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain, 1744 StoresToEmit[i].first, 1745 getValue(StoresToEmit[i].second), 1746 DAG.getSrcValue(StoresToEmit[i].second))); 1747 if (!OutChains.empty()) 1748 Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, OutChains); 1749 DAG.setRoot(Chain); 1750} 1751 1752 1753void SelectionDAGLowering::visitMalloc(MallocInst &I) { 1754 SDOperand Src = getValue(I.getOperand(0)); 1755 1756 MVT::ValueType IntPtr = TLI.getPointerTy(); 1757 1758 if (IntPtr < Src.getValueType()) 1759 Src = DAG.getNode(ISD::TRUNCATE, IntPtr, Src); 1760 else if (IntPtr > Src.getValueType()) 1761 Src = DAG.getNode(ISD::ZERO_EXTEND, IntPtr, Src); 1762 1763 // Scale the source by the type size. 1764 uint64_t ElementSize = TD.getTypeSize(I.getType()->getElementType()); 1765 Src = DAG.getNode(ISD::MUL, Src.getValueType(), 1766 Src, getIntPtrConstant(ElementSize)); 1767 1768 std::vector<std::pair<SDOperand, const Type*> > Args; 1769 Args.push_back(std::make_pair(Src, TLI.getTargetData().getIntPtrType())); 1770 1771 std::pair<SDOperand,SDOperand> Result = 1772 TLI.LowerCallTo(getRoot(), I.getType(), false, CallingConv::C, true, 1773 DAG.getExternalSymbol("malloc", IntPtr), 1774 Args, DAG); 1775 setValue(&I, Result.first); // Pointers always fit in registers 1776 DAG.setRoot(Result.second); 1777} 1778 1779void SelectionDAGLowering::visitFree(FreeInst &I) { 1780 std::vector<std::pair<SDOperand, const Type*> > Args; 1781 Args.push_back(std::make_pair(getValue(I.getOperand(0)), 1782 TLI.getTargetData().getIntPtrType())); 1783 MVT::ValueType IntPtr = TLI.getPointerTy(); 1784 std::pair<SDOperand,SDOperand> Result = 1785 TLI.LowerCallTo(getRoot(), Type::VoidTy, false, CallingConv::C, true, 1786 DAG.getExternalSymbol("free", IntPtr), Args, DAG); 1787 DAG.setRoot(Result.second); 1788} 1789 1790// InsertAtEndOfBasicBlock - This method should be implemented by targets that 1791// mark instructions with the 'usesCustomDAGSchedInserter' flag. These 1792// instructions are special in various ways, which require special support to 1793// insert. The specified MachineInstr is created but not inserted into any 1794// basic blocks, and the scheduler passes ownership of it to this method. 1795MachineBasicBlock *TargetLowering::InsertAtEndOfBasicBlock(MachineInstr *MI, 1796 MachineBasicBlock *MBB) { 1797 std::cerr << "If a target marks an instruction with " 1798 "'usesCustomDAGSchedInserter', it must implement " 1799 "TargetLowering::InsertAtEndOfBasicBlock!\n"; 1800 abort(); 1801 return 0; 1802} 1803 1804void SelectionDAGLowering::visitVAStart(CallInst &I) { 1805 DAG.setRoot(DAG.getNode(ISD::VASTART, MVT::Other, getRoot(), 1806 getValue(I.getOperand(1)), 1807 DAG.getSrcValue(I.getOperand(1)))); 1808} 1809 1810void SelectionDAGLowering::visitVAArg(VAArgInst &I) { 1811 SDOperand V = DAG.getVAArg(TLI.getValueType(I.getType()), getRoot(), 1812 getValue(I.getOperand(0)), 1813 DAG.getSrcValue(I.getOperand(0))); 1814 setValue(&I, V); 1815 DAG.setRoot(V.getValue(1)); 1816} 1817 1818void SelectionDAGLowering::visitVAEnd(CallInst &I) { 1819 DAG.setRoot(DAG.getNode(ISD::VAEND, MVT::Other, getRoot(), 1820 getValue(I.getOperand(1)), 1821 DAG.getSrcValue(I.getOperand(1)))); 1822} 1823 1824void SelectionDAGLowering::visitVACopy(CallInst &I) { 1825 DAG.setRoot(DAG.getNode(ISD::VACOPY, MVT::Other, getRoot(), 1826 getValue(I.getOperand(1)), 1827 getValue(I.getOperand(2)), 1828 DAG.getSrcValue(I.getOperand(1)), 1829 DAG.getSrcValue(I.getOperand(2)))); 1830} 1831 1832// It is always conservatively correct for llvm.returnaddress and 1833// llvm.frameaddress to return 0. 1834std::pair<SDOperand, SDOperand> 1835TargetLowering::LowerFrameReturnAddress(bool isFrameAddr, SDOperand Chain, 1836 unsigned Depth, SelectionDAG &DAG) { 1837 return std::make_pair(DAG.getConstant(0, getPointerTy()), Chain); 1838} 1839 1840SDOperand TargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) { 1841 assert(0 && "LowerOperation not implemented for this target!"); 1842 abort(); 1843 return SDOperand(); 1844} 1845 1846SDOperand TargetLowering::CustomPromoteOperation(SDOperand Op, 1847 SelectionDAG &DAG) { 1848 assert(0 && "CustomPromoteOperation not implemented for this target!"); 1849 abort(); 1850 return SDOperand(); 1851} 1852 1853void SelectionDAGLowering::visitFrameReturnAddress(CallInst &I, bool isFrame) { 1854 unsigned Depth = (unsigned)cast<ConstantUInt>(I.getOperand(1))->getValue(); 1855 std::pair<SDOperand,SDOperand> Result = 1856 TLI.LowerFrameReturnAddress(isFrame, getRoot(), Depth, DAG); 1857 setValue(&I, Result.first); 1858 DAG.setRoot(Result.second); 1859} 1860 1861/// getMemsetValue - Vectorized representation of the memset value 1862/// operand. 1863static SDOperand getMemsetValue(SDOperand Value, MVT::ValueType VT, 1864 SelectionDAG &DAG) { 1865 MVT::ValueType CurVT = VT; 1866 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) { 1867 uint64_t Val = C->getValue() & 255; 1868 unsigned Shift = 8; 1869 while (CurVT != MVT::i8) { 1870 Val = (Val << Shift) | Val; 1871 Shift <<= 1; 1872 CurVT = (MVT::ValueType)((unsigned)CurVT - 1); 1873 } 1874 return DAG.getConstant(Val, VT); 1875 } else { 1876 Value = DAG.getNode(ISD::ZERO_EXTEND, VT, Value); 1877 unsigned Shift = 8; 1878 while (CurVT != MVT::i8) { 1879 Value = 1880 DAG.getNode(ISD::OR, VT, 1881 DAG.getNode(ISD::SHL, VT, Value, 1882 DAG.getConstant(Shift, MVT::i8)), Value); 1883 Shift <<= 1; 1884 CurVT = (MVT::ValueType)((unsigned)CurVT - 1); 1885 } 1886 1887 return Value; 1888 } 1889} 1890 1891/// getMemsetStringVal - Similar to getMemsetValue. Except this is only 1892/// used when a memcpy is turned into a memset when the source is a constant 1893/// string ptr. 1894static SDOperand getMemsetStringVal(MVT::ValueType VT, 1895 SelectionDAG &DAG, TargetLowering &TLI, 1896 std::string &Str, unsigned Offset) { 1897 MVT::ValueType CurVT = VT; 1898 uint64_t Val = 0; 1899 unsigned MSB = getSizeInBits(VT) / 8; 1900 if (TLI.isLittleEndian()) 1901 Offset = Offset + MSB - 1; 1902 for (unsigned i = 0; i != MSB; ++i) { 1903 Val = (Val << 8) | Str[Offset]; 1904 Offset += TLI.isLittleEndian() ? -1 : 1; 1905 } 1906 return DAG.getConstant(Val, VT); 1907} 1908 1909/// getMemBasePlusOffset - Returns base and offset node for the 1910static SDOperand getMemBasePlusOffset(SDOperand Base, unsigned Offset, 1911 SelectionDAG &DAG, TargetLowering &TLI) { 1912 MVT::ValueType VT = Base.getValueType(); 1913 return DAG.getNode(ISD::ADD, VT, Base, DAG.getConstant(Offset, VT)); 1914} 1915 1916/// MeetsMaxMemopRequirement - Determines if the number of memory ops required 1917/// to replace the memset / memcpy is below the threshold. It also returns the 1918/// types of the sequence of memory ops to perform memset / memcpy. 1919static bool MeetsMaxMemopRequirement(std::vector<MVT::ValueType> &MemOps, 1920 unsigned Limit, uint64_t Size, 1921 unsigned Align, TargetLowering &TLI) { 1922 MVT::ValueType VT; 1923 1924 if (TLI.allowsUnalignedMemoryAccesses()) { 1925 VT = MVT::i64; 1926 } else { 1927 switch (Align & 7) { 1928 case 0: 1929 VT = MVT::i64; 1930 break; 1931 case 4: 1932 VT = MVT::i32; 1933 break; 1934 case 2: 1935 VT = MVT::i16; 1936 break; 1937 default: 1938 VT = MVT::i8; 1939 break; 1940 } 1941 } 1942 1943 MVT::ValueType LVT = MVT::i64; 1944 while (!TLI.isTypeLegal(LVT)) 1945 LVT = (MVT::ValueType)((unsigned)LVT - 1); 1946 assert(MVT::isInteger(LVT)); 1947 1948 if (VT > LVT) 1949 VT = LVT; 1950 1951 unsigned NumMemOps = 0; 1952 while (Size != 0) { 1953 unsigned VTSize = getSizeInBits(VT) / 8; 1954 while (VTSize > Size) { 1955 VT = (MVT::ValueType)((unsigned)VT - 1); 1956 VTSize >>= 1; 1957 } 1958 assert(MVT::isInteger(VT)); 1959 1960 if (++NumMemOps > Limit) 1961 return false; 1962 MemOps.push_back(VT); 1963 Size -= VTSize; 1964 } 1965 1966 return true; 1967} 1968 1969void SelectionDAGLowering::visitMemIntrinsic(CallInst &I, unsigned Op) { 1970 SDOperand Op1 = getValue(I.getOperand(1)); 1971 SDOperand Op2 = getValue(I.getOperand(2)); 1972 SDOperand Op3 = getValue(I.getOperand(3)); 1973 SDOperand Op4 = getValue(I.getOperand(4)); 1974 unsigned Align = (unsigned)cast<ConstantSDNode>(Op4)->getValue(); 1975 if (Align == 0) Align = 1; 1976 1977 if (ConstantSDNode *Size = dyn_cast<ConstantSDNode>(Op3)) { 1978 std::vector<MVT::ValueType> MemOps; 1979 1980 // Expand memset / memcpy to a series of load / store ops 1981 // if the size operand falls below a certain threshold. 1982 std::vector<SDOperand> OutChains; 1983 switch (Op) { 1984 default: break; // Do nothing for now. 1985 case ISD::MEMSET: { 1986 if (MeetsMaxMemopRequirement(MemOps, TLI.getMaxStoresPerMemset(), 1987 Size->getValue(), Align, TLI)) { 1988 unsigned NumMemOps = MemOps.size(); 1989 unsigned Offset = 0; 1990 for (unsigned i = 0; i < NumMemOps; i++) { 1991 MVT::ValueType VT = MemOps[i]; 1992 unsigned VTSize = getSizeInBits(VT) / 8; 1993 SDOperand Value = getMemsetValue(Op2, VT, DAG); 1994 SDOperand Store = DAG.getNode(ISD::STORE, MVT::Other, getRoot(), 1995 Value, 1996 getMemBasePlusOffset(Op1, Offset, DAG, TLI), 1997 DAG.getSrcValue(I.getOperand(1), Offset)); 1998 OutChains.push_back(Store); 1999 Offset += VTSize; 2000 } 2001 } 2002 break; 2003 } 2004 case ISD::MEMCPY: { 2005 if (MeetsMaxMemopRequirement(MemOps, TLI.getMaxStoresPerMemcpy(), 2006 Size->getValue(), Align, TLI)) { 2007 unsigned NumMemOps = MemOps.size(); 2008 unsigned SrcOff = 0, DstOff = 0, SrcDelta = 0; 2009 GlobalAddressSDNode *G = NULL; 2010 std::string Str; 2011 bool CopyFromStr = false; 2012 2013 if (Op2.getOpcode() == ISD::GlobalAddress) 2014 G = cast<GlobalAddressSDNode>(Op2); 2015 else if (Op2.getOpcode() == ISD::ADD && 2016 Op2.getOperand(0).getOpcode() == ISD::GlobalAddress && 2017 Op2.getOperand(1).getOpcode() == ISD::Constant) { 2018 G = cast<GlobalAddressSDNode>(Op2.getOperand(0)); 2019 SrcDelta = cast<ConstantSDNode>(Op2.getOperand(1))->getValue(); 2020 } 2021 if (G) { 2022 GlobalVariable *GV = dyn_cast<GlobalVariable>(G->getGlobal()); 2023 if (GV) { 2024 Str = GV->getStringValue(false); 2025 if (!Str.empty()) { 2026 CopyFromStr = true; 2027 SrcOff += SrcDelta; 2028 } 2029 } 2030 } 2031 2032 for (unsigned i = 0; i < NumMemOps; i++) { 2033 MVT::ValueType VT = MemOps[i]; 2034 unsigned VTSize = getSizeInBits(VT) / 8; 2035 SDOperand Value, Chain, Store; 2036 2037 if (CopyFromStr) { 2038 Value = getMemsetStringVal(VT, DAG, TLI, Str, SrcOff); 2039 Chain = getRoot(); 2040 Store = 2041 DAG.getNode(ISD::STORE, MVT::Other, Chain, Value, 2042 getMemBasePlusOffset(Op1, DstOff, DAG, TLI), 2043 DAG.getSrcValue(I.getOperand(1), DstOff)); 2044 } else { 2045 Value = DAG.getLoad(VT, getRoot(), 2046 getMemBasePlusOffset(Op2, SrcOff, DAG, TLI), 2047 DAG.getSrcValue(I.getOperand(2), SrcOff)); 2048 Chain = Value.getValue(1); 2049 Store = 2050 DAG.getNode(ISD::STORE, MVT::Other, Chain, Value, 2051 getMemBasePlusOffset(Op1, DstOff, DAG, TLI), 2052 DAG.getSrcValue(I.getOperand(1), DstOff)); 2053 } 2054 OutChains.push_back(Store); 2055 SrcOff += VTSize; 2056 DstOff += VTSize; 2057 } 2058 } 2059 break; 2060 } 2061 } 2062 2063 if (!OutChains.empty()) { 2064 DAG.setRoot(DAG.getNode(ISD::TokenFactor, MVT::Other, OutChains)); 2065 return; 2066 } 2067 } 2068 2069 std::vector<SDOperand> Ops; 2070 Ops.push_back(getRoot()); 2071 Ops.push_back(Op1); 2072 Ops.push_back(Op2); 2073 Ops.push_back(Op3); 2074 Ops.push_back(Op4); 2075 DAG.setRoot(DAG.getNode(Op, MVT::Other, Ops)); 2076} 2077 2078//===----------------------------------------------------------------------===// 2079// SelectionDAGISel code 2080//===----------------------------------------------------------------------===// 2081 2082unsigned SelectionDAGISel::MakeReg(MVT::ValueType VT) { 2083 return RegMap->createVirtualRegister(TLI.getRegClassFor(VT)); 2084} 2085 2086void SelectionDAGISel::getAnalysisUsage(AnalysisUsage &AU) const { 2087 // FIXME: we only modify the CFG to split critical edges. This 2088 // updates dom and loop info. 2089} 2090 2091 2092/// InsertGEPComputeCode - Insert code into BB to compute Ptr+PtrOffset, 2093/// casting to the type of GEPI. 2094static Value *InsertGEPComputeCode(Value *&V, BasicBlock *BB, Instruction *GEPI, 2095 Value *Ptr, Value *PtrOffset) { 2096 if (V) return V; // Already computed. 2097 2098 BasicBlock::iterator InsertPt; 2099 if (BB == GEPI->getParent()) { 2100 // If insert into the GEP's block, insert right after the GEP. 2101 InsertPt = GEPI; 2102 ++InsertPt; 2103 } else { 2104 // Otherwise, insert at the top of BB, after any PHI nodes 2105 InsertPt = BB->begin(); 2106 while (isa<PHINode>(InsertPt)) ++InsertPt; 2107 } 2108 2109 // If Ptr is itself a cast, but in some other BB, emit a copy of the cast into 2110 // BB so that there is only one value live across basic blocks (the cast 2111 // operand). 2112 if (CastInst *CI = dyn_cast<CastInst>(Ptr)) 2113 if (CI->getParent() != BB && isa<PointerType>(CI->getOperand(0)->getType())) 2114 Ptr = new CastInst(CI->getOperand(0), CI->getType(), "", InsertPt); 2115 2116 // Add the offset, cast it to the right type. 2117 Ptr = BinaryOperator::createAdd(Ptr, PtrOffset, "", InsertPt); 2118 Ptr = new CastInst(Ptr, GEPI->getType(), "", InsertPt); 2119 return V = Ptr; 2120} 2121 2122 2123/// OptimizeGEPExpression - Since we are doing basic-block-at-a-time instruction 2124/// selection, we want to be a bit careful about some things. In particular, if 2125/// we have a GEP instruction that is used in a different block than it is 2126/// defined, the addressing expression of the GEP cannot be folded into loads or 2127/// stores that use it. In this case, decompose the GEP and move constant 2128/// indices into blocks that use it. 2129static void OptimizeGEPExpression(GetElementPtrInst *GEPI, 2130 const TargetData &TD) { 2131 // If this GEP is only used inside the block it is defined in, there is no 2132 // need to rewrite it. 2133 bool isUsedOutsideDefBB = false; 2134 BasicBlock *DefBB = GEPI->getParent(); 2135 for (Value::use_iterator UI = GEPI->use_begin(), E = GEPI->use_end(); 2136 UI != E; ++UI) { 2137 if (cast<Instruction>(*UI)->getParent() != DefBB) { 2138 isUsedOutsideDefBB = true; 2139 break; 2140 } 2141 } 2142 if (!isUsedOutsideDefBB) return; 2143 2144 // If this GEP has no non-zero constant indices, there is nothing we can do, 2145 // ignore it. 2146 bool hasConstantIndex = false; 2147 for (GetElementPtrInst::op_iterator OI = GEPI->op_begin()+1, 2148 E = GEPI->op_end(); OI != E; ++OI) { 2149 if (ConstantInt *CI = dyn_cast<ConstantInt>(*OI)) 2150 if (CI->getRawValue()) { 2151 hasConstantIndex = true; 2152 break; 2153 } 2154 } 2155 // If this is a GEP &Alloca, 0, 0, forward subst the frame index into uses. 2156 if (!hasConstantIndex && !isa<AllocaInst>(GEPI->getOperand(0))) return; 2157 2158 // Otherwise, decompose the GEP instruction into multiplies and adds. Sum the 2159 // constant offset (which we now know is non-zero) and deal with it later. 2160 uint64_t ConstantOffset = 0; 2161 const Type *UIntPtrTy = TD.getIntPtrType(); 2162 Value *Ptr = new CastInst(GEPI->getOperand(0), UIntPtrTy, "", GEPI); 2163 const Type *Ty = GEPI->getOperand(0)->getType(); 2164 2165 for (GetElementPtrInst::op_iterator OI = GEPI->op_begin()+1, 2166 E = GEPI->op_end(); OI != E; ++OI) { 2167 Value *Idx = *OI; 2168 if (const StructType *StTy = dyn_cast<StructType>(Ty)) { 2169 unsigned Field = cast<ConstantUInt>(Idx)->getValue(); 2170 if (Field) 2171 ConstantOffset += TD.getStructLayout(StTy)->MemberOffsets[Field]; 2172 Ty = StTy->getElementType(Field); 2173 } else { 2174 Ty = cast<SequentialType>(Ty)->getElementType(); 2175 2176 // Handle constant subscripts. 2177 if (ConstantInt *CI = dyn_cast<ConstantInt>(Idx)) { 2178 if (CI->getRawValue() == 0) continue; 2179 2180 if (ConstantSInt *CSI = dyn_cast<ConstantSInt>(CI)) 2181 ConstantOffset += (int64_t)TD.getTypeSize(Ty)*CSI->getValue(); 2182 else 2183 ConstantOffset+=TD.getTypeSize(Ty)*cast<ConstantUInt>(CI)->getValue(); 2184 continue; 2185 } 2186 2187 // Ptr = Ptr + Idx * ElementSize; 2188 2189 // Cast Idx to UIntPtrTy if needed. 2190 Idx = new CastInst(Idx, UIntPtrTy, "", GEPI); 2191 2192 uint64_t ElementSize = TD.getTypeSize(Ty); 2193 // Mask off bits that should not be set. 2194 ElementSize &= ~0ULL >> (64-UIntPtrTy->getPrimitiveSizeInBits()); 2195 Constant *SizeCst = ConstantUInt::get(UIntPtrTy, ElementSize); 2196 2197 // Multiply by the element size and add to the base. 2198 Idx = BinaryOperator::createMul(Idx, SizeCst, "", GEPI); 2199 Ptr = BinaryOperator::createAdd(Ptr, Idx, "", GEPI); 2200 } 2201 } 2202 2203 // Make sure that the offset fits in uintptr_t. 2204 ConstantOffset &= ~0ULL >> (64-UIntPtrTy->getPrimitiveSizeInBits()); 2205 Constant *PtrOffset = ConstantUInt::get(UIntPtrTy, ConstantOffset); 2206 2207 // Okay, we have now emitted all of the variable index parts to the BB that 2208 // the GEP is defined in. Loop over all of the using instructions, inserting 2209 // an "add Ptr, ConstantOffset" into each block that uses it and update the 2210 // instruction to use the newly computed value, making GEPI dead. When the 2211 // user is a load or store instruction address, we emit the add into the user 2212 // block, otherwise we use a canonical version right next to the gep (these 2213 // won't be foldable as addresses, so we might as well share the computation). 2214 2215 std::map<BasicBlock*,Value*> InsertedExprs; 2216 while (!GEPI->use_empty()) { 2217 Instruction *User = cast<Instruction>(GEPI->use_back()); 2218 2219 // If this use is not foldable into the addressing mode, use a version 2220 // emitted in the GEP block. 2221 Value *NewVal; 2222 if (!isa<LoadInst>(User) && 2223 (!isa<StoreInst>(User) || User->getOperand(0) == GEPI)) { 2224 NewVal = InsertGEPComputeCode(InsertedExprs[DefBB], DefBB, GEPI, 2225 Ptr, PtrOffset); 2226 } else { 2227 // Otherwise, insert the code in the User's block so it can be folded into 2228 // any users in that block. 2229 NewVal = InsertGEPComputeCode(InsertedExprs[User->getParent()], 2230 User->getParent(), GEPI, 2231 Ptr, PtrOffset); 2232 } 2233 User->replaceUsesOfWith(GEPI, NewVal); 2234 } 2235 2236 // Finally, the GEP is dead, remove it. 2237 GEPI->eraseFromParent(); 2238} 2239 2240bool SelectionDAGISel::runOnFunction(Function &Fn) { 2241 MachineFunction &MF = MachineFunction::construct(&Fn, TLI.getTargetMachine()); 2242 RegMap = MF.getSSARegMap(); 2243 DEBUG(std::cerr << "\n\n\n=== " << Fn.getName() << "\n"); 2244 2245 // First, split all critical edges for PHI nodes with incoming values that are 2246 // constants, this way the load of the constant into a vreg will not be placed 2247 // into MBBs that are used some other way. 2248 // 2249 // In this pass we also look for GEP instructions that are used across basic 2250 // blocks and rewrites them to improve basic-block-at-a-time selection. 2251 // 2252 for (Function::iterator BB = Fn.begin(), E = Fn.end(); BB != E; ++BB) { 2253 PHINode *PN; 2254 BasicBlock::iterator BBI; 2255 for (BBI = BB->begin(); (PN = dyn_cast<PHINode>(BBI)); ++BBI) 2256 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) 2257 if (isa<Constant>(PN->getIncomingValue(i))) 2258 SplitCriticalEdge(PN->getIncomingBlock(i), BB); 2259 2260 for (BasicBlock::iterator E = BB->end(); BBI != E; ) 2261 if (GetElementPtrInst *GEPI = dyn_cast<GetElementPtrInst>(BBI++)) 2262 OptimizeGEPExpression(GEPI, TLI.getTargetData()); 2263 } 2264 2265 FunctionLoweringInfo FuncInfo(TLI, Fn, MF); 2266 2267 for (Function::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I) 2268 SelectBasicBlock(I, MF, FuncInfo); 2269 2270 return true; 2271} 2272 2273 2274SDOperand SelectionDAGISel:: 2275CopyValueToVirtualRegister(SelectionDAGLowering &SDL, Value *V, unsigned Reg) { 2276 SDOperand Op = SDL.getValue(V); 2277 assert((Op.getOpcode() != ISD::CopyFromReg || 2278 cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) && 2279 "Copy from a reg to the same reg!"); 2280 2281 // If this type is not legal, we must make sure to not create an invalid 2282 // register use. 2283 MVT::ValueType SrcVT = Op.getValueType(); 2284 MVT::ValueType DestVT = TLI.getTypeToTransformTo(SrcVT); 2285 SelectionDAG &DAG = SDL.DAG; 2286 if (SrcVT == DestVT) { 2287 return DAG.getCopyToReg(SDL.getRoot(), Reg, Op); 2288 } else if (SrcVT < DestVT) { 2289 // The src value is promoted to the register. 2290 if (MVT::isFloatingPoint(SrcVT)) 2291 Op = DAG.getNode(ISD::FP_EXTEND, DestVT, Op); 2292 else 2293 Op = DAG.getNode(ISD::ANY_EXTEND, DestVT, Op); 2294 return DAG.getCopyToReg(SDL.getRoot(), Reg, Op); 2295 } else { 2296 // The src value is expanded into multiple registers. 2297 SDOperand Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DestVT, 2298 Op, DAG.getConstant(0, MVT::i32)); 2299 SDOperand Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DestVT, 2300 Op, DAG.getConstant(1, MVT::i32)); 2301 Op = DAG.getCopyToReg(SDL.getRoot(), Reg, Lo); 2302 return DAG.getCopyToReg(Op, Reg+1, Hi); 2303 } 2304} 2305 2306void SelectionDAGISel:: 2307LowerArguments(BasicBlock *BB, SelectionDAGLowering &SDL, 2308 std::vector<SDOperand> &UnorderedChains) { 2309 // If this is the entry block, emit arguments. 2310 Function &F = *BB->getParent(); 2311 FunctionLoweringInfo &FuncInfo = SDL.FuncInfo; 2312 SDOperand OldRoot = SDL.DAG.getRoot(); 2313 std::vector<SDOperand> Args = TLI.LowerArguments(F, SDL.DAG); 2314 2315 unsigned a = 0; 2316 for (Function::arg_iterator AI = F.arg_begin(), E = F.arg_end(); 2317 AI != E; ++AI, ++a) 2318 if (!AI->use_empty()) { 2319 SDL.setValue(AI, Args[a]); 2320 2321 // If this argument is live outside of the entry block, insert a copy from 2322 // whereever we got it to the vreg that other BB's will reference it as. 2323 if (FuncInfo.ValueMap.count(AI)) { 2324 SDOperand Copy = 2325 CopyValueToVirtualRegister(SDL, AI, FuncInfo.ValueMap[AI]); 2326 UnorderedChains.push_back(Copy); 2327 } 2328 } 2329 2330 // Next, if the function has live ins that need to be copied into vregs, 2331 // emit the copies now, into the top of the block. 2332 MachineFunction &MF = SDL.DAG.getMachineFunction(); 2333 if (MF.livein_begin() != MF.livein_end()) { 2334 SSARegMap *RegMap = MF.getSSARegMap(); 2335 const MRegisterInfo &MRI = *MF.getTarget().getRegisterInfo(); 2336 for (MachineFunction::livein_iterator LI = MF.livein_begin(), 2337 E = MF.livein_end(); LI != E; ++LI) 2338 if (LI->second) 2339 MRI.copyRegToReg(*MF.begin(), MF.begin()->end(), LI->second, 2340 LI->first, RegMap->getRegClass(LI->second)); 2341 } 2342 2343 // Finally, if the target has anything special to do, allow it to do so. 2344 EmitFunctionEntryCode(F, SDL.DAG.getMachineFunction()); 2345} 2346 2347 2348void SelectionDAGISel::BuildSelectionDAG(SelectionDAG &DAG, BasicBlock *LLVMBB, 2349 std::vector<std::pair<MachineInstr*, unsigned> > &PHINodesToUpdate, 2350 FunctionLoweringInfo &FuncInfo) { 2351 SelectionDAGLowering SDL(DAG, TLI, FuncInfo); 2352 2353 std::vector<SDOperand> UnorderedChains; 2354 2355 // Lower any arguments needed in this block if this is the entry block. 2356 if (LLVMBB == &LLVMBB->getParent()->front()) 2357 LowerArguments(LLVMBB, SDL, UnorderedChains); 2358 2359 BB = FuncInfo.MBBMap[LLVMBB]; 2360 SDL.setCurrentBasicBlock(BB); 2361 2362 // Lower all of the non-terminator instructions. 2363 for (BasicBlock::iterator I = LLVMBB->begin(), E = --LLVMBB->end(); 2364 I != E; ++I) 2365 SDL.visit(*I); 2366 2367 // Ensure that all instructions which are used outside of their defining 2368 // blocks are available as virtual registers. 2369 for (BasicBlock::iterator I = LLVMBB->begin(), E = LLVMBB->end(); I != E;++I) 2370 if (!I->use_empty() && !isa<PHINode>(I)) { 2371 std::map<const Value*, unsigned>::iterator VMI =FuncInfo.ValueMap.find(I); 2372 if (VMI != FuncInfo.ValueMap.end()) 2373 UnorderedChains.push_back( 2374 CopyValueToVirtualRegister(SDL, I, VMI->second)); 2375 } 2376 2377 // Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to 2378 // ensure constants are generated when needed. Remember the virtual registers 2379 // that need to be added to the Machine PHI nodes as input. We cannot just 2380 // directly add them, because expansion might result in multiple MBB's for one 2381 // BB. As such, the start of the BB might correspond to a different MBB than 2382 // the end. 2383 // 2384 2385 // Emit constants only once even if used by multiple PHI nodes. 2386 std::map<Constant*, unsigned> ConstantsOut; 2387 2388 // Check successor nodes PHI nodes that expect a constant to be available from 2389 // this block. 2390 TerminatorInst *TI = LLVMBB->getTerminator(); 2391 for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) { 2392 BasicBlock *SuccBB = TI->getSuccessor(succ); 2393 MachineBasicBlock::iterator MBBI = FuncInfo.MBBMap[SuccBB]->begin(); 2394 PHINode *PN; 2395 2396 // At this point we know that there is a 1-1 correspondence between LLVM PHI 2397 // nodes and Machine PHI nodes, but the incoming operands have not been 2398 // emitted yet. 2399 for (BasicBlock::iterator I = SuccBB->begin(); 2400 (PN = dyn_cast<PHINode>(I)); ++I) 2401 if (!PN->use_empty()) { 2402 unsigned Reg; 2403 Value *PHIOp = PN->getIncomingValueForBlock(LLVMBB); 2404 if (Constant *C = dyn_cast<Constant>(PHIOp)) { 2405 unsigned &RegOut = ConstantsOut[C]; 2406 if (RegOut == 0) { 2407 RegOut = FuncInfo.CreateRegForValue(C); 2408 UnorderedChains.push_back( 2409 CopyValueToVirtualRegister(SDL, C, RegOut)); 2410 } 2411 Reg = RegOut; 2412 } else { 2413 Reg = FuncInfo.ValueMap[PHIOp]; 2414 if (Reg == 0) { 2415 assert(isa<AllocaInst>(PHIOp) && 2416 FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) && 2417 "Didn't codegen value into a register!??"); 2418 Reg = FuncInfo.CreateRegForValue(PHIOp); 2419 UnorderedChains.push_back( 2420 CopyValueToVirtualRegister(SDL, PHIOp, Reg)); 2421 } 2422 } 2423 2424 // Remember that this register needs to added to the machine PHI node as 2425 // the input for this MBB. 2426 unsigned NumElements = 2427 TLI.getNumElements(TLI.getValueType(PN->getType())); 2428 for (unsigned i = 0, e = NumElements; i != e; ++i) 2429 PHINodesToUpdate.push_back(std::make_pair(MBBI++, Reg+i)); 2430 } 2431 } 2432 ConstantsOut.clear(); 2433 2434 // Turn all of the unordered chains into one factored node. 2435 if (!UnorderedChains.empty()) { 2436 SDOperand Root = SDL.getRoot(); 2437 if (Root.getOpcode() != ISD::EntryToken) { 2438 unsigned i = 0, e = UnorderedChains.size(); 2439 for (; i != e; ++i) { 2440 assert(UnorderedChains[i].Val->getNumOperands() > 1); 2441 if (UnorderedChains[i].Val->getOperand(0) == Root) 2442 break; // Don't add the root if we already indirectly depend on it. 2443 } 2444 2445 if (i == e) 2446 UnorderedChains.push_back(Root); 2447 } 2448 DAG.setRoot(DAG.getNode(ISD::TokenFactor, MVT::Other, UnorderedChains)); 2449 } 2450 2451 // Lower the terminator after the copies are emitted. 2452 SDL.visit(*LLVMBB->getTerminator()); 2453 2454 // Make sure the root of the DAG is up-to-date. 2455 DAG.setRoot(SDL.getRoot()); 2456} 2457 2458void SelectionDAGISel::SelectBasicBlock(BasicBlock *LLVMBB, MachineFunction &MF, 2459 FunctionLoweringInfo &FuncInfo) { 2460 SelectionDAG DAG(TLI, MF, getAnalysisToUpdate<MachineDebugInfo>()); 2461 CurDAG = &DAG; 2462 std::vector<std::pair<MachineInstr*, unsigned> > PHINodesToUpdate; 2463 2464 // First step, lower LLVM code to some DAG. This DAG may use operations and 2465 // types that are not supported by the target. 2466 BuildSelectionDAG(DAG, LLVMBB, PHINodesToUpdate, FuncInfo); 2467 2468 // Run the DAG combiner in pre-legalize mode. 2469 DAG.Combine(false); 2470 2471 DEBUG(std::cerr << "Lowered selection DAG:\n"); 2472 DEBUG(DAG.dump()); 2473 2474 // Second step, hack on the DAG until it only uses operations and types that 2475 // the target supports. 2476 DAG.Legalize(); 2477 2478 DEBUG(std::cerr << "Legalized selection DAG:\n"); 2479 DEBUG(DAG.dump()); 2480 2481 // Run the DAG combiner in post-legalize mode. 2482 DAG.Combine(true); 2483 2484 if (ViewISelDAGs) DAG.viewGraph(); 2485 2486 // Third, instruction select all of the operations to machine code, adding the 2487 // code to the MachineBasicBlock. 2488 InstructionSelectBasicBlock(DAG); 2489 2490 DEBUG(std::cerr << "Selected machine code:\n"); 2491 DEBUG(BB->dump()); 2492 2493 // Next, now that we know what the last MBB the LLVM BB expanded is, update 2494 // PHI nodes in successors. 2495 for (unsigned i = 0, e = PHINodesToUpdate.size(); i != e; ++i) { 2496 MachineInstr *PHI = PHINodesToUpdate[i].first; 2497 assert(PHI->getOpcode() == TargetInstrInfo::PHI && 2498 "This is not a machine PHI node that we are updating!"); 2499 PHI->addRegOperand(PHINodesToUpdate[i].second); 2500 PHI->addMachineBasicBlockOperand(BB); 2501 } 2502 2503 // Finally, add the CFG edges from the last selected MBB to the successor 2504 // MBBs. 2505 TerminatorInst *TI = LLVMBB->getTerminator(); 2506 for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i) { 2507 MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[TI->getSuccessor(i)]; 2508 BB->addSuccessor(Succ0MBB); 2509 } 2510} 2511 2512//===----------------------------------------------------------------------===// 2513/// ScheduleAndEmitDAG - Pick a safe ordering and emit instructions for each 2514/// target node in the graph. 2515void SelectionDAGISel::ScheduleAndEmitDAG(SelectionDAG &DAG) { 2516 if (ViewSchedDAGs) DAG.viewGraph(); 2517 ScheduleDAG *SL = NULL; 2518 2519 switch (ISHeuristic) { 2520 default: assert(0 && "Unrecognized scheduling heuristic"); 2521 case defaultScheduling: 2522 if (TLI.getSchedulingPreference() == TargetLowering::SchedulingForLatency) 2523 SL = createSimpleDAGScheduler(noScheduling, DAG, BB); 2524 else /* TargetLowering::SchedulingForRegPressure */ 2525 SL = createBURRListDAGScheduler(DAG, BB); 2526 break; 2527 case noScheduling: 2528 SL = createBFS_DAGScheduler(DAG, BB); 2529 break; 2530 case simpleScheduling: 2531 SL = createSimpleDAGScheduler(false, DAG, BB); 2532 break; 2533 case simpleNoItinScheduling: 2534 SL = createSimpleDAGScheduler(true, DAG, BB); 2535 break; 2536 case listSchedulingBURR: 2537 SL = createBURRListDAGScheduler(DAG, BB); 2538 break; 2539 case listSchedulingTD: 2540 SL = createTDListDAGScheduler(DAG, BB, CreateTargetHazardRecognizer()); 2541 break; 2542 } 2543 BB = SL->Run(); 2544 delete SL; 2545} 2546 2547HazardRecognizer *SelectionDAGISel::CreateTargetHazardRecognizer() { 2548 return new HazardRecognizer(); 2549} 2550 2551/// SelectInlineAsmMemoryOperands - Calls to this are automatically generated 2552/// by tblgen. Others should not call it. 2553void SelectionDAGISel:: 2554SelectInlineAsmMemoryOperands(std::vector<SDOperand> &Ops, SelectionDAG &DAG) { 2555 std::vector<SDOperand> InOps; 2556 std::swap(InOps, Ops); 2557 2558 Ops.push_back(InOps[0]); // input chain. 2559 Ops.push_back(InOps[1]); // input asm string. 2560 2561 const char *AsmStr = cast<ExternalSymbolSDNode>(InOps[1])->getSymbol(); 2562 unsigned i = 2, e = InOps.size(); 2563 if (InOps[e-1].getValueType() == MVT::Flag) 2564 --e; // Don't process a flag operand if it is here. 2565 2566 while (i != e) { 2567 unsigned Flags = cast<ConstantSDNode>(InOps[i])->getValue(); 2568 if ((Flags & 7) != 4 /*MEM*/) { 2569 // Just skip over this operand, copying the operands verbatim. 2570 Ops.insert(Ops.end(), InOps.begin()+i, InOps.begin()+i+(Flags >> 3) + 1); 2571 i += (Flags >> 3) + 1; 2572 } else { 2573 assert((Flags >> 3) == 1 && "Memory operand with multiple values?"); 2574 // Otherwise, this is a memory operand. Ask the target to select it. 2575 std::vector<SDOperand> SelOps; 2576 if (SelectInlineAsmMemoryOperand(InOps[i+1], 'm', SelOps, DAG)) { 2577 std::cerr << "Could not match memory address. Inline asm failure!\n"; 2578 exit(1); 2579 } 2580 2581 // Add this to the output node. 2582 Ops.push_back(DAG.getConstant(4/*MEM*/ | (SelOps.size() << 3), MVT::i32)); 2583 Ops.insert(Ops.end(), SelOps.begin(), SelOps.end()); 2584 i += 2; 2585 } 2586 } 2587 2588 // Add the flag input back if present. 2589 if (e != InOps.size()) 2590 Ops.push_back(InOps.back()); 2591} 2592