SelectionDAG.cpp revision 59d3ae6cdc4316ad338cd848251f33a236ccb36c
1//===-- SelectionDAG.cpp - Implement the SelectionDAG data structures -----===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This implements the SelectionDAG class. 11// 12//===----------------------------------------------------------------------===// 13 14#include "llvm/CodeGen/SelectionDAG.h" 15#include "SDNodeDbgValue.h" 16#include "llvm/ADT/SetVector.h" 17#include "llvm/ADT/SmallPtrSet.h" 18#include "llvm/ADT/SmallSet.h" 19#include "llvm/ADT/SmallVector.h" 20#include "llvm/ADT/StringExtras.h" 21#include "llvm/Analysis/TargetTransformInfo.h" 22#include "llvm/Analysis/ValueTracking.h" 23#include "llvm/Assembly/Writer.h" 24#include "llvm/CodeGen/MachineBasicBlock.h" 25#include "llvm/CodeGen/MachineConstantPool.h" 26#include "llvm/CodeGen/MachineFrameInfo.h" 27#include "llvm/CodeGen/MachineModuleInfo.h" 28#include "llvm/DebugInfo.h" 29#include "llvm/IR/CallingConv.h" 30#include "llvm/IR/Constants.h" 31#include "llvm/IR/DataLayout.h" 32#include "llvm/IR/DerivedTypes.h" 33#include "llvm/IR/Function.h" 34#include "llvm/IR/GlobalAlias.h" 35#include "llvm/IR/GlobalVariable.h" 36#include "llvm/IR/Intrinsics.h" 37#include "llvm/Support/CommandLine.h" 38#include "llvm/Support/Debug.h" 39#include "llvm/Support/ErrorHandling.h" 40#include "llvm/Support/ManagedStatic.h" 41#include "llvm/Support/MathExtras.h" 42#include "llvm/Support/Mutex.h" 43#include "llvm/Support/raw_ostream.h" 44#include "llvm/Target/TargetInstrInfo.h" 45#include "llvm/Target/TargetIntrinsicInfo.h" 46#include "llvm/Target/TargetLowering.h" 47#include "llvm/Target/TargetMachine.h" 48#include "llvm/Target/TargetOptions.h" 49#include "llvm/Target/TargetRegisterInfo.h" 50#include "llvm/Target/TargetSelectionDAGInfo.h" 51#include <algorithm> 52#include <cmath> 53using namespace llvm; 54 55/// makeVTList - Return an instance of the SDVTList struct initialized with the 56/// specified members. 57static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) { 58 SDVTList Res = {VTs, NumVTs}; 59 return Res; 60} 61 62// Default null implementations of the callbacks. 63void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {} 64void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {} 65 66//===----------------------------------------------------------------------===// 67// ConstantFPSDNode Class 68//===----------------------------------------------------------------------===// 69 70/// isExactlyValue - We don't rely on operator== working on double values, as 71/// it returns true for things that are clearly not equal, like -0.0 and 0.0. 72/// As such, this method can be used to do an exact bit-for-bit comparison of 73/// two floating point values. 74bool ConstantFPSDNode::isExactlyValue(const APFloat& V) const { 75 return getValueAPF().bitwiseIsEqual(V); 76} 77 78bool ConstantFPSDNode::isValueValidForType(EVT VT, 79 const APFloat& Val) { 80 assert(VT.isFloatingPoint() && "Can only convert between FP types"); 81 82 // convert modifies in place, so make a copy. 83 APFloat Val2 = APFloat(Val); 84 bool losesInfo; 85 (void) Val2.convert(SelectionDAG::EVTToAPFloatSemantics(VT), 86 APFloat::rmNearestTiesToEven, 87 &losesInfo); 88 return !losesInfo; 89} 90 91//===----------------------------------------------------------------------===// 92// ISD Namespace 93//===----------------------------------------------------------------------===// 94 95/// isBuildVectorAllOnes - Return true if the specified node is a 96/// BUILD_VECTOR where all of the elements are ~0 or undef. 97bool ISD::isBuildVectorAllOnes(const SDNode *N) { 98 // Look through a bit convert. 99 if (N->getOpcode() == ISD::BITCAST) 100 N = N->getOperand(0).getNode(); 101 102 if (N->getOpcode() != ISD::BUILD_VECTOR) return false; 103 104 unsigned i = 0, e = N->getNumOperands(); 105 106 // Skip over all of the undef values. 107 while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF) 108 ++i; 109 110 // Do not accept an all-undef vector. 111 if (i == e) return false; 112 113 // Do not accept build_vectors that aren't all constants or which have non-~0 114 // elements. We have to be a bit careful here, as the type of the constant 115 // may not be the same as the type of the vector elements due to type 116 // legalization (the elements are promoted to a legal type for the target and 117 // a vector of a type may be legal when the base element type is not). 118 // We only want to check enough bits to cover the vector elements, because 119 // we care if the resultant vector is all ones, not whether the individual 120 // constants are. 121 SDValue NotZero = N->getOperand(i); 122 unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits(); 123 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) { 124 if (CN->getAPIntValue().countTrailingOnes() < EltSize) 125 return false; 126 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) { 127 if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize) 128 return false; 129 } else 130 return false; 131 132 // Okay, we have at least one ~0 value, check to see if the rest match or are 133 // undefs. Even with the above element type twiddling, this should be OK, as 134 // the same type legalization should have applied to all the elements. 135 for (++i; i != e; ++i) 136 if (N->getOperand(i) != NotZero && 137 N->getOperand(i).getOpcode() != ISD::UNDEF) 138 return false; 139 return true; 140} 141 142 143/// isBuildVectorAllZeros - Return true if the specified node is a 144/// BUILD_VECTOR where all of the elements are 0 or undef. 145bool ISD::isBuildVectorAllZeros(const SDNode *N) { 146 // Look through a bit convert. 147 if (N->getOpcode() == ISD::BITCAST) 148 N = N->getOperand(0).getNode(); 149 150 if (N->getOpcode() != ISD::BUILD_VECTOR) return false; 151 152 unsigned i = 0, e = N->getNumOperands(); 153 154 // Skip over all of the undef values. 155 while (i != e && N->getOperand(i).getOpcode() == ISD::UNDEF) 156 ++i; 157 158 // Do not accept an all-undef vector. 159 if (i == e) return false; 160 161 // Do not accept build_vectors that aren't all constants or which have non-0 162 // elements. 163 SDValue Zero = N->getOperand(i); 164 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Zero)) { 165 if (!CN->isNullValue()) 166 return false; 167 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Zero)) { 168 if (!CFPN->getValueAPF().isPosZero()) 169 return false; 170 } else 171 return false; 172 173 // Okay, we have at least one 0 value, check to see if the rest match or are 174 // undefs. 175 for (++i; i != e; ++i) 176 if (N->getOperand(i) != Zero && 177 N->getOperand(i).getOpcode() != ISD::UNDEF) 178 return false; 179 return true; 180} 181 182/// isScalarToVector - Return true if the specified node is a 183/// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low 184/// element is not an undef. 185bool ISD::isScalarToVector(const SDNode *N) { 186 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) 187 return true; 188 189 if (N->getOpcode() != ISD::BUILD_VECTOR) 190 return false; 191 if (N->getOperand(0).getOpcode() == ISD::UNDEF) 192 return false; 193 unsigned NumElems = N->getNumOperands(); 194 if (NumElems == 1) 195 return false; 196 for (unsigned i = 1; i < NumElems; ++i) { 197 SDValue V = N->getOperand(i); 198 if (V.getOpcode() != ISD::UNDEF) 199 return false; 200 } 201 return true; 202} 203 204/// allOperandsUndef - Return true if the node has at least one operand 205/// and all operands of the specified node are ISD::UNDEF. 206bool ISD::allOperandsUndef(const SDNode *N) { 207 // Return false if the node has no operands. 208 // This is "logically inconsistent" with the definition of "all" but 209 // is probably the desired behavior. 210 if (N->getNumOperands() == 0) 211 return false; 212 213 for (unsigned i = 0, e = N->getNumOperands(); i != e ; ++i) 214 if (N->getOperand(i).getOpcode() != ISD::UNDEF) 215 return false; 216 217 return true; 218} 219 220/// getSetCCSwappedOperands - Return the operation corresponding to (Y op X) 221/// when given the operation for (X op Y). 222ISD::CondCode ISD::getSetCCSwappedOperands(ISD::CondCode Operation) { 223 // To perform this operation, we just need to swap the L and G bits of the 224 // operation. 225 unsigned OldL = (Operation >> 2) & 1; 226 unsigned OldG = (Operation >> 1) & 1; 227 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits 228 (OldL << 1) | // New G bit 229 (OldG << 2)); // New L bit. 230} 231 232/// getSetCCInverse - Return the operation corresponding to !(X op Y), where 233/// 'op' is a valid SetCC operation. 234ISD::CondCode ISD::getSetCCInverse(ISD::CondCode Op, bool isInteger) { 235 unsigned Operation = Op; 236 if (isInteger) 237 Operation ^= 7; // Flip L, G, E bits, but not U. 238 else 239 Operation ^= 15; // Flip all of the condition bits. 240 241 if (Operation > ISD::SETTRUE2) 242 Operation &= ~8; // Don't let N and U bits get set. 243 244 return ISD::CondCode(Operation); 245} 246 247 248/// isSignedOp - For an integer comparison, return 1 if the comparison is a 249/// signed operation and 2 if the result is an unsigned comparison. Return zero 250/// if the operation does not depend on the sign of the input (setne and seteq). 251static int isSignedOp(ISD::CondCode Opcode) { 252 switch (Opcode) { 253 default: llvm_unreachable("Illegal integer setcc operation!"); 254 case ISD::SETEQ: 255 case ISD::SETNE: return 0; 256 case ISD::SETLT: 257 case ISD::SETLE: 258 case ISD::SETGT: 259 case ISD::SETGE: return 1; 260 case ISD::SETULT: 261 case ISD::SETULE: 262 case ISD::SETUGT: 263 case ISD::SETUGE: return 2; 264 } 265} 266 267/// getSetCCOrOperation - Return the result of a logical OR between different 268/// comparisons of identical values: ((X op1 Y) | (X op2 Y)). This function 269/// returns SETCC_INVALID if it is not possible to represent the resultant 270/// comparison. 271ISD::CondCode ISD::getSetCCOrOperation(ISD::CondCode Op1, ISD::CondCode Op2, 272 bool isInteger) { 273 if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) 274 // Cannot fold a signed integer setcc with an unsigned integer setcc. 275 return ISD::SETCC_INVALID; 276 277 unsigned Op = Op1 | Op2; // Combine all of the condition bits. 278 279 // If the N and U bits get set then the resultant comparison DOES suddenly 280 // care about orderedness, and is true when ordered. 281 if (Op > ISD::SETTRUE2) 282 Op &= ~16; // Clear the U bit if the N bit is set. 283 284 // Canonicalize illegal integer setcc's. 285 if (isInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT 286 Op = ISD::SETNE; 287 288 return ISD::CondCode(Op); 289} 290 291/// getSetCCAndOperation - Return the result of a logical AND between different 292/// comparisons of identical values: ((X op1 Y) & (X op2 Y)). This 293/// function returns zero if it is not possible to represent the resultant 294/// comparison. 295ISD::CondCode ISD::getSetCCAndOperation(ISD::CondCode Op1, ISD::CondCode Op2, 296 bool isInteger) { 297 if (isInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3) 298 // Cannot fold a signed setcc with an unsigned setcc. 299 return ISD::SETCC_INVALID; 300 301 // Combine all of the condition bits. 302 ISD::CondCode Result = ISD::CondCode(Op1 & Op2); 303 304 // Canonicalize illegal integer setcc's. 305 if (isInteger) { 306 switch (Result) { 307 default: break; 308 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT 309 case ISD::SETOEQ: // SETEQ & SETU[LG]E 310 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE 311 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE 312 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE 313 } 314 } 315 316 return Result; 317} 318 319//===----------------------------------------------------------------------===// 320// SDNode Profile Support 321//===----------------------------------------------------------------------===// 322 323/// AddNodeIDOpcode - Add the node opcode to the NodeID data. 324/// 325static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) { 326 ID.AddInteger(OpC); 327} 328 329/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them 330/// solely with their pointer. 331static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList) { 332 ID.AddPointer(VTList.VTs); 333} 334 335/// AddNodeIDOperands - Various routines for adding operands to the NodeID data. 336/// 337static void AddNodeIDOperands(FoldingSetNodeID &ID, 338 const SDValue *Ops, unsigned NumOps) { 339 for (; NumOps; --NumOps, ++Ops) { 340 ID.AddPointer(Ops->getNode()); 341 ID.AddInteger(Ops->getResNo()); 342 } 343} 344 345/// AddNodeIDOperands - Various routines for adding operands to the NodeID data. 346/// 347static void AddNodeIDOperands(FoldingSetNodeID &ID, 348 const SDUse *Ops, unsigned NumOps) { 349 for (; NumOps; --NumOps, ++Ops) { 350 ID.AddPointer(Ops->getNode()); 351 ID.AddInteger(Ops->getResNo()); 352 } 353} 354 355static void AddNodeIDNode(FoldingSetNodeID &ID, 356 unsigned short OpC, SDVTList VTList, 357 const SDValue *OpList, unsigned N) { 358 AddNodeIDOpcode(ID, OpC); 359 AddNodeIDValueTypes(ID, VTList); 360 AddNodeIDOperands(ID, OpList, N); 361} 362 363/// AddNodeIDCustom - If this is an SDNode with special info, add this info to 364/// the NodeID data. 365static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { 366 switch (N->getOpcode()) { 367 case ISD::TargetExternalSymbol: 368 case ISD::ExternalSymbol: 369 llvm_unreachable("Should only be used on nodes with operands"); 370 default: break; // Normal nodes don't need extra info. 371 case ISD::TargetConstant: 372 case ISD::Constant: 373 ID.AddPointer(cast<ConstantSDNode>(N)->getConstantIntValue()); 374 break; 375 case ISD::TargetConstantFP: 376 case ISD::ConstantFP: { 377 ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue()); 378 break; 379 } 380 case ISD::TargetGlobalAddress: 381 case ISD::GlobalAddress: 382 case ISD::TargetGlobalTLSAddress: 383 case ISD::GlobalTLSAddress: { 384 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N); 385 ID.AddPointer(GA->getGlobal()); 386 ID.AddInteger(GA->getOffset()); 387 ID.AddInteger(GA->getTargetFlags()); 388 ID.AddInteger(GA->getAddressSpace()); 389 break; 390 } 391 case ISD::BasicBlock: 392 ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock()); 393 break; 394 case ISD::Register: 395 ID.AddInteger(cast<RegisterSDNode>(N)->getReg()); 396 break; 397 case ISD::RegisterMask: 398 ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask()); 399 break; 400 case ISD::SRCVALUE: 401 ID.AddPointer(cast<SrcValueSDNode>(N)->getValue()); 402 break; 403 case ISD::FrameIndex: 404 case ISD::TargetFrameIndex: 405 ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex()); 406 break; 407 case ISD::JumpTable: 408 case ISD::TargetJumpTable: 409 ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex()); 410 ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags()); 411 break; 412 case ISD::ConstantPool: 413 case ISD::TargetConstantPool: { 414 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N); 415 ID.AddInteger(CP->getAlignment()); 416 ID.AddInteger(CP->getOffset()); 417 if (CP->isMachineConstantPoolEntry()) 418 CP->getMachineCPVal()->addSelectionDAGCSEId(ID); 419 else 420 ID.AddPointer(CP->getConstVal()); 421 ID.AddInteger(CP->getTargetFlags()); 422 break; 423 } 424 case ISD::TargetIndex: { 425 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N); 426 ID.AddInteger(TI->getIndex()); 427 ID.AddInteger(TI->getOffset()); 428 ID.AddInteger(TI->getTargetFlags()); 429 break; 430 } 431 case ISD::LOAD: { 432 const LoadSDNode *LD = cast<LoadSDNode>(N); 433 ID.AddInteger(LD->getMemoryVT().getRawBits()); 434 ID.AddInteger(LD->getRawSubclassData()); 435 ID.AddInteger(LD->getPointerInfo().getAddrSpace()); 436 break; 437 } 438 case ISD::STORE: { 439 const StoreSDNode *ST = cast<StoreSDNode>(N); 440 ID.AddInteger(ST->getMemoryVT().getRawBits()); 441 ID.AddInteger(ST->getRawSubclassData()); 442 ID.AddInteger(ST->getPointerInfo().getAddrSpace()); 443 break; 444 } 445 case ISD::ATOMIC_CMP_SWAP: 446 case ISD::ATOMIC_SWAP: 447 case ISD::ATOMIC_LOAD_ADD: 448 case ISD::ATOMIC_LOAD_SUB: 449 case ISD::ATOMIC_LOAD_AND: 450 case ISD::ATOMIC_LOAD_OR: 451 case ISD::ATOMIC_LOAD_XOR: 452 case ISD::ATOMIC_LOAD_NAND: 453 case ISD::ATOMIC_LOAD_MIN: 454 case ISD::ATOMIC_LOAD_MAX: 455 case ISD::ATOMIC_LOAD_UMIN: 456 case ISD::ATOMIC_LOAD_UMAX: 457 case ISD::ATOMIC_LOAD: 458 case ISD::ATOMIC_STORE: { 459 const AtomicSDNode *AT = cast<AtomicSDNode>(N); 460 ID.AddInteger(AT->getMemoryVT().getRawBits()); 461 ID.AddInteger(AT->getRawSubclassData()); 462 ID.AddInteger(AT->getPointerInfo().getAddrSpace()); 463 break; 464 } 465 case ISD::PREFETCH: { 466 const MemSDNode *PF = cast<MemSDNode>(N); 467 ID.AddInteger(PF->getPointerInfo().getAddrSpace()); 468 break; 469 } 470 case ISD::VECTOR_SHUFFLE: { 471 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N); 472 for (unsigned i = 0, e = N->getValueType(0).getVectorNumElements(); 473 i != e; ++i) 474 ID.AddInteger(SVN->getMaskElt(i)); 475 break; 476 } 477 case ISD::TargetBlockAddress: 478 case ISD::BlockAddress: { 479 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N); 480 ID.AddPointer(BA->getBlockAddress()); 481 ID.AddInteger(BA->getOffset()); 482 ID.AddInteger(BA->getTargetFlags()); 483 break; 484 } 485 } // end switch (N->getOpcode()) 486 487 // Target specific memory nodes could also have address spaces to check. 488 if (N->isTargetMemoryOpcode()) 489 ID.AddInteger(cast<MemSDNode>(N)->getPointerInfo().getAddrSpace()); 490} 491 492/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID 493/// data. 494static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) { 495 AddNodeIDOpcode(ID, N->getOpcode()); 496 // Add the return value info. 497 AddNodeIDValueTypes(ID, N->getVTList()); 498 // Add the operand info. 499 AddNodeIDOperands(ID, N->op_begin(), N->getNumOperands()); 500 501 // Handle SDNode leafs with special info. 502 AddNodeIDCustom(ID, N); 503} 504 505/// encodeMemSDNodeFlags - Generic routine for computing a value for use in 506/// the CSE map that carries volatility, temporalness, indexing mode, and 507/// extension/truncation information. 508/// 509static inline unsigned 510encodeMemSDNodeFlags(int ConvType, ISD::MemIndexedMode AM, bool isVolatile, 511 bool isNonTemporal, bool isInvariant) { 512 assert((ConvType & 3) == ConvType && 513 "ConvType may not require more than 2 bits!"); 514 assert((AM & 7) == AM && 515 "AM may not require more than 3 bits!"); 516 return ConvType | 517 (AM << 2) | 518 (isVolatile << 5) | 519 (isNonTemporal << 6) | 520 (isInvariant << 7); 521} 522 523//===----------------------------------------------------------------------===// 524// SelectionDAG Class 525//===----------------------------------------------------------------------===// 526 527/// doNotCSE - Return true if CSE should not be performed for this node. 528static bool doNotCSE(SDNode *N) { 529 if (N->getValueType(0) == MVT::Glue) 530 return true; // Never CSE anything that produces a flag. 531 532 switch (N->getOpcode()) { 533 default: break; 534 case ISD::HANDLENODE: 535 case ISD::EH_LABEL: 536 return true; // Never CSE these nodes. 537 } 538 539 // Check that remaining values produced are not flags. 540 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i) 541 if (N->getValueType(i) == MVT::Glue) 542 return true; // Never CSE anything that produces a flag. 543 544 return false; 545} 546 547/// RemoveDeadNodes - This method deletes all unreachable nodes in the 548/// SelectionDAG. 549void SelectionDAG::RemoveDeadNodes() { 550 // Create a dummy node (which is not added to allnodes), that adds a reference 551 // to the root node, preventing it from being deleted. 552 HandleSDNode Dummy(getRoot()); 553 554 SmallVector<SDNode*, 128> DeadNodes; 555 556 // Add all obviously-dead nodes to the DeadNodes worklist. 557 for (allnodes_iterator I = allnodes_begin(), E = allnodes_end(); I != E; ++I) 558 if (I->use_empty()) 559 DeadNodes.push_back(I); 560 561 RemoveDeadNodes(DeadNodes); 562 563 // If the root changed (e.g. it was a dead load, update the root). 564 setRoot(Dummy.getValue()); 565} 566 567/// RemoveDeadNodes - This method deletes the unreachable nodes in the 568/// given list, and any nodes that become unreachable as a result. 569void SelectionDAG::RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes) { 570 571 // Process the worklist, deleting the nodes and adding their uses to the 572 // worklist. 573 while (!DeadNodes.empty()) { 574 SDNode *N = DeadNodes.pop_back_val(); 575 576 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 577 DUL->NodeDeleted(N, 0); 578 579 // Take the node out of the appropriate CSE map. 580 RemoveNodeFromCSEMaps(N); 581 582 // Next, brutally remove the operand list. This is safe to do, as there are 583 // no cycles in the graph. 584 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { 585 SDUse &Use = *I++; 586 SDNode *Operand = Use.getNode(); 587 Use.set(SDValue()); 588 589 // Now that we removed this operand, see if there are no uses of it left. 590 if (Operand->use_empty()) 591 DeadNodes.push_back(Operand); 592 } 593 594 DeallocateNode(N); 595 } 596} 597 598void SelectionDAG::RemoveDeadNode(SDNode *N){ 599 SmallVector<SDNode*, 16> DeadNodes(1, N); 600 601 // Create a dummy node that adds a reference to the root node, preventing 602 // it from being deleted. (This matters if the root is an operand of the 603 // dead node.) 604 HandleSDNode Dummy(getRoot()); 605 606 RemoveDeadNodes(DeadNodes); 607} 608 609void SelectionDAG::DeleteNode(SDNode *N) { 610 // First take this out of the appropriate CSE map. 611 RemoveNodeFromCSEMaps(N); 612 613 // Finally, remove uses due to operands of this node, remove from the 614 // AllNodes list, and delete the node. 615 DeleteNodeNotInCSEMaps(N); 616} 617 618void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) { 619 assert(N != AllNodes.begin() && "Cannot delete the entry node!"); 620 assert(N->use_empty() && "Cannot delete a node that is not dead!"); 621 622 // Drop all of the operands and decrement used node's use counts. 623 N->DropOperands(); 624 625 DeallocateNode(N); 626} 627 628void SelectionDAG::DeallocateNode(SDNode *N) { 629 if (N->OperandsNeedDelete) 630 delete[] N->OperandList; 631 632 // Set the opcode to DELETED_NODE to help catch bugs when node 633 // memory is reallocated. 634 N->NodeType = ISD::DELETED_NODE; 635 636 NodeAllocator.Deallocate(AllNodes.remove(N)); 637 638 // If any of the SDDbgValue nodes refer to this SDNode, invalidate them. 639 ArrayRef<SDDbgValue*> DbgVals = DbgInfo->getSDDbgValues(N); 640 for (unsigned i = 0, e = DbgVals.size(); i != e; ++i) 641 DbgVals[i]->setIsInvalidated(); 642} 643 644/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that 645/// correspond to it. This is useful when we're about to delete or repurpose 646/// the node. We don't want future request for structurally identical nodes 647/// to return N anymore. 648bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) { 649 bool Erased = false; 650 switch (N->getOpcode()) { 651 case ISD::HANDLENODE: return false; // noop. 652 case ISD::CONDCODE: 653 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] && 654 "Cond code doesn't exist!"); 655 Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != 0; 656 CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = 0; 657 break; 658 case ISD::ExternalSymbol: 659 Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol()); 660 break; 661 case ISD::TargetExternalSymbol: { 662 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N); 663 Erased = TargetExternalSymbols.erase( 664 std::pair<std::string,unsigned char>(ESN->getSymbol(), 665 ESN->getTargetFlags())); 666 break; 667 } 668 case ISD::VALUETYPE: { 669 EVT VT = cast<VTSDNode>(N)->getVT(); 670 if (VT.isExtended()) { 671 Erased = ExtendedValueTypeNodes.erase(VT); 672 } else { 673 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != 0; 674 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = 0; 675 } 676 break; 677 } 678 default: 679 // Remove it from the CSE Map. 680 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!"); 681 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!"); 682 Erased = CSEMap.RemoveNode(N); 683 break; 684 } 685#ifndef NDEBUG 686 // Verify that the node was actually in one of the CSE maps, unless it has a 687 // flag result (which cannot be CSE'd) or is one of the special cases that are 688 // not subject to CSE. 689 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue && 690 !N->isMachineOpcode() && !doNotCSE(N)) { 691 N->dump(this); 692 dbgs() << "\n"; 693 llvm_unreachable("Node is not in map!"); 694 } 695#endif 696 return Erased; 697} 698 699/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE 700/// maps and modified in place. Add it back to the CSE maps, unless an identical 701/// node already exists, in which case transfer all its users to the existing 702/// node. This transfer can potentially trigger recursive merging. 703/// 704void 705SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) { 706 // For node types that aren't CSE'd, just act as if no identical node 707 // already exists. 708 if (!doNotCSE(N)) { 709 SDNode *Existing = CSEMap.GetOrInsertNode(N); 710 if (Existing != N) { 711 // If there was already an existing matching node, use ReplaceAllUsesWith 712 // to replace the dead one with the existing one. This can cause 713 // recursive merging of other unrelated nodes down the line. 714 ReplaceAllUsesWith(N, Existing); 715 716 // N is now dead. Inform the listeners and delete it. 717 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 718 DUL->NodeDeleted(N, Existing); 719 DeleteNodeNotInCSEMaps(N); 720 return; 721 } 722 } 723 724 // If the node doesn't already exist, we updated it. Inform listeners. 725 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) 726 DUL->NodeUpdated(N); 727} 728 729/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 730/// were replaced with those specified. If this node is never memoized, 731/// return null, otherwise return a pointer to the slot it would take. If a 732/// node already exists with these operands, the slot will be non-null. 733SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op, 734 void *&InsertPos) { 735 if (doNotCSE(N)) 736 return 0; 737 738 SDValue Ops[] = { Op }; 739 FoldingSetNodeID ID; 740 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 1); 741 AddNodeIDCustom(ID, N); 742 SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 743 return Node; 744} 745 746/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 747/// were replaced with those specified. If this node is never memoized, 748/// return null, otherwise return a pointer to the slot it would take. If a 749/// node already exists with these operands, the slot will be non-null. 750SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, 751 SDValue Op1, SDValue Op2, 752 void *&InsertPos) { 753 if (doNotCSE(N)) 754 return 0; 755 756 SDValue Ops[] = { Op1, Op2 }; 757 FoldingSetNodeID ID; 758 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, 2); 759 AddNodeIDCustom(ID, N); 760 SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 761 return Node; 762} 763 764 765/// FindModifiedNodeSlot - Find a slot for the specified node if its operands 766/// were replaced with those specified. If this node is never memoized, 767/// return null, otherwise return a pointer to the slot it would take. If a 768/// node already exists with these operands, the slot will be non-null. 769SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, 770 const SDValue *Ops,unsigned NumOps, 771 void *&InsertPos) { 772 if (doNotCSE(N)) 773 return 0; 774 775 FoldingSetNodeID ID; 776 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops, NumOps); 777 AddNodeIDCustom(ID, N); 778 SDNode *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); 779 return Node; 780} 781 782#ifndef NDEBUG 783/// VerifyNodeCommon - Sanity check the given node. Aborts if it is invalid. 784static void VerifyNodeCommon(SDNode *N) { 785 switch (N->getOpcode()) { 786 default: 787 break; 788 case ISD::BUILD_PAIR: { 789 EVT VT = N->getValueType(0); 790 assert(N->getNumValues() == 1 && "Too many results!"); 791 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) && 792 "Wrong return type!"); 793 assert(N->getNumOperands() == 2 && "Wrong number of operands!"); 794 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() && 795 "Mismatched operand types!"); 796 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() && 797 "Wrong operand type!"); 798 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() && 799 "Wrong return type size"); 800 break; 801 } 802 case ISD::BUILD_VECTOR: { 803 assert(N->getNumValues() == 1 && "Too many results!"); 804 assert(N->getValueType(0).isVector() && "Wrong return type!"); 805 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() && 806 "Wrong number of operands!"); 807 EVT EltVT = N->getValueType(0).getVectorElementType(); 808 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ++I) { 809 assert((I->getValueType() == EltVT || 810 (EltVT.isInteger() && I->getValueType().isInteger() && 811 EltVT.bitsLE(I->getValueType()))) && 812 "Wrong operand type!"); 813 assert(I->getValueType() == N->getOperand(0).getValueType() && 814 "Operands must all have the same type"); 815 } 816 break; 817 } 818 } 819} 820 821/// VerifySDNode - Sanity check the given SDNode. Aborts if it is invalid. 822static void VerifySDNode(SDNode *N) { 823 // The SDNode allocators cannot be used to allocate nodes with fields that are 824 // not present in an SDNode! 825 assert(!isa<MemSDNode>(N) && "Bad MemSDNode!"); 826 assert(!isa<ShuffleVectorSDNode>(N) && "Bad ShuffleVectorSDNode!"); 827 assert(!isa<ConstantSDNode>(N) && "Bad ConstantSDNode!"); 828 assert(!isa<ConstantFPSDNode>(N) && "Bad ConstantFPSDNode!"); 829 assert(!isa<GlobalAddressSDNode>(N) && "Bad GlobalAddressSDNode!"); 830 assert(!isa<FrameIndexSDNode>(N) && "Bad FrameIndexSDNode!"); 831 assert(!isa<JumpTableSDNode>(N) && "Bad JumpTableSDNode!"); 832 assert(!isa<ConstantPoolSDNode>(N) && "Bad ConstantPoolSDNode!"); 833 assert(!isa<BasicBlockSDNode>(N) && "Bad BasicBlockSDNode!"); 834 assert(!isa<SrcValueSDNode>(N) && "Bad SrcValueSDNode!"); 835 assert(!isa<MDNodeSDNode>(N) && "Bad MDNodeSDNode!"); 836 assert(!isa<RegisterSDNode>(N) && "Bad RegisterSDNode!"); 837 assert(!isa<BlockAddressSDNode>(N) && "Bad BlockAddressSDNode!"); 838 assert(!isa<EHLabelSDNode>(N) && "Bad EHLabelSDNode!"); 839 assert(!isa<ExternalSymbolSDNode>(N) && "Bad ExternalSymbolSDNode!"); 840 assert(!isa<CondCodeSDNode>(N) && "Bad CondCodeSDNode!"); 841 assert(!isa<CvtRndSatSDNode>(N) && "Bad CvtRndSatSDNode!"); 842 assert(!isa<VTSDNode>(N) && "Bad VTSDNode!"); 843 assert(!isa<MachineSDNode>(N) && "Bad MachineSDNode!"); 844 845 VerifyNodeCommon(N); 846} 847 848/// VerifyMachineNode - Sanity check the given MachineNode. Aborts if it is 849/// invalid. 850static void VerifyMachineNode(SDNode *N) { 851 // The MachineNode allocators cannot be used to allocate nodes with fields 852 // that are not present in a MachineNode! 853 // Currently there are no such nodes. 854 855 VerifyNodeCommon(N); 856} 857#endif // NDEBUG 858 859/// getEVTAlignment - Compute the default alignment value for the 860/// given type. 861/// 862unsigned SelectionDAG::getEVTAlignment(EVT VT) const { 863 Type *Ty = VT == MVT::iPTR ? 864 PointerType::get(Type::getInt8Ty(*getContext()), 0) : 865 VT.getTypeForEVT(*getContext()); 866 867 return TM.getTargetLowering()->getDataLayout()->getABITypeAlignment(Ty); 868} 869 870// EntryNode could meaningfully have debug info if we can find it... 871SelectionDAG::SelectionDAG(const TargetMachine &tm, CodeGenOpt::Level OL) 872 : TM(tm), TSI(*tm.getSelectionDAGInfo()), TTI(0), OptLevel(OL), 873 EntryNode(ISD::EntryToken, 0, DebugLoc(), getVTList(MVT::Other)), 874 Root(getEntryNode()), UpdateListeners(0) { 875 AllNodes.push_back(&EntryNode); 876 DbgInfo = new SDDbgInfo(); 877} 878 879void SelectionDAG::init(MachineFunction &mf, const TargetTransformInfo *tti) { 880 MF = &mf; 881 TTI = tti; 882 Context = &mf.getFunction()->getContext(); 883} 884 885SelectionDAG::~SelectionDAG() { 886 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners"); 887 allnodes_clear(); 888 delete DbgInfo; 889} 890 891void SelectionDAG::allnodes_clear() { 892 assert(&*AllNodes.begin() == &EntryNode); 893 AllNodes.remove(AllNodes.begin()); 894 while (!AllNodes.empty()) 895 DeallocateNode(AllNodes.begin()); 896} 897 898void SelectionDAG::clear() { 899 allnodes_clear(); 900 OperandAllocator.Reset(); 901 CSEMap.clear(); 902 903 ExtendedValueTypeNodes.clear(); 904 ExternalSymbols.clear(); 905 TargetExternalSymbols.clear(); 906 std::fill(CondCodeNodes.begin(), CondCodeNodes.end(), 907 static_cast<CondCodeSDNode*>(0)); 908 std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(), 909 static_cast<SDNode*>(0)); 910 911 EntryNode.UseList = 0; 912 AllNodes.push_back(&EntryNode); 913 Root = getEntryNode(); 914 DbgInfo->clear(); 915} 916 917SDValue SelectionDAG::getAnyExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) { 918 return VT.bitsGT(Op.getValueType()) ? 919 getNode(ISD::ANY_EXTEND, DL, VT, Op) : 920 getNode(ISD::TRUNCATE, DL, VT, Op); 921} 922 923SDValue SelectionDAG::getSExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) { 924 return VT.bitsGT(Op.getValueType()) ? 925 getNode(ISD::SIGN_EXTEND, DL, VT, Op) : 926 getNode(ISD::TRUNCATE, DL, VT, Op); 927} 928 929SDValue SelectionDAG::getZExtOrTrunc(SDValue Op, SDLoc DL, EVT VT) { 930 return VT.bitsGT(Op.getValueType()) ? 931 getNode(ISD::ZERO_EXTEND, DL, VT, Op) : 932 getNode(ISD::TRUNCATE, DL, VT, Op); 933} 934 935SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, SDLoc DL, EVT VT) { 936 assert(!VT.isVector() && 937 "getZeroExtendInReg should use the vector element type instead of " 938 "the vector type!"); 939 if (Op.getValueType() == VT) return Op; 940 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 941 APInt Imm = APInt::getLowBitsSet(BitWidth, 942 VT.getSizeInBits()); 943 return getNode(ISD::AND, DL, Op.getValueType(), Op, 944 getConstant(Imm, Op.getValueType())); 945} 946 947/// getNOT - Create a bitwise NOT operation as (XOR Val, -1). 948/// 949SDValue SelectionDAG::getNOT(SDLoc DL, SDValue Val, EVT VT) { 950 EVT EltVT = VT.getScalarType(); 951 SDValue NegOne = 952 getConstant(APInt::getAllOnesValue(EltVT.getSizeInBits()), VT); 953 return getNode(ISD::XOR, DL, VT, Val, NegOne); 954} 955 956SDValue SelectionDAG::getConstant(uint64_t Val, EVT VT, bool isT) { 957 EVT EltVT = VT.getScalarType(); 958 assert((EltVT.getSizeInBits() >= 64 || 959 (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) && 960 "getConstant with a uint64_t value that doesn't fit in the type!"); 961 return getConstant(APInt(EltVT.getSizeInBits(), Val), VT, isT); 962} 963 964SDValue SelectionDAG::getConstant(const APInt &Val, EVT VT, bool isT) { 965 return getConstant(*ConstantInt::get(*Context, Val), VT, isT); 966} 967 968SDValue SelectionDAG::getConstant(const ConstantInt &Val, EVT VT, bool isT) { 969 assert(VT.isInteger() && "Cannot create FP integer constant!"); 970 971 EVT EltVT = VT.getScalarType(); 972 const ConstantInt *Elt = &Val; 973 974 const TargetLowering *TLI = TM.getTargetLowering(); 975 976 // In some cases the vector type is legal but the element type is illegal and 977 // needs to be promoted, for example v8i8 on ARM. In this case, promote the 978 // inserted value (the type does not need to match the vector element type). 979 // Any extra bits introduced will be truncated away. 980 if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) == 981 TargetLowering::TypePromoteInteger) { 982 EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT); 983 APInt NewVal = Elt->getValue().zext(EltVT.getSizeInBits()); 984 Elt = ConstantInt::get(*getContext(), NewVal); 985 } 986 987 assert(Elt->getBitWidth() == EltVT.getSizeInBits() && 988 "APInt size does not match type size!"); 989 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant; 990 FoldingSetNodeID ID; 991 AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0); 992 ID.AddPointer(Elt); 993 void *IP = 0; 994 SDNode *N = NULL; 995 if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) 996 if (!VT.isVector()) 997 return SDValue(N, 0); 998 999 if (!N) { 1000 N = new (NodeAllocator) ConstantSDNode(isT, Elt, EltVT); 1001 CSEMap.InsertNode(N, IP); 1002 AllNodes.push_back(N); 1003 } 1004 1005 SDValue Result(N, 0); 1006 if (VT.isVector()) { 1007 SmallVector<SDValue, 8> Ops; 1008 Ops.assign(VT.getVectorNumElements(), Result); 1009 Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, &Ops[0], Ops.size()); 1010 } 1011 return Result; 1012} 1013 1014SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, bool isTarget) { 1015 return getConstant(Val, TM.getTargetLowering()->getPointerTy(), isTarget); 1016} 1017 1018 1019SDValue SelectionDAG::getConstantFP(const APFloat& V, EVT VT, bool isTarget) { 1020 return getConstantFP(*ConstantFP::get(*getContext(), V), VT, isTarget); 1021} 1022 1023SDValue SelectionDAG::getConstantFP(const ConstantFP& V, EVT VT, bool isTarget){ 1024 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!"); 1025 1026 EVT EltVT = VT.getScalarType(); 1027 1028 // Do the map lookup using the actual bit pattern for the floating point 1029 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and 1030 // we don't have issues with SNANs. 1031 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP; 1032 FoldingSetNodeID ID; 1033 AddNodeIDNode(ID, Opc, getVTList(EltVT), 0, 0); 1034 ID.AddPointer(&V); 1035 void *IP = 0; 1036 SDNode *N = NULL; 1037 if ((N = CSEMap.FindNodeOrInsertPos(ID, IP))) 1038 if (!VT.isVector()) 1039 return SDValue(N, 0); 1040 1041 if (!N) { 1042 N = new (NodeAllocator) ConstantFPSDNode(isTarget, &V, EltVT); 1043 CSEMap.InsertNode(N, IP); 1044 AllNodes.push_back(N); 1045 } 1046 1047 SDValue Result(N, 0); 1048 if (VT.isVector()) { 1049 SmallVector<SDValue, 8> Ops; 1050 Ops.assign(VT.getVectorNumElements(), Result); 1051 // FIXME SDLoc info might be appropriate here 1052 Result = getNode(ISD::BUILD_VECTOR, SDLoc(), VT, &Ops[0], Ops.size()); 1053 } 1054 return Result; 1055} 1056 1057SDValue SelectionDAG::getConstantFP(double Val, EVT VT, bool isTarget) { 1058 EVT EltVT = VT.getScalarType(); 1059 if (EltVT==MVT::f32) 1060 return getConstantFP(APFloat((float)Val), VT, isTarget); 1061 else if (EltVT==MVT::f64) 1062 return getConstantFP(APFloat(Val), VT, isTarget); 1063 else if (EltVT==MVT::f80 || EltVT==MVT::f128 || EltVT==MVT::ppcf128 || 1064 EltVT==MVT::f16) { 1065 bool ignored; 1066 APFloat apf = APFloat(Val); 1067 apf.convert(EVTToAPFloatSemantics(EltVT), APFloat::rmNearestTiesToEven, 1068 &ignored); 1069 return getConstantFP(apf, VT, isTarget); 1070 } else 1071 llvm_unreachable("Unsupported type in getConstantFP"); 1072} 1073 1074SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, SDLoc DL, 1075 EVT VT, int64_t Offset, 1076 bool isTargetGA, 1077 unsigned char TargetFlags) { 1078 assert((TargetFlags == 0 || isTargetGA) && 1079 "Cannot set target flags on target-independent globals"); 1080 1081 // Truncate (with sign-extension) the offset value to the pointer size. 1082 unsigned BitWidth = TM.getTargetLowering()->getPointerTy().getSizeInBits(); 1083 if (BitWidth < 64) 1084 Offset = SignExtend64(Offset, BitWidth); 1085 1086 const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV); 1087 if (!GVar) { 1088 // If GV is an alias then use the aliasee for determining thread-localness. 1089 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV)) 1090 GVar = dyn_cast_or_null<GlobalVariable>(GA->resolveAliasedGlobal(false)); 1091 } 1092 1093 unsigned Opc; 1094 if (GVar && GVar->isThreadLocal()) 1095 Opc = isTargetGA ? ISD::TargetGlobalTLSAddress : ISD::GlobalTLSAddress; 1096 else 1097 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress; 1098 1099 FoldingSetNodeID ID; 1100 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1101 ID.AddPointer(GV); 1102 ID.AddInteger(Offset); 1103 ID.AddInteger(TargetFlags); 1104 ID.AddInteger(GV->getType()->getAddressSpace()); 1105 void *IP = 0; 1106 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1107 return SDValue(E, 0); 1108 1109 SDNode *N = new (NodeAllocator) GlobalAddressSDNode(Opc, DL.getIROrder(), 1110 DL.getDebugLoc(), GV, VT, 1111 Offset, TargetFlags); 1112 CSEMap.InsertNode(N, IP); 1113 AllNodes.push_back(N); 1114 return SDValue(N, 0); 1115} 1116 1117SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) { 1118 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex; 1119 FoldingSetNodeID ID; 1120 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1121 ID.AddInteger(FI); 1122 void *IP = 0; 1123 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1124 return SDValue(E, 0); 1125 1126 SDNode *N = new (NodeAllocator) FrameIndexSDNode(FI, VT, isTarget); 1127 CSEMap.InsertNode(N, IP); 1128 AllNodes.push_back(N); 1129 return SDValue(N, 0); 1130} 1131 1132SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget, 1133 unsigned char TargetFlags) { 1134 assert((TargetFlags == 0 || isTarget) && 1135 "Cannot set target flags on target-independent jump tables"); 1136 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable; 1137 FoldingSetNodeID ID; 1138 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1139 ID.AddInteger(JTI); 1140 ID.AddInteger(TargetFlags); 1141 void *IP = 0; 1142 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1143 return SDValue(E, 0); 1144 1145 SDNode *N = new (NodeAllocator) JumpTableSDNode(JTI, VT, isTarget, 1146 TargetFlags); 1147 CSEMap.InsertNode(N, IP); 1148 AllNodes.push_back(N); 1149 return SDValue(N, 0); 1150} 1151 1152SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT, 1153 unsigned Alignment, int Offset, 1154 bool isTarget, 1155 unsigned char TargetFlags) { 1156 assert((TargetFlags == 0 || isTarget) && 1157 "Cannot set target flags on target-independent globals"); 1158 if (Alignment == 0) 1159 Alignment = 1160 TM.getTargetLowering()->getDataLayout()->getPrefTypeAlignment(C->getType()); 1161 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; 1162 FoldingSetNodeID ID; 1163 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1164 ID.AddInteger(Alignment); 1165 ID.AddInteger(Offset); 1166 ID.AddPointer(C); 1167 ID.AddInteger(TargetFlags); 1168 void *IP = 0; 1169 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1170 return SDValue(E, 0); 1171 1172 SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset, 1173 Alignment, TargetFlags); 1174 CSEMap.InsertNode(N, IP); 1175 AllNodes.push_back(N); 1176 return SDValue(N, 0); 1177} 1178 1179 1180SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT, 1181 unsigned Alignment, int Offset, 1182 bool isTarget, 1183 unsigned char TargetFlags) { 1184 assert((TargetFlags == 0 || isTarget) && 1185 "Cannot set target flags on target-independent globals"); 1186 if (Alignment == 0) 1187 Alignment = 1188 TM.getTargetLowering()->getDataLayout()->getPrefTypeAlignment(C->getType()); 1189 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; 1190 FoldingSetNodeID ID; 1191 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1192 ID.AddInteger(Alignment); 1193 ID.AddInteger(Offset); 1194 C->addSelectionDAGCSEId(ID); 1195 ID.AddInteger(TargetFlags); 1196 void *IP = 0; 1197 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1198 return SDValue(E, 0); 1199 1200 SDNode *N = new (NodeAllocator) ConstantPoolSDNode(isTarget, C, VT, Offset, 1201 Alignment, TargetFlags); 1202 CSEMap.InsertNode(N, IP); 1203 AllNodes.push_back(N); 1204 return SDValue(N, 0); 1205} 1206 1207SDValue SelectionDAG::getTargetIndex(int Index, EVT VT, int64_t Offset, 1208 unsigned char TargetFlags) { 1209 FoldingSetNodeID ID; 1210 AddNodeIDNode(ID, ISD::TargetIndex, getVTList(VT), 0, 0); 1211 ID.AddInteger(Index); 1212 ID.AddInteger(Offset); 1213 ID.AddInteger(TargetFlags); 1214 void *IP = 0; 1215 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1216 return SDValue(E, 0); 1217 1218 SDNode *N = new (NodeAllocator) TargetIndexSDNode(Index, VT, Offset, 1219 TargetFlags); 1220 CSEMap.InsertNode(N, IP); 1221 AllNodes.push_back(N); 1222 return SDValue(N, 0); 1223} 1224 1225SDValue SelectionDAG::getBasicBlock(MachineBasicBlock *MBB) { 1226 FoldingSetNodeID ID; 1227 AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), 0, 0); 1228 ID.AddPointer(MBB); 1229 void *IP = 0; 1230 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1231 return SDValue(E, 0); 1232 1233 SDNode *N = new (NodeAllocator) BasicBlockSDNode(MBB); 1234 CSEMap.InsertNode(N, IP); 1235 AllNodes.push_back(N); 1236 return SDValue(N, 0); 1237} 1238 1239SDValue SelectionDAG::getValueType(EVT VT) { 1240 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >= 1241 ValueTypeNodes.size()) 1242 ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1); 1243 1244 SDNode *&N = VT.isExtended() ? 1245 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy]; 1246 1247 if (N) return SDValue(N, 0); 1248 N = new (NodeAllocator) VTSDNode(VT); 1249 AllNodes.push_back(N); 1250 return SDValue(N, 0); 1251} 1252 1253SDValue SelectionDAG::getExternalSymbol(const char *Sym, EVT VT) { 1254 SDNode *&N = ExternalSymbols[Sym]; 1255 if (N) return SDValue(N, 0); 1256 N = new (NodeAllocator) ExternalSymbolSDNode(false, Sym, 0, VT); 1257 AllNodes.push_back(N); 1258 return SDValue(N, 0); 1259} 1260 1261SDValue SelectionDAG::getTargetExternalSymbol(const char *Sym, EVT VT, 1262 unsigned char TargetFlags) { 1263 SDNode *&N = 1264 TargetExternalSymbols[std::pair<std::string,unsigned char>(Sym, 1265 TargetFlags)]; 1266 if (N) return SDValue(N, 0); 1267 N = new (NodeAllocator) ExternalSymbolSDNode(true, Sym, TargetFlags, VT); 1268 AllNodes.push_back(N); 1269 return SDValue(N, 0); 1270} 1271 1272SDValue SelectionDAG::getCondCode(ISD::CondCode Cond) { 1273 if ((unsigned)Cond >= CondCodeNodes.size()) 1274 CondCodeNodes.resize(Cond+1); 1275 1276 if (CondCodeNodes[Cond] == 0) { 1277 CondCodeSDNode *N = new (NodeAllocator) CondCodeSDNode(Cond); 1278 CondCodeNodes[Cond] = N; 1279 AllNodes.push_back(N); 1280 } 1281 1282 return SDValue(CondCodeNodes[Cond], 0); 1283} 1284 1285// commuteShuffle - swaps the values of N1 and N2, and swaps all indices in 1286// the shuffle mask M that point at N1 to point at N2, and indices that point 1287// N2 to point at N1. 1288static void commuteShuffle(SDValue &N1, SDValue &N2, SmallVectorImpl<int> &M) { 1289 std::swap(N1, N2); 1290 int NElts = M.size(); 1291 for (int i = 0; i != NElts; ++i) { 1292 if (M[i] >= NElts) 1293 M[i] -= NElts; 1294 else if (M[i] >= 0) 1295 M[i] += NElts; 1296 } 1297} 1298 1299SDValue SelectionDAG::getVectorShuffle(EVT VT, SDLoc dl, SDValue N1, 1300 SDValue N2, const int *Mask) { 1301 assert(VT == N1.getValueType() && VT == N2.getValueType() && 1302 "Invalid VECTOR_SHUFFLE"); 1303 1304 // Canonicalize shuffle undef, undef -> undef 1305 if (N1.getOpcode() == ISD::UNDEF && N2.getOpcode() == ISD::UNDEF) 1306 return getUNDEF(VT); 1307 1308 // Validate that all indices in Mask are within the range of the elements 1309 // input to the shuffle. 1310 unsigned NElts = VT.getVectorNumElements(); 1311 SmallVector<int, 8> MaskVec; 1312 for (unsigned i = 0; i != NElts; ++i) { 1313 assert(Mask[i] < (int)(NElts * 2) && "Index out of range"); 1314 MaskVec.push_back(Mask[i]); 1315 } 1316 1317 // Canonicalize shuffle v, v -> v, undef 1318 if (N1 == N2) { 1319 N2 = getUNDEF(VT); 1320 for (unsigned i = 0; i != NElts; ++i) 1321 if (MaskVec[i] >= (int)NElts) MaskVec[i] -= NElts; 1322 } 1323 1324 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask. 1325 if (N1.getOpcode() == ISD::UNDEF) 1326 commuteShuffle(N1, N2, MaskVec); 1327 1328 // Canonicalize all index into lhs, -> shuffle lhs, undef 1329 // Canonicalize all index into rhs, -> shuffle rhs, undef 1330 bool AllLHS = true, AllRHS = true; 1331 bool N2Undef = N2.getOpcode() == ISD::UNDEF; 1332 for (unsigned i = 0; i != NElts; ++i) { 1333 if (MaskVec[i] >= (int)NElts) { 1334 if (N2Undef) 1335 MaskVec[i] = -1; 1336 else 1337 AllLHS = false; 1338 } else if (MaskVec[i] >= 0) { 1339 AllRHS = false; 1340 } 1341 } 1342 if (AllLHS && AllRHS) 1343 return getUNDEF(VT); 1344 if (AllLHS && !N2Undef) 1345 N2 = getUNDEF(VT); 1346 if (AllRHS) { 1347 N1 = getUNDEF(VT); 1348 commuteShuffle(N1, N2, MaskVec); 1349 } 1350 1351 // If Identity shuffle return that node. 1352 bool Identity = true; 1353 for (unsigned i = 0; i != NElts; ++i) { 1354 if (MaskVec[i] >= 0 && MaskVec[i] != (int)i) Identity = false; 1355 } 1356 if (Identity && NElts) 1357 return N1; 1358 1359 FoldingSetNodeID ID; 1360 SDValue Ops[2] = { N1, N2 }; 1361 AddNodeIDNode(ID, ISD::VECTOR_SHUFFLE, getVTList(VT), Ops, 2); 1362 for (unsigned i = 0; i != NElts; ++i) 1363 ID.AddInteger(MaskVec[i]); 1364 1365 void* IP = 0; 1366 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1367 return SDValue(E, 0); 1368 1369 // Allocate the mask array for the node out of the BumpPtrAllocator, since 1370 // SDNode doesn't have access to it. This memory will be "leaked" when 1371 // the node is deallocated, but recovered when the NodeAllocator is released. 1372 int *MaskAlloc = OperandAllocator.Allocate<int>(NElts); 1373 memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int)); 1374 1375 ShuffleVectorSDNode *N = 1376 new (NodeAllocator) ShuffleVectorSDNode(VT, dl.getIROrder(), 1377 dl.getDebugLoc(), N1, N2, 1378 MaskAlloc); 1379 CSEMap.InsertNode(N, IP); 1380 AllNodes.push_back(N); 1381 return SDValue(N, 0); 1382} 1383 1384SDValue SelectionDAG::getConvertRndSat(EVT VT, SDLoc dl, 1385 SDValue Val, SDValue DTy, 1386 SDValue STy, SDValue Rnd, SDValue Sat, 1387 ISD::CvtCode Code) { 1388 // If the src and dest types are the same and the conversion is between 1389 // integer types of the same sign or two floats, no conversion is necessary. 1390 if (DTy == STy && 1391 (Code == ISD::CVT_UU || Code == ISD::CVT_SS || Code == ISD::CVT_FF)) 1392 return Val; 1393 1394 FoldingSetNodeID ID; 1395 SDValue Ops[] = { Val, DTy, STy, Rnd, Sat }; 1396 AddNodeIDNode(ID, ISD::CONVERT_RNDSAT, getVTList(VT), &Ops[0], 5); 1397 void* IP = 0; 1398 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1399 return SDValue(E, 0); 1400 1401 CvtRndSatSDNode *N = new (NodeAllocator) CvtRndSatSDNode(VT, dl.getIROrder(), 1402 dl.getDebugLoc(), 1403 Ops, 5, Code); 1404 CSEMap.InsertNode(N, IP); 1405 AllNodes.push_back(N); 1406 return SDValue(N, 0); 1407} 1408 1409SDValue SelectionDAG::getRegister(unsigned RegNo, EVT VT) { 1410 FoldingSetNodeID ID; 1411 AddNodeIDNode(ID, ISD::Register, getVTList(VT), 0, 0); 1412 ID.AddInteger(RegNo); 1413 void *IP = 0; 1414 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1415 return SDValue(E, 0); 1416 1417 SDNode *N = new (NodeAllocator) RegisterSDNode(RegNo, VT); 1418 CSEMap.InsertNode(N, IP); 1419 AllNodes.push_back(N); 1420 return SDValue(N, 0); 1421} 1422 1423SDValue SelectionDAG::getRegisterMask(const uint32_t *RegMask) { 1424 FoldingSetNodeID ID; 1425 AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), 0, 0); 1426 ID.AddPointer(RegMask); 1427 void *IP = 0; 1428 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1429 return SDValue(E, 0); 1430 1431 SDNode *N = new (NodeAllocator) RegisterMaskSDNode(RegMask); 1432 CSEMap.InsertNode(N, IP); 1433 AllNodes.push_back(N); 1434 return SDValue(N, 0); 1435} 1436 1437SDValue SelectionDAG::getEHLabel(SDLoc dl, SDValue Root, MCSymbol *Label) { 1438 FoldingSetNodeID ID; 1439 SDValue Ops[] = { Root }; 1440 AddNodeIDNode(ID, ISD::EH_LABEL, getVTList(MVT::Other), &Ops[0], 1); 1441 ID.AddPointer(Label); 1442 void *IP = 0; 1443 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1444 return SDValue(E, 0); 1445 1446 SDNode *N = new (NodeAllocator) EHLabelSDNode(dl.getIROrder(), 1447 dl.getDebugLoc(), Root, Label); 1448 CSEMap.InsertNode(N, IP); 1449 AllNodes.push_back(N); 1450 return SDValue(N, 0); 1451} 1452 1453 1454SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT, 1455 int64_t Offset, 1456 bool isTarget, 1457 unsigned char TargetFlags) { 1458 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress; 1459 1460 FoldingSetNodeID ID; 1461 AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0); 1462 ID.AddPointer(BA); 1463 ID.AddInteger(Offset); 1464 ID.AddInteger(TargetFlags); 1465 void *IP = 0; 1466 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1467 return SDValue(E, 0); 1468 1469 SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, Offset, 1470 TargetFlags); 1471 CSEMap.InsertNode(N, IP); 1472 AllNodes.push_back(N); 1473 return SDValue(N, 0); 1474} 1475 1476SDValue SelectionDAG::getSrcValue(const Value *V) { 1477 assert((!V || V->getType()->isPointerTy()) && 1478 "SrcValue is not a pointer?"); 1479 1480 FoldingSetNodeID ID; 1481 AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), 0, 0); 1482 ID.AddPointer(V); 1483 1484 void *IP = 0; 1485 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1486 return SDValue(E, 0); 1487 1488 SDNode *N = new (NodeAllocator) SrcValueSDNode(V); 1489 CSEMap.InsertNode(N, IP); 1490 AllNodes.push_back(N); 1491 return SDValue(N, 0); 1492} 1493 1494/// getMDNode - Return an MDNodeSDNode which holds an MDNode. 1495SDValue SelectionDAG::getMDNode(const MDNode *MD) { 1496 FoldingSetNodeID ID; 1497 AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), 0, 0); 1498 ID.AddPointer(MD); 1499 1500 void *IP = 0; 1501 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1502 return SDValue(E, 0); 1503 1504 SDNode *N = new (NodeAllocator) MDNodeSDNode(MD); 1505 CSEMap.InsertNode(N, IP); 1506 AllNodes.push_back(N); 1507 return SDValue(N, 0); 1508} 1509 1510/// getAddrSpaceCast - Return an AddrSpaceCastSDNode. 1511SDValue SelectionDAG::getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr, 1512 unsigned SrcAS, unsigned DestAS) { 1513 SDValue Ops[] = {Ptr}; 1514 FoldingSetNodeID ID; 1515 AddNodeIDNode(ID, ISD::ADDRSPACECAST, getVTList(VT), &Ops[0], 1); 1516 ID.AddInteger(SrcAS); 1517 ID.AddInteger(DestAS); 1518 1519 void *IP = 0; 1520 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 1521 return SDValue(E, 0); 1522 1523 SDNode *N = new (NodeAllocator) AddrSpaceCastSDNode(dl.getIROrder(), 1524 dl.getDebugLoc(), 1525 VT, Ptr, SrcAS, DestAS); 1526 CSEMap.InsertNode(N, IP); 1527 AllNodes.push_back(N); 1528 return SDValue(N, 0); 1529} 1530 1531/// getShiftAmountOperand - Return the specified value casted to 1532/// the target's desired shift amount type. 1533SDValue SelectionDAG::getShiftAmountOperand(EVT LHSTy, SDValue Op) { 1534 EVT OpTy = Op.getValueType(); 1535 EVT ShTy = TM.getTargetLowering()->getShiftAmountTy(LHSTy); 1536 if (OpTy == ShTy || OpTy.isVector()) return Op; 1537 1538 ISD::NodeType Opcode = OpTy.bitsGT(ShTy) ? ISD::TRUNCATE : ISD::ZERO_EXTEND; 1539 return getNode(Opcode, SDLoc(Op), ShTy, Op); 1540} 1541 1542/// CreateStackTemporary - Create a stack temporary, suitable for holding the 1543/// specified value type. 1544SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) { 1545 MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo(); 1546 unsigned ByteSize = VT.getStoreSize(); 1547 Type *Ty = VT.getTypeForEVT(*getContext()); 1548 const TargetLowering *TLI = TM.getTargetLowering(); 1549 unsigned StackAlign = 1550 std::max((unsigned)TLI->getDataLayout()->getPrefTypeAlignment(Ty), minAlign); 1551 1552 int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false); 1553 return getFrameIndex(FrameIdx, TLI->getPointerTy()); 1554} 1555 1556/// CreateStackTemporary - Create a stack temporary suitable for holding 1557/// either of the specified value types. 1558SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) { 1559 unsigned Bytes = std::max(VT1.getStoreSizeInBits(), 1560 VT2.getStoreSizeInBits())/8; 1561 Type *Ty1 = VT1.getTypeForEVT(*getContext()); 1562 Type *Ty2 = VT2.getTypeForEVT(*getContext()); 1563 const TargetLowering *TLI = TM.getTargetLowering(); 1564 const DataLayout *TD = TLI->getDataLayout(); 1565 unsigned Align = std::max(TD->getPrefTypeAlignment(Ty1), 1566 TD->getPrefTypeAlignment(Ty2)); 1567 1568 MachineFrameInfo *FrameInfo = getMachineFunction().getFrameInfo(); 1569 int FrameIdx = FrameInfo->CreateStackObject(Bytes, Align, false); 1570 return getFrameIndex(FrameIdx, TLI->getPointerTy()); 1571} 1572 1573SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, 1574 SDValue N2, ISD::CondCode Cond, SDLoc dl) { 1575 // These setcc operations always fold. 1576 switch (Cond) { 1577 default: break; 1578 case ISD::SETFALSE: 1579 case ISD::SETFALSE2: return getConstant(0, VT); 1580 case ISD::SETTRUE: 1581 case ISD::SETTRUE2: { 1582 const TargetLowering *TLI = TM.getTargetLowering(); 1583 TargetLowering::BooleanContent Cnt = TLI->getBooleanContents(VT.isVector()); 1584 return getConstant( 1585 Cnt == TargetLowering::ZeroOrNegativeOneBooleanContent ? -1ULL : 1, VT); 1586 } 1587 1588 case ISD::SETOEQ: 1589 case ISD::SETOGT: 1590 case ISD::SETOGE: 1591 case ISD::SETOLT: 1592 case ISD::SETOLE: 1593 case ISD::SETONE: 1594 case ISD::SETO: 1595 case ISD::SETUO: 1596 case ISD::SETUEQ: 1597 case ISD::SETUNE: 1598 assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!"); 1599 break; 1600 } 1601 1602 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode())) { 1603 const APInt &C2 = N2C->getAPIntValue(); 1604 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) { 1605 const APInt &C1 = N1C->getAPIntValue(); 1606 1607 switch (Cond) { 1608 default: llvm_unreachable("Unknown integer setcc!"); 1609 case ISD::SETEQ: return getConstant(C1 == C2, VT); 1610 case ISD::SETNE: return getConstant(C1 != C2, VT); 1611 case ISD::SETULT: return getConstant(C1.ult(C2), VT); 1612 case ISD::SETUGT: return getConstant(C1.ugt(C2), VT); 1613 case ISD::SETULE: return getConstant(C1.ule(C2), VT); 1614 case ISD::SETUGE: return getConstant(C1.uge(C2), VT); 1615 case ISD::SETLT: return getConstant(C1.slt(C2), VT); 1616 case ISD::SETGT: return getConstant(C1.sgt(C2), VT); 1617 case ISD::SETLE: return getConstant(C1.sle(C2), VT); 1618 case ISD::SETGE: return getConstant(C1.sge(C2), VT); 1619 } 1620 } 1621 } 1622 if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1.getNode())) { 1623 if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2.getNode())) { 1624 APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF()); 1625 switch (Cond) { 1626 default: break; 1627 case ISD::SETEQ: if (R==APFloat::cmpUnordered) 1628 return getUNDEF(VT); 1629 // fall through 1630 case ISD::SETOEQ: return getConstant(R==APFloat::cmpEqual, VT); 1631 case ISD::SETNE: if (R==APFloat::cmpUnordered) 1632 return getUNDEF(VT); 1633 // fall through 1634 case ISD::SETONE: return getConstant(R==APFloat::cmpGreaterThan || 1635 R==APFloat::cmpLessThan, VT); 1636 case ISD::SETLT: if (R==APFloat::cmpUnordered) 1637 return getUNDEF(VT); 1638 // fall through 1639 case ISD::SETOLT: return getConstant(R==APFloat::cmpLessThan, VT); 1640 case ISD::SETGT: if (R==APFloat::cmpUnordered) 1641 return getUNDEF(VT); 1642 // fall through 1643 case ISD::SETOGT: return getConstant(R==APFloat::cmpGreaterThan, VT); 1644 case ISD::SETLE: if (R==APFloat::cmpUnordered) 1645 return getUNDEF(VT); 1646 // fall through 1647 case ISD::SETOLE: return getConstant(R==APFloat::cmpLessThan || 1648 R==APFloat::cmpEqual, VT); 1649 case ISD::SETGE: if (R==APFloat::cmpUnordered) 1650 return getUNDEF(VT); 1651 // fall through 1652 case ISD::SETOGE: return getConstant(R==APFloat::cmpGreaterThan || 1653 R==APFloat::cmpEqual, VT); 1654 case ISD::SETO: return getConstant(R!=APFloat::cmpUnordered, VT); 1655 case ISD::SETUO: return getConstant(R==APFloat::cmpUnordered, VT); 1656 case ISD::SETUEQ: return getConstant(R==APFloat::cmpUnordered || 1657 R==APFloat::cmpEqual, VT); 1658 case ISD::SETUNE: return getConstant(R!=APFloat::cmpEqual, VT); 1659 case ISD::SETULT: return getConstant(R==APFloat::cmpUnordered || 1660 R==APFloat::cmpLessThan, VT); 1661 case ISD::SETUGT: return getConstant(R==APFloat::cmpGreaterThan || 1662 R==APFloat::cmpUnordered, VT); 1663 case ISD::SETULE: return getConstant(R!=APFloat::cmpGreaterThan, VT); 1664 case ISD::SETUGE: return getConstant(R!=APFloat::cmpLessThan, VT); 1665 } 1666 } else { 1667 // Ensure that the constant occurs on the RHS. 1668 ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond); 1669 MVT CompVT = N1.getValueType().getSimpleVT(); 1670 if (!TM.getTargetLowering()->isCondCodeLegal(SwappedCond, CompVT)) 1671 return SDValue(); 1672 1673 return getSetCC(dl, VT, N2, N1, SwappedCond); 1674 } 1675 } 1676 1677 // Could not fold it. 1678 return SDValue(); 1679} 1680 1681/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We 1682/// use this predicate to simplify operations downstream. 1683bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const { 1684 // This predicate is not safe for vector operations. 1685 if (Op.getValueType().isVector()) 1686 return false; 1687 1688 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 1689 return MaskedValueIsZero(Op, APInt::getSignBit(BitWidth), Depth); 1690} 1691 1692/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use 1693/// this predicate to simplify operations downstream. Mask is known to be zero 1694/// for bits that V cannot have. 1695bool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask, 1696 unsigned Depth) const { 1697 APInt KnownZero, KnownOne; 1698 ComputeMaskedBits(Op, KnownZero, KnownOne, Depth); 1699 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1700 return (KnownZero & Mask) == Mask; 1701} 1702 1703/// ComputeMaskedBits - Determine which of the bits specified in Mask are 1704/// known to be either zero or one and return them in the KnownZero/KnownOne 1705/// bitsets. This code only analyzes bits in Mask, in order to short-circuit 1706/// processing. 1707void SelectionDAG::ComputeMaskedBits(SDValue Op, APInt &KnownZero, 1708 APInt &KnownOne, unsigned Depth) const { 1709 const TargetLowering *TLI = TM.getTargetLowering(); 1710 unsigned BitWidth = Op.getValueType().getScalarType().getSizeInBits(); 1711 1712 KnownZero = KnownOne = APInt(BitWidth, 0); // Don't know anything. 1713 if (Depth == 6) 1714 return; // Limit search depth. 1715 1716 APInt KnownZero2, KnownOne2; 1717 1718 switch (Op.getOpcode()) { 1719 case ISD::Constant: 1720 // We know all of the bits for a constant! 1721 KnownOne = cast<ConstantSDNode>(Op)->getAPIntValue(); 1722 KnownZero = ~KnownOne; 1723 return; 1724 case ISD::AND: 1725 // If either the LHS or the RHS are Zero, the result is zero. 1726 ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 1727 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1728 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1729 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1730 1731 // Output known-1 bits are only known if set in both the LHS & RHS. 1732 KnownOne &= KnownOne2; 1733 // Output known-0 are known to be clear if zero in either the LHS | RHS. 1734 KnownZero |= KnownZero2; 1735 return; 1736 case ISD::OR: 1737 ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 1738 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1739 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1740 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1741 1742 // Output known-0 bits are only known if clear in both the LHS & RHS. 1743 KnownZero &= KnownZero2; 1744 // Output known-1 are known to be set if set in either the LHS | RHS. 1745 KnownOne |= KnownOne2; 1746 return; 1747 case ISD::XOR: { 1748 ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 1749 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1750 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1751 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1752 1753 // Output known-0 bits are known if clear or set in both the LHS & RHS. 1754 APInt KnownZeroOut = (KnownZero & KnownZero2) | (KnownOne & KnownOne2); 1755 // Output known-1 are known to be set if set in only one of the LHS, RHS. 1756 KnownOne = (KnownZero & KnownOne2) | (KnownOne & KnownZero2); 1757 KnownZero = KnownZeroOut; 1758 return; 1759 } 1760 case ISD::MUL: { 1761 ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 1762 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1763 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1764 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1765 1766 // If low bits are zero in either operand, output low known-0 bits. 1767 // Also compute a conserative estimate for high known-0 bits. 1768 // More trickiness is possible, but this is sufficient for the 1769 // interesting case of alignment computation. 1770 KnownOne.clearAllBits(); 1771 unsigned TrailZ = KnownZero.countTrailingOnes() + 1772 KnownZero2.countTrailingOnes(); 1773 unsigned LeadZ = std::max(KnownZero.countLeadingOnes() + 1774 KnownZero2.countLeadingOnes(), 1775 BitWidth) - BitWidth; 1776 1777 TrailZ = std::min(TrailZ, BitWidth); 1778 LeadZ = std::min(LeadZ, BitWidth); 1779 KnownZero = APInt::getLowBitsSet(BitWidth, TrailZ) | 1780 APInt::getHighBitsSet(BitWidth, LeadZ); 1781 return; 1782 } 1783 case ISD::UDIV: { 1784 // For the purposes of computing leading zeros we can conservatively 1785 // treat a udiv as a logical right shift by the power of 2 known to 1786 // be less than the denominator. 1787 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 1788 unsigned LeadZ = KnownZero2.countLeadingOnes(); 1789 1790 KnownOne2.clearAllBits(); 1791 KnownZero2.clearAllBits(); 1792 ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 1793 unsigned RHSUnknownLeadingOnes = KnownOne2.countLeadingZeros(); 1794 if (RHSUnknownLeadingOnes != BitWidth) 1795 LeadZ = std::min(BitWidth, 1796 LeadZ + BitWidth - RHSUnknownLeadingOnes - 1); 1797 1798 KnownZero = APInt::getHighBitsSet(BitWidth, LeadZ); 1799 return; 1800 } 1801 case ISD::SELECT: 1802 ComputeMaskedBits(Op.getOperand(2), KnownZero, KnownOne, Depth+1); 1803 ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 1804 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1805 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1806 1807 // Only known if known in both the LHS and RHS. 1808 KnownOne &= KnownOne2; 1809 KnownZero &= KnownZero2; 1810 return; 1811 case ISD::SELECT_CC: 1812 ComputeMaskedBits(Op.getOperand(3), KnownZero, KnownOne, Depth+1); 1813 ComputeMaskedBits(Op.getOperand(2), KnownZero2, KnownOne2, Depth+1); 1814 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1815 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 1816 1817 // Only known if known in both the LHS and RHS. 1818 KnownOne &= KnownOne2; 1819 KnownZero &= KnownZero2; 1820 return; 1821 case ISD::SADDO: 1822 case ISD::UADDO: 1823 case ISD::SSUBO: 1824 case ISD::USUBO: 1825 case ISD::SMULO: 1826 case ISD::UMULO: 1827 if (Op.getResNo() != 1) 1828 return; 1829 // The boolean result conforms to getBooleanContents. Fall through. 1830 case ISD::SETCC: 1831 // If we know the result of a setcc has the top bits zero, use this info. 1832 if (TLI->getBooleanContents(Op.getValueType().isVector()) == 1833 TargetLowering::ZeroOrOneBooleanContent && BitWidth > 1) 1834 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1); 1835 return; 1836 case ISD::SHL: 1837 // (shl X, C1) & C2 == 0 iff (X & C2 >>u C1) == 0 1838 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 1839 unsigned ShAmt = SA->getZExtValue(); 1840 1841 // If the shift count is an invalid immediate, don't do anything. 1842 if (ShAmt >= BitWidth) 1843 return; 1844 1845 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1846 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1847 KnownZero <<= ShAmt; 1848 KnownOne <<= ShAmt; 1849 // low bits known zero. 1850 KnownZero |= APInt::getLowBitsSet(BitWidth, ShAmt); 1851 } 1852 return; 1853 case ISD::SRL: 1854 // (ushr X, C1) & C2 == 0 iff (-1 >> C1) & C2 == 0 1855 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 1856 unsigned ShAmt = SA->getZExtValue(); 1857 1858 // If the shift count is an invalid immediate, don't do anything. 1859 if (ShAmt >= BitWidth) 1860 return; 1861 1862 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1863 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1864 KnownZero = KnownZero.lshr(ShAmt); 1865 KnownOne = KnownOne.lshr(ShAmt); 1866 1867 APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt); 1868 KnownZero |= HighBits; // High bits known zero. 1869 } 1870 return; 1871 case ISD::SRA: 1872 if (ConstantSDNode *SA = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 1873 unsigned ShAmt = SA->getZExtValue(); 1874 1875 // If the shift count is an invalid immediate, don't do anything. 1876 if (ShAmt >= BitWidth) 1877 return; 1878 1879 // If any of the demanded bits are produced by the sign extension, we also 1880 // demand the input sign bit. 1881 APInt HighBits = APInt::getHighBitsSet(BitWidth, ShAmt); 1882 1883 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1884 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1885 KnownZero = KnownZero.lshr(ShAmt); 1886 KnownOne = KnownOne.lshr(ShAmt); 1887 1888 // Handle the sign bits. 1889 APInt SignBit = APInt::getSignBit(BitWidth); 1890 SignBit = SignBit.lshr(ShAmt); // Adjust to where it is now in the mask. 1891 1892 if (KnownZero.intersects(SignBit)) { 1893 KnownZero |= HighBits; // New bits are known zero. 1894 } else if (KnownOne.intersects(SignBit)) { 1895 KnownOne |= HighBits; // New bits are known one. 1896 } 1897 } 1898 return; 1899 case ISD::SIGN_EXTEND_INREG: { 1900 EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 1901 unsigned EBits = EVT.getScalarType().getSizeInBits(); 1902 1903 // Sign extension. Compute the demanded bits in the result that are not 1904 // present in the input. 1905 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - EBits); 1906 1907 APInt InSignBit = APInt::getSignBit(EBits); 1908 APInt InputDemandedBits = APInt::getLowBitsSet(BitWidth, EBits); 1909 1910 // If the sign extended bits are demanded, we know that the sign 1911 // bit is demanded. 1912 InSignBit = InSignBit.zext(BitWidth); 1913 if (NewBits.getBoolValue()) 1914 InputDemandedBits |= InSignBit; 1915 1916 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1917 KnownOne &= InputDemandedBits; 1918 KnownZero &= InputDemandedBits; 1919 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 1920 1921 // If the sign bit of the input is known set or clear, then we know the 1922 // top bits of the result. 1923 if (KnownZero.intersects(InSignBit)) { // Input sign bit known clear 1924 KnownZero |= NewBits; 1925 KnownOne &= ~NewBits; 1926 } else if (KnownOne.intersects(InSignBit)) { // Input sign bit known set 1927 KnownOne |= NewBits; 1928 KnownZero &= ~NewBits; 1929 } else { // Input sign bit unknown 1930 KnownZero &= ~NewBits; 1931 KnownOne &= ~NewBits; 1932 } 1933 return; 1934 } 1935 case ISD::CTTZ: 1936 case ISD::CTTZ_ZERO_UNDEF: 1937 case ISD::CTLZ: 1938 case ISD::CTLZ_ZERO_UNDEF: 1939 case ISD::CTPOP: { 1940 unsigned LowBits = Log2_32(BitWidth)+1; 1941 KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - LowBits); 1942 KnownOne.clearAllBits(); 1943 return; 1944 } 1945 case ISD::LOAD: { 1946 LoadSDNode *LD = cast<LoadSDNode>(Op); 1947 // If this is a ZEXTLoad and we are looking at the loaded value. 1948 if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) { 1949 EVT VT = LD->getMemoryVT(); 1950 unsigned MemBits = VT.getScalarType().getSizeInBits(); 1951 KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits); 1952 } else if (const MDNode *Ranges = LD->getRanges()) { 1953 computeMaskedBitsLoad(*Ranges, KnownZero); 1954 } 1955 return; 1956 } 1957 case ISD::ZERO_EXTEND: { 1958 EVT InVT = Op.getOperand(0).getValueType(); 1959 unsigned InBits = InVT.getScalarType().getSizeInBits(); 1960 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits); 1961 KnownZero = KnownZero.trunc(InBits); 1962 KnownOne = KnownOne.trunc(InBits); 1963 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1964 KnownZero = KnownZero.zext(BitWidth); 1965 KnownOne = KnownOne.zext(BitWidth); 1966 KnownZero |= NewBits; 1967 return; 1968 } 1969 case ISD::SIGN_EXTEND: { 1970 EVT InVT = Op.getOperand(0).getValueType(); 1971 unsigned InBits = InVT.getScalarType().getSizeInBits(); 1972 APInt NewBits = APInt::getHighBitsSet(BitWidth, BitWidth - InBits); 1973 1974 KnownZero = KnownZero.trunc(InBits); 1975 KnownOne = KnownOne.trunc(InBits); 1976 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 1977 1978 // Note if the sign bit is known to be zero or one. 1979 bool SignBitKnownZero = KnownZero.isNegative(); 1980 bool SignBitKnownOne = KnownOne.isNegative(); 1981 assert(!(SignBitKnownZero && SignBitKnownOne) && 1982 "Sign bit can't be known to be both zero and one!"); 1983 1984 KnownZero = KnownZero.zext(BitWidth); 1985 KnownOne = KnownOne.zext(BitWidth); 1986 1987 // If the sign bit is known zero or one, the top bits match. 1988 if (SignBitKnownZero) 1989 KnownZero |= NewBits; 1990 else if (SignBitKnownOne) 1991 KnownOne |= NewBits; 1992 return; 1993 } 1994 case ISD::ANY_EXTEND: { 1995 EVT InVT = Op.getOperand(0).getValueType(); 1996 unsigned InBits = InVT.getScalarType().getSizeInBits(); 1997 KnownZero = KnownZero.trunc(InBits); 1998 KnownOne = KnownOne.trunc(InBits); 1999 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2000 KnownZero = KnownZero.zext(BitWidth); 2001 KnownOne = KnownOne.zext(BitWidth); 2002 return; 2003 } 2004 case ISD::TRUNCATE: { 2005 EVT InVT = Op.getOperand(0).getValueType(); 2006 unsigned InBits = InVT.getScalarType().getSizeInBits(); 2007 KnownZero = KnownZero.zext(InBits); 2008 KnownOne = KnownOne.zext(InBits); 2009 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2010 assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 2011 KnownZero = KnownZero.trunc(BitWidth); 2012 KnownOne = KnownOne.trunc(BitWidth); 2013 break; 2014 } 2015 case ISD::AssertZext: { 2016 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT(); 2017 APInt InMask = APInt::getLowBitsSet(BitWidth, VT.getSizeInBits()); 2018 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2019 KnownZero |= (~InMask); 2020 KnownOne &= (~KnownZero); 2021 return; 2022 } 2023 case ISD::FGETSIGN: 2024 // All bits are zero except the low bit. 2025 KnownZero = APInt::getHighBitsSet(BitWidth, BitWidth - 1); 2026 return; 2027 2028 case ISD::SUB: { 2029 if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) { 2030 // We know that the top bits of C-X are clear if X contains less bits 2031 // than C (i.e. no wrap-around can happen). For example, 20-X is 2032 // positive if we can prove that X is >= 0 and < 16. 2033 if (CLHS->getAPIntValue().isNonNegative()) { 2034 unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros(); 2035 // NLZ can't be BitWidth with no sign bit 2036 APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1); 2037 ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 2038 2039 // If all of the MaskV bits are known to be zero, then we know the 2040 // output top bits are zero, because we now know that the output is 2041 // from [0-C]. 2042 if ((KnownZero2 & MaskV) == MaskV) { 2043 unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros(); 2044 // Top bits known zero. 2045 KnownZero = APInt::getHighBitsSet(BitWidth, NLZ2); 2046 } 2047 } 2048 } 2049 } 2050 // fall through 2051 case ISD::ADD: 2052 case ISD::ADDE: { 2053 // Output known-0 bits are known if clear or set in both the low clear bits 2054 // common to both LHS & RHS. For example, 8+(X<<3) is known to have the 2055 // low 3 bits clear. 2056 ComputeMaskedBits(Op.getOperand(0), KnownZero2, KnownOne2, Depth+1); 2057 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 2058 unsigned KnownZeroOut = KnownZero2.countTrailingOnes(); 2059 2060 ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 2061 assert((KnownZero2 & KnownOne2) == 0 && "Bits known to be one AND zero?"); 2062 KnownZeroOut = std::min(KnownZeroOut, 2063 KnownZero2.countTrailingOnes()); 2064 2065 if (Op.getOpcode() == ISD::ADD) { 2066 KnownZero |= APInt::getLowBitsSet(BitWidth, KnownZeroOut); 2067 return; 2068 } 2069 2070 // With ADDE, a carry bit may be added in, so we can only use this 2071 // information if we know (at least) that the low two bits are clear. We 2072 // then return to the caller that the low bit is unknown but that other bits 2073 // are known zero. 2074 if (KnownZeroOut >= 2) // ADDE 2075 KnownZero |= APInt::getBitsSet(BitWidth, 1, KnownZeroOut); 2076 return; 2077 } 2078 case ISD::SREM: 2079 if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2080 const APInt &RA = Rem->getAPIntValue().abs(); 2081 if (RA.isPowerOf2()) { 2082 APInt LowBits = RA - 1; 2083 ComputeMaskedBits(Op.getOperand(0), KnownZero2,KnownOne2,Depth+1); 2084 2085 // The low bits of the first operand are unchanged by the srem. 2086 KnownZero = KnownZero2 & LowBits; 2087 KnownOne = KnownOne2 & LowBits; 2088 2089 // If the first operand is non-negative or has all low bits zero, then 2090 // the upper bits are all zero. 2091 if (KnownZero2[BitWidth-1] || ((KnownZero2 & LowBits) == LowBits)) 2092 KnownZero |= ~LowBits; 2093 2094 // If the first operand is negative and not all low bits are zero, then 2095 // the upper bits are all one. 2096 if (KnownOne2[BitWidth-1] && ((KnownOne2 & LowBits) != 0)) 2097 KnownOne |= ~LowBits; 2098 assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?"); 2099 } 2100 } 2101 return; 2102 case ISD::UREM: { 2103 if (ConstantSDNode *Rem = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2104 const APInt &RA = Rem->getAPIntValue(); 2105 if (RA.isPowerOf2()) { 2106 APInt LowBits = (RA - 1); 2107 KnownZero |= ~LowBits; 2108 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne,Depth+1); 2109 assert((KnownZero & KnownOne) == 0&&"Bits known to be one AND zero?"); 2110 break; 2111 } 2112 } 2113 2114 // Since the result is less than or equal to either operand, any leading 2115 // zero bits in either operand must also exist in the result. 2116 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2117 ComputeMaskedBits(Op.getOperand(1), KnownZero2, KnownOne2, Depth+1); 2118 2119 uint32_t Leaders = std::max(KnownZero.countLeadingOnes(), 2120 KnownZero2.countLeadingOnes()); 2121 KnownOne.clearAllBits(); 2122 KnownZero = APInt::getHighBitsSet(BitWidth, Leaders); 2123 return; 2124 } 2125 case ISD::FrameIndex: 2126 case ISD::TargetFrameIndex: 2127 if (unsigned Align = InferPtrAlignment(Op)) { 2128 // The low bits are known zero if the pointer is aligned. 2129 KnownZero = APInt::getLowBitsSet(BitWidth, Log2_32(Align)); 2130 return; 2131 } 2132 break; 2133 2134 default: 2135 if (Op.getOpcode() < ISD::BUILTIN_OP_END) 2136 break; 2137 // Fallthrough 2138 case ISD::INTRINSIC_WO_CHAIN: 2139 case ISD::INTRINSIC_W_CHAIN: 2140 case ISD::INTRINSIC_VOID: 2141 // Allow the target to implement this method for its nodes. 2142 TLI->computeMaskedBitsForTargetNode(Op, KnownZero, KnownOne, *this, Depth); 2143 return; 2144 } 2145} 2146 2147/// ComputeNumSignBits - Return the number of times the sign bit of the 2148/// register is replicated into the other bits. We know that at least 1 bit 2149/// is always equal to the sign bit (itself), but other cases can give us 2150/// information. For example, immediately after an "SRA X, 2", we know that 2151/// the top 3 bits are all equal to each other, so we return 3. 2152unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, unsigned Depth) const{ 2153 const TargetLowering *TLI = TM.getTargetLowering(); 2154 EVT VT = Op.getValueType(); 2155 assert(VT.isInteger() && "Invalid VT!"); 2156 unsigned VTBits = VT.getScalarType().getSizeInBits(); 2157 unsigned Tmp, Tmp2; 2158 unsigned FirstAnswer = 1; 2159 2160 if (Depth == 6) 2161 return 1; // Limit search depth. 2162 2163 switch (Op.getOpcode()) { 2164 default: break; 2165 case ISD::AssertSext: 2166 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits(); 2167 return VTBits-Tmp+1; 2168 case ISD::AssertZext: 2169 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits(); 2170 return VTBits-Tmp; 2171 2172 case ISD::Constant: { 2173 const APInt &Val = cast<ConstantSDNode>(Op)->getAPIntValue(); 2174 return Val.getNumSignBits(); 2175 } 2176 2177 case ISD::SIGN_EXTEND: 2178 Tmp = 2179 VTBits-Op.getOperand(0).getValueType().getScalarType().getSizeInBits(); 2180 return ComputeNumSignBits(Op.getOperand(0), Depth+1) + Tmp; 2181 2182 case ISD::SIGN_EXTEND_INREG: 2183 // Max of the input and what this extends. 2184 Tmp = 2185 cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarType().getSizeInBits(); 2186 Tmp = VTBits-Tmp+1; 2187 2188 Tmp2 = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2189 return std::max(Tmp, Tmp2); 2190 2191 case ISD::SRA: 2192 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2193 // SRA X, C -> adds C sign bits. 2194 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2195 Tmp += C->getZExtValue(); 2196 if (Tmp > VTBits) Tmp = VTBits; 2197 } 2198 return Tmp; 2199 case ISD::SHL: 2200 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2201 // shl destroys sign bits. 2202 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2203 if (C->getZExtValue() >= VTBits || // Bad shift. 2204 C->getZExtValue() >= Tmp) break; // Shifted all sign bits out. 2205 return Tmp - C->getZExtValue(); 2206 } 2207 break; 2208 case ISD::AND: 2209 case ISD::OR: 2210 case ISD::XOR: // NOT is handled here. 2211 // Logical binary ops preserve the number of sign bits at the worst. 2212 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2213 if (Tmp != 1) { 2214 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2215 FirstAnswer = std::min(Tmp, Tmp2); 2216 // We computed what we know about the sign bits as our first 2217 // answer. Now proceed to the generic code that uses 2218 // ComputeMaskedBits, and pick whichever answer is better. 2219 } 2220 break; 2221 2222 case ISD::SELECT: 2223 Tmp = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2224 if (Tmp == 1) return 1; // Early out. 2225 Tmp2 = ComputeNumSignBits(Op.getOperand(2), Depth+1); 2226 return std::min(Tmp, Tmp2); 2227 2228 case ISD::SADDO: 2229 case ISD::UADDO: 2230 case ISD::SSUBO: 2231 case ISD::USUBO: 2232 case ISD::SMULO: 2233 case ISD::UMULO: 2234 if (Op.getResNo() != 1) 2235 break; 2236 // The boolean result conforms to getBooleanContents. Fall through. 2237 case ISD::SETCC: 2238 // If setcc returns 0/-1, all bits are sign bits. 2239 if (TLI->getBooleanContents(Op.getValueType().isVector()) == 2240 TargetLowering::ZeroOrNegativeOneBooleanContent) 2241 return VTBits; 2242 break; 2243 case ISD::ROTL: 2244 case ISD::ROTR: 2245 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) { 2246 unsigned RotAmt = C->getZExtValue() & (VTBits-1); 2247 2248 // Handle rotate right by N like a rotate left by 32-N. 2249 if (Op.getOpcode() == ISD::ROTR) 2250 RotAmt = (VTBits-RotAmt) & (VTBits-1); 2251 2252 // If we aren't rotating out all of the known-in sign bits, return the 2253 // number that are left. This handles rotl(sext(x), 1) for example. 2254 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2255 if (Tmp > RotAmt+1) return Tmp-RotAmt; 2256 } 2257 break; 2258 case ISD::ADD: 2259 // Add can have at most one carry bit. Thus we know that the output 2260 // is, at worst, one more bit than the inputs. 2261 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2262 if (Tmp == 1) return 1; // Early out. 2263 2264 // Special case decrementing a value (ADD X, -1): 2265 if (ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(Op.getOperand(1))) 2266 if (CRHS->isAllOnesValue()) { 2267 APInt KnownZero, KnownOne; 2268 ComputeMaskedBits(Op.getOperand(0), KnownZero, KnownOne, Depth+1); 2269 2270 // If the input is known to be 0 or 1, the output is 0/-1, which is all 2271 // sign bits set. 2272 if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue()) 2273 return VTBits; 2274 2275 // If we are subtracting one from a positive number, there is no carry 2276 // out of the result. 2277 if (KnownZero.isNegative()) 2278 return Tmp; 2279 } 2280 2281 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2282 if (Tmp2 == 1) return 1; 2283 return std::min(Tmp, Tmp2)-1; 2284 2285 case ISD::SUB: 2286 Tmp2 = ComputeNumSignBits(Op.getOperand(1), Depth+1); 2287 if (Tmp2 == 1) return 1; 2288 2289 // Handle NEG. 2290 if (ConstantSDNode *CLHS = dyn_cast<ConstantSDNode>(Op.getOperand(0))) 2291 if (CLHS->isNullValue()) { 2292 APInt KnownZero, KnownOne; 2293 ComputeMaskedBits(Op.getOperand(1), KnownZero, KnownOne, Depth+1); 2294 // If the input is known to be 0 or 1, the output is 0/-1, which is all 2295 // sign bits set. 2296 if ((KnownZero | APInt(VTBits, 1)).isAllOnesValue()) 2297 return VTBits; 2298 2299 // If the input is known to be positive (the sign bit is known clear), 2300 // the output of the NEG has the same number of sign bits as the input. 2301 if (KnownZero.isNegative()) 2302 return Tmp2; 2303 2304 // Otherwise, we treat this like a SUB. 2305 } 2306 2307 // Sub can have at most one carry bit. Thus we know that the output 2308 // is, at worst, one more bit than the inputs. 2309 Tmp = ComputeNumSignBits(Op.getOperand(0), Depth+1); 2310 if (Tmp == 1) return 1; // Early out. 2311 return std::min(Tmp, Tmp2)-1; 2312 case ISD::TRUNCATE: 2313 // FIXME: it's tricky to do anything useful for this, but it is an important 2314 // case for targets like X86. 2315 break; 2316 } 2317 2318 // If we are looking at the loaded value of the SDNode. 2319 if (Op.getResNo() == 0) { 2320 // Handle LOADX separately here. EXTLOAD case will fallthrough. 2321 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) { 2322 unsigned ExtType = LD->getExtensionType(); 2323 switch (ExtType) { 2324 default: break; 2325 case ISD::SEXTLOAD: // '17' bits known 2326 Tmp = LD->getMemoryVT().getScalarType().getSizeInBits(); 2327 return VTBits-Tmp+1; 2328 case ISD::ZEXTLOAD: // '16' bits known 2329 Tmp = LD->getMemoryVT().getScalarType().getSizeInBits(); 2330 return VTBits-Tmp; 2331 } 2332 } 2333 } 2334 2335 // Allow the target to implement this method for its nodes. 2336 if (Op.getOpcode() >= ISD::BUILTIN_OP_END || 2337 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN || 2338 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN || 2339 Op.getOpcode() == ISD::INTRINSIC_VOID) { 2340 unsigned NumBits = TLI->ComputeNumSignBitsForTargetNode(Op, Depth); 2341 if (NumBits > 1) FirstAnswer = std::max(FirstAnswer, NumBits); 2342 } 2343 2344 // Finally, if we can prove that the top bits of the result are 0's or 1's, 2345 // use this information. 2346 APInt KnownZero, KnownOne; 2347 ComputeMaskedBits(Op, KnownZero, KnownOne, Depth); 2348 2349 APInt Mask; 2350 if (KnownZero.isNegative()) { // sign bit is 0 2351 Mask = KnownZero; 2352 } else if (KnownOne.isNegative()) { // sign bit is 1; 2353 Mask = KnownOne; 2354 } else { 2355 // Nothing known. 2356 return FirstAnswer; 2357 } 2358 2359 // Okay, we know that the sign bit in Mask is set. Use CLZ to determine 2360 // the number of identical bits in the top of the input value. 2361 Mask = ~Mask; 2362 Mask <<= Mask.getBitWidth()-VTBits; 2363 // Return # leading zeros. We use 'min' here in case Val was zero before 2364 // shifting. We don't want to return '64' as for an i32 "0". 2365 return std::max(FirstAnswer, std::min(VTBits, Mask.countLeadingZeros())); 2366} 2367 2368/// isBaseWithConstantOffset - Return true if the specified operand is an 2369/// ISD::ADD with a ConstantSDNode on the right-hand side, or if it is an 2370/// ISD::OR with a ConstantSDNode that is guaranteed to have the same 2371/// semantics as an ADD. This handles the equivalence: 2372/// X|Cst == X+Cst iff X&Cst = 0. 2373bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const { 2374 if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) || 2375 !isa<ConstantSDNode>(Op.getOperand(1))) 2376 return false; 2377 2378 if (Op.getOpcode() == ISD::OR && 2379 !MaskedValueIsZero(Op.getOperand(0), 2380 cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue())) 2381 return false; 2382 2383 return true; 2384} 2385 2386 2387bool SelectionDAG::isKnownNeverNaN(SDValue Op) const { 2388 // If we're told that NaNs won't happen, assume they won't. 2389 if (getTarget().Options.NoNaNsFPMath) 2390 return true; 2391 2392 // If the value is a constant, we can obviously see if it is a NaN or not. 2393 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) 2394 return !C->getValueAPF().isNaN(); 2395 2396 // TODO: Recognize more cases here. 2397 2398 return false; 2399} 2400 2401bool SelectionDAG::isKnownNeverZero(SDValue Op) const { 2402 // If the value is a constant, we can obviously see if it is a zero or not. 2403 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) 2404 return !C->isZero(); 2405 2406 // TODO: Recognize more cases here. 2407 switch (Op.getOpcode()) { 2408 default: break; 2409 case ISD::OR: 2410 if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) 2411 return !C->isNullValue(); 2412 break; 2413 } 2414 2415 return false; 2416} 2417 2418bool SelectionDAG::isEqualTo(SDValue A, SDValue B) const { 2419 // Check the obvious case. 2420 if (A == B) return true; 2421 2422 // For for negative and positive zero. 2423 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A)) 2424 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B)) 2425 if (CA->isZero() && CB->isZero()) return true; 2426 2427 // Otherwise they may not be equal. 2428 return false; 2429} 2430 2431/// getNode - Gets or creates the specified node. 2432/// 2433SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT) { 2434 FoldingSetNodeID ID; 2435 AddNodeIDNode(ID, Opcode, getVTList(VT), 0, 0); 2436 void *IP = 0; 2437 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 2438 return SDValue(E, 0); 2439 2440 SDNode *N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), 2441 DL.getDebugLoc(), getVTList(VT)); 2442 CSEMap.InsertNode(N, IP); 2443 2444 AllNodes.push_back(N); 2445#ifndef NDEBUG 2446 VerifySDNode(N); 2447#endif 2448 return SDValue(N, 0); 2449} 2450 2451SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, 2452 EVT VT, SDValue Operand) { 2453 // Constant fold unary operations with an integer constant operand. 2454 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Operand.getNode())) { 2455 const APInt &Val = C->getAPIntValue(); 2456 switch (Opcode) { 2457 default: break; 2458 case ISD::SIGN_EXTEND: 2459 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), VT); 2460 case ISD::ANY_EXTEND: 2461 case ISD::ZERO_EXTEND: 2462 case ISD::TRUNCATE: 2463 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), VT); 2464 case ISD::UINT_TO_FP: 2465 case ISD::SINT_TO_FP: { 2466 APFloat apf(EVTToAPFloatSemantics(VT), 2467 APInt::getNullValue(VT.getSizeInBits())); 2468 (void)apf.convertFromAPInt(Val, 2469 Opcode==ISD::SINT_TO_FP, 2470 APFloat::rmNearestTiesToEven); 2471 return getConstantFP(apf, VT); 2472 } 2473 case ISD::BITCAST: 2474 if (VT == MVT::f32 && C->getValueType(0) == MVT::i32) 2475 return getConstantFP(APFloat(APFloat::IEEEsingle, Val), VT); 2476 else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64) 2477 return getConstantFP(APFloat(APFloat::IEEEdouble, Val), VT); 2478 break; 2479 case ISD::BSWAP: 2480 return getConstant(Val.byteSwap(), VT); 2481 case ISD::CTPOP: 2482 return getConstant(Val.countPopulation(), VT); 2483 case ISD::CTLZ: 2484 case ISD::CTLZ_ZERO_UNDEF: 2485 return getConstant(Val.countLeadingZeros(), VT); 2486 case ISD::CTTZ: 2487 case ISD::CTTZ_ZERO_UNDEF: 2488 return getConstant(Val.countTrailingZeros(), VT); 2489 } 2490 } 2491 2492 // Constant fold unary operations with a floating point constant operand. 2493 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand.getNode())) { 2494 APFloat V = C->getValueAPF(); // make copy 2495 switch (Opcode) { 2496 case ISD::FNEG: 2497 V.changeSign(); 2498 return getConstantFP(V, VT); 2499 case ISD::FABS: 2500 V.clearSign(); 2501 return getConstantFP(V, VT); 2502 case ISD::FCEIL: { 2503 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive); 2504 if (fs == APFloat::opOK || fs == APFloat::opInexact) 2505 return getConstantFP(V, VT); 2506 break; 2507 } 2508 case ISD::FTRUNC: { 2509 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero); 2510 if (fs == APFloat::opOK || fs == APFloat::opInexact) 2511 return getConstantFP(V, VT); 2512 break; 2513 } 2514 case ISD::FFLOOR: { 2515 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative); 2516 if (fs == APFloat::opOK || fs == APFloat::opInexact) 2517 return getConstantFP(V, VT); 2518 break; 2519 } 2520 case ISD::FP_EXTEND: { 2521 bool ignored; 2522 // This can return overflow, underflow, or inexact; we don't care. 2523 // FIXME need to be more flexible about rounding mode. 2524 (void)V.convert(EVTToAPFloatSemantics(VT), 2525 APFloat::rmNearestTiesToEven, &ignored); 2526 return getConstantFP(V, VT); 2527 } 2528 case ISD::FP_TO_SINT: 2529 case ISD::FP_TO_UINT: { 2530 integerPart x[2]; 2531 bool ignored; 2532 assert(integerPartWidth >= 64); 2533 // FIXME need to be more flexible about rounding mode. 2534 APFloat::opStatus s = V.convertToInteger(x, VT.getSizeInBits(), 2535 Opcode==ISD::FP_TO_SINT, 2536 APFloat::rmTowardZero, &ignored); 2537 if (s==APFloat::opInvalidOp) // inexact is OK, in fact usual 2538 break; 2539 APInt api(VT.getSizeInBits(), x); 2540 return getConstant(api, VT); 2541 } 2542 case ISD::BITCAST: 2543 if (VT == MVT::i32 && C->getValueType(0) == MVT::f32) 2544 return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), VT); 2545 else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64) 2546 return getConstant(V.bitcastToAPInt().getZExtValue(), VT); 2547 break; 2548 } 2549 } 2550 2551 unsigned OpOpcode = Operand.getNode()->getOpcode(); 2552 switch (Opcode) { 2553 case ISD::TokenFactor: 2554 case ISD::MERGE_VALUES: 2555 case ISD::CONCAT_VECTORS: 2556 return Operand; // Factor, merge or concat of one node? No need. 2557 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node"); 2558 case ISD::FP_EXTEND: 2559 assert(VT.isFloatingPoint() && 2560 Operand.getValueType().isFloatingPoint() && "Invalid FP cast!"); 2561 if (Operand.getValueType() == VT) return Operand; // noop conversion. 2562 assert((!VT.isVector() || 2563 VT.getVectorNumElements() == 2564 Operand.getValueType().getVectorNumElements()) && 2565 "Vector element count mismatch!"); 2566 if (Operand.getOpcode() == ISD::UNDEF) 2567 return getUNDEF(VT); 2568 break; 2569 case ISD::SIGN_EXTEND: 2570 assert(VT.isInteger() && Operand.getValueType().isInteger() && 2571 "Invalid SIGN_EXTEND!"); 2572 if (Operand.getValueType() == VT) return Operand; // noop extension 2573 assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 2574 "Invalid sext node, dst < src!"); 2575 assert((!VT.isVector() || 2576 VT.getVectorNumElements() == 2577 Operand.getValueType().getVectorNumElements()) && 2578 "Vector element count mismatch!"); 2579 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) 2580 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 2581 else if (OpOpcode == ISD::UNDEF) 2582 // sext(undef) = 0, because the top bits will all be the same. 2583 return getConstant(0, VT); 2584 break; 2585 case ISD::ZERO_EXTEND: 2586 assert(VT.isInteger() && Operand.getValueType().isInteger() && 2587 "Invalid ZERO_EXTEND!"); 2588 if (Operand.getValueType() == VT) return Operand; // noop extension 2589 assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 2590 "Invalid zext node, dst < src!"); 2591 assert((!VT.isVector() || 2592 VT.getVectorNumElements() == 2593 Operand.getValueType().getVectorNumElements()) && 2594 "Vector element count mismatch!"); 2595 if (OpOpcode == ISD::ZERO_EXTEND) // (zext (zext x)) -> (zext x) 2596 return getNode(ISD::ZERO_EXTEND, DL, VT, 2597 Operand.getNode()->getOperand(0)); 2598 else if (OpOpcode == ISD::UNDEF) 2599 // zext(undef) = 0, because the top bits will be zero. 2600 return getConstant(0, VT); 2601 break; 2602 case ISD::ANY_EXTEND: 2603 assert(VT.isInteger() && Operand.getValueType().isInteger() && 2604 "Invalid ANY_EXTEND!"); 2605 if (Operand.getValueType() == VT) return Operand; // noop extension 2606 assert(Operand.getValueType().getScalarType().bitsLT(VT.getScalarType()) && 2607 "Invalid anyext node, dst < src!"); 2608 assert((!VT.isVector() || 2609 VT.getVectorNumElements() == 2610 Operand.getValueType().getVectorNumElements()) && 2611 "Vector element count mismatch!"); 2612 2613 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || 2614 OpOpcode == ISD::ANY_EXTEND) 2615 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x) 2616 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 2617 else if (OpOpcode == ISD::UNDEF) 2618 return getUNDEF(VT); 2619 2620 // (ext (trunx x)) -> x 2621 if (OpOpcode == ISD::TRUNCATE) { 2622 SDValue OpOp = Operand.getNode()->getOperand(0); 2623 if (OpOp.getValueType() == VT) 2624 return OpOp; 2625 } 2626 break; 2627 case ISD::TRUNCATE: 2628 assert(VT.isInteger() && Operand.getValueType().isInteger() && 2629 "Invalid TRUNCATE!"); 2630 if (Operand.getValueType() == VT) return Operand; // noop truncate 2631 assert(Operand.getValueType().getScalarType().bitsGT(VT.getScalarType()) && 2632 "Invalid truncate node, src < dst!"); 2633 assert((!VT.isVector() || 2634 VT.getVectorNumElements() == 2635 Operand.getValueType().getVectorNumElements()) && 2636 "Vector element count mismatch!"); 2637 if (OpOpcode == ISD::TRUNCATE) 2638 return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0)); 2639 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND || 2640 OpOpcode == ISD::ANY_EXTEND) { 2641 // If the source is smaller than the dest, we still need an extend. 2642 if (Operand.getNode()->getOperand(0).getValueType().getScalarType() 2643 .bitsLT(VT.getScalarType())) 2644 return getNode(OpOpcode, DL, VT, Operand.getNode()->getOperand(0)); 2645 if (Operand.getNode()->getOperand(0).getValueType().bitsGT(VT)) 2646 return getNode(ISD::TRUNCATE, DL, VT, Operand.getNode()->getOperand(0)); 2647 return Operand.getNode()->getOperand(0); 2648 } 2649 if (OpOpcode == ISD::UNDEF) 2650 return getUNDEF(VT); 2651 break; 2652 case ISD::BITCAST: 2653 // Basic sanity checking. 2654 assert(VT.getSizeInBits() == Operand.getValueType().getSizeInBits() 2655 && "Cannot BITCAST between types of different sizes!"); 2656 if (VT == Operand.getValueType()) return Operand; // noop conversion. 2657 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x) 2658 return getNode(ISD::BITCAST, DL, VT, Operand.getOperand(0)); 2659 if (OpOpcode == ISD::UNDEF) 2660 return getUNDEF(VT); 2661 break; 2662 case ISD::SCALAR_TO_VECTOR: 2663 assert(VT.isVector() && !Operand.getValueType().isVector() && 2664 (VT.getVectorElementType() == Operand.getValueType() || 2665 (VT.getVectorElementType().isInteger() && 2666 Operand.getValueType().isInteger() && 2667 VT.getVectorElementType().bitsLE(Operand.getValueType()))) && 2668 "Illegal SCALAR_TO_VECTOR node!"); 2669 if (OpOpcode == ISD::UNDEF) 2670 return getUNDEF(VT); 2671 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined. 2672 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT && 2673 isa<ConstantSDNode>(Operand.getOperand(1)) && 2674 Operand.getConstantOperandVal(1) == 0 && 2675 Operand.getOperand(0).getValueType() == VT) 2676 return Operand.getOperand(0); 2677 break; 2678 case ISD::FNEG: 2679 // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0 2680 if (getTarget().Options.UnsafeFPMath && OpOpcode == ISD::FSUB) 2681 return getNode(ISD::FSUB, DL, VT, Operand.getNode()->getOperand(1), 2682 Operand.getNode()->getOperand(0)); 2683 if (OpOpcode == ISD::FNEG) // --X -> X 2684 return Operand.getNode()->getOperand(0); 2685 break; 2686 case ISD::FABS: 2687 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X) 2688 return getNode(ISD::FABS, DL, VT, Operand.getNode()->getOperand(0)); 2689 break; 2690 } 2691 2692 SDNode *N; 2693 SDVTList VTs = getVTList(VT); 2694 if (VT != MVT::Glue) { // Don't CSE flag producing nodes 2695 FoldingSetNodeID ID; 2696 SDValue Ops[1] = { Operand }; 2697 AddNodeIDNode(ID, Opcode, VTs, Ops, 1); 2698 void *IP = 0; 2699 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 2700 return SDValue(E, 0); 2701 2702 N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(), 2703 DL.getDebugLoc(), VTs, Operand); 2704 CSEMap.InsertNode(N, IP); 2705 } else { 2706 N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(), 2707 DL.getDebugLoc(), VTs, Operand); 2708 } 2709 2710 AllNodes.push_back(N); 2711#ifndef NDEBUG 2712 VerifySDNode(N); 2713#endif 2714 return SDValue(N, 0); 2715} 2716 2717SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, EVT VT, 2718 SDNode *Cst1, SDNode *Cst2) { 2719 SmallVector<std::pair<ConstantSDNode *, ConstantSDNode *>, 4> Inputs; 2720 SmallVector<SDValue, 4> Outputs; 2721 EVT SVT = VT.getScalarType(); 2722 2723 ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1); 2724 ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2); 2725 if (Scalar1 && Scalar2) { 2726 // Scalar instruction. 2727 Inputs.push_back(std::make_pair(Scalar1, Scalar2)); 2728 } else { 2729 // For vectors extract each constant element into Inputs so we can constant 2730 // fold them individually. 2731 BuildVectorSDNode *BV1 = dyn_cast<BuildVectorSDNode>(Cst1); 2732 BuildVectorSDNode *BV2 = dyn_cast<BuildVectorSDNode>(Cst2); 2733 if (!BV1 || !BV2) 2734 return SDValue(); 2735 2736 assert(BV1->getNumOperands() == BV2->getNumOperands() && "Out of sync!"); 2737 2738 for (unsigned I = 0, E = BV1->getNumOperands(); I != E; ++I) { 2739 ConstantSDNode *V1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I)); 2740 ConstantSDNode *V2 = dyn_cast<ConstantSDNode>(BV2->getOperand(I)); 2741 if (!V1 || !V2) // Not a constant, bail. 2742 return SDValue(); 2743 2744 // Avoid BUILD_VECTOR nodes that perform implicit truncation. 2745 // FIXME: This is valid and could be handled by truncating the APInts. 2746 if (V1->getValueType(0) != SVT || V2->getValueType(0) != SVT) 2747 return SDValue(); 2748 2749 Inputs.push_back(std::make_pair(V1, V2)); 2750 } 2751 } 2752 2753 // We have a number of constant values, constant fold them element by element. 2754 for (unsigned I = 0, E = Inputs.size(); I != E; ++I) { 2755 const APInt &C1 = Inputs[I].first->getAPIntValue(); 2756 const APInt &C2 = Inputs[I].second->getAPIntValue(); 2757 2758 switch (Opcode) { 2759 case ISD::ADD: 2760 Outputs.push_back(getConstant(C1 + C2, SVT)); 2761 break; 2762 case ISD::SUB: 2763 Outputs.push_back(getConstant(C1 - C2, SVT)); 2764 break; 2765 case ISD::MUL: 2766 Outputs.push_back(getConstant(C1 * C2, SVT)); 2767 break; 2768 case ISD::UDIV: 2769 if (!C2.getBoolValue()) 2770 return SDValue(); 2771 Outputs.push_back(getConstant(C1.udiv(C2), SVT)); 2772 break; 2773 case ISD::UREM: 2774 if (!C2.getBoolValue()) 2775 return SDValue(); 2776 Outputs.push_back(getConstant(C1.urem(C2), SVT)); 2777 break; 2778 case ISD::SDIV: 2779 if (!C2.getBoolValue()) 2780 return SDValue(); 2781 Outputs.push_back(getConstant(C1.sdiv(C2), SVT)); 2782 break; 2783 case ISD::SREM: 2784 if (!C2.getBoolValue()) 2785 return SDValue(); 2786 Outputs.push_back(getConstant(C1.srem(C2), SVT)); 2787 break; 2788 case ISD::AND: 2789 Outputs.push_back(getConstant(C1 & C2, SVT)); 2790 break; 2791 case ISD::OR: 2792 Outputs.push_back(getConstant(C1 | C2, SVT)); 2793 break; 2794 case ISD::XOR: 2795 Outputs.push_back(getConstant(C1 ^ C2, SVT)); 2796 break; 2797 case ISD::SHL: 2798 Outputs.push_back(getConstant(C1 << C2, SVT)); 2799 break; 2800 case ISD::SRL: 2801 Outputs.push_back(getConstant(C1.lshr(C2), SVT)); 2802 break; 2803 case ISD::SRA: 2804 Outputs.push_back(getConstant(C1.ashr(C2), SVT)); 2805 break; 2806 case ISD::ROTL: 2807 Outputs.push_back(getConstant(C1.rotl(C2), SVT)); 2808 break; 2809 case ISD::ROTR: 2810 Outputs.push_back(getConstant(C1.rotr(C2), SVT)); 2811 break; 2812 default: 2813 return SDValue(); 2814 } 2815 } 2816 2817 // Handle the scalar case first. 2818 if (Scalar1 && Scalar2) 2819 return Outputs.back(); 2820 2821 // Otherwise build a big vector out of the scalar elements we generated. 2822 return getNode(ISD::BUILD_VECTOR, SDLoc(), VT, Outputs.data(), 2823 Outputs.size()); 2824} 2825 2826SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, 2827 SDValue N2) { 2828 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode()); 2829 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2.getNode()); 2830 switch (Opcode) { 2831 default: break; 2832 case ISD::TokenFactor: 2833 assert(VT == MVT::Other && N1.getValueType() == MVT::Other && 2834 N2.getValueType() == MVT::Other && "Invalid token factor!"); 2835 // Fold trivial token factors. 2836 if (N1.getOpcode() == ISD::EntryToken) return N2; 2837 if (N2.getOpcode() == ISD::EntryToken) return N1; 2838 if (N1 == N2) return N1; 2839 break; 2840 case ISD::CONCAT_VECTORS: 2841 // Concat of UNDEFs is UNDEF. 2842 if (N1.getOpcode() == ISD::UNDEF && 2843 N2.getOpcode() == ISD::UNDEF) 2844 return getUNDEF(VT); 2845 2846 // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to 2847 // one big BUILD_VECTOR. 2848 if (N1.getOpcode() == ISD::BUILD_VECTOR && 2849 N2.getOpcode() == ISD::BUILD_VECTOR) { 2850 SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(), 2851 N1.getNode()->op_end()); 2852 Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end()); 2853 return getNode(ISD::BUILD_VECTOR, DL, VT, &Elts[0], Elts.size()); 2854 } 2855 break; 2856 case ISD::AND: 2857 assert(VT.isInteger() && "This operator does not apply to FP types!"); 2858 assert(N1.getValueType() == N2.getValueType() && 2859 N1.getValueType() == VT && "Binary operator types must match!"); 2860 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's 2861 // worth handling here. 2862 if (N2C && N2C->isNullValue()) 2863 return N2; 2864 if (N2C && N2C->isAllOnesValue()) // X & -1 -> X 2865 return N1; 2866 break; 2867 case ISD::OR: 2868 case ISD::XOR: 2869 case ISD::ADD: 2870 case ISD::SUB: 2871 assert(VT.isInteger() && "This operator does not apply to FP types!"); 2872 assert(N1.getValueType() == N2.getValueType() && 2873 N1.getValueType() == VT && "Binary operator types must match!"); 2874 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so 2875 // it's worth handling here. 2876 if (N2C && N2C->isNullValue()) 2877 return N1; 2878 break; 2879 case ISD::UDIV: 2880 case ISD::UREM: 2881 case ISD::MULHU: 2882 case ISD::MULHS: 2883 case ISD::MUL: 2884 case ISD::SDIV: 2885 case ISD::SREM: 2886 assert(VT.isInteger() && "This operator does not apply to FP types!"); 2887 assert(N1.getValueType() == N2.getValueType() && 2888 N1.getValueType() == VT && "Binary operator types must match!"); 2889 break; 2890 case ISD::FADD: 2891 case ISD::FSUB: 2892 case ISD::FMUL: 2893 case ISD::FDIV: 2894 case ISD::FREM: 2895 if (getTarget().Options.UnsafeFPMath) { 2896 if (Opcode == ISD::FADD) { 2897 // 0+x --> x 2898 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1)) 2899 if (CFP->getValueAPF().isZero()) 2900 return N2; 2901 // x+0 --> x 2902 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2)) 2903 if (CFP->getValueAPF().isZero()) 2904 return N1; 2905 } else if (Opcode == ISD::FSUB) { 2906 // x-0 --> x 2907 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2)) 2908 if (CFP->getValueAPF().isZero()) 2909 return N1; 2910 } else if (Opcode == ISD::FMUL) { 2911 ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1); 2912 SDValue V = N2; 2913 2914 // If the first operand isn't the constant, try the second 2915 if (!CFP) { 2916 CFP = dyn_cast<ConstantFPSDNode>(N2); 2917 V = N1; 2918 } 2919 2920 if (CFP) { 2921 // 0*x --> 0 2922 if (CFP->isZero()) 2923 return SDValue(CFP,0); 2924 // 1*x --> x 2925 if (CFP->isExactlyValue(1.0)) 2926 return V; 2927 } 2928 } 2929 } 2930 assert(VT.isFloatingPoint() && "This operator only applies to FP types!"); 2931 assert(N1.getValueType() == N2.getValueType() && 2932 N1.getValueType() == VT && "Binary operator types must match!"); 2933 break; 2934 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match. 2935 assert(N1.getValueType() == VT && 2936 N1.getValueType().isFloatingPoint() && 2937 N2.getValueType().isFloatingPoint() && 2938 "Invalid FCOPYSIGN!"); 2939 break; 2940 case ISD::SHL: 2941 case ISD::SRA: 2942 case ISD::SRL: 2943 case ISD::ROTL: 2944 case ISD::ROTR: 2945 assert(VT == N1.getValueType() && 2946 "Shift operators return type must be the same as their first arg"); 2947 assert(VT.isInteger() && N2.getValueType().isInteger() && 2948 "Shifts only work on integers"); 2949 assert((!VT.isVector() || VT == N2.getValueType()) && 2950 "Vector shift amounts must be in the same as their first arg"); 2951 // Verify that the shift amount VT is bit enough to hold valid shift 2952 // amounts. This catches things like trying to shift an i1024 value by an 2953 // i8, which is easy to fall into in generic code that uses 2954 // TLI.getShiftAmount(). 2955 assert(N2.getValueType().getSizeInBits() >= 2956 Log2_32_Ceil(N1.getValueType().getSizeInBits()) && 2957 "Invalid use of small shift amount with oversized value!"); 2958 2959 // Always fold shifts of i1 values so the code generator doesn't need to 2960 // handle them. Since we know the size of the shift has to be less than the 2961 // size of the value, the shift/rotate count is guaranteed to be zero. 2962 if (VT == MVT::i1) 2963 return N1; 2964 if (N2C && N2C->isNullValue()) 2965 return N1; 2966 break; 2967 case ISD::FP_ROUND_INREG: { 2968 EVT EVT = cast<VTSDNode>(N2)->getVT(); 2969 assert(VT == N1.getValueType() && "Not an inreg round!"); 2970 assert(VT.isFloatingPoint() && EVT.isFloatingPoint() && 2971 "Cannot FP_ROUND_INREG integer types"); 2972 assert(EVT.isVector() == VT.isVector() && 2973 "FP_ROUND_INREG type should be vector iff the operand " 2974 "type is vector!"); 2975 assert((!EVT.isVector() || 2976 EVT.getVectorNumElements() == VT.getVectorNumElements()) && 2977 "Vector element counts must match in FP_ROUND_INREG"); 2978 assert(EVT.bitsLE(VT) && "Not rounding down!"); 2979 (void)EVT; 2980 if (cast<VTSDNode>(N2)->getVT() == VT) return N1; // Not actually rounding. 2981 break; 2982 } 2983 case ISD::FP_ROUND: 2984 assert(VT.isFloatingPoint() && 2985 N1.getValueType().isFloatingPoint() && 2986 VT.bitsLE(N1.getValueType()) && 2987 isa<ConstantSDNode>(N2) && "Invalid FP_ROUND!"); 2988 if (N1.getValueType() == VT) return N1; // noop conversion. 2989 break; 2990 case ISD::AssertSext: 2991 case ISD::AssertZext: { 2992 EVT EVT = cast<VTSDNode>(N2)->getVT(); 2993 assert(VT == N1.getValueType() && "Not an inreg extend!"); 2994 assert(VT.isInteger() && EVT.isInteger() && 2995 "Cannot *_EXTEND_INREG FP types"); 2996 assert(!EVT.isVector() && 2997 "AssertSExt/AssertZExt type should be the vector element type " 2998 "rather than the vector type!"); 2999 assert(EVT.bitsLE(VT) && "Not extending!"); 3000 if (VT == EVT) return N1; // noop assertion. 3001 break; 3002 } 3003 case ISD::SIGN_EXTEND_INREG: { 3004 EVT EVT = cast<VTSDNode>(N2)->getVT(); 3005 assert(VT == N1.getValueType() && "Not an inreg extend!"); 3006 assert(VT.isInteger() && EVT.isInteger() && 3007 "Cannot *_EXTEND_INREG FP types"); 3008 assert(EVT.isVector() == VT.isVector() && 3009 "SIGN_EXTEND_INREG type should be vector iff the operand " 3010 "type is vector!"); 3011 assert((!EVT.isVector() || 3012 EVT.getVectorNumElements() == VT.getVectorNumElements()) && 3013 "Vector element counts must match in SIGN_EXTEND_INREG"); 3014 assert(EVT.bitsLE(VT) && "Not extending!"); 3015 if (EVT == VT) return N1; // Not actually extending 3016 3017 if (N1C) { 3018 APInt Val = N1C->getAPIntValue(); 3019 unsigned FromBits = EVT.getScalarType().getSizeInBits(); 3020 Val <<= Val.getBitWidth()-FromBits; 3021 Val = Val.ashr(Val.getBitWidth()-FromBits); 3022 return getConstant(Val, VT); 3023 } 3024 break; 3025 } 3026 case ISD::EXTRACT_VECTOR_ELT: 3027 // EXTRACT_VECTOR_ELT of an UNDEF is an UNDEF. 3028 if (N1.getOpcode() == ISD::UNDEF) 3029 return getUNDEF(VT); 3030 3031 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is 3032 // expanding copies of large vectors from registers. 3033 if (N2C && 3034 N1.getOpcode() == ISD::CONCAT_VECTORS && 3035 N1.getNumOperands() > 0) { 3036 unsigned Factor = 3037 N1.getOperand(0).getValueType().getVectorNumElements(); 3038 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, 3039 N1.getOperand(N2C->getZExtValue() / Factor), 3040 getConstant(N2C->getZExtValue() % Factor, 3041 N2.getValueType())); 3042 } 3043 3044 // EXTRACT_VECTOR_ELT of BUILD_VECTOR is often formed while lowering is 3045 // expanding large vector constants. 3046 if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) { 3047 SDValue Elt = N1.getOperand(N2C->getZExtValue()); 3048 3049 if (VT != Elt.getValueType()) 3050 // If the vector element type is not legal, the BUILD_VECTOR operands 3051 // are promoted and implicitly truncated, and the result implicitly 3052 // extended. Make that explicit here. 3053 Elt = getAnyExtOrTrunc(Elt, DL, VT); 3054 3055 return Elt; 3056 } 3057 3058 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector 3059 // operations are lowered to scalars. 3060 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) { 3061 // If the indices are the same, return the inserted element else 3062 // if the indices are known different, extract the element from 3063 // the original vector. 3064 SDValue N1Op2 = N1.getOperand(2); 3065 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2.getNode()); 3066 3067 if (N1Op2C && N2C) { 3068 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) { 3069 if (VT == N1.getOperand(1).getValueType()) 3070 return N1.getOperand(1); 3071 else 3072 return getSExtOrTrunc(N1.getOperand(1), DL, VT); 3073 } 3074 3075 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2); 3076 } 3077 } 3078 break; 3079 case ISD::EXTRACT_ELEMENT: 3080 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!"); 3081 assert(!N1.getValueType().isVector() && !VT.isVector() && 3082 (N1.getValueType().isInteger() == VT.isInteger()) && 3083 N1.getValueType() != VT && 3084 "Wrong types for EXTRACT_ELEMENT!"); 3085 3086 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding 3087 // 64-bit integers into 32-bit parts. Instead of building the extract of 3088 // the BUILD_PAIR, only to have legalize rip it apart, just do it now. 3089 if (N1.getOpcode() == ISD::BUILD_PAIR) 3090 return N1.getOperand(N2C->getZExtValue()); 3091 3092 // EXTRACT_ELEMENT of a constant int is also very common. 3093 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) { 3094 unsigned ElementSize = VT.getSizeInBits(); 3095 unsigned Shift = ElementSize * N2C->getZExtValue(); 3096 APInt ShiftedVal = C->getAPIntValue().lshr(Shift); 3097 return getConstant(ShiftedVal.trunc(ElementSize), VT); 3098 } 3099 break; 3100 case ISD::EXTRACT_SUBVECTOR: { 3101 SDValue Index = N2; 3102 if (VT.isSimple() && N1.getValueType().isSimple()) { 3103 assert(VT.isVector() && N1.getValueType().isVector() && 3104 "Extract subvector VTs must be a vectors!"); 3105 assert(VT.getVectorElementType() == 3106 N1.getValueType().getVectorElementType() && 3107 "Extract subvector VTs must have the same element type!"); 3108 assert(VT.getSimpleVT() <= N1.getSimpleValueType() && 3109 "Extract subvector must be from larger vector to smaller vector!"); 3110 3111 if (isa<ConstantSDNode>(Index.getNode())) { 3112 assert((VT.getVectorNumElements() + 3113 cast<ConstantSDNode>(Index.getNode())->getZExtValue() 3114 <= N1.getValueType().getVectorNumElements()) 3115 && "Extract subvector overflow!"); 3116 } 3117 3118 // Trivial extraction. 3119 if (VT.getSimpleVT() == N1.getSimpleValueType()) 3120 return N1; 3121 } 3122 break; 3123 } 3124 } 3125 3126 // Perform trivial constant folding. 3127 SDValue SV = FoldConstantArithmetic(Opcode, VT, N1.getNode(), N2.getNode()); 3128 if (SV.getNode()) return SV; 3129 3130 // Canonicalize constant to RHS if commutative. 3131 if (N1C && !N2C && isCommutativeBinOp(Opcode)) { 3132 std::swap(N1C, N2C); 3133 std::swap(N1, N2); 3134 } 3135 3136 // Constant fold FP operations. 3137 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1.getNode()); 3138 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2.getNode()); 3139 if (N1CFP) { 3140 if (!N2CFP && isCommutativeBinOp(Opcode)) { 3141 // Canonicalize constant to RHS if commutative. 3142 std::swap(N1CFP, N2CFP); 3143 std::swap(N1, N2); 3144 } else if (N2CFP) { 3145 APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF(); 3146 APFloat::opStatus s; 3147 switch (Opcode) { 3148 case ISD::FADD: 3149 s = V1.add(V2, APFloat::rmNearestTiesToEven); 3150 if (s != APFloat::opInvalidOp) 3151 return getConstantFP(V1, VT); 3152 break; 3153 case ISD::FSUB: 3154 s = V1.subtract(V2, APFloat::rmNearestTiesToEven); 3155 if (s!=APFloat::opInvalidOp) 3156 return getConstantFP(V1, VT); 3157 break; 3158 case ISD::FMUL: 3159 s = V1.multiply(V2, APFloat::rmNearestTiesToEven); 3160 if (s!=APFloat::opInvalidOp) 3161 return getConstantFP(V1, VT); 3162 break; 3163 case ISD::FDIV: 3164 s = V1.divide(V2, APFloat::rmNearestTiesToEven); 3165 if (s!=APFloat::opInvalidOp && s!=APFloat::opDivByZero) 3166 return getConstantFP(V1, VT); 3167 break; 3168 case ISD::FREM : 3169 s = V1.mod(V2, APFloat::rmNearestTiesToEven); 3170 if (s!=APFloat::opInvalidOp && s!=APFloat::opDivByZero) 3171 return getConstantFP(V1, VT); 3172 break; 3173 case ISD::FCOPYSIGN: 3174 V1.copySign(V2); 3175 return getConstantFP(V1, VT); 3176 default: break; 3177 } 3178 } 3179 3180 if (Opcode == ISD::FP_ROUND) { 3181 APFloat V = N1CFP->getValueAPF(); // make copy 3182 bool ignored; 3183 // This can return overflow, underflow, or inexact; we don't care. 3184 // FIXME need to be more flexible about rounding mode. 3185 (void)V.convert(EVTToAPFloatSemantics(VT), 3186 APFloat::rmNearestTiesToEven, &ignored); 3187 return getConstantFP(V, VT); 3188 } 3189 } 3190 3191 // Canonicalize an UNDEF to the RHS, even over a constant. 3192 if (N1.getOpcode() == ISD::UNDEF) { 3193 if (isCommutativeBinOp(Opcode)) { 3194 std::swap(N1, N2); 3195 } else { 3196 switch (Opcode) { 3197 case ISD::FP_ROUND_INREG: 3198 case ISD::SIGN_EXTEND_INREG: 3199 case ISD::SUB: 3200 case ISD::FSUB: 3201 case ISD::FDIV: 3202 case ISD::FREM: 3203 case ISD::SRA: 3204 return N1; // fold op(undef, arg2) -> undef 3205 case ISD::UDIV: 3206 case ISD::SDIV: 3207 case ISD::UREM: 3208 case ISD::SREM: 3209 case ISD::SRL: 3210 case ISD::SHL: 3211 if (!VT.isVector()) 3212 return getConstant(0, VT); // fold op(undef, arg2) -> 0 3213 // For vectors, we can't easily build an all zero vector, just return 3214 // the LHS. 3215 return N2; 3216 } 3217 } 3218 } 3219 3220 // Fold a bunch of operators when the RHS is undef. 3221 if (N2.getOpcode() == ISD::UNDEF) { 3222 switch (Opcode) { 3223 case ISD::XOR: 3224 if (N1.getOpcode() == ISD::UNDEF) 3225 // Handle undef ^ undef -> 0 special case. This is a common 3226 // idiom (misuse). 3227 return getConstant(0, VT); 3228 // fallthrough 3229 case ISD::ADD: 3230 case ISD::ADDC: 3231 case ISD::ADDE: 3232 case ISD::SUB: 3233 case ISD::UDIV: 3234 case ISD::SDIV: 3235 case ISD::UREM: 3236 case ISD::SREM: 3237 return N2; // fold op(arg1, undef) -> undef 3238 case ISD::FADD: 3239 case ISD::FSUB: 3240 case ISD::FMUL: 3241 case ISD::FDIV: 3242 case ISD::FREM: 3243 if (getTarget().Options.UnsafeFPMath) 3244 return N2; 3245 break; 3246 case ISD::MUL: 3247 case ISD::AND: 3248 case ISD::SRL: 3249 case ISD::SHL: 3250 if (!VT.isVector()) 3251 return getConstant(0, VT); // fold op(arg1, undef) -> 0 3252 // For vectors, we can't easily build an all zero vector, just return 3253 // the LHS. 3254 return N1; 3255 case ISD::OR: 3256 if (!VT.isVector()) 3257 return getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT); 3258 // For vectors, we can't easily build an all one vector, just return 3259 // the LHS. 3260 return N1; 3261 case ISD::SRA: 3262 return N1; 3263 } 3264 } 3265 3266 // Memoize this node if possible. 3267 SDNode *N; 3268 SDVTList VTs = getVTList(VT); 3269 if (VT != MVT::Glue) { 3270 SDValue Ops[] = { N1, N2 }; 3271 FoldingSetNodeID ID; 3272 AddNodeIDNode(ID, Opcode, VTs, Ops, 2); 3273 void *IP = 0; 3274 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 3275 return SDValue(E, 0); 3276 3277 N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(), 3278 DL.getDebugLoc(), VTs, N1, N2); 3279 CSEMap.InsertNode(N, IP); 3280 } else { 3281 N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(), 3282 DL.getDebugLoc(), VTs, N1, N2); 3283 } 3284 3285 AllNodes.push_back(N); 3286#ifndef NDEBUG 3287 VerifySDNode(N); 3288#endif 3289 return SDValue(N, 0); 3290} 3291 3292SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 3293 SDValue N1, SDValue N2, SDValue N3) { 3294 // Perform various simplifications. 3295 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1.getNode()); 3296 switch (Opcode) { 3297 case ISD::FMA: { 3298 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1); 3299 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2); 3300 ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3); 3301 if (N1CFP && N2CFP && N3CFP) { 3302 APFloat V1 = N1CFP->getValueAPF(); 3303 const APFloat &V2 = N2CFP->getValueAPF(); 3304 const APFloat &V3 = N3CFP->getValueAPF(); 3305 APFloat::opStatus s = 3306 V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven); 3307 if (s != APFloat::opInvalidOp) 3308 return getConstantFP(V1, VT); 3309 } 3310 break; 3311 } 3312 case ISD::CONCAT_VECTORS: 3313 // A CONCAT_VECTOR with all operands BUILD_VECTOR can be simplified to 3314 // one big BUILD_VECTOR. 3315 if (N1.getOpcode() == ISD::BUILD_VECTOR && 3316 N2.getOpcode() == ISD::BUILD_VECTOR && 3317 N3.getOpcode() == ISD::BUILD_VECTOR) { 3318 SmallVector<SDValue, 16> Elts(N1.getNode()->op_begin(), 3319 N1.getNode()->op_end()); 3320 Elts.append(N2.getNode()->op_begin(), N2.getNode()->op_end()); 3321 Elts.append(N3.getNode()->op_begin(), N3.getNode()->op_end()); 3322 return getNode(ISD::BUILD_VECTOR, DL, VT, &Elts[0], Elts.size()); 3323 } 3324 break; 3325 case ISD::SETCC: { 3326 // Use FoldSetCC to simplify SETCC's. 3327 SDValue Simp = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL); 3328 if (Simp.getNode()) return Simp; 3329 break; 3330 } 3331 case ISD::SELECT: 3332 if (N1C) { 3333 if (N1C->getZExtValue()) 3334 return N2; // select true, X, Y -> X 3335 return N3; // select false, X, Y -> Y 3336 } 3337 3338 if (N2 == N3) return N2; // select C, X, X -> X 3339 break; 3340 case ISD::VECTOR_SHUFFLE: 3341 llvm_unreachable("should use getVectorShuffle constructor!"); 3342 case ISD::INSERT_SUBVECTOR: { 3343 SDValue Index = N3; 3344 if (VT.isSimple() && N1.getValueType().isSimple() 3345 && N2.getValueType().isSimple()) { 3346 assert(VT.isVector() && N1.getValueType().isVector() && 3347 N2.getValueType().isVector() && 3348 "Insert subvector VTs must be a vectors"); 3349 assert(VT == N1.getValueType() && 3350 "Dest and insert subvector source types must match!"); 3351 assert(N2.getSimpleValueType() <= N1.getSimpleValueType() && 3352 "Insert subvector must be from smaller vector to larger vector!"); 3353 if (isa<ConstantSDNode>(Index.getNode())) { 3354 assert((N2.getValueType().getVectorNumElements() + 3355 cast<ConstantSDNode>(Index.getNode())->getZExtValue() 3356 <= VT.getVectorNumElements()) 3357 && "Insert subvector overflow!"); 3358 } 3359 3360 // Trivial insertion. 3361 if (VT.getSimpleVT() == N2.getSimpleValueType()) 3362 return N2; 3363 } 3364 break; 3365 } 3366 case ISD::BITCAST: 3367 // Fold bit_convert nodes from a type to themselves. 3368 if (N1.getValueType() == VT) 3369 return N1; 3370 break; 3371 } 3372 3373 // Memoize node if it doesn't produce a flag. 3374 SDNode *N; 3375 SDVTList VTs = getVTList(VT); 3376 if (VT != MVT::Glue) { 3377 SDValue Ops[] = { N1, N2, N3 }; 3378 FoldingSetNodeID ID; 3379 AddNodeIDNode(ID, Opcode, VTs, Ops, 3); 3380 void *IP = 0; 3381 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 3382 return SDValue(E, 0); 3383 3384 N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(), 3385 DL.getDebugLoc(), VTs, N1, N2, N3); 3386 CSEMap.InsertNode(N, IP); 3387 } else { 3388 N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(), 3389 DL.getDebugLoc(), VTs, N1, N2, N3); 3390 } 3391 3392 AllNodes.push_back(N); 3393#ifndef NDEBUG 3394 VerifySDNode(N); 3395#endif 3396 return SDValue(N, 0); 3397} 3398 3399SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 3400 SDValue N1, SDValue N2, SDValue N3, 3401 SDValue N4) { 3402 SDValue Ops[] = { N1, N2, N3, N4 }; 3403 return getNode(Opcode, DL, VT, Ops, 4); 3404} 3405 3406SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 3407 SDValue N1, SDValue N2, SDValue N3, 3408 SDValue N4, SDValue N5) { 3409 SDValue Ops[] = { N1, N2, N3, N4, N5 }; 3410 return getNode(Opcode, DL, VT, Ops, 5); 3411} 3412 3413/// getStackArgumentTokenFactor - Compute a TokenFactor to force all 3414/// the incoming stack arguments to be loaded from the stack. 3415SDValue SelectionDAG::getStackArgumentTokenFactor(SDValue Chain) { 3416 SmallVector<SDValue, 8> ArgChains; 3417 3418 // Include the original chain at the beginning of the list. When this is 3419 // used by target LowerCall hooks, this helps legalize find the 3420 // CALLSEQ_BEGIN node. 3421 ArgChains.push_back(Chain); 3422 3423 // Add a chain value for each stack argument. 3424 for (SDNode::use_iterator U = getEntryNode().getNode()->use_begin(), 3425 UE = getEntryNode().getNode()->use_end(); U != UE; ++U) 3426 if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U)) 3427 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr())) 3428 if (FI->getIndex() < 0) 3429 ArgChains.push_back(SDValue(L, 1)); 3430 3431 // Build a tokenfactor for all the chains. 3432 return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, 3433 &ArgChains[0], ArgChains.size()); 3434} 3435 3436/// getMemsetValue - Vectorized representation of the memset value 3437/// operand. 3438static SDValue getMemsetValue(SDValue Value, EVT VT, SelectionDAG &DAG, 3439 SDLoc dl) { 3440 assert(Value.getOpcode() != ISD::UNDEF); 3441 3442 unsigned NumBits = VT.getScalarType().getSizeInBits(); 3443 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) { 3444 assert(C->getAPIntValue().getBitWidth() == 8); 3445 APInt Val = APInt::getSplat(NumBits, C->getAPIntValue()); 3446 if (VT.isInteger()) 3447 return DAG.getConstant(Val, VT); 3448 return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), VT); 3449 } 3450 3451 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Value); 3452 if (NumBits > 8) { 3453 // Use a multiplication with 0x010101... to extend the input to the 3454 // required length. 3455 APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01)); 3456 Value = DAG.getNode(ISD::MUL, dl, VT, Value, DAG.getConstant(Magic, VT)); 3457 } 3458 3459 return Value; 3460} 3461 3462/// getMemsetStringVal - Similar to getMemsetValue. Except this is only 3463/// used when a memcpy is turned into a memset when the source is a constant 3464/// string ptr. 3465static SDValue getMemsetStringVal(EVT VT, SDLoc dl, SelectionDAG &DAG, 3466 const TargetLowering &TLI, StringRef Str) { 3467 // Handle vector with all elements zero. 3468 if (Str.empty()) { 3469 if (VT.isInteger()) 3470 return DAG.getConstant(0, VT); 3471 else if (VT == MVT::f32 || VT == MVT::f64) 3472 return DAG.getConstantFP(0.0, VT); 3473 else if (VT.isVector()) { 3474 unsigned NumElts = VT.getVectorNumElements(); 3475 MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64; 3476 return DAG.getNode(ISD::BITCAST, dl, VT, 3477 DAG.getConstant(0, EVT::getVectorVT(*DAG.getContext(), 3478 EltVT, NumElts))); 3479 } else 3480 llvm_unreachable("Expected type!"); 3481 } 3482 3483 assert(!VT.isVector() && "Can't handle vector type here!"); 3484 unsigned NumVTBits = VT.getSizeInBits(); 3485 unsigned NumVTBytes = NumVTBits / 8; 3486 unsigned NumBytes = std::min(NumVTBytes, unsigned(Str.size())); 3487 3488 APInt Val(NumVTBits, 0); 3489 if (TLI.isLittleEndian()) { 3490 for (unsigned i = 0; i != NumBytes; ++i) 3491 Val |= (uint64_t)(unsigned char)Str[i] << i*8; 3492 } else { 3493 for (unsigned i = 0; i != NumBytes; ++i) 3494 Val |= (uint64_t)(unsigned char)Str[i] << (NumVTBytes-i-1)*8; 3495 } 3496 3497 // If the "cost" of materializing the integer immediate is 1 or free, then 3498 // it is cost effective to turn the load into the immediate. 3499 const TargetTransformInfo *TTI = DAG.getTargetTransformInfo(); 3500 if (TTI->getIntImmCost(Val, VT.getTypeForEVT(*DAG.getContext())) < 2) 3501 return DAG.getConstant(Val, VT); 3502 return SDValue(0, 0); 3503} 3504 3505/// getMemBasePlusOffset - Returns base and offset node for the 3506/// 3507static SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, SDLoc dl, 3508 SelectionDAG &DAG) { 3509 EVT VT = Base.getValueType(); 3510 return DAG.getNode(ISD::ADD, dl, 3511 VT, Base, DAG.getConstant(Offset, VT)); 3512} 3513 3514/// isMemSrcFromString - Returns true if memcpy source is a string constant. 3515/// 3516static bool isMemSrcFromString(SDValue Src, StringRef &Str) { 3517 unsigned SrcDelta = 0; 3518 GlobalAddressSDNode *G = NULL; 3519 if (Src.getOpcode() == ISD::GlobalAddress) 3520 G = cast<GlobalAddressSDNode>(Src); 3521 else if (Src.getOpcode() == ISD::ADD && 3522 Src.getOperand(0).getOpcode() == ISD::GlobalAddress && 3523 Src.getOperand(1).getOpcode() == ISD::Constant) { 3524 G = cast<GlobalAddressSDNode>(Src.getOperand(0)); 3525 SrcDelta = cast<ConstantSDNode>(Src.getOperand(1))->getZExtValue(); 3526 } 3527 if (!G) 3528 return false; 3529 3530 return getConstantStringInfo(G->getGlobal(), Str, SrcDelta, false); 3531} 3532 3533/// FindOptimalMemOpLowering - Determines the optimial series memory ops 3534/// to replace the memset / memcpy. Return true if the number of memory ops 3535/// is below the threshold. It returns the types of the sequence of 3536/// memory ops to perform memset / memcpy by reference. 3537static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps, 3538 unsigned Limit, uint64_t Size, 3539 unsigned DstAlign, unsigned SrcAlign, 3540 bool IsMemset, 3541 bool ZeroMemset, 3542 bool MemcpyStrSrc, 3543 bool AllowOverlap, 3544 SelectionDAG &DAG, 3545 const TargetLowering &TLI) { 3546 assert((SrcAlign == 0 || SrcAlign >= DstAlign) && 3547 "Expecting memcpy / memset source to meet alignment requirement!"); 3548 // If 'SrcAlign' is zero, that means the memory operation does not need to 3549 // load the value, i.e. memset or memcpy from constant string. Otherwise, 3550 // it's the inferred alignment of the source. 'DstAlign', on the other hand, 3551 // is the specified alignment of the memory operation. If it is zero, that 3552 // means it's possible to change the alignment of the destination. 3553 // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does 3554 // not need to be loaded. 3555 EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign, 3556 IsMemset, ZeroMemset, MemcpyStrSrc, 3557 DAG.getMachineFunction()); 3558 3559 if (VT == MVT::Other) { 3560 if (DstAlign >= TLI.getDataLayout()->getPointerPrefAlignment() || 3561 TLI.allowsUnalignedMemoryAccesses(VT)) { 3562 VT = TLI.getPointerTy(); 3563 } else { 3564 switch (DstAlign & 7) { 3565 case 0: VT = MVT::i64; break; 3566 case 4: VT = MVT::i32; break; 3567 case 2: VT = MVT::i16; break; 3568 default: VT = MVT::i8; break; 3569 } 3570 } 3571 3572 MVT LVT = MVT::i64; 3573 while (!TLI.isTypeLegal(LVT)) 3574 LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1); 3575 assert(LVT.isInteger()); 3576 3577 if (VT.bitsGT(LVT)) 3578 VT = LVT; 3579 } 3580 3581 unsigned NumMemOps = 0; 3582 while (Size != 0) { 3583 unsigned VTSize = VT.getSizeInBits() / 8; 3584 while (VTSize > Size) { 3585 // For now, only use non-vector load / store's for the left-over pieces. 3586 EVT NewVT = VT; 3587 unsigned NewVTSize; 3588 3589 bool Found = false; 3590 if (VT.isVector() || VT.isFloatingPoint()) { 3591 NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32; 3592 if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) && 3593 TLI.isSafeMemOpType(NewVT.getSimpleVT())) 3594 Found = true; 3595 else if (NewVT == MVT::i64 && 3596 TLI.isOperationLegalOrCustom(ISD::STORE, MVT::f64) && 3597 TLI.isSafeMemOpType(MVT::f64)) { 3598 // i64 is usually not legal on 32-bit targets, but f64 may be. 3599 NewVT = MVT::f64; 3600 Found = true; 3601 } 3602 } 3603 3604 if (!Found) { 3605 do { 3606 NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1); 3607 if (NewVT == MVT::i8) 3608 break; 3609 } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT())); 3610 } 3611 NewVTSize = NewVT.getSizeInBits() / 8; 3612 3613 // If the new VT cannot cover all of the remaining bits, then consider 3614 // issuing a (or a pair of) unaligned and overlapping load / store. 3615 // FIXME: Only does this for 64-bit or more since we don't have proper 3616 // cost model for unaligned load / store. 3617 bool Fast; 3618 if (NumMemOps && AllowOverlap && 3619 VTSize >= 8 && NewVTSize < Size && 3620 TLI.allowsUnalignedMemoryAccesses(VT, &Fast) && Fast) 3621 VTSize = Size; 3622 else { 3623 VT = NewVT; 3624 VTSize = NewVTSize; 3625 } 3626 } 3627 3628 if (++NumMemOps > Limit) 3629 return false; 3630 3631 MemOps.push_back(VT); 3632 Size -= VTSize; 3633 } 3634 3635 return true; 3636} 3637 3638static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, SDLoc dl, 3639 SDValue Chain, SDValue Dst, 3640 SDValue Src, uint64_t Size, 3641 unsigned Align, bool isVol, 3642 bool AlwaysInline, 3643 MachinePointerInfo DstPtrInfo, 3644 MachinePointerInfo SrcPtrInfo) { 3645 // Turn a memcpy of undef to nop. 3646 if (Src.getOpcode() == ISD::UNDEF) 3647 return Chain; 3648 3649 // Expand memcpy to a series of load and store ops if the size operand falls 3650 // below a certain threshold. 3651 // TODO: In the AlwaysInline case, if the size is big then generate a loop 3652 // rather than maybe a humongous number of loads and stores. 3653 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3654 std::vector<EVT> MemOps; 3655 bool DstAlignCanChange = false; 3656 MachineFunction &MF = DAG.getMachineFunction(); 3657 MachineFrameInfo *MFI = MF.getFrameInfo(); 3658 bool OptSize = 3659 MF.getFunction()->getAttributes(). 3660 hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize); 3661 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 3662 if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 3663 DstAlignCanChange = true; 3664 unsigned SrcAlign = DAG.InferPtrAlignment(Src); 3665 if (Align > SrcAlign) 3666 SrcAlign = Align; 3667 StringRef Str; 3668 bool CopyFromStr = isMemSrcFromString(Src, Str); 3669 bool isZeroStr = CopyFromStr && Str.empty(); 3670 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize); 3671 3672 if (!FindOptimalMemOpLowering(MemOps, Limit, Size, 3673 (DstAlignCanChange ? 0 : Align), 3674 (isZeroStr ? 0 : SrcAlign), 3675 false, false, CopyFromStr, true, DAG, TLI)) 3676 return SDValue(); 3677 3678 if (DstAlignCanChange) { 3679 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 3680 unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty); 3681 3682 // Don't promote to an alignment that would require dynamic stack 3683 // realignment. 3684 const TargetRegisterInfo *TRI = MF.getTarget().getRegisterInfo(); 3685 if (!TRI->needsStackRealignment(MF)) 3686 while (NewAlign > Align && 3687 TLI.getDataLayout()->exceedsNaturalStackAlignment(NewAlign)) 3688 NewAlign /= 2; 3689 3690 if (NewAlign > Align) { 3691 // Give the stack frame object a larger alignment if needed. 3692 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 3693 MFI->setObjectAlignment(FI->getIndex(), NewAlign); 3694 Align = NewAlign; 3695 } 3696 } 3697 3698 SmallVector<SDValue, 8> OutChains; 3699 unsigned NumMemOps = MemOps.size(); 3700 uint64_t SrcOff = 0, DstOff = 0; 3701 for (unsigned i = 0; i != NumMemOps; ++i) { 3702 EVT VT = MemOps[i]; 3703 unsigned VTSize = VT.getSizeInBits() / 8; 3704 SDValue Value, Store; 3705 3706 if (VTSize > Size) { 3707 // Issuing an unaligned load / store pair that overlaps with the previous 3708 // pair. Adjust the offset accordingly. 3709 assert(i == NumMemOps-1 && i != 0); 3710 SrcOff -= VTSize - Size; 3711 DstOff -= VTSize - Size; 3712 } 3713 3714 if (CopyFromStr && 3715 (isZeroStr || (VT.isInteger() && !VT.isVector()))) { 3716 // It's unlikely a store of a vector immediate can be done in a single 3717 // instruction. It would require a load from a constantpool first. 3718 // We only handle zero vectors here. 3719 // FIXME: Handle other cases where store of vector immediate is done in 3720 // a single instruction. 3721 Value = getMemsetStringVal(VT, dl, DAG, TLI, Str.substr(SrcOff)); 3722 if (Value.getNode()) 3723 Store = DAG.getStore(Chain, dl, Value, 3724 getMemBasePlusOffset(Dst, DstOff, dl, DAG), 3725 DstPtrInfo.getWithOffset(DstOff), isVol, 3726 false, Align); 3727 } 3728 3729 if (!Store.getNode()) { 3730 // The type might not be legal for the target. This should only happen 3731 // if the type is smaller than a legal type, as on PPC, so the right 3732 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify 3733 // to Load/Store if NVT==VT. 3734 // FIXME does the case above also need this? 3735 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3736 assert(NVT.bitsGE(VT)); 3737 Value = DAG.getExtLoad(ISD::EXTLOAD, dl, NVT, Chain, 3738 getMemBasePlusOffset(Src, SrcOff, dl, DAG), 3739 SrcPtrInfo.getWithOffset(SrcOff), VT, isVol, false, 3740 MinAlign(SrcAlign, SrcOff)); 3741 Store = DAG.getTruncStore(Chain, dl, Value, 3742 getMemBasePlusOffset(Dst, DstOff, dl, DAG), 3743 DstPtrInfo.getWithOffset(DstOff), VT, isVol, 3744 false, Align); 3745 } 3746 OutChains.push_back(Store); 3747 SrcOff += VTSize; 3748 DstOff += VTSize; 3749 Size -= VTSize; 3750 } 3751 3752 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3753 &OutChains[0], OutChains.size()); 3754} 3755 3756static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, SDLoc dl, 3757 SDValue Chain, SDValue Dst, 3758 SDValue Src, uint64_t Size, 3759 unsigned Align, bool isVol, 3760 bool AlwaysInline, 3761 MachinePointerInfo DstPtrInfo, 3762 MachinePointerInfo SrcPtrInfo) { 3763 // Turn a memmove of undef to nop. 3764 if (Src.getOpcode() == ISD::UNDEF) 3765 return Chain; 3766 3767 // Expand memmove to a series of load and store ops if the size operand falls 3768 // below a certain threshold. 3769 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3770 std::vector<EVT> MemOps; 3771 bool DstAlignCanChange = false; 3772 MachineFunction &MF = DAG.getMachineFunction(); 3773 MachineFrameInfo *MFI = MF.getFrameInfo(); 3774 bool OptSize = MF.getFunction()->getAttributes(). 3775 hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize); 3776 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 3777 if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 3778 DstAlignCanChange = true; 3779 unsigned SrcAlign = DAG.InferPtrAlignment(Src); 3780 if (Align > SrcAlign) 3781 SrcAlign = Align; 3782 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize); 3783 3784 if (!FindOptimalMemOpLowering(MemOps, Limit, Size, 3785 (DstAlignCanChange ? 0 : Align), SrcAlign, 3786 false, false, false, false, DAG, TLI)) 3787 return SDValue(); 3788 3789 if (DstAlignCanChange) { 3790 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 3791 unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty); 3792 if (NewAlign > Align) { 3793 // Give the stack frame object a larger alignment if needed. 3794 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 3795 MFI->setObjectAlignment(FI->getIndex(), NewAlign); 3796 Align = NewAlign; 3797 } 3798 } 3799 3800 uint64_t SrcOff = 0, DstOff = 0; 3801 SmallVector<SDValue, 8> LoadValues; 3802 SmallVector<SDValue, 8> LoadChains; 3803 SmallVector<SDValue, 8> OutChains; 3804 unsigned NumMemOps = MemOps.size(); 3805 for (unsigned i = 0; i < NumMemOps; i++) { 3806 EVT VT = MemOps[i]; 3807 unsigned VTSize = VT.getSizeInBits() / 8; 3808 SDValue Value; 3809 3810 Value = DAG.getLoad(VT, dl, Chain, 3811 getMemBasePlusOffset(Src, SrcOff, dl, DAG), 3812 SrcPtrInfo.getWithOffset(SrcOff), isVol, 3813 false, false, SrcAlign); 3814 LoadValues.push_back(Value); 3815 LoadChains.push_back(Value.getValue(1)); 3816 SrcOff += VTSize; 3817 } 3818 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3819 &LoadChains[0], LoadChains.size()); 3820 OutChains.clear(); 3821 for (unsigned i = 0; i < NumMemOps; i++) { 3822 EVT VT = MemOps[i]; 3823 unsigned VTSize = VT.getSizeInBits() / 8; 3824 SDValue Store; 3825 3826 Store = DAG.getStore(Chain, dl, LoadValues[i], 3827 getMemBasePlusOffset(Dst, DstOff, dl, DAG), 3828 DstPtrInfo.getWithOffset(DstOff), isVol, false, Align); 3829 OutChains.push_back(Store); 3830 DstOff += VTSize; 3831 } 3832 3833 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3834 &OutChains[0], OutChains.size()); 3835} 3836 3837/// \brief Lower the call to 'memset' intrinsic function into a series of store 3838/// operations. 3839/// 3840/// \param DAG Selection DAG where lowered code is placed. 3841/// \param dl Link to corresponding IR location. 3842/// \param Chain Control flow dependency. 3843/// \param Dst Pointer to destination memory location. 3844/// \param Src Value of byte to write into the memory. 3845/// \param Size Number of bytes to write. 3846/// \param Align Alignment of the destination in bytes. 3847/// \param isVol True if destination is volatile. 3848/// \param DstPtrInfo IR information on the memory pointer. 3849/// \returns New head in the control flow, if lowering was successful, empty 3850/// SDValue otherwise. 3851/// 3852/// The function tries to replace 'llvm.memset' intrinsic with several store 3853/// operations and value calculation code. This is usually profitable for small 3854/// memory size. 3855static SDValue getMemsetStores(SelectionDAG &DAG, SDLoc dl, 3856 SDValue Chain, SDValue Dst, 3857 SDValue Src, uint64_t Size, 3858 unsigned Align, bool isVol, 3859 MachinePointerInfo DstPtrInfo) { 3860 // Turn a memset of undef to nop. 3861 if (Src.getOpcode() == ISD::UNDEF) 3862 return Chain; 3863 3864 // Expand memset to a series of load/store ops if the size operand 3865 // falls below a certain threshold. 3866 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3867 std::vector<EVT> MemOps; 3868 bool DstAlignCanChange = false; 3869 MachineFunction &MF = DAG.getMachineFunction(); 3870 MachineFrameInfo *MFI = MF.getFrameInfo(); 3871 bool OptSize = MF.getFunction()->getAttributes(). 3872 hasAttribute(AttributeSet::FunctionIndex, Attribute::OptimizeForSize); 3873 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst); 3874 if (FI && !MFI->isFixedObjectIndex(FI->getIndex())) 3875 DstAlignCanChange = true; 3876 bool IsZeroVal = 3877 isa<ConstantSDNode>(Src) && cast<ConstantSDNode>(Src)->isNullValue(); 3878 if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize), 3879 Size, (DstAlignCanChange ? 0 : Align), 0, 3880 true, IsZeroVal, false, true, DAG, TLI)) 3881 return SDValue(); 3882 3883 if (DstAlignCanChange) { 3884 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext()); 3885 unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty); 3886 if (NewAlign > Align) { 3887 // Give the stack frame object a larger alignment if needed. 3888 if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign) 3889 MFI->setObjectAlignment(FI->getIndex(), NewAlign); 3890 Align = NewAlign; 3891 } 3892 } 3893 3894 SmallVector<SDValue, 8> OutChains; 3895 uint64_t DstOff = 0; 3896 unsigned NumMemOps = MemOps.size(); 3897 3898 // Find the largest store and generate the bit pattern for it. 3899 EVT LargestVT = MemOps[0]; 3900 for (unsigned i = 1; i < NumMemOps; i++) 3901 if (MemOps[i].bitsGT(LargestVT)) 3902 LargestVT = MemOps[i]; 3903 SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl); 3904 3905 for (unsigned i = 0; i < NumMemOps; i++) { 3906 EVT VT = MemOps[i]; 3907 unsigned VTSize = VT.getSizeInBits() / 8; 3908 if (VTSize > Size) { 3909 // Issuing an unaligned load / store pair that overlaps with the previous 3910 // pair. Adjust the offset accordingly. 3911 assert(i == NumMemOps-1 && i != 0); 3912 DstOff -= VTSize - Size; 3913 } 3914 3915 // If this store is smaller than the largest store see whether we can get 3916 // the smaller value for free with a truncate. 3917 SDValue Value = MemSetValue; 3918 if (VT.bitsLT(LargestVT)) { 3919 if (!LargestVT.isVector() && !VT.isVector() && 3920 TLI.isTruncateFree(LargestVT, VT)) 3921 Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue); 3922 else 3923 Value = getMemsetValue(Src, VT, DAG, dl); 3924 } 3925 assert(Value.getValueType() == VT && "Value with wrong type."); 3926 SDValue Store = DAG.getStore(Chain, dl, Value, 3927 getMemBasePlusOffset(Dst, DstOff, dl, DAG), 3928 DstPtrInfo.getWithOffset(DstOff), 3929 isVol, false, Align); 3930 OutChains.push_back(Store); 3931 DstOff += VT.getSizeInBits() / 8; 3932 Size -= VTSize; 3933 } 3934 3935 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 3936 &OutChains[0], OutChains.size()); 3937} 3938 3939SDValue SelectionDAG::getMemcpy(SDValue Chain, SDLoc dl, SDValue Dst, 3940 SDValue Src, SDValue Size, 3941 unsigned Align, bool isVol, bool AlwaysInline, 3942 MachinePointerInfo DstPtrInfo, 3943 MachinePointerInfo SrcPtrInfo) { 3944 assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); 3945 3946 // Check to see if we should lower the memcpy to loads and stores first. 3947 // For cases within the target-specified limits, this is the best choice. 3948 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 3949 if (ConstantSize) { 3950 // Memcpy with size zero? Just return the original chain. 3951 if (ConstantSize->isNullValue()) 3952 return Chain; 3953 3954 SDValue Result = getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, 3955 ConstantSize->getZExtValue(),Align, 3956 isVol, false, DstPtrInfo, SrcPtrInfo); 3957 if (Result.getNode()) 3958 return Result; 3959 } 3960 3961 // Then check to see if we should lower the memcpy with target-specific 3962 // code. If the target chooses to do this, this is the next best. 3963 SDValue Result = 3964 TSI.EmitTargetCodeForMemcpy(*this, dl, Chain, Dst, Src, Size, Align, 3965 isVol, AlwaysInline, 3966 DstPtrInfo, SrcPtrInfo); 3967 if (Result.getNode()) 3968 return Result; 3969 3970 // If we really need inline code and the target declined to provide it, 3971 // use a (potentially long) sequence of loads and stores. 3972 if (AlwaysInline) { 3973 assert(ConstantSize && "AlwaysInline requires a constant size!"); 3974 return getMemcpyLoadsAndStores(*this, dl, Chain, Dst, Src, 3975 ConstantSize->getZExtValue(), Align, isVol, 3976 true, DstPtrInfo, SrcPtrInfo); 3977 } 3978 3979 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc 3980 // memcpy is not guaranteed to be safe. libc memcpys aren't required to 3981 // respect volatile, so they may do things like read or write memory 3982 // beyond the given memory regions. But fixing this isn't easy, and most 3983 // people don't care. 3984 3985 const TargetLowering *TLI = TM.getTargetLowering(); 3986 3987 // Emit a library call. 3988 TargetLowering::ArgListTy Args; 3989 TargetLowering::ArgListEntry Entry; 3990 Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext()); 3991 Entry.Node = Dst; Args.push_back(Entry); 3992 Entry.Node = Src; Args.push_back(Entry); 3993 Entry.Node = Size; Args.push_back(Entry); 3994 // FIXME: pass in SDLoc 3995 TargetLowering:: 3996 CallLoweringInfo CLI(Chain, Type::getVoidTy(*getContext()), 3997 false, false, false, false, 0, 3998 TLI->getLibcallCallingConv(RTLIB::MEMCPY), 3999 /*isTailCall=*/false, 4000 /*doesNotReturn=*/false, /*isReturnValueUsed=*/false, 4001 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY), 4002 TLI->getPointerTy()), 4003 Args, *this, dl); 4004 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); 4005 4006 return CallResult.second; 4007} 4008 4009SDValue SelectionDAG::getMemmove(SDValue Chain, SDLoc dl, SDValue Dst, 4010 SDValue Src, SDValue Size, 4011 unsigned Align, bool isVol, 4012 MachinePointerInfo DstPtrInfo, 4013 MachinePointerInfo SrcPtrInfo) { 4014 assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); 4015 4016 // Check to see if we should lower the memmove to loads and stores first. 4017 // For cases within the target-specified limits, this is the best choice. 4018 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 4019 if (ConstantSize) { 4020 // Memmove with size zero? Just return the original chain. 4021 if (ConstantSize->isNullValue()) 4022 return Chain; 4023 4024 SDValue Result = 4025 getMemmoveLoadsAndStores(*this, dl, Chain, Dst, Src, 4026 ConstantSize->getZExtValue(), Align, isVol, 4027 false, DstPtrInfo, SrcPtrInfo); 4028 if (Result.getNode()) 4029 return Result; 4030 } 4031 4032 // Then check to see if we should lower the memmove with target-specific 4033 // code. If the target chooses to do this, this is the next best. 4034 SDValue Result = 4035 TSI.EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size, Align, isVol, 4036 DstPtrInfo, SrcPtrInfo); 4037 if (Result.getNode()) 4038 return Result; 4039 4040 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may 4041 // not be safe. See memcpy above for more details. 4042 4043 const TargetLowering *TLI = TM.getTargetLowering(); 4044 4045 // Emit a library call. 4046 TargetLowering::ArgListTy Args; 4047 TargetLowering::ArgListEntry Entry; 4048 Entry.Ty = TLI->getDataLayout()->getIntPtrType(*getContext()); 4049 Entry.Node = Dst; Args.push_back(Entry); 4050 Entry.Node = Src; Args.push_back(Entry); 4051 Entry.Node = Size; Args.push_back(Entry); 4052 // FIXME: pass in SDLoc 4053 TargetLowering:: 4054 CallLoweringInfo CLI(Chain, Type::getVoidTy(*getContext()), 4055 false, false, false, false, 0, 4056 TLI->getLibcallCallingConv(RTLIB::MEMMOVE), 4057 /*isTailCall=*/false, 4058 /*doesNotReturn=*/false, /*isReturnValueUsed=*/false, 4059 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE), 4060 TLI->getPointerTy()), 4061 Args, *this, dl); 4062 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); 4063 4064 return CallResult.second; 4065} 4066 4067SDValue SelectionDAG::getMemset(SDValue Chain, SDLoc dl, SDValue Dst, 4068 SDValue Src, SDValue Size, 4069 unsigned Align, bool isVol, 4070 MachinePointerInfo DstPtrInfo) { 4071 assert(Align && "The SDAG layer expects explicit alignment and reserves 0"); 4072 4073 // Check to see if we should lower the memset to stores first. 4074 // For cases within the target-specified limits, this is the best choice. 4075 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size); 4076 if (ConstantSize) { 4077 // Memset with size zero? Just return the original chain. 4078 if (ConstantSize->isNullValue()) 4079 return Chain; 4080 4081 SDValue Result = 4082 getMemsetStores(*this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), 4083 Align, isVol, DstPtrInfo); 4084 4085 if (Result.getNode()) 4086 return Result; 4087 } 4088 4089 // Then check to see if we should lower the memset with target-specific 4090 // code. If the target chooses to do this, this is the next best. 4091 SDValue Result = 4092 TSI.EmitTargetCodeForMemset(*this, dl, Chain, Dst, Src, Size, Align, isVol, 4093 DstPtrInfo); 4094 if (Result.getNode()) 4095 return Result; 4096 4097 // Emit a library call. 4098 const TargetLowering *TLI = TM.getTargetLowering(); 4099 Type *IntPtrTy = TLI->getDataLayout()->getIntPtrType(*getContext()); 4100 TargetLowering::ArgListTy Args; 4101 TargetLowering::ArgListEntry Entry; 4102 Entry.Node = Dst; Entry.Ty = IntPtrTy; 4103 Args.push_back(Entry); 4104 // Extend or truncate the argument to be an i32 value for the call. 4105 if (Src.getValueType().bitsGT(MVT::i32)) 4106 Src = getNode(ISD::TRUNCATE, dl, MVT::i32, Src); 4107 else 4108 Src = getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Src); 4109 Entry.Node = Src; 4110 Entry.Ty = Type::getInt32Ty(*getContext()); 4111 Entry.isSExt = true; 4112 Args.push_back(Entry); 4113 Entry.Node = Size; 4114 Entry.Ty = IntPtrTy; 4115 Entry.isSExt = false; 4116 Args.push_back(Entry); 4117 // FIXME: pass in SDLoc 4118 TargetLowering:: 4119 CallLoweringInfo CLI(Chain, Type::getVoidTy(*getContext()), 4120 false, false, false, false, 0, 4121 TLI->getLibcallCallingConv(RTLIB::MEMSET), 4122 /*isTailCall=*/false, 4123 /*doesNotReturn*/false, /*isReturnValueUsed=*/false, 4124 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET), 4125 TLI->getPointerTy()), 4126 Args, *this, dl); 4127 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI); 4128 4129 return CallResult.second; 4130} 4131 4132SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4133 SDVTList VTList, SDValue* Ops, unsigned NumOps, 4134 MachineMemOperand *MMO, 4135 AtomicOrdering Ordering, 4136 SynchronizationScope SynchScope) { 4137 FoldingSetNodeID ID; 4138 ID.AddInteger(MemVT.getRawBits()); 4139 AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 4140 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4141 void* IP = 0; 4142 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4143 cast<AtomicSDNode>(E)->refineAlignment(MMO); 4144 return SDValue(E, 0); 4145 } 4146 4147 // Allocate the operands array for the node out of the BumpPtrAllocator, since 4148 // SDNode doesn't have access to it. This memory will be "leaked" when 4149 // the node is deallocated, but recovered when the allocator is released. 4150 // If the number of operands is less than 5 we use AtomicSDNode's internal 4151 // storage. 4152 SDUse *DynOps = NumOps > 4 ? OperandAllocator.Allocate<SDUse>(NumOps) : 0; 4153 4154 SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl.getIROrder(), 4155 dl.getDebugLoc(), VTList, MemVT, 4156 Ops, DynOps, NumOps, MMO, 4157 Ordering, SynchScope); 4158 CSEMap.InsertNode(N, IP); 4159 AllNodes.push_back(N); 4160 return SDValue(N, 0); 4161} 4162 4163SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4164 SDValue Chain, SDValue Ptr, SDValue Cmp, 4165 SDValue Swp, MachinePointerInfo PtrInfo, 4166 unsigned Alignment, 4167 AtomicOrdering Ordering, 4168 SynchronizationScope SynchScope) { 4169 if (Alignment == 0) // Ensure that codegen never sees alignment 0 4170 Alignment = getEVTAlignment(MemVT); 4171 4172 MachineFunction &MF = getMachineFunction(); 4173 4174 // All atomics are load and store, except for ATMOIC_LOAD and ATOMIC_STORE. 4175 // For now, atomics are considered to be volatile always. 4176 // FIXME: Volatile isn't really correct; we should keep track of atomic 4177 // orderings in the memoperand. 4178 unsigned Flags = MachineMemOperand::MOVolatile; 4179 if (Opcode != ISD::ATOMIC_STORE) 4180 Flags |= MachineMemOperand::MOLoad; 4181 if (Opcode != ISD::ATOMIC_LOAD) 4182 Flags |= MachineMemOperand::MOStore; 4183 4184 MachineMemOperand *MMO = 4185 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment); 4186 4187 return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO, 4188 Ordering, SynchScope); 4189} 4190 4191SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4192 SDValue Chain, 4193 SDValue Ptr, SDValue Cmp, 4194 SDValue Swp, MachineMemOperand *MMO, 4195 AtomicOrdering Ordering, 4196 SynchronizationScope SynchScope) { 4197 assert(Opcode == ISD::ATOMIC_CMP_SWAP && "Invalid Atomic Op"); 4198 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types"); 4199 4200 EVT VT = Cmp.getValueType(); 4201 4202 SDVTList VTs = getVTList(VT, MVT::Other); 4203 SDValue Ops[] = {Chain, Ptr, Cmp, Swp}; 4204 return getAtomic(Opcode, dl, MemVT, VTs, Ops, 4, MMO, Ordering, SynchScope); 4205} 4206 4207SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4208 SDValue Chain, 4209 SDValue Ptr, SDValue Val, 4210 const Value* PtrVal, 4211 unsigned Alignment, 4212 AtomicOrdering Ordering, 4213 SynchronizationScope SynchScope) { 4214 if (Alignment == 0) // Ensure that codegen never sees alignment 0 4215 Alignment = getEVTAlignment(MemVT); 4216 4217 MachineFunction &MF = getMachineFunction(); 4218 // An atomic store does not load. An atomic load does not store. 4219 // (An atomicrmw obviously both loads and stores.) 4220 // For now, atomics are considered to be volatile always, and they are 4221 // chained as such. 4222 // FIXME: Volatile isn't really correct; we should keep track of atomic 4223 // orderings in the memoperand. 4224 unsigned Flags = MachineMemOperand::MOVolatile; 4225 if (Opcode != ISD::ATOMIC_STORE) 4226 Flags |= MachineMemOperand::MOLoad; 4227 if (Opcode != ISD::ATOMIC_LOAD) 4228 Flags |= MachineMemOperand::MOStore; 4229 4230 MachineMemOperand *MMO = 4231 MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, 4232 MemVT.getStoreSize(), Alignment); 4233 4234 return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO, 4235 Ordering, SynchScope); 4236} 4237 4238SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4239 SDValue Chain, 4240 SDValue Ptr, SDValue Val, 4241 MachineMemOperand *MMO, 4242 AtomicOrdering Ordering, 4243 SynchronizationScope SynchScope) { 4244 assert((Opcode == ISD::ATOMIC_LOAD_ADD || 4245 Opcode == ISD::ATOMIC_LOAD_SUB || 4246 Opcode == ISD::ATOMIC_LOAD_AND || 4247 Opcode == ISD::ATOMIC_LOAD_OR || 4248 Opcode == ISD::ATOMIC_LOAD_XOR || 4249 Opcode == ISD::ATOMIC_LOAD_NAND || 4250 Opcode == ISD::ATOMIC_LOAD_MIN || 4251 Opcode == ISD::ATOMIC_LOAD_MAX || 4252 Opcode == ISD::ATOMIC_LOAD_UMIN || 4253 Opcode == ISD::ATOMIC_LOAD_UMAX || 4254 Opcode == ISD::ATOMIC_SWAP || 4255 Opcode == ISD::ATOMIC_STORE) && 4256 "Invalid Atomic Op"); 4257 4258 EVT VT = Val.getValueType(); 4259 4260 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) : 4261 getVTList(VT, MVT::Other); 4262 SDValue Ops[] = {Chain, Ptr, Val}; 4263 return getAtomic(Opcode, dl, MemVT, VTs, Ops, 3, MMO, Ordering, SynchScope); 4264} 4265 4266SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4267 EVT VT, SDValue Chain, 4268 SDValue Ptr, 4269 const Value* PtrVal, 4270 unsigned Alignment, 4271 AtomicOrdering Ordering, 4272 SynchronizationScope SynchScope) { 4273 if (Alignment == 0) // Ensure that codegen never sees alignment 0 4274 Alignment = getEVTAlignment(MemVT); 4275 4276 MachineFunction &MF = getMachineFunction(); 4277 // An atomic store does not load. An atomic load does not store. 4278 // (An atomicrmw obviously both loads and stores.) 4279 // For now, atomics are considered to be volatile always, and they are 4280 // chained as such. 4281 // FIXME: Volatile isn't really correct; we should keep track of atomic 4282 // orderings in the memoperand. 4283 unsigned Flags = MachineMemOperand::MOVolatile; 4284 if (Opcode != ISD::ATOMIC_STORE) 4285 Flags |= MachineMemOperand::MOLoad; 4286 if (Opcode != ISD::ATOMIC_LOAD) 4287 Flags |= MachineMemOperand::MOStore; 4288 4289 MachineMemOperand *MMO = 4290 MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, 4291 MemVT.getStoreSize(), Alignment); 4292 4293 return getAtomic(Opcode, dl, MemVT, VT, Chain, Ptr, MMO, 4294 Ordering, SynchScope); 4295} 4296 4297SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, 4298 EVT VT, SDValue Chain, 4299 SDValue Ptr, 4300 MachineMemOperand *MMO, 4301 AtomicOrdering Ordering, 4302 SynchronizationScope SynchScope) { 4303 assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op"); 4304 4305 SDVTList VTs = getVTList(VT, MVT::Other); 4306 SDValue Ops[] = {Chain, Ptr}; 4307 return getAtomic(Opcode, dl, MemVT, VTs, Ops, 2, MMO, Ordering, SynchScope); 4308} 4309 4310/// getMergeValues - Create a MERGE_VALUES node from the given operands. 4311SDValue SelectionDAG::getMergeValues(const SDValue *Ops, unsigned NumOps, 4312 SDLoc dl) { 4313 if (NumOps == 1) 4314 return Ops[0]; 4315 4316 SmallVector<EVT, 4> VTs; 4317 VTs.reserve(NumOps); 4318 for (unsigned i = 0; i < NumOps; ++i) 4319 VTs.push_back(Ops[i].getValueType()); 4320 return getNode(ISD::MERGE_VALUES, dl, getVTList(&VTs[0], NumOps), 4321 Ops, NumOps); 4322} 4323 4324SDValue 4325SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, 4326 const EVT *VTs, unsigned NumVTs, 4327 const SDValue *Ops, unsigned NumOps, 4328 EVT MemVT, MachinePointerInfo PtrInfo, 4329 unsigned Align, bool Vol, 4330 bool ReadMem, bool WriteMem) { 4331 return getMemIntrinsicNode(Opcode, dl, makeVTList(VTs, NumVTs), Ops, NumOps, 4332 MemVT, PtrInfo, Align, Vol, 4333 ReadMem, WriteMem); 4334} 4335 4336SDValue 4337SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList, 4338 const SDValue *Ops, unsigned NumOps, 4339 EVT MemVT, MachinePointerInfo PtrInfo, 4340 unsigned Align, bool Vol, 4341 bool ReadMem, bool WriteMem) { 4342 if (Align == 0) // Ensure that codegen never sees alignment 0 4343 Align = getEVTAlignment(MemVT); 4344 4345 MachineFunction &MF = getMachineFunction(); 4346 unsigned Flags = 0; 4347 if (WriteMem) 4348 Flags |= MachineMemOperand::MOStore; 4349 if (ReadMem) 4350 Flags |= MachineMemOperand::MOLoad; 4351 if (Vol) 4352 Flags |= MachineMemOperand::MOVolatile; 4353 MachineMemOperand *MMO = 4354 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Align); 4355 4356 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, NumOps, MemVT, MMO); 4357} 4358 4359SDValue 4360SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList, 4361 const SDValue *Ops, unsigned NumOps, 4362 EVT MemVT, MachineMemOperand *MMO) { 4363 assert((Opcode == ISD::INTRINSIC_VOID || 4364 Opcode == ISD::INTRINSIC_W_CHAIN || 4365 Opcode == ISD::PREFETCH || 4366 Opcode == ISD::LIFETIME_START || 4367 Opcode == ISD::LIFETIME_END || 4368 (Opcode <= INT_MAX && 4369 (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) && 4370 "Opcode is not a memory-accessing opcode!"); 4371 4372 // Memoize the node unless it returns a flag. 4373 MemIntrinsicSDNode *N; 4374 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 4375 FoldingSetNodeID ID; 4376 AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 4377 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4378 void *IP = 0; 4379 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4380 cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO); 4381 return SDValue(E, 0); 4382 } 4383 4384 N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(), 4385 dl.getDebugLoc(), VTList, Ops, 4386 NumOps, MemVT, MMO); 4387 CSEMap.InsertNode(N, IP); 4388 } else { 4389 N = new (NodeAllocator) MemIntrinsicSDNode(Opcode, dl.getIROrder(), 4390 dl.getDebugLoc(), VTList, Ops, 4391 NumOps, MemVT, MMO); 4392 } 4393 AllNodes.push_back(N); 4394 return SDValue(N, 0); 4395} 4396 4397/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a 4398/// MachinePointerInfo record from it. This is particularly useful because the 4399/// code generator has many cases where it doesn't bother passing in a 4400/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". 4401static MachinePointerInfo InferPointerInfo(SDValue Ptr, int64_t Offset = 0) { 4402 // If this is FI+Offset, we can model it. 4403 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) 4404 return MachinePointerInfo::getFixedStack(FI->getIndex(), Offset); 4405 4406 // If this is (FI+Offset1)+Offset2, we can model it. 4407 if (Ptr.getOpcode() != ISD::ADD || 4408 !isa<ConstantSDNode>(Ptr.getOperand(1)) || 4409 !isa<FrameIndexSDNode>(Ptr.getOperand(0))) 4410 return MachinePointerInfo(); 4411 4412 int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); 4413 return MachinePointerInfo::getFixedStack(FI, Offset+ 4414 cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue()); 4415} 4416 4417/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a 4418/// MachinePointerInfo record from it. This is particularly useful because the 4419/// code generator has many cases where it doesn't bother passing in a 4420/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst". 4421static MachinePointerInfo InferPointerInfo(SDValue Ptr, SDValue OffsetOp) { 4422 // If the 'Offset' value isn't a constant, we can't handle this. 4423 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp)) 4424 return InferPointerInfo(Ptr, OffsetNode->getSExtValue()); 4425 if (OffsetOp.getOpcode() == ISD::UNDEF) 4426 return InferPointerInfo(Ptr); 4427 return MachinePointerInfo(); 4428} 4429 4430 4431SDValue 4432SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 4433 EVT VT, SDLoc dl, SDValue Chain, 4434 SDValue Ptr, SDValue Offset, 4435 MachinePointerInfo PtrInfo, EVT MemVT, 4436 bool isVolatile, bool isNonTemporal, bool isInvariant, 4437 unsigned Alignment, const MDNode *TBAAInfo, 4438 const MDNode *Ranges) { 4439 assert(Chain.getValueType() == MVT::Other && 4440 "Invalid chain type"); 4441 if (Alignment == 0) // Ensure that codegen never sees alignment 0 4442 Alignment = getEVTAlignment(VT); 4443 4444 unsigned Flags = MachineMemOperand::MOLoad; 4445 if (isVolatile) 4446 Flags |= MachineMemOperand::MOVolatile; 4447 if (isNonTemporal) 4448 Flags |= MachineMemOperand::MONonTemporal; 4449 if (isInvariant) 4450 Flags |= MachineMemOperand::MOInvariant; 4451 4452 // If we don't have a PtrInfo, infer the trivial frame index case to simplify 4453 // clients. 4454 if (PtrInfo.V == 0) 4455 PtrInfo = InferPointerInfo(Ptr, Offset); 4456 4457 MachineFunction &MF = getMachineFunction(); 4458 MachineMemOperand *MMO = 4459 MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment, 4460 TBAAInfo, Ranges); 4461 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO); 4462} 4463 4464SDValue 4465SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, 4466 EVT VT, SDLoc dl, SDValue Chain, 4467 SDValue Ptr, SDValue Offset, EVT MemVT, 4468 MachineMemOperand *MMO) { 4469 if (VT == MemVT) { 4470 ExtType = ISD::NON_EXTLOAD; 4471 } else if (ExtType == ISD::NON_EXTLOAD) { 4472 assert(VT == MemVT && "Non-extending load from different memory type!"); 4473 } else { 4474 // Extending load. 4475 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) && 4476 "Should only be an extending load, not truncating!"); 4477 assert(VT.isInteger() == MemVT.isInteger() && 4478 "Cannot convert from FP to Int or Int -> FP!"); 4479 assert(VT.isVector() == MemVT.isVector() && 4480 "Cannot use trunc store to convert to or from a vector!"); 4481 assert((!VT.isVector() || 4482 VT.getVectorNumElements() == MemVT.getVectorNumElements()) && 4483 "Cannot use trunc store to change the number of vector elements!"); 4484 } 4485 4486 bool Indexed = AM != ISD::UNINDEXED; 4487 assert((Indexed || Offset.getOpcode() == ISD::UNDEF) && 4488 "Unindexed load with an offset!"); 4489 4490 SDVTList VTs = Indexed ? 4491 getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other); 4492 SDValue Ops[] = { Chain, Ptr, Offset }; 4493 FoldingSetNodeID ID; 4494 AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3); 4495 ID.AddInteger(MemVT.getRawBits()); 4496 ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(), 4497 MMO->isNonTemporal(), 4498 MMO->isInvariant())); 4499 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4500 void *IP = 0; 4501 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4502 cast<LoadSDNode>(E)->refineAlignment(MMO); 4503 return SDValue(E, 0); 4504 } 4505 SDNode *N = new (NodeAllocator) LoadSDNode(Ops, dl.getIROrder(), 4506 dl.getDebugLoc(), VTs, AM, ExtType, 4507 MemVT, MMO); 4508 CSEMap.InsertNode(N, IP); 4509 AllNodes.push_back(N); 4510 return SDValue(N, 0); 4511} 4512 4513SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl, 4514 SDValue Chain, SDValue Ptr, 4515 MachinePointerInfo PtrInfo, 4516 bool isVolatile, bool isNonTemporal, 4517 bool isInvariant, unsigned Alignment, 4518 const MDNode *TBAAInfo, 4519 const MDNode *Ranges) { 4520 SDValue Undef = getUNDEF(Ptr.getValueType()); 4521 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, 4522 PtrInfo, VT, isVolatile, isNonTemporal, isInvariant, Alignment, 4523 TBAAInfo, Ranges); 4524} 4525 4526SDValue SelectionDAG::getLoad(EVT VT, SDLoc dl, 4527 SDValue Chain, SDValue Ptr, 4528 MachineMemOperand *MMO) { 4529 SDValue Undef = getUNDEF(Ptr.getValueType()); 4530 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef, 4531 VT, MMO); 4532} 4533 4534SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT, 4535 SDValue Chain, SDValue Ptr, 4536 MachinePointerInfo PtrInfo, EVT MemVT, 4537 bool isVolatile, bool isNonTemporal, 4538 unsigned Alignment, const MDNode *TBAAInfo) { 4539 SDValue Undef = getUNDEF(Ptr.getValueType()); 4540 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, 4541 PtrInfo, MemVT, isVolatile, isNonTemporal, false, Alignment, 4542 TBAAInfo); 4543} 4544 4545 4546SDValue SelectionDAG::getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT, 4547 SDValue Chain, SDValue Ptr, EVT MemVT, 4548 MachineMemOperand *MMO) { 4549 SDValue Undef = getUNDEF(Ptr.getValueType()); 4550 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, 4551 MemVT, MMO); 4552} 4553 4554SDValue 4555SelectionDAG::getIndexedLoad(SDValue OrigLoad, SDLoc dl, SDValue Base, 4556 SDValue Offset, ISD::MemIndexedMode AM) { 4557 LoadSDNode *LD = cast<LoadSDNode>(OrigLoad); 4558 assert(LD->getOffset().getOpcode() == ISD::UNDEF && 4559 "Load is already a indexed load!"); 4560 return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl, 4561 LD->getChain(), Base, Offset, LD->getPointerInfo(), 4562 LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(), 4563 false, LD->getAlignment()); 4564} 4565 4566SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val, 4567 SDValue Ptr, MachinePointerInfo PtrInfo, 4568 bool isVolatile, bool isNonTemporal, 4569 unsigned Alignment, const MDNode *TBAAInfo) { 4570 assert(Chain.getValueType() == MVT::Other && 4571 "Invalid chain type"); 4572 if (Alignment == 0) // Ensure that codegen never sees alignment 0 4573 Alignment = getEVTAlignment(Val.getValueType()); 4574 4575 unsigned Flags = MachineMemOperand::MOStore; 4576 if (isVolatile) 4577 Flags |= MachineMemOperand::MOVolatile; 4578 if (isNonTemporal) 4579 Flags |= MachineMemOperand::MONonTemporal; 4580 4581 if (PtrInfo.V == 0) 4582 PtrInfo = InferPointerInfo(Ptr); 4583 4584 MachineFunction &MF = getMachineFunction(); 4585 MachineMemOperand *MMO = 4586 MF.getMachineMemOperand(PtrInfo, Flags, 4587 Val.getValueType().getStoreSize(), Alignment, 4588 TBAAInfo); 4589 4590 return getStore(Chain, dl, Val, Ptr, MMO); 4591} 4592 4593SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val, 4594 SDValue Ptr, MachineMemOperand *MMO) { 4595 assert(Chain.getValueType() == MVT::Other && 4596 "Invalid chain type"); 4597 EVT VT = Val.getValueType(); 4598 SDVTList VTs = getVTList(MVT::Other); 4599 SDValue Undef = getUNDEF(Ptr.getValueType()); 4600 SDValue Ops[] = { Chain, Val, Ptr, Undef }; 4601 FoldingSetNodeID ID; 4602 AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 4603 ID.AddInteger(VT.getRawBits()); 4604 ID.AddInteger(encodeMemSDNodeFlags(false, ISD::UNINDEXED, MMO->isVolatile(), 4605 MMO->isNonTemporal(), MMO->isInvariant())); 4606 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4607 void *IP = 0; 4608 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4609 cast<StoreSDNode>(E)->refineAlignment(MMO); 4610 return SDValue(E, 0); 4611 } 4612 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(), 4613 dl.getDebugLoc(), VTs, 4614 ISD::UNINDEXED, false, VT, MMO); 4615 CSEMap.InsertNode(N, IP); 4616 AllNodes.push_back(N); 4617 return SDValue(N, 0); 4618} 4619 4620SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val, 4621 SDValue Ptr, MachinePointerInfo PtrInfo, 4622 EVT SVT,bool isVolatile, bool isNonTemporal, 4623 unsigned Alignment, 4624 const MDNode *TBAAInfo) { 4625 assert(Chain.getValueType() == MVT::Other && 4626 "Invalid chain type"); 4627 if (Alignment == 0) // Ensure that codegen never sees alignment 0 4628 Alignment = getEVTAlignment(SVT); 4629 4630 unsigned Flags = MachineMemOperand::MOStore; 4631 if (isVolatile) 4632 Flags |= MachineMemOperand::MOVolatile; 4633 if (isNonTemporal) 4634 Flags |= MachineMemOperand::MONonTemporal; 4635 4636 if (PtrInfo.V == 0) 4637 PtrInfo = InferPointerInfo(Ptr); 4638 4639 MachineFunction &MF = getMachineFunction(); 4640 MachineMemOperand *MMO = 4641 MF.getMachineMemOperand(PtrInfo, Flags, SVT.getStoreSize(), Alignment, 4642 TBAAInfo); 4643 4644 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO); 4645} 4646 4647SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val, 4648 SDValue Ptr, EVT SVT, 4649 MachineMemOperand *MMO) { 4650 EVT VT = Val.getValueType(); 4651 4652 assert(Chain.getValueType() == MVT::Other && 4653 "Invalid chain type"); 4654 if (VT == SVT) 4655 return getStore(Chain, dl, Val, Ptr, MMO); 4656 4657 assert(SVT.getScalarType().bitsLT(VT.getScalarType()) && 4658 "Should only be a truncating store, not extending!"); 4659 assert(VT.isInteger() == SVT.isInteger() && 4660 "Can't do FP-INT conversion!"); 4661 assert(VT.isVector() == SVT.isVector() && 4662 "Cannot use trunc store to convert to or from a vector!"); 4663 assert((!VT.isVector() || 4664 VT.getVectorNumElements() == SVT.getVectorNumElements()) && 4665 "Cannot use trunc store to change the number of vector elements!"); 4666 4667 SDVTList VTs = getVTList(MVT::Other); 4668 SDValue Undef = getUNDEF(Ptr.getValueType()); 4669 SDValue Ops[] = { Chain, Val, Ptr, Undef }; 4670 FoldingSetNodeID ID; 4671 AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 4672 ID.AddInteger(SVT.getRawBits()); 4673 ID.AddInteger(encodeMemSDNodeFlags(true, ISD::UNINDEXED, MMO->isVolatile(), 4674 MMO->isNonTemporal(), MMO->isInvariant())); 4675 ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); 4676 void *IP = 0; 4677 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 4678 cast<StoreSDNode>(E)->refineAlignment(MMO); 4679 return SDValue(E, 0); 4680 } 4681 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(), 4682 dl.getDebugLoc(), VTs, 4683 ISD::UNINDEXED, true, SVT, MMO); 4684 CSEMap.InsertNode(N, IP); 4685 AllNodes.push_back(N); 4686 return SDValue(N, 0); 4687} 4688 4689SDValue 4690SelectionDAG::getIndexedStore(SDValue OrigStore, SDLoc dl, SDValue Base, 4691 SDValue Offset, ISD::MemIndexedMode AM) { 4692 StoreSDNode *ST = cast<StoreSDNode>(OrigStore); 4693 assert(ST->getOffset().getOpcode() == ISD::UNDEF && 4694 "Store is already a indexed store!"); 4695 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other); 4696 SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset }; 4697 FoldingSetNodeID ID; 4698 AddNodeIDNode(ID, ISD::STORE, VTs, Ops, 4); 4699 ID.AddInteger(ST->getMemoryVT().getRawBits()); 4700 ID.AddInteger(ST->getRawSubclassData()); 4701 ID.AddInteger(ST->getPointerInfo().getAddrSpace()); 4702 void *IP = 0; 4703 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 4704 return SDValue(E, 0); 4705 4706 SDNode *N = new (NodeAllocator) StoreSDNode(Ops, dl.getIROrder(), 4707 dl.getDebugLoc(), VTs, AM, 4708 ST->isTruncatingStore(), 4709 ST->getMemoryVT(), 4710 ST->getMemOperand()); 4711 CSEMap.InsertNode(N, IP); 4712 AllNodes.push_back(N); 4713 return SDValue(N, 0); 4714} 4715 4716SDValue SelectionDAG::getVAArg(EVT VT, SDLoc dl, 4717 SDValue Chain, SDValue Ptr, 4718 SDValue SV, 4719 unsigned Align) { 4720 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, MVT::i32) }; 4721 return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops, 4); 4722} 4723 4724SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 4725 const SDUse *Ops, unsigned NumOps) { 4726 switch (NumOps) { 4727 case 0: return getNode(Opcode, DL, VT); 4728 case 1: return getNode(Opcode, DL, VT, Ops[0]); 4729 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]); 4730 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); 4731 default: break; 4732 } 4733 4734 // Copy from an SDUse array into an SDValue array for use with 4735 // the regular getNode logic. 4736 SmallVector<SDValue, 8> NewOps(Ops, Ops + NumOps); 4737 return getNode(Opcode, DL, VT, &NewOps[0], NumOps); 4738} 4739 4740SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, 4741 const SDValue *Ops, unsigned NumOps) { 4742 switch (NumOps) { 4743 case 0: return getNode(Opcode, DL, VT); 4744 case 1: return getNode(Opcode, DL, VT, Ops[0]); 4745 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]); 4746 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]); 4747 default: break; 4748 } 4749 4750 switch (Opcode) { 4751 default: break; 4752 case ISD::SELECT_CC: { 4753 assert(NumOps == 5 && "SELECT_CC takes 5 operands!"); 4754 assert(Ops[0].getValueType() == Ops[1].getValueType() && 4755 "LHS and RHS of condition must have same type!"); 4756 assert(Ops[2].getValueType() == Ops[3].getValueType() && 4757 "True and False arms of SelectCC must have same type!"); 4758 assert(Ops[2].getValueType() == VT && 4759 "select_cc node must be of same type as true and false value!"); 4760 break; 4761 } 4762 case ISD::BR_CC: { 4763 assert(NumOps == 5 && "BR_CC takes 5 operands!"); 4764 assert(Ops[2].getValueType() == Ops[3].getValueType() && 4765 "LHS/RHS of comparison should match types!"); 4766 break; 4767 } 4768 } 4769 4770 // Memoize nodes. 4771 SDNode *N; 4772 SDVTList VTs = getVTList(VT); 4773 4774 if (VT != MVT::Glue) { 4775 FoldingSetNodeID ID; 4776 AddNodeIDNode(ID, Opcode, VTs, Ops, NumOps); 4777 void *IP = 0; 4778 4779 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 4780 return SDValue(E, 0); 4781 4782 N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), 4783 VTs, Ops, NumOps); 4784 CSEMap.InsertNode(N, IP); 4785 } else { 4786 N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), 4787 VTs, Ops, NumOps); 4788 } 4789 4790 AllNodes.push_back(N); 4791#ifndef NDEBUG 4792 VerifySDNode(N); 4793#endif 4794 return SDValue(N, 0); 4795} 4796 4797SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, 4798 ArrayRef<EVT> ResultTys, 4799 const SDValue *Ops, unsigned NumOps) { 4800 return getNode(Opcode, DL, getVTList(&ResultTys[0], ResultTys.size()), 4801 Ops, NumOps); 4802} 4803 4804SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, 4805 const EVT *VTs, unsigned NumVTs, 4806 const SDValue *Ops, unsigned NumOps) { 4807 if (NumVTs == 1) 4808 return getNode(Opcode, DL, VTs[0], Ops, NumOps); 4809 return getNode(Opcode, DL, makeVTList(VTs, NumVTs), Ops, NumOps); 4810} 4811 4812SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4813 const SDValue *Ops, unsigned NumOps) { 4814 if (VTList.NumVTs == 1) 4815 return getNode(Opcode, DL, VTList.VTs[0], Ops, NumOps); 4816 4817#if 0 4818 switch (Opcode) { 4819 // FIXME: figure out how to safely handle things like 4820 // int foo(int x) { return 1 << (x & 255); } 4821 // int bar() { return foo(256); } 4822 case ISD::SRA_PARTS: 4823 case ISD::SRL_PARTS: 4824 case ISD::SHL_PARTS: 4825 if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG && 4826 cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1) 4827 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); 4828 else if (N3.getOpcode() == ISD::AND) 4829 if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) { 4830 // If the and is only masking out bits that cannot effect the shift, 4831 // eliminate the and. 4832 unsigned NumBits = VT.getScalarType().getSizeInBits()*2; 4833 if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1) 4834 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0)); 4835 } 4836 break; 4837 } 4838#endif 4839 4840 // Memoize the node unless it returns a flag. 4841 SDNode *N; 4842 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 4843 FoldingSetNodeID ID; 4844 AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 4845 void *IP = 0; 4846 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 4847 return SDValue(E, 0); 4848 4849 if (NumOps == 1) { 4850 N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(), 4851 DL.getDebugLoc(), VTList, Ops[0]); 4852 } else if (NumOps == 2) { 4853 N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(), 4854 DL.getDebugLoc(), VTList, Ops[0], 4855 Ops[1]); 4856 } else if (NumOps == 3) { 4857 N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(), 4858 DL.getDebugLoc(), VTList, Ops[0], 4859 Ops[1], Ops[2]); 4860 } else { 4861 N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), 4862 VTList, Ops, NumOps); 4863 } 4864 CSEMap.InsertNode(N, IP); 4865 } else { 4866 if (NumOps == 1) { 4867 N = new (NodeAllocator) UnarySDNode(Opcode, DL.getIROrder(), 4868 DL.getDebugLoc(), VTList, Ops[0]); 4869 } else if (NumOps == 2) { 4870 N = new (NodeAllocator) BinarySDNode(Opcode, DL.getIROrder(), 4871 DL.getDebugLoc(), VTList, Ops[0], 4872 Ops[1]); 4873 } else if (NumOps == 3) { 4874 N = new (NodeAllocator) TernarySDNode(Opcode, DL.getIROrder(), 4875 DL.getDebugLoc(), VTList, Ops[0], 4876 Ops[1], Ops[2]); 4877 } else { 4878 N = new (NodeAllocator) SDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), 4879 VTList, Ops, NumOps); 4880 } 4881 } 4882 AllNodes.push_back(N); 4883#ifndef NDEBUG 4884 VerifySDNode(N); 4885#endif 4886 return SDValue(N, 0); 4887} 4888 4889SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList) { 4890 return getNode(Opcode, DL, VTList, 0, 0); 4891} 4892 4893SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4894 SDValue N1) { 4895 SDValue Ops[] = { N1 }; 4896 return getNode(Opcode, DL, VTList, Ops, 1); 4897} 4898 4899SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4900 SDValue N1, SDValue N2) { 4901 SDValue Ops[] = { N1, N2 }; 4902 return getNode(Opcode, DL, VTList, Ops, 2); 4903} 4904 4905SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4906 SDValue N1, SDValue N2, SDValue N3) { 4907 SDValue Ops[] = { N1, N2, N3 }; 4908 return getNode(Opcode, DL, VTList, Ops, 3); 4909} 4910 4911SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4912 SDValue N1, SDValue N2, SDValue N3, 4913 SDValue N4) { 4914 SDValue Ops[] = { N1, N2, N3, N4 }; 4915 return getNode(Opcode, DL, VTList, Ops, 4); 4916} 4917 4918SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, 4919 SDValue N1, SDValue N2, SDValue N3, 4920 SDValue N4, SDValue N5) { 4921 SDValue Ops[] = { N1, N2, N3, N4, N5 }; 4922 return getNode(Opcode, DL, VTList, Ops, 5); 4923} 4924 4925SDVTList SelectionDAG::getVTList(EVT VT) { 4926 return makeVTList(SDNode::getValueTypeList(VT), 1); 4927} 4928 4929SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2) { 4930 FoldingSetNodeID ID; 4931 ID.AddInteger(2U); 4932 ID.AddInteger(VT1.getRawBits()); 4933 ID.AddInteger(VT2.getRawBits()); 4934 4935 void *IP = 0; 4936 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 4937 if (Result == NULL) { 4938 EVT *Array = Allocator.Allocate<EVT>(2); 4939 Array[0] = VT1; 4940 Array[1] = VT2; 4941 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2); 4942 VTListMap.InsertNode(Result, IP); 4943 } 4944 return Result->getSDVTList(); 4945} 4946 4947SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3) { 4948 FoldingSetNodeID ID; 4949 ID.AddInteger(3U); 4950 ID.AddInteger(VT1.getRawBits()); 4951 ID.AddInteger(VT2.getRawBits()); 4952 ID.AddInteger(VT3.getRawBits()); 4953 4954 void *IP = 0; 4955 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 4956 if (Result == NULL) { 4957 EVT *Array = Allocator.Allocate<EVT>(3); 4958 Array[0] = VT1; 4959 Array[1] = VT2; 4960 Array[2] = VT3; 4961 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3); 4962 VTListMap.InsertNode(Result, IP); 4963 } 4964 return Result->getSDVTList(); 4965} 4966 4967SDVTList SelectionDAG::getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4) { 4968 FoldingSetNodeID ID; 4969 ID.AddInteger(4U); 4970 ID.AddInteger(VT1.getRawBits()); 4971 ID.AddInteger(VT2.getRawBits()); 4972 ID.AddInteger(VT3.getRawBits()); 4973 ID.AddInteger(VT4.getRawBits()); 4974 4975 void *IP = 0; 4976 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 4977 if (Result == NULL) { 4978 EVT *Array = Allocator.Allocate<EVT>(4); 4979 Array[0] = VT1; 4980 Array[1] = VT2; 4981 Array[2] = VT3; 4982 Array[3] = VT4; 4983 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4); 4984 VTListMap.InsertNode(Result, IP); 4985 } 4986 return Result->getSDVTList(); 4987} 4988 4989SDVTList SelectionDAG::getVTList(const EVT *VTs, unsigned NumVTs) { 4990 FoldingSetNodeID ID; 4991 ID.AddInteger(NumVTs); 4992 for (unsigned index = 0; index < NumVTs; index++) { 4993 ID.AddInteger(VTs[index].getRawBits()); 4994 } 4995 4996 void *IP = 0; 4997 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP); 4998 if (Result == NULL) { 4999 EVT *Array = Allocator.Allocate<EVT>(NumVTs); 5000 std::copy(VTs, VTs + NumVTs, Array); 5001 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs); 5002 VTListMap.InsertNode(Result, IP); 5003 } 5004 return Result->getSDVTList(); 5005} 5006 5007 5008/// UpdateNodeOperands - *Mutate* the specified node in-place to have the 5009/// specified operands. If the resultant node already exists in the DAG, 5010/// this does not modify the specified node, instead it returns the node that 5011/// already exists. If the resultant node does not exist in the DAG, the 5012/// input node is returned. As a degenerate case, if you specify the same 5013/// input operands as the node already has, the input node is returned. 5014SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op) { 5015 assert(N->getNumOperands() == 1 && "Update with wrong number of operands"); 5016 5017 // Check to see if there is no change. 5018 if (Op == N->getOperand(0)) return N; 5019 5020 // See if the modified node already exists. 5021 void *InsertPos = 0; 5022 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos)) 5023 return Existing; 5024 5025 // Nope it doesn't. Remove the node from its current place in the maps. 5026 if (InsertPos) 5027 if (!RemoveNodeFromCSEMaps(N)) 5028 InsertPos = 0; 5029 5030 // Now we update the operands. 5031 N->OperandList[0].set(Op); 5032 5033 // If this gets put into a CSE map, add it. 5034 if (InsertPos) CSEMap.InsertNode(N, InsertPos); 5035 return N; 5036} 5037 5038SDNode *SelectionDAG::UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2) { 5039 assert(N->getNumOperands() == 2 && "Update with wrong number of operands"); 5040 5041 // Check to see if there is no change. 5042 if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1)) 5043 return N; // No operands changed, just return the input node. 5044 5045 // See if the modified node already exists. 5046 void *InsertPos = 0; 5047 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos)) 5048 return Existing; 5049 5050 // Nope it doesn't. Remove the node from its current place in the maps. 5051 if (InsertPos) 5052 if (!RemoveNodeFromCSEMaps(N)) 5053 InsertPos = 0; 5054 5055 // Now we update the operands. 5056 if (N->OperandList[0] != Op1) 5057 N->OperandList[0].set(Op1); 5058 if (N->OperandList[1] != Op2) 5059 N->OperandList[1].set(Op2); 5060 5061 // If this gets put into a CSE map, add it. 5062 if (InsertPos) CSEMap.InsertNode(N, InsertPos); 5063 return N; 5064} 5065 5066SDNode *SelectionDAG:: 5067UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, SDValue Op3) { 5068 SDValue Ops[] = { Op1, Op2, Op3 }; 5069 return UpdateNodeOperands(N, Ops, 3); 5070} 5071 5072SDNode *SelectionDAG:: 5073UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 5074 SDValue Op3, SDValue Op4) { 5075 SDValue Ops[] = { Op1, Op2, Op3, Op4 }; 5076 return UpdateNodeOperands(N, Ops, 4); 5077} 5078 5079SDNode *SelectionDAG:: 5080UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2, 5081 SDValue Op3, SDValue Op4, SDValue Op5) { 5082 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 }; 5083 return UpdateNodeOperands(N, Ops, 5); 5084} 5085 5086SDNode *SelectionDAG:: 5087UpdateNodeOperands(SDNode *N, const SDValue *Ops, unsigned NumOps) { 5088 assert(N->getNumOperands() == NumOps && 5089 "Update with wrong number of operands"); 5090 5091 // Check to see if there is no change. 5092 bool AnyChange = false; 5093 for (unsigned i = 0; i != NumOps; ++i) { 5094 if (Ops[i] != N->getOperand(i)) { 5095 AnyChange = true; 5096 break; 5097 } 5098 } 5099 5100 // No operands changed, just return the input node. 5101 if (!AnyChange) return N; 5102 5103 // See if the modified node already exists. 5104 void *InsertPos = 0; 5105 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, NumOps, InsertPos)) 5106 return Existing; 5107 5108 // Nope it doesn't. Remove the node from its current place in the maps. 5109 if (InsertPos) 5110 if (!RemoveNodeFromCSEMaps(N)) 5111 InsertPos = 0; 5112 5113 // Now we update the operands. 5114 for (unsigned i = 0; i != NumOps; ++i) 5115 if (N->OperandList[i] != Ops[i]) 5116 N->OperandList[i].set(Ops[i]); 5117 5118 // If this gets put into a CSE map, add it. 5119 if (InsertPos) CSEMap.InsertNode(N, InsertPos); 5120 return N; 5121} 5122 5123/// DropOperands - Release the operands and set this node to have 5124/// zero operands. 5125void SDNode::DropOperands() { 5126 // Unlike the code in MorphNodeTo that does this, we don't need to 5127 // watch for dead nodes here. 5128 for (op_iterator I = op_begin(), E = op_end(); I != E; ) { 5129 SDUse &Use = *I++; 5130 Use.set(SDValue()); 5131 } 5132} 5133 5134/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a 5135/// machine opcode. 5136/// 5137SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5138 EVT VT) { 5139 SDVTList VTs = getVTList(VT); 5140 return SelectNodeTo(N, MachineOpc, VTs, 0, 0); 5141} 5142 5143SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5144 EVT VT, SDValue Op1) { 5145 SDVTList VTs = getVTList(VT); 5146 SDValue Ops[] = { Op1 }; 5147 return SelectNodeTo(N, MachineOpc, VTs, Ops, 1); 5148} 5149 5150SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5151 EVT VT, SDValue Op1, 5152 SDValue Op2) { 5153 SDVTList VTs = getVTList(VT); 5154 SDValue Ops[] = { Op1, Op2 }; 5155 return SelectNodeTo(N, MachineOpc, VTs, Ops, 2); 5156} 5157 5158SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5159 EVT VT, SDValue Op1, 5160 SDValue Op2, SDValue Op3) { 5161 SDVTList VTs = getVTList(VT); 5162 SDValue Ops[] = { Op1, Op2, Op3 }; 5163 return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 5164} 5165 5166SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5167 EVT VT, const SDValue *Ops, 5168 unsigned NumOps) { 5169 SDVTList VTs = getVTList(VT); 5170 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 5171} 5172 5173SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5174 EVT VT1, EVT VT2, const SDValue *Ops, 5175 unsigned NumOps) { 5176 SDVTList VTs = getVTList(VT1, VT2); 5177 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 5178} 5179 5180SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5181 EVT VT1, EVT VT2) { 5182 SDVTList VTs = getVTList(VT1, VT2); 5183 return SelectNodeTo(N, MachineOpc, VTs, (SDValue *)0, 0); 5184} 5185 5186SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5187 EVT VT1, EVT VT2, EVT VT3, 5188 const SDValue *Ops, unsigned NumOps) { 5189 SDVTList VTs = getVTList(VT1, VT2, VT3); 5190 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 5191} 5192 5193SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5194 EVT VT1, EVT VT2, EVT VT3, EVT VT4, 5195 const SDValue *Ops, unsigned NumOps) { 5196 SDVTList VTs = getVTList(VT1, VT2, VT3, VT4); 5197 return SelectNodeTo(N, MachineOpc, VTs, Ops, NumOps); 5198} 5199 5200SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5201 EVT VT1, EVT VT2, 5202 SDValue Op1) { 5203 SDVTList VTs = getVTList(VT1, VT2); 5204 SDValue Ops[] = { Op1 }; 5205 return SelectNodeTo(N, MachineOpc, VTs, Ops, 1); 5206} 5207 5208SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5209 EVT VT1, EVT VT2, 5210 SDValue Op1, SDValue Op2) { 5211 SDVTList VTs = getVTList(VT1, VT2); 5212 SDValue Ops[] = { Op1, Op2 }; 5213 return SelectNodeTo(N, MachineOpc, VTs, Ops, 2); 5214} 5215 5216SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5217 EVT VT1, EVT VT2, 5218 SDValue Op1, SDValue Op2, 5219 SDValue Op3) { 5220 SDVTList VTs = getVTList(VT1, VT2); 5221 SDValue Ops[] = { Op1, Op2, Op3 }; 5222 return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 5223} 5224 5225SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5226 EVT VT1, EVT VT2, EVT VT3, 5227 SDValue Op1, SDValue Op2, 5228 SDValue Op3) { 5229 SDVTList VTs = getVTList(VT1, VT2, VT3); 5230 SDValue Ops[] = { Op1, Op2, Op3 }; 5231 return SelectNodeTo(N, MachineOpc, VTs, Ops, 3); 5232} 5233 5234SDNode *SelectionDAG::SelectNodeTo(SDNode *N, unsigned MachineOpc, 5235 SDVTList VTs, const SDValue *Ops, 5236 unsigned NumOps) { 5237 N = MorphNodeTo(N, ~MachineOpc, VTs, Ops, NumOps); 5238 // Reset the NodeID to -1. 5239 N->setNodeId(-1); 5240 return N; 5241} 5242 5243/// UpdadeSDLocOnMergedSDNode - If the opt level is -O0 then it throws away 5244/// the line number information on the merged node since it is not possible to 5245/// preserve the information that operation is associated with multiple lines. 5246/// This will make the debugger working better at -O0, were there is a higher 5247/// probability having other instructions associated with that line. 5248/// 5249/// For IROrder, we keep the smaller of the two 5250SDNode *SelectionDAG::UpdadeSDLocOnMergedSDNode(SDNode *N, SDLoc OLoc) { 5251 DebugLoc NLoc = N->getDebugLoc(); 5252 if (!(NLoc.isUnknown()) && (OptLevel == CodeGenOpt::None) && 5253 (OLoc.getDebugLoc() != NLoc)) { 5254 N->setDebugLoc(DebugLoc()); 5255 } 5256 unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder()); 5257 N->setIROrder(Order); 5258 return N; 5259} 5260 5261/// MorphNodeTo - This *mutates* the specified node to have the specified 5262/// return type, opcode, and operands. 5263/// 5264/// Note that MorphNodeTo returns the resultant node. If there is already a 5265/// node of the specified opcode and operands, it returns that node instead of 5266/// the current one. Note that the SDLoc need not be the same. 5267/// 5268/// Using MorphNodeTo is faster than creating a new node and swapping it in 5269/// with ReplaceAllUsesWith both because it often avoids allocating a new 5270/// node, and because it doesn't require CSE recalculation for any of 5271/// the node's users. 5272/// 5273SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, 5274 SDVTList VTs, const SDValue *Ops, 5275 unsigned NumOps) { 5276 // If an identical node already exists, use it. 5277 void *IP = 0; 5278 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) { 5279 FoldingSetNodeID ID; 5280 AddNodeIDNode(ID, Opc, VTs, Ops, NumOps); 5281 if (SDNode *ON = CSEMap.FindNodeOrInsertPos(ID, IP)) 5282 return UpdadeSDLocOnMergedSDNode(ON, SDLoc(N)); 5283 } 5284 5285 if (!RemoveNodeFromCSEMaps(N)) 5286 IP = 0; 5287 5288 // Start the morphing. 5289 N->NodeType = Opc; 5290 N->ValueList = VTs.VTs; 5291 N->NumValues = VTs.NumVTs; 5292 5293 // Clear the operands list, updating used nodes to remove this from their 5294 // use list. Keep track of any operands that become dead as a result. 5295 SmallPtrSet<SDNode*, 16> DeadNodeSet; 5296 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) { 5297 SDUse &Use = *I++; 5298 SDNode *Used = Use.getNode(); 5299 Use.set(SDValue()); 5300 if (Used->use_empty()) 5301 DeadNodeSet.insert(Used); 5302 } 5303 5304 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) { 5305 // Initialize the memory references information. 5306 MN->setMemRefs(0, 0); 5307 // If NumOps is larger than the # of operands we can have in a 5308 // MachineSDNode, reallocate the operand list. 5309 if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) { 5310 if (MN->OperandsNeedDelete) 5311 delete[] MN->OperandList; 5312 if (NumOps > array_lengthof(MN->LocalOperands)) 5313 // We're creating a final node that will live unmorphed for the 5314 // remainder of the current SelectionDAG iteration, so we can allocate 5315 // the operands directly out of a pool with no recycling metadata. 5316 MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps), 5317 Ops, NumOps); 5318 else 5319 MN->InitOperands(MN->LocalOperands, Ops, NumOps); 5320 MN->OperandsNeedDelete = false; 5321 } else 5322 MN->InitOperands(MN->OperandList, Ops, NumOps); 5323 } else { 5324 // If NumOps is larger than the # of operands we currently have, reallocate 5325 // the operand list. 5326 if (NumOps > N->NumOperands) { 5327 if (N->OperandsNeedDelete) 5328 delete[] N->OperandList; 5329 N->InitOperands(new SDUse[NumOps], Ops, NumOps); 5330 N->OperandsNeedDelete = true; 5331 } else 5332 N->InitOperands(N->OperandList, Ops, NumOps); 5333 } 5334 5335 // Delete any nodes that are still dead after adding the uses for the 5336 // new operands. 5337 if (!DeadNodeSet.empty()) { 5338 SmallVector<SDNode *, 16> DeadNodes; 5339 for (SmallPtrSet<SDNode *, 16>::iterator I = DeadNodeSet.begin(), 5340 E = DeadNodeSet.end(); I != E; ++I) 5341 if ((*I)->use_empty()) 5342 DeadNodes.push_back(*I); 5343 RemoveDeadNodes(DeadNodes); 5344 } 5345 5346 if (IP) 5347 CSEMap.InsertNode(N, IP); // Memoize the new node. 5348 return N; 5349} 5350 5351 5352/// getMachineNode - These are used for target selectors to create a new node 5353/// with specified return type(s), MachineInstr opcode, and operands. 5354/// 5355/// Note that getMachineNode returns the resultant node. If there is already a 5356/// node of the specified opcode and operands, it returns that node instead of 5357/// the current one. 5358MachineSDNode * 5359SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT) { 5360 SDVTList VTs = getVTList(VT); 5361 return getMachineNode(Opcode, dl, VTs, None); 5362} 5363 5364MachineSDNode * 5365SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, SDValue Op1) { 5366 SDVTList VTs = getVTList(VT); 5367 SDValue Ops[] = { Op1 }; 5368 return getMachineNode(Opcode, dl, VTs, Ops); 5369} 5370 5371MachineSDNode * 5372SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, 5373 SDValue Op1, SDValue Op2) { 5374 SDVTList VTs = getVTList(VT); 5375 SDValue Ops[] = { Op1, Op2 }; 5376 return getMachineNode(Opcode, dl, VTs, Ops); 5377} 5378 5379MachineSDNode * 5380SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, 5381 SDValue Op1, SDValue Op2, SDValue Op3) { 5382 SDVTList VTs = getVTList(VT); 5383 SDValue Ops[] = { Op1, Op2, Op3 }; 5384 return getMachineNode(Opcode, dl, VTs, Ops); 5385} 5386 5387MachineSDNode * 5388SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT, 5389 ArrayRef<SDValue> Ops) { 5390 SDVTList VTs = getVTList(VT); 5391 return getMachineNode(Opcode, dl, VTs, Ops); 5392} 5393 5394MachineSDNode * 5395SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1, EVT VT2) { 5396 SDVTList VTs = getVTList(VT1, VT2); 5397 return getMachineNode(Opcode, dl, VTs, None); 5398} 5399 5400MachineSDNode * 5401SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5402 EVT VT1, EVT VT2, SDValue Op1) { 5403 SDVTList VTs = getVTList(VT1, VT2); 5404 SDValue Ops[] = { Op1 }; 5405 return getMachineNode(Opcode, dl, VTs, Ops); 5406} 5407 5408MachineSDNode * 5409SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5410 EVT VT1, EVT VT2, SDValue Op1, SDValue Op2) { 5411 SDVTList VTs = getVTList(VT1, VT2); 5412 SDValue Ops[] = { Op1, Op2 }; 5413 return getMachineNode(Opcode, dl, VTs, Ops); 5414} 5415 5416MachineSDNode * 5417SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5418 EVT VT1, EVT VT2, SDValue Op1, 5419 SDValue Op2, SDValue Op3) { 5420 SDVTList VTs = getVTList(VT1, VT2); 5421 SDValue Ops[] = { Op1, Op2, Op3 }; 5422 return getMachineNode(Opcode, dl, VTs, Ops); 5423} 5424 5425MachineSDNode * 5426SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5427 EVT VT1, EVT VT2, 5428 ArrayRef<SDValue> Ops) { 5429 SDVTList VTs = getVTList(VT1, VT2); 5430 return getMachineNode(Opcode, dl, VTs, Ops); 5431} 5432 5433MachineSDNode * 5434SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5435 EVT VT1, EVT VT2, EVT VT3, 5436 SDValue Op1, SDValue Op2) { 5437 SDVTList VTs = getVTList(VT1, VT2, VT3); 5438 SDValue Ops[] = { Op1, Op2 }; 5439 return getMachineNode(Opcode, dl, VTs, Ops); 5440} 5441 5442MachineSDNode * 5443SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5444 EVT VT1, EVT VT2, EVT VT3, 5445 SDValue Op1, SDValue Op2, SDValue Op3) { 5446 SDVTList VTs = getVTList(VT1, VT2, VT3); 5447 SDValue Ops[] = { Op1, Op2, Op3 }; 5448 return getMachineNode(Opcode, dl, VTs, Ops); 5449} 5450 5451MachineSDNode * 5452SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5453 EVT VT1, EVT VT2, EVT VT3, 5454 ArrayRef<SDValue> Ops) { 5455 SDVTList VTs = getVTList(VT1, VT2, VT3); 5456 return getMachineNode(Opcode, dl, VTs, Ops); 5457} 5458 5459MachineSDNode * 5460SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, EVT VT1, 5461 EVT VT2, EVT VT3, EVT VT4, 5462 ArrayRef<SDValue> Ops) { 5463 SDVTList VTs = getVTList(VT1, VT2, VT3, VT4); 5464 return getMachineNode(Opcode, dl, VTs, Ops); 5465} 5466 5467MachineSDNode * 5468SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, 5469 ArrayRef<EVT> ResultTys, 5470 ArrayRef<SDValue> Ops) { 5471 SDVTList VTs = getVTList(&ResultTys[0], ResultTys.size()); 5472 return getMachineNode(Opcode, dl, VTs, Ops); 5473} 5474 5475MachineSDNode * 5476SelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs, 5477 ArrayRef<SDValue> OpsArray) { 5478 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue; 5479 MachineSDNode *N; 5480 void *IP = 0; 5481 const SDValue *Ops = OpsArray.data(); 5482 unsigned NumOps = OpsArray.size(); 5483 5484 if (DoCSE) { 5485 FoldingSetNodeID ID; 5486 AddNodeIDNode(ID, ~Opcode, VTs, Ops, NumOps); 5487 IP = 0; 5488 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) { 5489 return cast<MachineSDNode>(UpdadeSDLocOnMergedSDNode(E, DL)); 5490 } 5491 } 5492 5493 // Allocate a new MachineSDNode. 5494 N = new (NodeAllocator) MachineSDNode(~Opcode, DL.getIROrder(), 5495 DL.getDebugLoc(), VTs); 5496 5497 // Initialize the operands list. 5498 if (NumOps > array_lengthof(N->LocalOperands)) 5499 // We're creating a final node that will live unmorphed for the 5500 // remainder of the current SelectionDAG iteration, so we can allocate 5501 // the operands directly out of a pool with no recycling metadata. 5502 N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps), 5503 Ops, NumOps); 5504 else 5505 N->InitOperands(N->LocalOperands, Ops, NumOps); 5506 N->OperandsNeedDelete = false; 5507 5508 if (DoCSE) 5509 CSEMap.InsertNode(N, IP); 5510 5511 AllNodes.push_back(N); 5512#ifndef NDEBUG 5513 VerifyMachineNode(N); 5514#endif 5515 return N; 5516} 5517 5518/// getTargetExtractSubreg - A convenience function for creating 5519/// TargetOpcode::EXTRACT_SUBREG nodes. 5520SDValue 5521SelectionDAG::getTargetExtractSubreg(int SRIdx, SDLoc DL, EVT VT, 5522 SDValue Operand) { 5523 SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32); 5524 SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, 5525 VT, Operand, SRIdxVal); 5526 return SDValue(Subreg, 0); 5527} 5528 5529/// getTargetInsertSubreg - A convenience function for creating 5530/// TargetOpcode::INSERT_SUBREG nodes. 5531SDValue 5532SelectionDAG::getTargetInsertSubreg(int SRIdx, SDLoc DL, EVT VT, 5533 SDValue Operand, SDValue Subreg) { 5534 SDValue SRIdxVal = getTargetConstant(SRIdx, MVT::i32); 5535 SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL, 5536 VT, Operand, Subreg, SRIdxVal); 5537 return SDValue(Result, 0); 5538} 5539 5540/// getNodeIfExists - Get the specified node if it's already available, or 5541/// else return NULL. 5542SDNode *SelectionDAG::getNodeIfExists(unsigned Opcode, SDVTList VTList, 5543 const SDValue *Ops, unsigned NumOps) { 5544 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { 5545 FoldingSetNodeID ID; 5546 AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); 5547 void *IP = 0; 5548 if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) 5549 return E; 5550 } 5551 return NULL; 5552} 5553 5554/// getDbgValue - Creates a SDDbgValue node. 5555/// 5556SDDbgValue * 5557SelectionDAG::getDbgValue(MDNode *MDPtr, SDNode *N, unsigned R, uint64_t Off, 5558 DebugLoc DL, unsigned O) { 5559 return new (Allocator) SDDbgValue(MDPtr, N, R, Off, DL, O); 5560} 5561 5562SDDbgValue * 5563SelectionDAG::getDbgValue(MDNode *MDPtr, const Value *C, uint64_t Off, 5564 DebugLoc DL, unsigned O) { 5565 return new (Allocator) SDDbgValue(MDPtr, C, Off, DL, O); 5566} 5567 5568SDDbgValue * 5569SelectionDAG::getDbgValue(MDNode *MDPtr, unsigned FI, uint64_t Off, 5570 DebugLoc DL, unsigned O) { 5571 return new (Allocator) SDDbgValue(MDPtr, FI, Off, DL, O); 5572} 5573 5574namespace { 5575 5576/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node 5577/// pointed to by a use iterator is deleted, increment the use iterator 5578/// so that it doesn't dangle. 5579/// 5580class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener { 5581 SDNode::use_iterator &UI; 5582 SDNode::use_iterator &UE; 5583 5584 virtual void NodeDeleted(SDNode *N, SDNode *E) { 5585 // Increment the iterator as needed. 5586 while (UI != UE && N == *UI) 5587 ++UI; 5588 } 5589 5590public: 5591 RAUWUpdateListener(SelectionDAG &d, 5592 SDNode::use_iterator &ui, 5593 SDNode::use_iterator &ue) 5594 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {} 5595}; 5596 5597} 5598 5599/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 5600/// This can cause recursive merging of nodes in the DAG. 5601/// 5602/// This version assumes From has a single result value. 5603/// 5604void SelectionDAG::ReplaceAllUsesWith(SDValue FromN, SDValue To) { 5605 SDNode *From = FromN.getNode(); 5606 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 && 5607 "Cannot replace with this method!"); 5608 assert(From != To.getNode() && "Cannot replace uses of with self"); 5609 5610 // Iterate over all the existing uses of From. New uses will be added 5611 // to the beginning of the use list, which we avoid visiting. 5612 // This specifically avoids visiting uses of From that arise while the 5613 // replacement is happening, because any such uses would be the result 5614 // of CSE: If an existing node looks like From after one of its operands 5615 // is replaced by To, we don't want to replace of all its users with To 5616 // too. See PR3018 for more info. 5617 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 5618 RAUWUpdateListener Listener(*this, UI, UE); 5619 while (UI != UE) { 5620 SDNode *User = *UI; 5621 5622 // This node is about to morph, remove its old self from the CSE maps. 5623 RemoveNodeFromCSEMaps(User); 5624 5625 // A user can appear in a use list multiple times, and when this 5626 // happens the uses are usually next to each other in the list. 5627 // To help reduce the number of CSE recomputations, process all 5628 // the uses of this user that we can find this way. 5629 do { 5630 SDUse &Use = UI.getUse(); 5631 ++UI; 5632 Use.set(To); 5633 } while (UI != UE && *UI == User); 5634 5635 // Now that we have modified User, add it back to the CSE maps. If it 5636 // already exists there, recursively merge the results together. 5637 AddModifiedNodeToCSEMaps(User); 5638 } 5639 5640 // If we just RAUW'd the root, take note. 5641 if (FromN == getRoot()) 5642 setRoot(To); 5643} 5644 5645/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 5646/// This can cause recursive merging of nodes in the DAG. 5647/// 5648/// This version assumes that for each value of From, there is a 5649/// corresponding value in To in the same position with the same type. 5650/// 5651void SelectionDAG::ReplaceAllUsesWith(SDNode *From, SDNode *To) { 5652#ifndef NDEBUG 5653 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) 5654 assert((!From->hasAnyUseOfValue(i) || 5655 From->getValueType(i) == To->getValueType(i)) && 5656 "Cannot use this version of ReplaceAllUsesWith!"); 5657#endif 5658 5659 // Handle the trivial case. 5660 if (From == To) 5661 return; 5662 5663 // Iterate over just the existing users of From. See the comments in 5664 // the ReplaceAllUsesWith above. 5665 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 5666 RAUWUpdateListener Listener(*this, UI, UE); 5667 while (UI != UE) { 5668 SDNode *User = *UI; 5669 5670 // This node is about to morph, remove its old self from the CSE maps. 5671 RemoveNodeFromCSEMaps(User); 5672 5673 // A user can appear in a use list multiple times, and when this 5674 // happens the uses are usually next to each other in the list. 5675 // To help reduce the number of CSE recomputations, process all 5676 // the uses of this user that we can find this way. 5677 do { 5678 SDUse &Use = UI.getUse(); 5679 ++UI; 5680 Use.setNode(To); 5681 } while (UI != UE && *UI == User); 5682 5683 // Now that we have modified User, add it back to the CSE maps. If it 5684 // already exists there, recursively merge the results together. 5685 AddModifiedNodeToCSEMaps(User); 5686 } 5687 5688 // If we just RAUW'd the root, take note. 5689 if (From == getRoot().getNode()) 5690 setRoot(SDValue(To, getRoot().getResNo())); 5691} 5692 5693/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead. 5694/// This can cause recursive merging of nodes in the DAG. 5695/// 5696/// This version can replace From with any result values. To must match the 5697/// number and types of values returned by From. 5698void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) { 5699 if (From->getNumValues() == 1) // Handle the simple case efficiently. 5700 return ReplaceAllUsesWith(SDValue(From, 0), To[0]); 5701 5702 // Iterate over just the existing users of From. See the comments in 5703 // the ReplaceAllUsesWith above. 5704 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end(); 5705 RAUWUpdateListener Listener(*this, UI, UE); 5706 while (UI != UE) { 5707 SDNode *User = *UI; 5708 5709 // This node is about to morph, remove its old self from the CSE maps. 5710 RemoveNodeFromCSEMaps(User); 5711 5712 // A user can appear in a use list multiple times, and when this 5713 // happens the uses are usually next to each other in the list. 5714 // To help reduce the number of CSE recomputations, process all 5715 // the uses of this user that we can find this way. 5716 do { 5717 SDUse &Use = UI.getUse(); 5718 const SDValue &ToOp = To[Use.getResNo()]; 5719 ++UI; 5720 Use.set(ToOp); 5721 } while (UI != UE && *UI == User); 5722 5723 // Now that we have modified User, add it back to the CSE maps. If it 5724 // already exists there, recursively merge the results together. 5725 AddModifiedNodeToCSEMaps(User); 5726 } 5727 5728 // If we just RAUW'd the root, take note. 5729 if (From == getRoot().getNode()) 5730 setRoot(SDValue(To[getRoot().getResNo()])); 5731} 5732 5733/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving 5734/// uses of other values produced by From.getNode() alone. The Deleted 5735/// vector is handled the same way as for ReplaceAllUsesWith. 5736void SelectionDAG::ReplaceAllUsesOfValueWith(SDValue From, SDValue To){ 5737 // Handle the really simple, really trivial case efficiently. 5738 if (From == To) return; 5739 5740 // Handle the simple, trivial, case efficiently. 5741 if (From.getNode()->getNumValues() == 1) { 5742 ReplaceAllUsesWith(From, To); 5743 return; 5744 } 5745 5746 // Iterate over just the existing users of From. See the comments in 5747 // the ReplaceAllUsesWith above. 5748 SDNode::use_iterator UI = From.getNode()->use_begin(), 5749 UE = From.getNode()->use_end(); 5750 RAUWUpdateListener Listener(*this, UI, UE); 5751 while (UI != UE) { 5752 SDNode *User = *UI; 5753 bool UserRemovedFromCSEMaps = false; 5754 5755 // A user can appear in a use list multiple times, and when this 5756 // happens the uses are usually next to each other in the list. 5757 // To help reduce the number of CSE recomputations, process all 5758 // the uses of this user that we can find this way. 5759 do { 5760 SDUse &Use = UI.getUse(); 5761 5762 // Skip uses of different values from the same node. 5763 if (Use.getResNo() != From.getResNo()) { 5764 ++UI; 5765 continue; 5766 } 5767 5768 // If this node hasn't been modified yet, it's still in the CSE maps, 5769 // so remove its old self from the CSE maps. 5770 if (!UserRemovedFromCSEMaps) { 5771 RemoveNodeFromCSEMaps(User); 5772 UserRemovedFromCSEMaps = true; 5773 } 5774 5775 ++UI; 5776 Use.set(To); 5777 } while (UI != UE && *UI == User); 5778 5779 // We are iterating over all uses of the From node, so if a use 5780 // doesn't use the specific value, no changes are made. 5781 if (!UserRemovedFromCSEMaps) 5782 continue; 5783 5784 // Now that we have modified User, add it back to the CSE maps. If it 5785 // already exists there, recursively merge the results together. 5786 AddModifiedNodeToCSEMaps(User); 5787 } 5788 5789 // If we just RAUW'd the root, take note. 5790 if (From == getRoot()) 5791 setRoot(To); 5792} 5793 5794namespace { 5795 /// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith 5796 /// to record information about a use. 5797 struct UseMemo { 5798 SDNode *User; 5799 unsigned Index; 5800 SDUse *Use; 5801 }; 5802 5803 /// operator< - Sort Memos by User. 5804 bool operator<(const UseMemo &L, const UseMemo &R) { 5805 return (intptr_t)L.User < (intptr_t)R.User; 5806 } 5807} 5808 5809/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving 5810/// uses of other values produced by From.getNode() alone. The same value 5811/// may appear in both the From and To list. The Deleted vector is 5812/// handled the same way as for ReplaceAllUsesWith. 5813void SelectionDAG::ReplaceAllUsesOfValuesWith(const SDValue *From, 5814 const SDValue *To, 5815 unsigned Num){ 5816 // Handle the simple, trivial case efficiently. 5817 if (Num == 1) 5818 return ReplaceAllUsesOfValueWith(*From, *To); 5819 5820 // Read up all the uses and make records of them. This helps 5821 // processing new uses that are introduced during the 5822 // replacement process. 5823 SmallVector<UseMemo, 4> Uses; 5824 for (unsigned i = 0; i != Num; ++i) { 5825 unsigned FromResNo = From[i].getResNo(); 5826 SDNode *FromNode = From[i].getNode(); 5827 for (SDNode::use_iterator UI = FromNode->use_begin(), 5828 E = FromNode->use_end(); UI != E; ++UI) { 5829 SDUse &Use = UI.getUse(); 5830 if (Use.getResNo() == FromResNo) { 5831 UseMemo Memo = { *UI, i, &Use }; 5832 Uses.push_back(Memo); 5833 } 5834 } 5835 } 5836 5837 // Sort the uses, so that all the uses from a given User are together. 5838 std::sort(Uses.begin(), Uses.end()); 5839 5840 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size(); 5841 UseIndex != UseIndexEnd; ) { 5842 // We know that this user uses some value of From. If it is the right 5843 // value, update it. 5844 SDNode *User = Uses[UseIndex].User; 5845 5846 // This node is about to morph, remove its old self from the CSE maps. 5847 RemoveNodeFromCSEMaps(User); 5848 5849 // The Uses array is sorted, so all the uses for a given User 5850 // are next to each other in the list. 5851 // To help reduce the number of CSE recomputations, process all 5852 // the uses of this user that we can find this way. 5853 do { 5854 unsigned i = Uses[UseIndex].Index; 5855 SDUse &Use = *Uses[UseIndex].Use; 5856 ++UseIndex; 5857 5858 Use.set(To[i]); 5859 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User); 5860 5861 // Now that we have modified User, add it back to the CSE maps. If it 5862 // already exists there, recursively merge the results together. 5863 AddModifiedNodeToCSEMaps(User); 5864 } 5865} 5866 5867/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG 5868/// based on their topological order. It returns the maximum id and a vector 5869/// of the SDNodes* in assigned order by reference. 5870unsigned SelectionDAG::AssignTopologicalOrder() { 5871 5872 unsigned DAGSize = 0; 5873 5874 // SortedPos tracks the progress of the algorithm. Nodes before it are 5875 // sorted, nodes after it are unsorted. When the algorithm completes 5876 // it is at the end of the list. 5877 allnodes_iterator SortedPos = allnodes_begin(); 5878 5879 // Visit all the nodes. Move nodes with no operands to the front of 5880 // the list immediately. Annotate nodes that do have operands with their 5881 // operand count. Before we do this, the Node Id fields of the nodes 5882 // may contain arbitrary values. After, the Node Id fields for nodes 5883 // before SortedPos will contain the topological sort index, and the 5884 // Node Id fields for nodes At SortedPos and after will contain the 5885 // count of outstanding operands. 5886 for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ) { 5887 SDNode *N = I++; 5888 checkForCycles(N); 5889 unsigned Degree = N->getNumOperands(); 5890 if (Degree == 0) { 5891 // A node with no uses, add it to the result array immediately. 5892 N->setNodeId(DAGSize++); 5893 allnodes_iterator Q = N; 5894 if (Q != SortedPos) 5895 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q)); 5896 assert(SortedPos != AllNodes.end() && "Overran node list"); 5897 ++SortedPos; 5898 } else { 5899 // Temporarily use the Node Id as scratch space for the degree count. 5900 N->setNodeId(Degree); 5901 } 5902 } 5903 5904 // Visit all the nodes. As we iterate, move nodes into sorted order, 5905 // such that by the time the end is reached all nodes will be sorted. 5906 for (allnodes_iterator I = allnodes_begin(),E = allnodes_end(); I != E; ++I) { 5907 SDNode *N = I; 5908 checkForCycles(N); 5909 // N is in sorted position, so all its uses have one less operand 5910 // that needs to be sorted. 5911 for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end(); 5912 UI != UE; ++UI) { 5913 SDNode *P = *UI; 5914 unsigned Degree = P->getNodeId(); 5915 assert(Degree != 0 && "Invalid node degree"); 5916 --Degree; 5917 if (Degree == 0) { 5918 // All of P's operands are sorted, so P may sorted now. 5919 P->setNodeId(DAGSize++); 5920 if (P != SortedPos) 5921 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P)); 5922 assert(SortedPos != AllNodes.end() && "Overran node list"); 5923 ++SortedPos; 5924 } else { 5925 // Update P's outstanding operand count. 5926 P->setNodeId(Degree); 5927 } 5928 } 5929 if (I == SortedPos) { 5930#ifndef NDEBUG 5931 SDNode *S = ++I; 5932 dbgs() << "Overran sorted position:\n"; 5933 S->dumprFull(); 5934#endif 5935 llvm_unreachable(0); 5936 } 5937 } 5938 5939 assert(SortedPos == AllNodes.end() && 5940 "Topological sort incomplete!"); 5941 assert(AllNodes.front().getOpcode() == ISD::EntryToken && 5942 "First node in topological sort is not the entry token!"); 5943 assert(AllNodes.front().getNodeId() == 0 && 5944 "First node in topological sort has non-zero id!"); 5945 assert(AllNodes.front().getNumOperands() == 0 && 5946 "First node in topological sort has operands!"); 5947 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 && 5948 "Last node in topologic sort has unexpected id!"); 5949 assert(AllNodes.back().use_empty() && 5950 "Last node in topologic sort has users!"); 5951 assert(DAGSize == allnodes_size() && "Node count mismatch!"); 5952 return DAGSize; 5953} 5954 5955/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the 5956/// value is produced by SD. 5957void SelectionDAG::AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter) { 5958 DbgInfo->add(DB, SD, isParameter); 5959 if (SD) 5960 SD->setHasDebugValue(true); 5961} 5962 5963/// TransferDbgValues - Transfer SDDbgValues. 5964void SelectionDAG::TransferDbgValues(SDValue From, SDValue To) { 5965 if (From == To || !From.getNode()->getHasDebugValue()) 5966 return; 5967 SDNode *FromNode = From.getNode(); 5968 SDNode *ToNode = To.getNode(); 5969 ArrayRef<SDDbgValue *> DVs = GetDbgValues(FromNode); 5970 SmallVector<SDDbgValue *, 2> ClonedDVs; 5971 for (ArrayRef<SDDbgValue *>::iterator I = DVs.begin(), E = DVs.end(); 5972 I != E; ++I) { 5973 SDDbgValue *Dbg = *I; 5974 if (Dbg->getKind() == SDDbgValue::SDNODE) { 5975 SDDbgValue *Clone = getDbgValue(Dbg->getMDPtr(), ToNode, To.getResNo(), 5976 Dbg->getOffset(), Dbg->getDebugLoc(), 5977 Dbg->getOrder()); 5978 ClonedDVs.push_back(Clone); 5979 } 5980 } 5981 for (SmallVectorImpl<SDDbgValue *>::iterator I = ClonedDVs.begin(), 5982 E = ClonedDVs.end(); I != E; ++I) 5983 AddDbgValue(*I, ToNode, false); 5984} 5985 5986//===----------------------------------------------------------------------===// 5987// SDNode Class 5988//===----------------------------------------------------------------------===// 5989 5990HandleSDNode::~HandleSDNode() { 5991 DropOperands(); 5992} 5993 5994GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order, 5995 DebugLoc DL, const GlobalValue *GA, 5996 EVT VT, int64_t o, unsigned char TF) 5997 : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) { 5998 TheGlobal = GA; 5999} 6000 6001AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT, 6002 SDValue X, unsigned SrcAS, 6003 unsigned DestAS) 6004 : UnarySDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT), X), 6005 SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {} 6006 6007MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, 6008 EVT memvt, MachineMemOperand *mmo) 6009 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) { 6010 SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(), 6011 MMO->isNonTemporal(), MMO->isInvariant()); 6012 assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!"); 6013 assert(isNonTemporal() == MMO->isNonTemporal() && 6014 "Non-temporal encoding error!"); 6015 assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!"); 6016} 6017 6018MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, 6019 const SDValue *Ops, unsigned NumOps, EVT memvt, 6020 MachineMemOperand *mmo) 6021 : SDNode(Opc, Order, dl, VTs, Ops, NumOps), 6022 MemoryVT(memvt), MMO(mmo) { 6023 SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(), 6024 MMO->isNonTemporal(), MMO->isInvariant()); 6025 assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!"); 6026 assert(memvt.getStoreSize() == MMO->getSize() && "Size mismatch!"); 6027} 6028 6029/// Profile - Gather unique data for the node. 6030/// 6031void SDNode::Profile(FoldingSetNodeID &ID) const { 6032 AddNodeIDNode(ID, this); 6033} 6034 6035namespace { 6036 struct EVTArray { 6037 std::vector<EVT> VTs; 6038 6039 EVTArray() { 6040 VTs.reserve(MVT::LAST_VALUETYPE); 6041 for (unsigned i = 0; i < MVT::LAST_VALUETYPE; ++i) 6042 VTs.push_back(MVT((MVT::SimpleValueType)i)); 6043 } 6044 }; 6045} 6046 6047static ManagedStatic<std::set<EVT, EVT::compareRawBits> > EVTs; 6048static ManagedStatic<EVTArray> SimpleVTArray; 6049static ManagedStatic<sys::SmartMutex<true> > VTMutex; 6050 6051/// getValueTypeList - Return a pointer to the specified value type. 6052/// 6053const EVT *SDNode::getValueTypeList(EVT VT) { 6054 if (VT.isExtended()) { 6055 sys::SmartScopedLock<true> Lock(*VTMutex); 6056 return &(*EVTs->insert(VT).first); 6057 } else { 6058 assert(VT.getSimpleVT() < MVT::LAST_VALUETYPE && 6059 "Value type out of range!"); 6060 return &SimpleVTArray->VTs[VT.getSimpleVT().SimpleTy]; 6061 } 6062} 6063 6064/// hasNUsesOfValue - Return true if there are exactly NUSES uses of the 6065/// indicated value. This method ignores uses of other values defined by this 6066/// operation. 6067bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const { 6068 assert(Value < getNumValues() && "Bad value!"); 6069 6070 // TODO: Only iterate over uses of a given value of the node 6071 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) { 6072 if (UI.getUse().getResNo() == Value) { 6073 if (NUses == 0) 6074 return false; 6075 --NUses; 6076 } 6077 } 6078 6079 // Found exactly the right number of uses? 6080 return NUses == 0; 6081} 6082 6083 6084/// hasAnyUseOfValue - Return true if there are any use of the indicated 6085/// value. This method ignores uses of other values defined by this operation. 6086bool SDNode::hasAnyUseOfValue(unsigned Value) const { 6087 assert(Value < getNumValues() && "Bad value!"); 6088 6089 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) 6090 if (UI.getUse().getResNo() == Value) 6091 return true; 6092 6093 return false; 6094} 6095 6096 6097/// isOnlyUserOf - Return true if this node is the only use of N. 6098/// 6099bool SDNode::isOnlyUserOf(SDNode *N) const { 6100 bool Seen = false; 6101 for (SDNode::use_iterator I = N->use_begin(), E = N->use_end(); I != E; ++I) { 6102 SDNode *User = *I; 6103 if (User == this) 6104 Seen = true; 6105 else 6106 return false; 6107 } 6108 6109 return Seen; 6110} 6111 6112/// isOperand - Return true if this node is an operand of N. 6113/// 6114bool SDValue::isOperandOf(SDNode *N) const { 6115 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 6116 if (*this == N->getOperand(i)) 6117 return true; 6118 return false; 6119} 6120 6121bool SDNode::isOperandOf(SDNode *N) const { 6122 for (unsigned i = 0, e = N->NumOperands; i != e; ++i) 6123 if (this == N->OperandList[i].getNode()) 6124 return true; 6125 return false; 6126} 6127 6128/// reachesChainWithoutSideEffects - Return true if this operand (which must 6129/// be a chain) reaches the specified operand without crossing any 6130/// side-effecting instructions on any chain path. In practice, this looks 6131/// through token factors and non-volatile loads. In order to remain efficient, 6132/// this only looks a couple of nodes in, it does not do an exhaustive search. 6133bool SDValue::reachesChainWithoutSideEffects(SDValue Dest, 6134 unsigned Depth) const { 6135 if (*this == Dest) return true; 6136 6137 // Don't search too deeply, we just want to be able to see through 6138 // TokenFactor's etc. 6139 if (Depth == 0) return false; 6140 6141 // If this is a token factor, all inputs to the TF happen in parallel. If any 6142 // of the operands of the TF does not reach dest, then we cannot do the xform. 6143 if (getOpcode() == ISD::TokenFactor) { 6144 for (unsigned i = 0, e = getNumOperands(); i != e; ++i) 6145 if (!getOperand(i).reachesChainWithoutSideEffects(Dest, Depth-1)) 6146 return false; 6147 return true; 6148 } 6149 6150 // Loads don't have side effects, look through them. 6151 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) { 6152 if (!Ld->isVolatile()) 6153 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1); 6154 } 6155 return false; 6156} 6157 6158/// hasPredecessor - Return true if N is a predecessor of this node. 6159/// N is either an operand of this node, or can be reached by recursively 6160/// traversing up the operands. 6161/// NOTE: This is an expensive method. Use it carefully. 6162bool SDNode::hasPredecessor(const SDNode *N) const { 6163 SmallPtrSet<const SDNode *, 32> Visited; 6164 SmallVector<const SDNode *, 16> Worklist; 6165 return hasPredecessorHelper(N, Visited, Worklist); 6166} 6167 6168bool 6169SDNode::hasPredecessorHelper(const SDNode *N, 6170 SmallPtrSet<const SDNode *, 32> &Visited, 6171 SmallVectorImpl<const SDNode *> &Worklist) const { 6172 if (Visited.empty()) { 6173 Worklist.push_back(this); 6174 } else { 6175 // Take a look in the visited set. If we've already encountered this node 6176 // we needn't search further. 6177 if (Visited.count(N)) 6178 return true; 6179 } 6180 6181 // Haven't visited N yet. Continue the search. 6182 while (!Worklist.empty()) { 6183 const SDNode *M = Worklist.pop_back_val(); 6184 for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) { 6185 SDNode *Op = M->getOperand(i).getNode(); 6186 if (Visited.insert(Op)) 6187 Worklist.push_back(Op); 6188 if (Op == N) 6189 return true; 6190 } 6191 } 6192 6193 return false; 6194} 6195 6196uint64_t SDNode::getConstantOperandVal(unsigned Num) const { 6197 assert(Num < NumOperands && "Invalid child # of SDNode!"); 6198 return cast<ConstantSDNode>(OperandList[Num])->getZExtValue(); 6199} 6200 6201SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) { 6202 assert(N->getNumValues() == 1 && 6203 "Can't unroll a vector with multiple results!"); 6204 6205 EVT VT = N->getValueType(0); 6206 unsigned NE = VT.getVectorNumElements(); 6207 EVT EltVT = VT.getVectorElementType(); 6208 SDLoc dl(N); 6209 6210 SmallVector<SDValue, 8> Scalars; 6211 SmallVector<SDValue, 4> Operands(N->getNumOperands()); 6212 6213 // If ResNE is 0, fully unroll the vector op. 6214 if (ResNE == 0) 6215 ResNE = NE; 6216 else if (NE > ResNE) 6217 NE = ResNE; 6218 6219 unsigned i; 6220 for (i= 0; i != NE; ++i) { 6221 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) { 6222 SDValue Operand = N->getOperand(j); 6223 EVT OperandVT = Operand.getValueType(); 6224 if (OperandVT.isVector()) { 6225 // A vector operand; extract a single element. 6226 const TargetLowering *TLI = TM.getTargetLowering(); 6227 EVT OperandEltVT = OperandVT.getVectorElementType(); 6228 Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, 6229 OperandEltVT, 6230 Operand, 6231 getConstant(i, TLI->getVectorIdxTy())); 6232 } else { 6233 // A scalar operand; just use it as is. 6234 Operands[j] = Operand; 6235 } 6236 } 6237 6238 switch (N->getOpcode()) { 6239 default: 6240 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, 6241 &Operands[0], Operands.size())); 6242 break; 6243 case ISD::VSELECT: 6244 Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, 6245 &Operands[0], Operands.size())); 6246 break; 6247 case ISD::SHL: 6248 case ISD::SRA: 6249 case ISD::SRL: 6250 case ISD::ROTL: 6251 case ISD::ROTR: 6252 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0], 6253 getShiftAmountOperand(Operands[0].getValueType(), 6254 Operands[1]))); 6255 break; 6256 case ISD::SIGN_EXTEND_INREG: 6257 case ISD::FP_ROUND_INREG: { 6258 EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType(); 6259 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, 6260 Operands[0], 6261 getValueType(ExtVT))); 6262 } 6263 } 6264 } 6265 6266 for (; i < ResNE; ++i) 6267 Scalars.push_back(getUNDEF(EltVT)); 6268 6269 return getNode(ISD::BUILD_VECTOR, dl, 6270 EVT::getVectorVT(*getContext(), EltVT, ResNE), 6271 &Scalars[0], Scalars.size()); 6272} 6273 6274 6275/// isConsecutiveLoad - Return true if LD is loading 'Bytes' bytes from a 6276/// location that is 'Dist' units away from the location that the 'Base' load 6277/// is loading from. 6278bool SelectionDAG::isConsecutiveLoad(LoadSDNode *LD, LoadSDNode *Base, 6279 unsigned Bytes, int Dist) const { 6280 if (LD->getChain() != Base->getChain()) 6281 return false; 6282 EVT VT = LD->getValueType(0); 6283 if (VT.getSizeInBits() / 8 != Bytes) 6284 return false; 6285 6286 SDValue Loc = LD->getOperand(1); 6287 SDValue BaseLoc = Base->getOperand(1); 6288 if (Loc.getOpcode() == ISD::FrameIndex) { 6289 if (BaseLoc.getOpcode() != ISD::FrameIndex) 6290 return false; 6291 const MachineFrameInfo *MFI = getMachineFunction().getFrameInfo(); 6292 int FI = cast<FrameIndexSDNode>(Loc)->getIndex(); 6293 int BFI = cast<FrameIndexSDNode>(BaseLoc)->getIndex(); 6294 int FS = MFI->getObjectSize(FI); 6295 int BFS = MFI->getObjectSize(BFI); 6296 if (FS != BFS || FS != (int)Bytes) return false; 6297 return MFI->getObjectOffset(FI) == (MFI->getObjectOffset(BFI) + Dist*Bytes); 6298 } 6299 6300 // Handle X+C 6301 if (isBaseWithConstantOffset(Loc) && Loc.getOperand(0) == BaseLoc && 6302 cast<ConstantSDNode>(Loc.getOperand(1))->getSExtValue() == Dist*Bytes) 6303 return true; 6304 6305 const GlobalValue *GV1 = NULL; 6306 const GlobalValue *GV2 = NULL; 6307 int64_t Offset1 = 0; 6308 int64_t Offset2 = 0; 6309 const TargetLowering *TLI = TM.getTargetLowering(); 6310 bool isGA1 = TLI->isGAPlusOffset(Loc.getNode(), GV1, Offset1); 6311 bool isGA2 = TLI->isGAPlusOffset(BaseLoc.getNode(), GV2, Offset2); 6312 if (isGA1 && isGA2 && GV1 == GV2) 6313 return Offset1 == (Offset2 + Dist*Bytes); 6314 return false; 6315} 6316 6317 6318/// InferPtrAlignment - Infer alignment of a load / store address. Return 0 if 6319/// it cannot be inferred. 6320unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const { 6321 // If this is a GlobalAddress + cst, return the alignment. 6322 const GlobalValue *GV; 6323 int64_t GVOffset = 0; 6324 const TargetLowering *TLI = TM.getTargetLowering(); 6325 if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) { 6326 unsigned PtrWidth = TLI->getPointerTy().getSizeInBits(); 6327 APInt KnownZero(PtrWidth, 0), KnownOne(PtrWidth, 0); 6328 llvm::ComputeMaskedBits(const_cast<GlobalValue*>(GV), KnownZero, KnownOne, 6329 TLI->getDataLayout()); 6330 unsigned AlignBits = KnownZero.countTrailingOnes(); 6331 unsigned Align = AlignBits ? 1 << std::min(31U, AlignBits) : 0; 6332 if (Align) 6333 return MinAlign(Align, GVOffset); 6334 } 6335 6336 // If this is a direct reference to a stack slot, use information about the 6337 // stack slot's alignment. 6338 int FrameIdx = 1 << 31; 6339 int64_t FrameOffset = 0; 6340 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) { 6341 FrameIdx = FI->getIndex(); 6342 } else if (isBaseWithConstantOffset(Ptr) && 6343 isa<FrameIndexSDNode>(Ptr.getOperand(0))) { 6344 // Handle FI+Cst 6345 FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex(); 6346 FrameOffset = Ptr.getConstantOperandVal(1); 6347 } 6348 6349 if (FrameIdx != (1 << 31)) { 6350 const MachineFrameInfo &MFI = *getMachineFunction().getFrameInfo(); 6351 unsigned FIInfoAlign = MinAlign(MFI.getObjectAlignment(FrameIdx), 6352 FrameOffset); 6353 return FIInfoAlign; 6354 } 6355 6356 return 0; 6357} 6358 6359// getAddressSpace - Return the address space this GlobalAddress belongs to. 6360unsigned GlobalAddressSDNode::getAddressSpace() const { 6361 return getGlobal()->getType()->getAddressSpace(); 6362} 6363 6364 6365Type *ConstantPoolSDNode::getType() const { 6366 if (isMachineConstantPoolEntry()) 6367 return Val.MachineCPVal->getType(); 6368 return Val.ConstVal->getType(); 6369} 6370 6371bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, 6372 APInt &SplatUndef, 6373 unsigned &SplatBitSize, 6374 bool &HasAnyUndefs, 6375 unsigned MinSplatBits, 6376 bool isBigEndian) { 6377 EVT VT = getValueType(0); 6378 assert(VT.isVector() && "Expected a vector type"); 6379 unsigned sz = VT.getSizeInBits(); 6380 if (MinSplatBits > sz) 6381 return false; 6382 6383 SplatValue = APInt(sz, 0); 6384 SplatUndef = APInt(sz, 0); 6385 6386 // Get the bits. Bits with undefined values (when the corresponding element 6387 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared 6388 // in SplatValue. If any of the values are not constant, give up and return 6389 // false. 6390 unsigned int nOps = getNumOperands(); 6391 assert(nOps > 0 && "isConstantSplat has 0-size build vector"); 6392 unsigned EltBitSize = VT.getVectorElementType().getSizeInBits(); 6393 6394 for (unsigned j = 0; j < nOps; ++j) { 6395 unsigned i = isBigEndian ? nOps-1-j : j; 6396 SDValue OpVal = getOperand(i); 6397 unsigned BitPos = j * EltBitSize; 6398 6399 if (OpVal.getOpcode() == ISD::UNDEF) 6400 SplatUndef |= APInt::getBitsSet(sz, BitPos, BitPos + EltBitSize); 6401 else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(OpVal)) 6402 SplatValue |= CN->getAPIntValue().zextOrTrunc(EltBitSize). 6403 zextOrTrunc(sz) << BitPos; 6404 else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(OpVal)) 6405 SplatValue |= CN->getValueAPF().bitcastToAPInt().zextOrTrunc(sz) <<BitPos; 6406 else 6407 return false; 6408 } 6409 6410 // The build_vector is all constants or undefs. Find the smallest element 6411 // size that splats the vector. 6412 6413 HasAnyUndefs = (SplatUndef != 0); 6414 while (sz > 8) { 6415 6416 unsigned HalfSize = sz / 2; 6417 APInt HighValue = SplatValue.lshr(HalfSize).trunc(HalfSize); 6418 APInt LowValue = SplatValue.trunc(HalfSize); 6419 APInt HighUndef = SplatUndef.lshr(HalfSize).trunc(HalfSize); 6420 APInt LowUndef = SplatUndef.trunc(HalfSize); 6421 6422 // If the two halves do not match (ignoring undef bits), stop here. 6423 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) || 6424 MinSplatBits > HalfSize) 6425 break; 6426 6427 SplatValue = HighValue | LowValue; 6428 SplatUndef = HighUndef & LowUndef; 6429 6430 sz = HalfSize; 6431 } 6432 6433 SplatBitSize = sz; 6434 return true; 6435} 6436 6437bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) { 6438 // Find the first non-undef value in the shuffle mask. 6439 unsigned i, e; 6440 for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i) 6441 /* search */; 6442 6443 assert(i != e && "VECTOR_SHUFFLE node with all undef indices!"); 6444 6445 // Make sure all remaining elements are either undef or the same as the first 6446 // non-undef value. 6447 for (int Idx = Mask[i]; i != e; ++i) 6448 if (Mask[i] >= 0 && Mask[i] != Idx) 6449 return false; 6450 return true; 6451} 6452 6453#ifdef XDEBUG 6454static void checkForCyclesHelper(const SDNode *N, 6455 SmallPtrSet<const SDNode*, 32> &Visited, 6456 SmallPtrSet<const SDNode*, 32> &Checked) { 6457 // If this node has already been checked, don't check it again. 6458 if (Checked.count(N)) 6459 return; 6460 6461 // If a node has already been visited on this depth-first walk, reject it as 6462 // a cycle. 6463 if (!Visited.insert(N)) { 6464 dbgs() << "Offending node:\n"; 6465 N->dumprFull(); 6466 errs() << "Detected cycle in SelectionDAG\n"; 6467 abort(); 6468 } 6469 6470 for(unsigned i = 0, e = N->getNumOperands(); i != e; ++i) 6471 checkForCyclesHelper(N->getOperand(i).getNode(), Visited, Checked); 6472 6473 Checked.insert(N); 6474 Visited.erase(N); 6475} 6476#endif 6477 6478void llvm::checkForCycles(const llvm::SDNode *N) { 6479#ifdef XDEBUG 6480 assert(N && "Checking nonexistent SDNode"); 6481 SmallPtrSet<const SDNode*, 32> visited; 6482 SmallPtrSet<const SDNode*, 32> checked; 6483 checkForCyclesHelper(N, visited, checked); 6484#endif 6485} 6486 6487void llvm::checkForCycles(const llvm::SelectionDAG *DAG) { 6488 checkForCycles(DAG->getRoot().getNode()); 6489} 6490